#include #include #include #include #include "tmtacq.h" #include "racqumem.h" #include "racqurw.h" #ifndef NOPCIECARD #include "racquwbin.h" #include "dmamgrv3.h" #endif #include "racquproc.h" #include "pciewrap.h" #include "brpaqu.h" #include "minifits.h" #include "resusage.h" #include "ctimer.h" #include "timing.h" #include "tarrinit.h" #include "fiosinit.h" using namespace std; using namespace SOPHYA; //----------------------------------------------------------------------- // Programme test acquisition BAORadio multi-thread // LAL - R. Ansari Juillet - 2008 // M.Taurigna 2009 // == Programme reste a la version de Mai 2009 (1er tests Pittsburgh) == //----------------------------------------------------------------------- /* Fonction d'arret par Ctl et ses pointeurs globaux */ static DataSaver *pDs = NULL; static DataSaver *pDs2 = NULL; static PCIEReaderChecker *pPcierc = NULL; static PCIEReaderChecker *pPcierc2 = NULL; static PCIEReader *pPcier = NULL; static PCIEReader *pPcier2 = NULL; static DataProc2C *pPr = NULL; static DataProc2C *pPr2 = NULL; static DataProcFFT2C *pPrfft = NULL; static DataProcFFT2C *pPrfft2 = NULL; static RAcqMemZoneMgr *pMmgr = NULL; static RAcqMemZoneMgr *pMmgr2 = NULL; void Stop(int s) { if (s == 9765) cout << " Stop after exception ..." << endl; else printf("............. MAIN ... receive signal %d \n",s); if (pPcierc != NULL) pPcierc->Stop(); if (pPcierc2 != NULL) pPcierc2->Stop(); if (pPcier != NULL) pPcier->Stop(); if (pPcier2 != NULL) pPcier2->Stop(); if (pPr !=NULL) pPr->Stop(); if (pPr2 !=NULL) pPr2->Stop(); if (pPrfft !=NULL) pPrfft->Stop(); if (pPrfft2 !=NULL) pPrfft2->Stop(); if (pDs != NULL) pDs->Stop(); if (pDs2 != NULL) pDs2->Stop(); if (pMmgr !=NULL) pMmgr->Stop(); if (pMmgr2 !=NULL) pMmgr2->Stop(); } #ifndef NOPCIECARD // Declaration des fonctions pour DMAMgr... firmware V51 PCIEWrapperInterface* CreatePCIEWrapperV5F1( UINT32 numcard , UINT32 patternsize, UINT32 dmasz, bool activpatt=false); void DeletePCIEWrapperV5F1(UINT32 numcard); PCIEWrapperInterface* CreatePCIEWrapperV5F2( UINT32 numcard , UINT32 patternsize, UINT32 dmasz, bool activpatt=false); void DeletePCIEWrapperV5F2(UINT32 numcard); #endif /* -------------------------------------------------------- */ // Fonctions appelees par le main void Usage(bool fgshort=true); int DecodeArgs(int narg, char* arg[]); void DecodeAcqMode(); int MonoCardAcq(); int MultiCardAcq(); //------------------ Parametres de controle ---------------------------- static string acqmode; static int fibre = 0; // =0 firmware pciexpress V496, =1 V51, fibre 1, =2 V51, fibre 2 static bool savesigfits = true; static bool fgnulldev4fits = false; static BRDataFmtConv swapall = BR_Copy; static bool monothr = false; static string OutPathName; static string ProcPathName; static bool singlecard = true; static int cardlist[4] = {1,1,1,1}; static int card = 1; static unsigned int sizeFrame = 4096; static unsigned int nbFrameDMA = 32; static int NbFiles=1; static int NBlocPerFile=10; static int NMaxProc = 0; // Nombre de blocs traites par le thread de calcul static uint_4 nZones = 4; // Nombre de zones memoires static uint_4 nPaqZone = 128; // 128 Paquets / zone memoire - valeur par defaut static bool fg_hard_ctrlc = false; static uint_4 PaqSZ = 4096; // Taille de paquets // uint_4 dmaSize = nbFrameDMA*PaqSZ ; // plantage static uint_4 dmaSize = 32*1024 ; static uint_4 patternSZ=0x400; // pas utilise avec la fibre static bool activate_pattern=false; // true -> on active le pattern du firmware au lieu de la fibre static uint_4 NMaxBloc = 10; static double nopciLossRate = 0.; // Taux de perte simule des paquets par TestPCIWrapperNODMA //---------------------------- FIN parametres de controle ----------------- //----------------------------------------------------------------------- //-------------------- le programme principal --------------------------- //----------------------------------------------------------------------- int main(int narg, char* arg[]) { int rc = 0; cout << " ============ BAORadio / Acquisition : tmtacq =================" << endl; cout << " ===============================================================" << endl; cout << " ========= " < No Processing " << endl; cout << " - DataDirPath : Subdirectory of /Raid " << endl; cout << " specify /dev/null for performance tests ( ./XZXZXZX/ then used for other files)" << endl; cout << " - AcqMode: Acquisition mode for the firmware V=496 (manages 64bits-byteswap)"< Single thread PCIEReaderChecker, copy packets" << endl; cout << " pattern: activate pattern, and write fits files "<1)&&(strcmp(arg[1],"-h")==0)) { Usage(false); return 1; } if (narg < 8) { Usage(true); return 1; } acqmode = "std"; fibre = 0; // firmware V496 if (narg>8) acqmode = arg[8]; savesigfits = true; // BRDataFmtConv swapall = BR_SwapAll; swapall = BR_Copy; monothr = false; DecodeAcqMode(); if (index(arg[1],',') != NULL) { sscanf(arg[1],"%d,%d", cardlist, cardlist+1); cout << " tmtacq[1]/MultiPCICard acquisition ... Cards= " << cardlist[0] << " ," << cardlist[1] << " FrameSize=PaqSize=" << sizeFrame << endl; singlecard = false; } else { card = atoi(arg[1]); cardlist[0] = cardlist[1] = card; singlecard = true; cout << "tmtacq[1]/MonoPCICard acquisition ... Card= " << card << " FrameSize=PaqSize=" << sizeFrame << endl; } sizeFrame = atoi(arg[2]); nbFrameDMA = atoi(arg[3]); NbFiles = atoi(arg[4]); NBlocPerFile = atoi(arg[5]); NMaxProc = atoi(arg[6]); // Nombre de blocs traites par le thread de calcul OutPathName = arg[7]; fgnulldev4fits = (OutPathName == "/dev/null") ? true : false; if (!fgnulldev4fits) { #ifdef NOPCIECARD OutPathName = OutPathName+"/"; #else OutPathName =string("/Raid/")+OutPathName+"/"; #endif ProcPathName = OutPathName; } cout << " DataDirpath=" << OutPathName << " SwapMode=" << BRPaquet::FmtConvToString(swapall) << " DataSaveMode=" << ((savesigfits)?"Yes/FitsFile":"NO") << endl; if (!fgnulldev4fits) { char cmd[512]; if (singlecard) { sprintf(cmd,"mkdir %s",OutPathName.c_str()); if (system(cmd) < 0) { cout << "tmtacq/Error: Can not create subdirectory " << OutPathName << " -> exit" << endl; return 2; } else cout << " tmtacq[1] - Executed command " << cmd << endl; } else { sprintf(cmd,"mkdir %s",OutPathName.c_str()); if (system(cmd) < 0) { cout << "tmtacq/Error: Can not create subdirectory " << OutPathName << " -> exit" << endl; return 2; } else cout << " tmtacq[1] - Executed command " << cmd << endl; sprintf(cmd,"mkdir %s/Card1",OutPathName.c_str()); if (system(cmd) < 0) { cout << "tmtacq/Error: Can not create subdirectory " << OutPathName << " -> exit" << endl; return 2; } else cout << " tmtacq[1] - Executed command " << cmd << endl; sprintf(cmd,"mkdir %s/Card2",OutPathName.c_str()); if (system(cmd) < 0) { cout << "tmtacq/Error: Can not create subdirectory " << OutPathName << " -> exit" << endl; return 2; } else cout << " tmtacq[1] - Executed command " << cmd << endl; } } else { ProcPathName = "./XZXZXZX/"; cout << " Using " << ProcPathName << " for other processed files ... " << endl; char cmd[512]; sprintf(cmd,"mkdir %s",ProcPathName.c_str()); system(cmd); if (!singlecard) { sprintf(cmd,"mkdir %s/Card1 %s/Card2",ProcPathName.c_str(),ProcPathName.c_str()); system(cmd); } } nZones = 4; // Nombre de zones memoires nPaqZone = 128; // 128 Paquets / zone memoire - valeur par defaut if (narg > 9) { int ia1, ia2; sscanf(arg[9],"%d,%d", &ia1, &ia2); nZones = ia1; nPaqZone = ia2; } fg_hard_ctrlc = false; struct sigaction act; if (narg > 10) // pour traiter eventuellement un arret brutal par CtlC mettre le 10eme arg a Y if ((*arg[10]=='y')||(*arg[10]=='Y')) fg_hard_ctrlc=true; if (!fg_hard_ctrlc) { act.sa_handler=Stop; sigaction(SIGINT,&act,NULL); } nopciLossRate = 0.; // Taux de perte simule des paquets par TestPCIWrapperNODMA if (narg > 11) nopciLossRate=atof(arg[11]); PaqSZ =sizeFrame; // Taille de paquets // uint_4 dmaSize = nbFrameDMA*PaqSZ ; // plantage dmaSize = nbFrameDMA*1024 ; patternSZ=(sizeFrame-40)/4; // pas utilise avec la fibre NMaxBloc = NbFiles*NBlocPerFile; cout << "tmtacq[2] - PaqSize = " << PaqSZ << " NbPaq/Zone=" << nPaqZone << " NZones=" << nZones << endl; cout << " NbFiles=" << NbFiles << " NBloc/File=" << NBlocPerFile << " -> NMaxBloc=" << NMaxBloc << endl; cout << "tmtacq[2] NbFrameDMA=" << nbFrameDMA << " DMASize=" << dmaSize << " bytes" << endl; cout << "tmtacq[2] NMaxProc=" << NMaxProc << endl; return 0; } /* --Nouvelle-Fonction-- */ void DecodeAcqMode() { if (acqmode.substr(0,3)=="f1.") { fibre=1, acqmode=acqmode.substr(3); cout << " DecodeAcqMode() -> firmware pci-express V51, fibre 1" << endl; } if (acqmode.substr(0,3)=="f2.") { fibre=2, acqmode=acqmode.substr(3); cout << " DecodeAcqMode() -> firmware pci-express V51, fibre 2" << endl; } if (acqmode == "nosw") swapall = BR_Copy ; if (acqmode == "nof") { swapall = BR_Copy ; savesigfits = false; } if (acqmode == "fft1c") swapall = BR_FFTOneChan; if (acqmode == "fft2c") swapall = BR_FFTTwoChan; if (acqmode == "fft1cnof") { swapall = BR_FFTOneChan; savesigfits = false; } if (acqmode == "fft2cnof") { swapall = BR_FFTTwoChan; savesigfits = false; } if (acqmode == "mono") { monothr = true; swapall = BR_Copy; } if (acqmode == "patmono") { monothr = true; swapall = BR_Copy; activate_pattern=true; } if (acqmode == "patnof") { savesigfits = false; swapall = BR_Copy; activate_pattern=true; } if (acqmode == "pattern") { savesigfits = true; swapall = BR_Copy; activate_pattern=true; } if (acqmode == "swapall") swapall = BR_SwapAll; if (acqmode == "fft1csw") swapall = BR_FFTOneChanSwapAll; if (acqmode == "fft2csw") swapall = BR_FFTTwoChanSwapAll; if (acqmode == "fft1cswnof") { swapall = BR_FFTOneChanSwapAll; savesigfits = false; } if (acqmode == "fft2cswnof") { swapall = BR_FFTTwoChanSwapAll; savesigfits = false; } if ((acqmode == "swh") || (acqmode == "mxs") || (acqmode == "monoswh") ) swapall = BR_SwapHDR; if ((acqmode == "swapallnof") || (acqmode == "mxs") ) savesigfits = false; if (acqmode == "monoswh") { monothr = true; swapall = BR_SwapHDR;; } if (acqmode == "monosw") { monothr = true; swapall = BR_SwapAll; } if (acqmode == "swap32") swapall = BR_Swap32 ; if (acqmode == "nof32") { swapall = BR_Swap32 ; savesigfits = false; } if (acqmode == "fft1c32") swapall = BR_FFTOneChanSwap32; if (acqmode == "fft2c32") swapall = BR_FFTTwoChanSwap32; if (acqmode == "fft1cnof32") { swapall = BR_FFTOneChanSwap32; savesigfits = false; } if (acqmode == "fft2cnof32") { swapall = BR_FFTTwoChanSwap32; savesigfits = false; } } /* --Nouvelle-Fonction-- */ int MonoCardAcq() { Timer tm("tmtacq/MonoCard"); cout << " ---- tmtacq/ MonoCardAcq() ------------- " << endl; PCIEWrapperInterface* pciwp=NULL; #ifdef NOPCIECARD TestPCIWrapperNODMA pciw(PaqSZ,nopciLossRate); pciwp = &pciw; #else DMAMgr* dmap=NULL; if (fibre == 1) { pciwp = CreatePCIEWrapperV5F1(card,patternSZ,dmaSize,activate_pattern); } else if (fibre == 2) { pciwp = CreatePCIEWrapperV5F2(card,patternSZ,dmaSize,activate_pattern); } else { dmap = new DMAMgr(card,patternSZ,dmaSize,activate_pattern); if (!activate_pattern) { if (! dmap->StatusFibre() ) { cout << " tmtacq[3] - fibre non accrochee -> exit " << endl; throw PCIEWException(" Fibre non accrochee "); } else cout << " tmtacq[3] - fibre accrochee OK " << endl; } pciwp = new PCIEWrapper(*dmap); } #endif RAcqMemZoneMgr mmgr(nZones, nPaqZone, PaqSZ); pMmgr =&mmgr; if (monothr) { cout << "tmtacq[4] single thread PCIE test PCIEReaderChecker ... "; PCIEReaderChecker pcirc(*pciwp,dmaSize,PaqSZ ,mmgr, NMaxBloc, swapall); pPcierc=&pcirc; pcirc.start(); sleep(1); pcirc.join(); cout << "tmtacq[5] - single thread PCIEReaderChecker finished " << endl; tm.Split("Single Thread Finished"); mmgr.Print(cout); return 0; } PCIEReader pcir(*pciwp,dmaSize,PaqSZ ,mmgr, NMaxBloc, swapall); pPcier= &pcir; DataSaver ds(mmgr, OutPathName, NbFiles, NBlocPerFile, savesigfits); pDs= &ds; //DataBinSaver ds(mmgr, OutPathName, NbFiles ,NBlocPerFile); // DataProc1C pr(mmgr, OutPathName, nmean, stepproc, 100); Pour processer un canal int stepproc = 2; int nmean = nPaqZone*NBlocPerFile/stepproc; DataProc2C pr(mmgr, ProcPathName, nmean, stepproc, NMaxProc); DataProcFFT2C prfft(mmgr, ProcPathName, nmean, stepproc, NMaxProc); tm.Split("Threads created"); cout << "tmtacq[4] - starting 3 threads pcir, ds, pr ... " << endl; /* char test[100]; cout << " to continue, x to exit ..." << endl; gets(test); if (test[0] == 'x') return 9; */ pcir.start(); ds.start(); if (NMaxProc>0) { // On ne demarre que si au moins NMaxProc>0 if (acqmode.substr(0,5)=="fft2c") { prfft.start(); pPrfft=&prfft;} else { pr.start(); pPr=≺} } cout << "tmtacq[5] - waiting for threads to finish ... " << endl; sleep(1); pcir.join(); ds.join(); sleep(1); mmgr.Stop(); if (NMaxProc>0) { if ( pPr != NULL) pr.join(); if ( pPrfft != NULL) prfft.join(); } cout << "tmtacq[6] - threads finished " << endl; tm.Split("Threads Finished"); mmgr.Print(cout); #ifndef NOPCIECARD if (fibre==0) { delete pciwp; delete dmap; } if (fibre==1) DeletePCIEWrapperV5F1(card); if (fibre==2) DeletePCIEWrapperV5F2(card); #endif return 0; } /* --Nouvelle-Fonction-- */ int MultiCardAcq() { Timer tm("tmtacq/MultiCard"); cout << " ---- tmtacq/ MultiCardAcq() ------------- " << endl; PCIEWrapperInterface* pciwp1=NULL; PCIEWrapperInterface* pciwp2=NULL; #ifdef NOPCIECARD TestPCIWrapperNODMA pciw1(PaqSZ,nopciLossRate); TestPCIWrapperNODMA pciw2(PaqSZ,nopciLossRate); pciwp1 = &pciw1; pciwp2 = &pciw2; #else DMAMgr* dmap1=NULL; DMAMgr* dmap2=NULL; if (fibre == 1) { pciwp1 = CreatePCIEWrapperV5F1(cardlist[0],patternSZ,dmaSize,activate_pattern); pciwp2 = CreatePCIEWrapperV5F1(cardlist[1],patternSZ,dmaSize,activate_pattern); } else if (fibre == 2) { pciwp1 = CreatePCIEWrapperV5F2(cardlist[0],patternSZ,dmaSize,activate_pattern); pciwp2 = CreatePCIEWrapperV5F2(cardlist[1],patternSZ,dmaSize,activate_pattern); } else { dmap1 = new DMAMgr(cardlist[0],patternSZ,dmaSize,activate_pattern); if (!activate_pattern) { if (! dmap1->StatusFibre() ) { cout << " tmtacq[3] - fibre non accrochee Card" << cardlist[0] << " -> exit " << endl; throw PCIEWException(" Fibre non accrochee 1"); } else cout << " tmtacq[3] - fibre accrochee OK Card " << cardlist[0] << endl; dmap2 = new DMAMgr(cardlist[1],patternSZ,dmaSize,activate_pattern); if (! dmap2->StatusFibre() ) { cout << " tmtacq[3] - fibre non accrochee Card" << cardlist[1] << " -> exit " << endl; throw PCIEWException(" Fibre non accrochee 2"); } else cout << " tmtacq[3] - fibre accrochee OK Card " << cardlist[1] << endl; } pciwp1 = new PCIEWrapper(*dmap1); pciwp2 = new PCIEWrapper(*dmap2); } /* cout << " ---- S to stop , C continue ... " << endl; char ans[64]; gets(ans); if ((ans[0] == 'S') || (ans[0] == 's')) return 5; */ #endif RAcqMemZoneMgr mmgr1(nZones, nPaqZone, PaqSZ); RAcqMemZoneMgr mmgr2(nZones, nPaqZone, PaqSZ); pMmgr =&mmgr1; pMmgr2 =&mmgr2; if (monothr) { cout << "tmtacq[4] single thread/Card PCIE test PCIEReaderChecker ... "; PCIEReaderChecker pcirc1(*pciwp1,dmaSize,PaqSZ ,mmgr1, NMaxBloc, swapall); PCIEReaderChecker pcirc2(*pciwp2,dmaSize,PaqSZ ,mmgr2, NMaxBloc, swapall); pPcierc=&pcirc1; pPcierc2=&pcirc2; pcirc1.start(); pcirc2.start(); sleep(1); pcirc1.join(); pcirc2.join(); cout << "tmtacq[5] - single thread/Card PCIEReaderChecker finished " << endl; tm.Split("Single Thread Finished"); mmgr1.Print(cout); mmgr2.Print(cout); return 0; } PCIEReader pcir1(*pciwp1,dmaSize,PaqSZ ,mmgr1, NMaxBloc, swapall); pPcier= &pcir1; PCIEReader pcir2(*pciwp2,dmaSize,PaqSZ ,mmgr2, NMaxBloc, swapall); pPcier2= &pcir2; char buff[32]; sprintf(buff, "Card%d", cardlist[0]); string path1 = OutPathName + buff; string ppath1 = ProcPathName + buff; DataSaver ds1(mmgr1, path1, NbFiles, NBlocPerFile, savesigfits); pDs= &ds1; sprintf(buff, "Card%d", cardlist[1]); string path2 = OutPathName + buff; string ppath2 = ProcPathName + buff; DataSaver ds2(mmgr2, path2, NbFiles, NBlocPerFile, savesigfits); pDs2= &ds2; //DataBinSaver ds(mmgr, OutPathName, NbFiles ,NBlocPerFile); // DataProc1C pr(mmgr, OutPathName, nmean, stepproc, 100); Pour processer un canal int stepproc = 2; int nmean = nPaqZone*NBlocPerFile/stepproc; DataProc2C pr1(mmgr1, ppath1, nmean, stepproc, NMaxProc); DataProcFFT2C prfft1(mmgr1, ppath1, nmean, stepproc, NMaxProc); DataProc2C pr2(mmgr2, ppath2, nmean, stepproc, NMaxProc); DataProcFFT2C prfft2(mmgr2, ppath2, nmean, stepproc, NMaxProc); tm.Split("Threads created"); cout << "tmtacq[4] - starting 3 threads pcir, ds, pr ... " << endl; /* char test[100]; cout << " to continue, x to exit ..." << endl; gets(test); if (test[0] == 'x') return 9; */ pcir1.start(); ds1.start(); pcir2.start(); ds2.start(); if (NMaxProc>0) { // On ne demarre que si au moins NMaxProc>0 if (acqmode.substr(0,5)=="fft2c") { prfft1.start(); pPrfft=&prfft1; prfft2.start(); pPrfft2=&prfft2; } else { pr1.start(); pPr=&pr1; pr2.start(); pPr2=&pr2; } } cout << "tmtacq[5] - waiting for threads to finish ... " << endl; sleep(1); pcir1.join(); ds1.join(); pcir2.join(); ds2.join(); sleep(1); mmgr1.Stop(); mmgr2.Stop(); if (NMaxProc>0) { if ( pPr != NULL) pr1.join(); if ( pPr2 != NULL) pr2.join(); if ( pPrfft != NULL) prfft1.join(); if ( pPrfft2 != NULL) prfft2.join(); } cout << "tmtacq[6] - threads finished " << endl; tm.Split("Threads Finished"); mmgr1.Print(cout); mmgr2.Print(cout); #ifndef NOPCIECARD if (fibre==0) { delete pciwp1; delete dmap1; delete pciwp2; delete dmap2; } if (fibre==1) { DeletePCIEWrapperV5F1(cardlist[0]); DeletePCIEWrapperV5F1(cardlist[1]); } if (fibre==2) { DeletePCIEWrapperV5F2(cardlist[0]); DeletePCIEWrapperV5F2(cardlist[1]); } #endif return 0; }