source: Sophya/trunk/ArchTOIPipe/Kernel/fitstoirdr.cc@ 2065

Last change on this file since 2065 was 2065, checked in by ansari, 23 years ago

Debug de setSepFlagFile (3) - Reza 18/6/2002
File size: 13.1 KB
Line 
1// ArchTOIPipe (C) CEA/DAPNIA/SPP IN2P3/LAL
2// Eric Aubourg
3// Christophe Magneville
4// Reza Ansari
5// $Id: fitstoirdr.cc,v 1.35 2002-06-18 19:15:04 ansari Exp $
6
7#include "fitstoirdr.h"
8#include "toimanager.h"
9#include <sched.h>
10
11
12FITSTOIReader::FITSTOIReader(string fn,int buff_sz) {
13 fname = fn;
14 allfn.push_back(fn);
15 Buff_Sz = (buff_sz>0) ? buff_sz: 1000;
16 cout<<"FITSTOIReader::FITSTOIReader"<<endl;
17 cout<<"FITSTOIReader::inited "<<inited<<" bsz="<<Buff_Sz<<endl;
18 name = "rdr";
19 fptr = NULL;
20 totnscount = 0;
21
22 implicitSN = false;
23 implicitSNStart = 0;
24
25 // Variables rajoutee pour gerer les fichiers de flag de LevelS Reza 18/6/2002
26 sepFlagfile = false;
27 fptrflg = NULL;
28}
29
30FITSTOIReader::~FITSTOIReader() {
31}
32
33void FITSTOIReader::setImplicitSN(long snStart) {
34 implicitSN = true;
35 implicitSNStart = snStart;
36}
37
38void FITSTOIReader::setBufferSize(int buffsz)
39{
40 Buff_Sz = (buffsz>0) ? buffsz: 1024;
41 return;
42}
43
44pthread_mutex_t fits_mutex = PTHREAD_MUTEX_INITIALIZER;
45void fits_lock();
46void fits_unlock();
47
48void fits_lock() {
49 pthread_mutex_lock(&fits_mutex);
50}
51void fits_unlock() {
52 pthread_mutex_unlock(&fits_mutex);
53}
54
55void FITSTOIReader::openFile(string fn) {
56 fits_lock();
57 if (fptr) {
58 fits_close_file(fptr,&fstatus);
59 fptr = NULL;
60 }
61 fname = fn;
62 cout << "FITSTOIReader::open FileName=" << fname << endl;
63 fstatus = 0;
64 // Open file
65 fits_open_file(&fptr,fname.c_str(),READONLY,&fstatus);
66 if (fstatus != 0) {
67 fits_report_error(stderr, fstatus);
68 fits_unlock();
69 throw IOExc("FITSTOIReader::openFile() fitsio error");
70 }
71
72 // Go to first extension, which should be a BINTABLE
73
74 int simple, bitpix, naxis;
75 long naxes;
76 long pcount, gcount;
77 int extend;
78 fits_read_imghdr(fptr, 1, &simple, &bitpix,
79 &naxis, &naxes, &pcount, &gcount, &extend, &fstatus);
80
81 fits_movabs_hdu(fptr, 2, NULL, &fstatus);
82
83 fits_get_num_cols(fptr,&ncols,&fstatus);
84 fits_get_num_rows(fptr,&nrows,&fstatus);
85
86 cout << "FITSTOIReader cols = " << ncols << " rows=" << nrows << endl;
87 if (implicitSN) {
88 firstSn = implicitSNStart;
89 } else {
90 int anyNul;
91 double y;
92 fits_read_col_dbl(fptr,1,1,1,1,0,&y,&anyNul,&fstatus);
93 firstSn = (int) (y+.1);
94 }
95
96 // Ouverture fichier de flag separe de LevelS (Reza 18/6/2002)
97 if (sepFlagfile) {
98 cout << " DBG - Opening separate flag file " << sepFlagFileName
99 << " NColFlags= " << sepFlagCols.size() << endl;
100 fits_open_file(&fptrflg,sepFlagFileName.c_str(),READONLY,&fstatus);
101 if (fstatus != 0) {
102 fits_report_error(stderr, fstatus);
103 fits_unlock();
104 throw IOExc("FITSTOIReader::openFile() - sepFlagfile open fitsio error");
105 }
106 fits_movabs_hdu(fptrflg, 2, NULL, &fstatus);
107 if (fstatus != 0) {
108 fits_report_error(stderr, fstatus);
109 fits_unlock();
110 throw IOExc("FITSTOIReader::openFile() - sepFlagfile fits_movabs_hdu(2) fitsio error");
111 }
112 long nrowsflg = 0;
113 fits_get_num_rows(fptrflg,&nrowsflg,&fstatus);
114 if (nrows != nrowsflg) {
115 cerr << " FITSTOIReader::openFile()/Error: Different NRows in flag and data files!" << endl;
116 fits_unlock();
117 throw ParmError("FITSTOIReader::openFile() Different NRows in flag and data files");
118 }
119 int ncolsflg = 0;
120 fits_get_num_cols(fptr,&ncolsflg,&fstatus);
121 if (ncolsflg < sepFlagCols.size()) {
122 cerr << " FITSTOIReader::openFile()/Error: ncolsflg (=" << ncolsflg
123 << ") < sepFlagCols.size() (=" << sepFlagCols.size() << ")" << endl;
124 fits_unlock();
125 throw ParmError("FITSTOIReader::openFile() Different NRows in flag and data files");
126 }
127 }
128 //
129 fits_unlock();
130}
131
132void FITSTOIReader::init() {
133
134 // Modif pour fichiers de flag separe de LevelS (Reza 18/6/2002)
135 if (sepFlagfile && (allfn.size() > 1) ) {
136 cerr << "FITSTOIReader::init()/Error- Multiple files and separate flag file not allowed !"<<endl;
137 throw ParmError("FITSTOIReader::init() Multiple files and separate flag file not allowed");
138 }
139
140 openFile(allfn.front());
141
142 fits_lock();
143 // si pas implicitSN, la premiere colonne est le sampleNum.
144 // Sinon, le samplenum est la fitsline + offset.
145 int itoi=-1;
146 int col1 = implicitSN ? 0 : 1;
147 for (int i=col1; i<ncols; i++) {
148 char templt[10];
149 sprintf(templt, "%d", i+1);
150 char colname[200];
151 int colnum;
152 fits_get_colname(fptr, CASESEN, templt, colname, &colnum, &fstatus);
153 cout << "FITSTOIReader col " << colname << endl;
154 // On verifie si c'est une colonne de flag
155 if (!strncmp(colname, "fg_", 3) || !strncmp(colname, "FLAG_", 5) ) {
156 colsinput[itoi].second=true;
157 } else {
158 declareOutput(colname);
159 itoi++;
160 colsinput[itoi] = pair<int,bool>(i,false);
161 }
162 }
163 fits_unlock();
164
165 snBegin = firstSn;
166 if (forcedMinIn > 0 && forcedMinIn > snBegin) {
167 snBegin = forcedMinIn;
168 }
169 openFile(allfn.back());
170 snEnd = firstSn+nrows-1;
171 if (forcedMaxIn > 0 && forcedMaxIn < snEnd) {
172 snEnd = forcedMaxIn;
173 }
174 cout << "FITSTOIReader range " << snBegin << " -> " << snEnd << endl;
175}
176
177int FITSTOIReader::calcMinOut() {
178 chkinit();
179 TOIManager* mgr = TOIManager::getManager();
180 int firstReq = mgr->getRequestedBegin();
181 return snBegin > firstReq ? snBegin : firstReq;
182}
183
184int FITSTOIReader::calcMaxOut() {
185 chkinit();
186 TOIManager* mgr = TOIManager::getManager();
187 int lastReq = mgr->getRequestedEnd();
188 return snEnd < lastReq ? snEnd : lastReq;
189}
190
191void FITSTOIReader::addFile(string fn) {
192 allfn.push_back(fn);
193}
194
195void FITSTOIReader::setFlagFile(string fn, vector<FlagToiDef> flags)
196{
197 if (flags.size() < 1) {
198 cerr << " FITSTOIReader::setFlagFile()/Error flag.size() = 0 ! " << endl;
199 throw ParmError("FITSTOIReader::setFlagFile() flag.size() = 0");
200 }
201
202 sepFlagfile = true;
203 sepFlagFileName = fn;
204 sepFlagCols = flags;
205}
206
207void FITSTOIReader::run() {
208 for (vector<string>::iterator i=allfn.begin(); i!=allfn.end(); i++) {
209 openFile(*i);
210 if (Buff_Sz > 1) run2(); // Lecture bufferise
211 else run1(); // Lecture un echantillon a la fois
212 }
213 fits_lock();
214 if (fptr) {
215 fits_close_file(fptr,&fstatus);
216 fptr = NULL;
217 }
218 if (sepFlagfile && fptrflg) {
219 fits_close_file(fptrflg,&fstatus);
220 fptrflg = NULL;
221 }
222 fits_unlock();
223}
224
225// run 1 : deprecated. NON MAINTENU. Incompatible avec implicit SN.
226// ^^^^^^^^^^^^ Reza , 13/5/2002 , Je viens de rajouter
227// implicitSNStart et switvh run1/run2
228// suivant buffersize
229
230void FITSTOIReader::run1() {
231 // Il faudrait optimiser en fonction de ce qui a ete demande comme samplenum,
232 // mais cela implique de gerer aussi bien echant uniforme que non.
233 // On pourrait aussi lire plusieurs elements d'un coup.
234 int ncols = outIx.size();
235 cout << "reader reading... NRows=" << nrows << " firstSn= "
236 << firstSn << endl;
237
238 double* tabdata = new double[getNOut()];
239 uint_8* tabflag = new uint_8[getNOut()];
240
241 for (long i=0; i<nrows; i++) {
242 int anyNul;
243 double y;
244 fits_lock();
245 long sn;
246 if (implicitSN) {
247 fits_read_col_dbl(fptr,1,i+1,1,1,0,&y,&anyNul,&fstatus);
248 sn = (long) (y+.1);
249 }
250 else sn = implicitSNStart+i;
251 //fits_unlock();
252 TOIManager* mgr = TOIManager::getManager();
253 if (sn > mgr->getRequestedEnd()) {fits_unlock(); break;}
254 if (sn < mgr->getRequestedBegin()) {fits_unlock(); continue;}
255 // if (sn < mgr->getRequestedBegin()+10) cout << "rdr out " << sn << endl;
256 int k;
257 for (k=0; k<getNOut(); k++) {
258 int j = colsinput[k].first;
259 if ( !checkOutputTOIIndex(k) ) continue; // Reza - Si TOI non connecte
260 //fits_lock();
261 fits_read_col_dbl(fptr,j+1,i+1,1,1,0,&y,&anyNul,&fstatus);
262 tabdata[k] = y;
263 long flg = 0;
264 if (colsinput[k].second) {
265 fits_read_col_lng(fptr,j+2,i+1,1,1,0,&flg,&anyNul,&fstatus);
266 }
267
268 if (sepFlagfile) { // Ajout Reza (18/6/2002) pour fichier de flags separe
269 unsigned char sflg;
270 flg = 0;
271 for(int skf=0; skf<sepFlagCols.size(); skf++) {
272 fits_read_col_byt(fptrflg, skf+1, i+1,1,1,0,&sflg,&anyNul,&fstatus);
273 if (sflg) flg |= sepFlagCols[skf];
274 }
275 } // Fin modif pour fichier de flags separe (18/6/2002)
276
277 tabflag[k] = flg;
278 // fits_unlock();
279 //putData(k, sn, y, flg);
280 }
281 fits_unlock();
282 for (k=0; k<getNOut(); k++) {
283 putData(k, sn, tabdata[k], tabflag[k]);
284 }
285 /* if (i%50==0) {
286 // fits_unlock();
287 sched_yield();
288 //fits_lock();
289 }*/
290 totnscount++;
291 }
292 //fits_unlock();
293 delete[] tabflag;
294 delete[] tabdata;
295 cout << "reader done reading... " << pthread_self() << endl;
296}
297
298void FITSTOIReader::run2()
299// ---- Version bufferisee (Christophe 20/10/2001)
300{
301 int ncols = outIx.size();
302 cout<<"reader reading... NRows="<<nrows<<" firstSn= " <<firstSn<<endl;
303
304 //////// Prepare buffer, allocate memory
305 double *samplenum = new double[Buff_Sz];
306 vector<double*> colval;
307 vector<long*> colflg;
308 for(int k=0; k<getNOut(); k++) {
309 colval.push_back(NULL);
310 colflg.push_back(NULL);
311 if(!checkOutputTOIIndex(k)) continue;
312 colval[k] = new double[Buff_Sz];
313 // modif || sepFlagfile pour fichier de flag separe (Reza 18/6/2002)
314 if((colsinput[k].second) || sepFlagfile) colflg[k] = new long[Buff_Sz];
315 }
316 uint_8 * tmpflg = new uint_8[Buff_Sz];
317
318 // Ajout Reza (18/6/2002) pour fichier de flags separe
319 unsigned char* stmpflg = NULL;
320 if (sepFlagfile) stmpflg = new unsigned char[Buff_Sz];
321 long nonzeroflg = 0;
322 long nbckflg = 0;
323 // Fin modif pour fichier de flags separe (18/6/2002)
324
325
326 TOIManager* mgr = TOIManager::getManager();
327
328 //////// Read data and put into pipe
329 long ideb=-1,ifin=-1;
330 for(long i=0; i<nrows; i+=Buff_Sz) {
331 fits_lock(); // lock en debut de boucle de lecture - Reza 13/05/2002
332 // faut-il lire dans le fichier fits ?
333 if(i<ideb || i>ifin) { // Toujours vrai avec le += Buff_Sz
334 ideb = i;
335 ifin = ideb+Buff_Sz-1;
336 if(ifin>=nrows) ifin=nrows-1;
337 long n = ifin-ideb+1;
338 int anyNul;
339 if (implicitSN) {
340 if (ideb+implicitSNStart > mgr->getRequestedEnd()
341 || (forcedMaxIn > 0 && ideb+implicitSNStart > forcedMaxIn)) {
342 fits_unlock(); // unlock avant de casser la boucle - Reza 16/05/2002
343 break;
344 }
345 if (ifin+implicitSNStart < mgr->getRequestedBegin()
346 || (forcedMinIn > 0 && ifin+implicitSNStart < forcedMinIn)) {
347 fits_unlock(); // unlock avant de continuer - Reza 16/05/2002
348 continue;
349 }
350 for (long j=0; j<Buff_Sz; j++) {
351 samplenum[j] = j+ideb+implicitSNStart;
352 }
353 //---- Pas la peine, fait en debut de boucle - Reza 13/05/2002 fits_lock();
354 } else {
355 //---- Pas la peine, fait en debut de boucle - Reza 13/05/2002 fits_lock();
356 fits_read_col_dbl(fptr,1,ideb+1,1,n,0,samplenum,&anyNul,&fstatus);
357 if (samplenum[0] > mgr->getRequestedEnd()
358 || (forcedMaxIn > 0 && samplenum[0] > forcedMaxIn)) {
359 fits_unlock(); // unlock avant de casser la boucle - Reza 13/05/2002
360 break;
361 }
362 if (samplenum[n-1] < mgr->getRequestedBegin()
363 || (forcedMinIn > 0 && samplenum[n-1] < forcedMinIn)) {
364 fits_unlock();
365 continue;
366 }
367 }
368 for(int k=0; k<getNOut(); k++) {
369 if(colval[k]==NULL) continue;
370 int j = colsinput[k].first;
371 fits_read_col_dbl(fptr,j+1,ideb+1,1,n,0,colval[k],&anyNul,&fstatus);
372
373 //--- Ajout Reza (18/6/2002) pour fichier de flags separe
374 if (sepFlagfile) {
375 int sjj;
376 for(sjj=0; sjj<n; sjj++) colflg[k][sjj] = 0;
377 for(int skf=0; skf<sepFlagCols.size(); skf++) {
378 fits_read_col_byt(fptrflg, skf+1, ideb+1,1,n,0,stmpflg,&anyNul,&fstatus);
379 if (fstatus !=0) {
380 fits_report_error(stderr,fstatus);
381 cerr << " RzBug - skf=" << skf << " ideb= " << ideb << endl;
382 }
383 for(sjj=0; sjj<n; sjj++) {
384 nbckflg++;
385 if (stmpflg[sjj]) {
386 nonzeroflg++;
387 colflg[k][sjj] |= sepFlagCols[skf];
388 }
389 }
390 }
391 } //--- Fin modif pour fichier de flags separe (18/6/2002)
392
393 if(colflg[k]==NULL) continue;
394 fits_read_col_lng(fptr,j+2,ideb+1,1,n,0,colflg[k],&anyNul,&fstatus);
395 }
396 if(fstatus!=0) {
397 fits_report_error(stderr,fstatus);
398 fits_unlock(); // On supprime le lock avant le throw - Reza 13/05/2002
399 throw RangeCheckError("FITSTOIReader::run2: Error Reading Fits file\n");
400 }
401 }
402 fits_unlock(); // unlock en fin de boucle de lecture - Reza 13/05/2002
403
404 long ip = i-ideb; // pointeurs dans les buffers
405 long sn = (long) (samplenum[ip]+.1);
406 if (ip == 0) {
407 for(int k=0; k<getNOut(); k++) {
408 if(colval[k]==NULL) continue;
409 if (colflg[k] != NULL) {
410 for (int ii=0; ii<Buff_Sz; ii++) {
411 tmpflg[ii] = colflg[k][ii];
412 }
413 putData(k, sn, ifin-ideb+1, colval[k], tmpflg);
414 } else {
415 putData(k, sn, ifin-ideb+1, colval[k]);
416 }
417 }
418 }
419 totnscount += (ifin-ideb+1);
420 }
421
422 //////// des-allocate buffers
423 delete [] samplenum;
424 delete [] tmpflg;
425 if (sepFlagfile) delete [] stmpflg;
426 {for(int k=0; k<getNOut(); k++) {
427 if(colval[k]!=NULL) delete [] colval[k];
428 if(colflg[k]!=NULL) delete [] colflg[k];
429 }}
430 colval.resize(0); colflg.resize(0);
431
432 cout << "reader (buffered) done reading... " << pthread_self() << endl;
433 cout << " --- CkRz: nbckflg=" << nbckflg << " nonzeroflg= " << nonzeroflg << endl;
434
435}
Note: See TracBrowser for help on using the repository browser.