source: BAORadio/AmasNancay/trunk/proc_specmfib.sh @ 627

Last change on this file since 627 was 627, checked in by campagne, 13 years ago

correct case esac structure (jec)
  • Property svn:executable set to *
File size: 16.3 KB
Line 
1#!/bin/sh -xvf
2#Basic tuning of system fuction used.
3AWK=/bin/awk
4SED=/bin/sed
5GREP=/bin/grep
6WC=/usr/bin/wc
7RM=/bin/rm
8CAT=/bin/cat
9TOUCH=/bin/touch
10DATE=/bin/date
11ECHO=/bin/echo
12LS=/bin/ls
13MKDIR=/bin/mkdir
14TR=/usr/bin/tr
15FIND=/usr/bin/find
16PRINTF=/usr/bin/printf
17XARGS=/usr/bin/xargs
18SORT=/bin/sort
19QSUB=qsub
20
21DefaultIFS=$' \t\n'
22IFS=$DefaultIFS
23
24
25scriptName="`basename $0`"
26$ECHO "Processing script ${scriptName} at `date`"
27######
28# Local functions
29######
30usage() {
31  echo "usage: $scriptName <source> <YYYYMMDD-directory> <type of process>"
32  echo "      -<source> as Abell85"
33  echo "      -<YYYYMMDD-directory> as 20110428"
34  echo "      -<type of process> as GAIN|CALIBON|CALIBOFF|ON|OFF"
35  echo "                         as GAINDS|ONDS"
36}
37######
38# argument test
39######
40
41#Process cmd line args: the -src option is mandatory (source name as Abell85)
42#action: gain|mspec (gain: gain-like doucble median filtering
43#                   mspec: mean+sigma wo filetring)
44action="gain"
45sourceRadio=
46dateSelected=
47typeofproc=
48#use -sim option to simulate processing (debug the script as if...)
49simulationMode=Off
50#
51#first and last cycle to process
52#
53firstCycle=
54lastCycle=
55while [ $# -gt 0 ]
56do
57  case "$1" in
58      -act)  action=$2;         shift;;
59      -src)  sourceRadio=$2;    shift;;
60      -date) dateSelected=$2;   shift;;
61      -type) typeofproc=$2;     shift;;
62      -fcycle) firstCycle=$2;   shift;;
63      -lcycle) lastCycle=$2;    shift;;
64      -sim)  simulationMode=On;;
65        -h)
66            echo >&2 \
67            "usage: $0 [-act <action> ] -src souce -date YYYYMMDD -type type [-fcycle firstCycle] [-lcycle lastCycle] [-sim to trig simulation mode]"
68            exit 1;;
69        *)  break;;     # terminate while loop
70    esac
71    shift
72done
73
74#Check input
75
76if [ "<${sourceRadio}>" = "<>" ]; then
77    $ECHO "FATAL: You have forgotten to select the source option (-src)"
78    exit 1
79fi
80
81if [ "<${dateSelected}>" = "<>" ]; then
82    $ECHO "FATAL: You have forgotten to select the date option (-date)"
83    exit 1
84fi
85
86if [ "<${typeofproc}>" = "<>" ]; then
87    $ECHO "FATAL: You have forgotten to select the type option (-type)"
88    exit 1
89
90fi
91
92
93srclower=`${ECHO} ${sourceRadio} | ${TR} "[:upper:]" "[:lower:]" `
94
95
96#Path to public backupable path
97publicpath="/afs/in2p3.fr/home/c/campagne/public"
98
99#temporary files to synchronize scripts
100tmppublicpath=${TMPPUBLICPATH}
101$LS -l ${tmppublicpath} > /dev/null
102
103
104#Path where the job will do temporary IO
105. ${SCRIPTPATH}/set_iojobpath.sh
106iojobpath=$(set_iojobpath)
107cd ${iojobpath}
108
109localpath="${sourceRadio}/${dateSelected}${srclower}"
110$MKDIR -p ./${localpath}
111cd ./${localpath}
112mainDirectory=`pwd`
113
114
115#Path of the utility scripts
116scriptpath=${SCRIPTPATH}
117
118
119#select file per cycle
120#$ECHO "Select files process..."
121#keep only lines with structure: cycle_number first,last,step
122#Get sca files
123$ECHO "DEBUG (${scriptName}): START call getscafiles.sh ${sourceRadio} ${dateSelected}"
124OUT1=./getScaStatus.$$
125#JEC 29/9/11 avoid finished stuff
126#$RM -f ${tmppublicpath}/getscafiles.finished
127${scriptpath}/getscafiles.sh ${sourceRadio} ${dateSelected} > ${OUT1} 2>&1
128#while [ ! -f "${tmppublicpath}/getscafiles.finished" ]; do
129#    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getscafiles.finished"
130#    date +%T
131#    sleep 30
132#done
133#$RM -f ${tmppublicpath}/getscafiles.finished
134$ECHO "DEBUG (${scriptName}): END"
135
136
137getScaStatus=`$GREP "^\(FATAL\|ERROR\)" ${OUT1}`
138
139if [ "<${getScaStatus}>" != "<>" ]; then
140  $ECHO "FATAL (${scriptName}): error to get sca file for $sourceRadio $dateSelected"
141  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
142  $CAT ${OUT1}
143  $ECHO "DEBUG (${scriptName}): END"
144  exit 1
145fi
146
147
148scaFileSummarized=`$FIND . -name "sca*.sum" -print`
149if [ "<${scaFileSummarized}>" = "<>" ]; then
150  $ECHO "FATAL (${scriptName}): strange sca file summarized not available: $sourceRadio $dateSelected"
151  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
152  $CAT ${OUT1}
153  $ECHO "DEBUG (${scriptName}): END"
154  exit 1
155fi
156$RM ${OUT1}
157
158scaFileSummarized=`basename $scaFileSummarized`
159
160#set Irods environment
161. /usr/local/shared/bin/irods_env.sh -noverbose
162#download header files to be used for signal files selction
163$MKDIR -p ./Fiber1
164dirFiberfile=`pwd`/Fiber1
165
166dirIrods="/baoradio/data/AmasNancay/${sourceRadio}/${dateSelected}${srclower}/Fiber1"
167
168listOfFiles=( `ils ${dirIrods} | $GREP -i "header" | $XARGS -i basename {} | $SORT -k1.7n` )
169IFS='
170'
171listOfFiles=( $( $PRINTF "%s\n" "${listOfFiles[@]}" | $AWK 'x[$0]++ == 0' ) )
172IFS=$DefaultIFS
173
174#
175$ECHO "DEBUG (${scriptName}): Download if necessary the headers"
176#
177for ifile in ${listOfFiles[@]}
178do
179#test if the headers have not yet been downloaded
180  if [ ! -e ${dirFiberfile}/${ifile} ]; then
181      OUT2=./getheaders.$$
182      iget -f -K ${dirIrods}/${ifile} ${dirFiberfile} >${OUT2} 2>&1
183      igetStatus=`$GREP "^ERROR" ${OUT2}`
184      if [ "<$igetStatus>" != "<>" ]; then
185          $ECHO "FATAL (${scriptName}): error while iget:"
186          $ECHO $igetStatus
187          exit 1
188      fi
189      $RM -f $OUT2
190  fi
191done
192#
193$ECHO "DEBUG (${scriptName}): START call select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc}"
194#
195#Get list of file to process
196headerFileSelected=./selectSignal.$$
197#JEC 29/9/11 avoid finished stuff
198#$RM -f ${tmppublicpath}/select.finished
199${scriptpath}/select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc} | $GREP "^#[0-9]" | $SED "s/^#//" > $headerFileSelected
200#while [ ! -f "${tmppublicpath}/select.finished" ]; do
201#    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/select.finished"
202#    date +%T
203#    sleep 30
204#done
205#$RM ${tmppublicpath}/select.finished
206#
207$ECHO "DEBUG (${scriptName}): END"
208#
209$ECHO "DEBUG (${scriptName}): START filling cycleArray, filesArray"
210#
211declare -a cycleArray
212declare -a filesArray
213$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
214IFS=$DefaultIFS
215$ECHO "DEBUG (${scriptName}): START cat $headerFileSelected"
216$CAT $headerFileSelected
217$ECHO "DEBUG (${scriptName}): END cat $headerFileSelected"
218
219while read v1 v2
220do
221 cycleArray[${#cycleArray[*]}]=$v1
222 filesArray[${#filesArray[*]}]=$v2
223done < $headerFileSelected
224
225if [ ${#cycleArray[*]} -eq 0 -o ${#filesArray[*]} -eq 0 ]; then
226   $ECHO "FATAL (${scriptName}): no header file selected (${#cycleArray[*]}, ${#filesArray[*]})"
227   exit 1
228else
229   $ECHO "DEBUG (${scriptName}): non zero elements found for cycleArray and filesArray" 
230fi
231
232#
233$ECHO "DEBUG (${scriptName}): number of elements <${#cycleArray[*]}> <${#filesArray[*]}>"
234$ECHO "DEBUG (${scriptName}): END"
235#
236# (timeShift)
237#Add 3sec to start ON / OFF
238#Substract 4sec to start DAB ON / OFF
239#
240#Take care: Cycle numbering starts at 1 in SCAFiles while Arrays index start at 0
241#
242#for GAIN the reference cycle takes the middle of the run
243#
244#nwinmean donne le nombre de fenetre tspwin utilise pour faire calculer la moyenne des medianes de chaque fenetre
245#This is the nimber of paquets of the time window which change according to processing type
246$ECHO "DEBUG (${scriptName}): START switch on $typeofproc"
247#
248case "$typeofproc" in
249    GAIN)
250      $ECHO "process signal files for GAIN phase"
251      ingain=""
252      freqfilter="-freqfilter -"
253      nwinmean="1"
254      tspwin="5120"
255      dirName="Off"
256      cyclebasename="gaincycle"
257      indexStartHour="6"
258      timeShift="-4"
259      timeDuration="14"
260      firstCycleId=`expr  ${#cycleArray[*]} / 2`
261      firstCycleId=`expr ${firstCycleId} - 1`
262      lastp1CycleId=`expr ${firstCycleId} + 1`
263      ;;
264    GAINDS)
265      $ECHO "process signal files for GAIN DRIFT SCAN phases"
266      ingain=""
267      freqfilter="-freqfilter -"
268      nwinmean="1"
269      tspwin="5120"
270      dirName="Off"
271      cyclebasename="gaincycle"
272      indexStartHour="2"
273      timeShift="-4"
274      timeDuration="14"
275      firstCycleId=`expr  ${#cycleArray[*]} / 2`
276      firstCycleId=`expr ${firstCycleId} - 1`
277      lastp1CycleId=`expr ${firstCycleId} + 1`
278      ;;
279    GAINNF)
280      $ECHO "process signal files for GAIN phase wo Freq FILTERING"
281      ingain=""
282      freqfilter="-freqfilter 4"
283      nwinmean="1"
284      tspwin="5120"
285      dirName="Off"
286      cyclebasename="gainf4cycle"
287      indexStartHour="6"
288      timeShift="-4"
289      timeDuration="14"
290      firstCycleId=`expr  ${#cycleArray[*]} / 2`
291      firstCycleId=`expr ${firstCycleId} - 1`
292      lastp1CycleId=`expr ${firstCycleId} + 1`
293      ;;
294    ON)
295      $ECHO "process signal files for ON phases"
296      ingain="-gain gain_${dateSelected}_${srclower}.fits"
297      freqfilter=""
298      nwinmean="5"
299      tspwin="5120"
300      dirName="On"
301      cyclebasename="datacycle"
302      indexStartHour="4"
303      timeShift="3"
304      timeDuration="30"
305      firstCycleId="0"
306      lastp1CycleId="${#cycleArray[@]}"
307      ;;
308    ONDS)
309      $ECHO "process signal files for ON Drift Scan phases"
310      ingain="-gain gain_${dateSelected}_${srclower}.fits"
311      freqfilter=""
312      nwinmean="1"
313      tspwin="5120"
314      dirName="On"
315      cyclebasename="datacycle"
316      indexStartHour="4"
317      timeShift="3"
318      timeDuration="170"
319      firstCycleId="0"
320      lastp1CycleId="${#cycleArray[@]}"
321      ;;
322    OFF)
323      $ECHO "process signal files for OFF phases"
324      ingain="-gain gain_${dateSelected}_${srclower}.fits"
325      freqfilter=""
326      nwinmean="5"
327      tspwin="5120"
328      dirName="Off"
329      cyclebasename="datacycle"
330      indexStartHour="8"
331      timeShift="3"
332      timeDuration="30"
333      firstCycleId="0"
334      lastp1CycleId="${#cycleArray[@]}"
335      ;;
336    MEANOFF)
337      $ECHO "process signal files for OFF phases (MEAN)"
338      ingain="-gain gain_${dateSelected}_${srclower}.fits"
339      freqfilter=""
340      nwinmean="25000"
341      tspwin="0"
342      dirName="Off"
343      cyclebasename="meancycle"
344      indexStartHour="8"
345      timeShift="3"
346      timeDuration="30"
347      firstCycleId="0"
348      lastp1CycleId="${#cycleArray[@]}"
349      ;;
350    MEANON)
351      $ECHO "process signal files for ON phases (MEAN)"
352      ingain="-gain gain_${dateSelected}_${srclower}.fits"
353      freqfilter=""
354      nwinmean="25000"
355      tspwin="0"
356      dirName="On"
357      cyclebasename="meancycle"
358      indexStartHour="4"
359      timeShift="3"
360      timeDuration="30"
361      firstCycleId="0"
362      lastp1CycleId="${#cycleArray[@]}"
363      ;;
364    CALIBON)
365      $ECHO "process signal files for CALIBRATION ON phases"
366      ingain="-gain gain_${dateSelected}_${srclower}.fits"
367      freqfilter="-freqfilter -"
368      nwinmean="1"
369      tspwin="1024"
370      dirName="On"
371      cyclebasename="calibcycle"
372      indexStartHour="2"
373      timeShift="-4"
374      timeDuration="14"
375      firstCycleId="0"
376      lastp1CycleId="${#cycleArray[@]}"
377      ;;
378    CALIBOFF)
379      $ECHO "process signal files for CALIBRATION OFF phases"
380      ingain="-gain gain_${dateSelected}_${srclower}.fits"
381      freqfilter="-freqfilter -"
382      nwinmean="1"
383      tspwin="1024"
384      dirName="Off"
385      cyclebasename="calibcycle"
386      indexStartHour="6"
387      timeShift="-4"
388      timeDuration="14"
389      firstCycleId="0"
390      lastp1CycleId="${#cycleArray[@]}"
391      ;;
392    *)
393    $ECHO "FATAL (${scriptName}): type of processing"
394    usage
395    exit 1
396    ;;
397esac
398#
399$ECHO "DEBUG (${scriptName}): END"
400#
401#
402#Compute time windows (Warning usage of GMT date)
403#
404$ECHO "DEBUG (${scriptName}): START compute time windows"
405OUT=./baotiming.$$
406${AWK} -v datesys=${DATE} -v type=${indexStartHour} -v shift=${timeShift} '
407BEGIN {FS=","}
408{
409  phaseStart=$type
410  cmd=sprintf("%s +%%s -d  \"%s\" ",datesys,phaseStart);
411  (cmd | getline phstart);
412  close(cmd);
413  newstart = phstart+shift;
414  cmd=sprintf("%s +%%T -d \"1970-01-01 %s sec GMT\"",datesys,newstart);
415  (cmd | getline newStart);
416  close(cmd);
417  print $1,newStart,phaseStart;
418}
419'  ${scaFileSummarized} > ${OUT}
420#
421$ECHO "DEBUG (${scriptName}): END"
422#
423$ECHO "DEBUG (${scriptName}): START cat ${OUT}"
424$CAT ${OUT}
425$ECHO "DEBUG (${scriptName}): END cat ${OUT}"
426#
427$ECHO "DEBUG (${scriptName}): START filling cycleArrayBis timeArray"
428#
429declare -a cycleArrayBis
430declare -a timeArray
431$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
432IFS=$DefaultIFS
433#select the selected cycles
434idtofind="0"
435while read v1 v2 v3
436do
437 if [ ${idtofind} -lt ${#cycleArray[*]} ]; then
438    if [ ${v1} = ${cycleArray[${idtofind}]} ]; then
439        cycleArrayBis[${#cycleArrayBis[*]}]=$v1
440        timeArray[${#timeArray[*]}]=$v2
441        idtofind=`expr ${idtofind} + 1`
442    fi
443 fi
444done < ${OUT}
445$ECHO "DEBUG (${scriptName}): number of elements ${#cycleArrayBis[*]} ${#timeArray[*]}"
446$ECHO "DEBUG (${scriptName}): END"
447
448
449#########################
450# specmfib exe
451#########################
452prg="/afs/in2p3.fr/throng/baoradio/Library/Progs/TAcq/Objs/specmfib"
453if [ ! -e $prg ]; then
454    $ECHO "FATAL (${scriptName}): $prg not found"
455    exit 1
456fi
457
458
459dirFiberfile=${iojobpath}/${localpath}
460
461curDir=`pwd`
462#
463$ECHO "DEBUG (${scriptName}): in principle $dirFiberfile == $curDir "
464#
465#force single channel
466forceSingle="-singlechan"
467#debug
468prtlevel="1"
469
470
471#Loop on cycles
472##
473$ECHO "DEBUG (${scriptName}): START loop on cycles Ids ${firstCycleId} to ${lastp1CycleId}"
474
475#write the script to be send to Batch
476#get gain file if necessary
477if [ ${typeofproc} != "GAIN" -a ${typeofproc} != "GAINDS" ]; then
478    OUT2=./igetStatus.$$
479    iget /baoradio/data/AmasNancay/${localpath}/gain_${dateSelected}_${srclower}.fits >${OUT2} 2>&1
480    igetStatus=`$GREP "^ERROR" ${OUT2}`
481    if [ "<${igetStatus}>" != "<>" ]; then
482        $ECHO "FATAL (${scriptName}): error while iget:"
483        $ECHO $igetStatus
484        exit 1
485    fi
486    $RM ${OUT2}
487fi   
488
489
490for ((i=${firstCycleId};i<${lastp1CycleId};i++)); do
491#
492
493####
494# Check daq status
495#JEC 1/10/11 Use generic baodaqstatus name
496####
497#get the daq current irod status
498#tag=`${DATE} +%F`
499#OUT1=${publicpath}/baodaqstatus-${tag}.txt
500OUT1=${publicpath}/baodaqstatus-current.txt
501
502if [ ! -e ${OUT1}  -o ! -r ${OUT1} ]; then
503    $ECHO "FATAL (${scriptName}): ${OUT1} has a problem"
504    exit 1
505#    $ECHO "We should bring the DAQ status more up to date, this take 1 or 2min..."
506#JEC 29/9/11 avoid finished stuff
507#    $RM -f ${tmppublicpath}/statusdaq.finished
508#    ${scriptpath}/statusdaq.sh > ${OUT1}
509#    while [ ! -f "${tmppublicpath}/statusdaq.finished" ]; do
510#       sleep 30
511#    done
512#    $RM ${tmppublicpath}/statusdaq.finished
513#protect against remove/rewriting   
514#    $CHMOD -v 444 ${OUT1}
515fi
516
517#time window selection
518    tmproc="-tmproc ${timeArray[${i}]},${timeDuration}"
519#cycle <number>
520    cycle="${cycleArray[${i}]}"
521    if [ "<${firstCycle}>" != "<>" -a "<${lastCycle}>" != "<>" ]; then
522        if [ ${cycle} -lt ${firstCycle} -o ${cycle} -gt ${lastCycle} ]; then
523            continue
524        fi
525    fi
526    $ECHO "DEBUG (${scriptName}): process cycle $cycle"
527
528#cycle <name><number>
529    cycle="${cyclebasename}${cycleArray[${i}]}"
530#signal files selection
531    infiles="${filesArray[${i}]}"
532#
533#Prepare BQS parameters: MEMORY, CPU...#
534    infilesArray=( `$ECHO $infiles | $AWK 'BEGIN {FS=","} {for(i=1;i<NF;i++)print $i;}'` )
535    IFS='
536    '
537    infilesArray=( $( printf "%s\n" "${infilesArray[@]}" | awk 'x[$0]++ == 0' ) )
538    IFS=$DefaultIFS
539    firstFile=${infilesArray[0]}
540    lastFile=${infilesArray[1]}
541#
542#
543#
544#get signals files
545    OUT3=./getsignals.$$
546    ${scriptpath}/getsignalfiles.sh ${sourceRadio} ${dateSelected} ${firstFile}  ${lastFile} > ${OUT3} 2>&1
547    getSignals=`$GREP "^FATAL" ${OUT3}`
548    if [ "<$getSignals>" != "<>" ]; then
549        $ECHO "FATAL (${scriptName}): error while get signals"
550        $ECHO $getSignals
551        exit 1
552    fi
553    $RM ${OUT3}
554
555#location of the signal file
556    fibfile="${curDir} 1,2"
557#
558    outDir=${curDir}/${dirName}/${cycle}
559    $MKDIR -p ${outDir}
560
561#
562# where ami...
563#
564    $ECHO "We are here `pwd`"
565    $LS -lrth
566    $FIND . -name "igetStatus.*" | $XARGS -i  $CAT {}
567    $LS ./Fiber1 | $WC -l
568    $LS ./Fiber2 | $WC -l
569    $LS -lR ${outDir}
570
571#Note: here act=gain means only that specmfib will use median-like algorithms (not only for gains...)
572    $prg -act ${action} $forceSingle $freqfilter  -prt $prtlevel -out ${outDir} -nmean $nwinmean $ingain $tmproc -tspwin ${tspwin},0,0 -in $infiles ${fibfile}
573
574    $ECHO    ${outDir}
575    $LS -lrt ${outDir}
576
577#save results to Irods (use option -f ONLY to force override)
578    emptydir=`$FIND ${outDir} -maxdepth 1 -type f | $WC -l`
579    if [ "<${emptydir}>" != "<0>" ]; then
580      iput -v -K -r -f ${curDir}/${dirName} /baoradio/data/AmasNancay/${localpath}
581    else
582      $ECHO "DEBUG (${scriptName}): directory ${outDir} is empty"
583    fi
584#
585#clean the signal files which can at the end of the day fill completly the scratch directory of the batch work/AmasNancayer
586    $RM -rf ${iojobpath}/${localpath}/Fiber1
587    $RM -rf ${iojobpath}/${localpath}/Fiber2
588#    $RM -rf ${iojobpath}/${localpath}/${typeofproc} JEC 25/10/11
589    $RM -rf ${curDir}/${dirName}
590#
591#
592
593#end loop on cycles
594done
595#
596$ECHO "DEBUG (${scriptName}): END loop on cycles"
597
598exit 0
Note: See TracBrowser for help on using the repository browser.