source: BAORadio/AmasNancay/trunk/proc_specmfib.sh @ 600

Last change on this file since 600 was 600, checked in by torrento, 13 years ago

Correct if condition in emptydir
  • Property svn:executable set to *
File size: 16.1 KB
Line 
1#!/bin/sh -xvf
2#Basic tuning of system fuction used.
3AWK=/bin/awk
4SED=/bin/sed
5GREP=/bin/grep
6WC=/usr/bin/wc
7RM=/bin/rm
8CAT=/bin/cat
9TOUCH=/bin/touch
10DATE=/bin/date
11ECHO=/bin/echo
12LS=/bin/ls
13MKDIR=/bin/mkdir
14TR=/usr/bin/tr
15FIND=/usr/bin/find
16PRINTF=/usr/bin/printf
17XARGS=/usr/bin/xargs
18SORT=/bin/sort
19QSUB=qsub
20
21DefaultIFS=$' \t\n'
22IFS=$DefaultIFS
23
24
25scriptName="`basename $0`"
26$ECHO "Processing script ${scriptName} at `date`"
27######
28# Local functions
29######
30usage() {
31  echo "usage: $scriptName <source> <YYYYMMDD-directory> <type of process>"
32  echo "      -<source> as Abell85"
33  echo "      -<YYYYMMDD-directory> as 20110428"
34  echo "      -<type of process> as GAIN|CALIBON|CALIBOFF|ON|OFF"
35}
36######
37# argument test
38######
39
40#Process cmd line args: the -src option is mandatory (source name as Abell85)
41#action: gain|mspec (gain: gain-like doucble median filtering
42#                   mspec: mean+sigma wo filetring)
43action="gain"
44sourceRadio=
45dateSelected=
46typeofproc=
47#use -sim option to simulate processing (debug the script as if...)
48simulationMode=Off
49#
50#first and last cycle to process
51#
52firstCycle=
53lastCycle=
54while [ $# -gt 0 ]
55do
56  case "$1" in
57      -act)  action=$2;         shift;;
58      -src)  sourceRadio=$2;    shift;;
59      -date) dateSelected=$2;   shift;;
60      -type) typeofproc=$2;     shift;;
61      -fcycle) firstCycle=$2;   shift;;
62      -lcycle) lastCycle=$2;    shift;;
63      -sim)  simulationMode=On;;
64        -h)
65            echo >&2 \
66            "usage: $0 [-act <action> ] -src souce -date YYYYMMDD -type type [-fcycle firstCycle] [-lcycle lastCycle] [-sim to trig simulation mode]"
67            exit 1;;
68        *)  break;;     # terminate while loop
69    esac
70    shift
71done
72
73#Check input
74
75if [ "<${sourceRadio}>" = "<>" ]; then
76    $ECHO "FATAL: You have forgotten to select the source option (-src)"
77    exit 1
78fi
79
80if [ "<${dateSelected}>" = "<>" ]; then
81    $ECHO "FATAL: You have forgotten to select the date option (-date)"
82    exit 1
83fi
84
85if [ "<${typeofproc}>" = "<>" ]; then
86    $ECHO "FATAL: You have forgotten to select the type option (-type)"
87    exit 1
88
89fi
90#else
91#    offtype=`$ECHO ${typeofproc} | $GREP -i OFF | $WC -l`
92#    ontype=`$ECHO ${typeofproc} | $GREP -i ON | $WC -l`
93#    if [ "<${offtype}>" = "<1>" -a  "<${ontype}>" = "<0>" ]; then
94#       datatype="OFF"
95#    elif [ "<${offtype}>" = "<0>" -a  "<${ontype}>" = "<1>" ]; then
96#       datatype="ON"
97#    else
98#       $ECHO "FATAL: not supported: -type ${typeofproc}"
99#       exit 1
100#    fi
101#fi
102
103
104srclower=`${ECHO} ${sourceRadio} | ${TR} "[:upper:]" "[:lower:]" `
105
106
107#Path to public backupable path
108publicpath="/afs/in2p3.fr/home/c/campagne/public"
109
110#temporary files to synchronize scripts
111tmppublicpath=${TMPPUBLICPATH}
112$LS -l ${tmppublicpath} > /dev/null
113
114
115#Path where the job will do temporary IO
116. ${SCRIPTPATH}/set_iojobpath.sh
117iojobpath=$(set_iojobpath)
118cd ${iojobpath}
119
120localpath="${sourceRadio}/${dateSelected}${srclower}"
121$MKDIR -p ./${localpath}
122cd ./${localpath}
123mainDirectory=`pwd`
124
125
126#Path of the utility scripts
127scriptpath=${SCRIPTPATH}
128
129
130#select file per cycle
131#$ECHO "Select files process..."
132#keep only lines with structure: cycle_number first,last,step
133#Get sca files
134$ECHO "DEBUG (${scriptName}): START call getscafiles.sh ${sourceRadio} ${dateSelected}"
135OUT1=./getScaStatus.$$
136#JEC 29/9/11 avoid finished stuff
137#$RM -f ${tmppublicpath}/getscafiles.finished
138${scriptpath}/getscafiles.sh ${sourceRadio} ${dateSelected} > ${OUT1} 2>&1
139#while [ ! -f "${tmppublicpath}/getscafiles.finished" ]; do
140#    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getscafiles.finished"
141#    date +%T
142#    sleep 30
143#done
144#$RM -f ${tmppublicpath}/getscafiles.finished
145$ECHO "DEBUG (${scriptName}): END"
146
147
148getScaStatus=`$GREP "^\(FATAL\|ERROR\)" ${OUT1}`
149
150if [ "<${getScaStatus}>" != "<>" ]; then
151  $ECHO "FATAL (${scriptName}): error to get sca file for $sourceRadio $dateSelected"
152  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
153  $CAT ${OUT1}
154  $ECHO "DEBUG (${scriptName}): END"
155  exit 1
156fi
157
158
159scaFileSummarized=`$FIND . -name "sca*.sum" -print`
160if [ "<${scaFileSummarized}>" = "<>" ]; then
161  $ECHO "FATAL (${scriptName}): strange sca file summarized not available: $sourceRadio $dateSelected"
162  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
163  $CAT ${OUT1}
164  $ECHO "DEBUG (${scriptName}): END"
165  exit 1
166fi
167$RM ${OUT1}
168
169scaFileSummarized=`basename $scaFileSummarized`
170
171#set Irods environment
172. /usr/local/shared/bin/irods_env.sh -noverbose
173#download header files to be used for signal files selction
174$MKDIR -p ./Fiber1
175dirFiberfile=`pwd`/Fiber1
176
177dirIrods="/baoradio/data/AmasNancay/${sourceRadio}/${dateSelected}${srclower}/Fiber1"
178
179listOfFiles=( `ils ${dirIrods} | $GREP -i "header" | $XARGS -i basename {} | $SORT -k1.7n` )
180IFS='
181'
182listOfFiles=( $( $PRINTF "%s\n" "${listOfFiles[@]}" | $AWK 'x[$0]++ == 0' ) )
183IFS=$DefaultIFS
184
185#
186$ECHO "DEBUG (${scriptName}): Download if necessary the headers"
187#
188for ifile in ${listOfFiles[@]}
189do
190#test if the headers have not yet been downloaded
191  if [ ! -e ${dirFiberfile}/${ifile} ]; then
192      OUT2=./getheaders.$$
193      iget -f -K ${dirIrods}/${ifile} ${dirFiberfile} >${OUT2} 2>&1
194      igetStatus=`$GREP "^ERROR" ${OUT2}`
195      if [ "<$igetStatus>" != "<>" ]; then
196          $ECHO "FATAL (${scriptName}): error while iget:"
197          $ECHO $igetStatus
198          exit 1
199      fi
200      $RM -f $OUT2
201  fi
202done
203#
204$ECHO "DEBUG (${scriptName}): START call select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc}"
205#
206#Get list of file to process
207headerFileSelected=./selectSignal.$$
208#JEC 29/9/11 avoid finished stuff
209#$RM -f ${tmppublicpath}/select.finished
210${scriptpath}/select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc} | $GREP "^#[0-9]" | $SED "s/^#//" > $headerFileSelected
211#while [ ! -f "${tmppublicpath}/select.finished" ]; do
212#    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/select.finished"
213#    date +%T
214#    sleep 30
215#done
216#$RM ${tmppublicpath}/select.finished
217#
218$ECHO "DEBUG (${scriptName}): END"
219#
220$ECHO "DEBUG (${scriptName}): START filling cycleArray, filesArray"
221#
222declare -a cycleArray
223declare -a filesArray
224$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
225IFS=$DefaultIFS
226$ECHO "DEBUG (${scriptName}): START cat $headerFileSelected"
227$CAT $headerFileSelected
228$ECHO "DEBUG (${scriptName}): END cat $headerFileSelected"
229
230while read v1 v2
231do
232 cycleArray[${#cycleArray[*]}]=$v1
233 filesArray[${#filesArray[*]}]=$v2
234done < $headerFileSelected
235
236if [ ${#cycleArray[*]} -eq 0 -o ${#filesArray[*]} -eq 0 ]; then
237   $ECHO "FATAL (${scriptName}): no header file selected (${#cycleArray[*]}, ${#filesArray[*]})"
238   exit 1
239else
240   $ECHO "DEBUG (${scriptName}): non zero elements found for cycleArray and filesArray" 
241fi
242
243#
244$ECHO "DEBUG (${scriptName}): number of elements <${#cycleArray[*]}> <${#filesArray[*]}>"
245$ECHO "DEBUG (${scriptName}): END"
246#
247# (timeShift)
248#Add 3sec to start ON / OFF
249#Substract 4sec to start DAB ON / OFF
250#
251#Take care: Cycle numbering starts at 1 in SCAFiles while Arrays index start at 0
252#
253#for GAIN the reference cycle takes the middle of the run
254#
255#nwinmean donne le nombre de fenetre tspwin utilise pour faire calculer la moyenne des medianes de chaque fenetre
256#This is the nimber of paquets of the time window which change according to processing type
257$ECHO "DEBUG (${scriptName}): START switch on $typeofproc"
258#
259case "$typeofproc" in
260    GAIN)
261      $ECHO "process signal files for GAIN phase"
262      ingain=""
263      freqfilter="-freqfilter -"
264      nwinmean="1"
265      tspwin="5120"
266      dirName="Off"
267      cyclebasename="gaincycle"
268      indexStartHour="6"
269      timeShift="-4"
270      timeDuration="14"
271      firstCycleId=`expr  ${#cycleArray[*]} / 2`
272      firstCycleId=`expr ${firstCycleId} - 1`
273      lastp1CycleId=`expr ${firstCycleId} + 1`
274      ;;
275    GAINNF)
276      $ECHO "process signal files for GAIN phase wo Freq FILTERING"
277      ingain=""
278      freqfilter="-freqfilter 4"
279      nwinmean="1"
280      tspwin="5120"
281      dirName="Off"
282      cyclebasename="gainf4cycle"
283      indexStartHour="6"
284      timeShift="-4"
285      timeDuration="14"
286      firstCycleId=`expr  ${#cycleArray[*]} / 2`
287      firstCycleId=`expr ${firstCycleId} - 1`
288      lastp1CycleId=`expr ${firstCycleId} + 1`
289      ;;
290    ON)
291      $ECHO "process signal files for ON phases"
292      ingain="-gain gain_${dateSelected}_${srclower}.fits"
293      freqfilter=""
294      nwinmean="5"
295      tspwin="5120"
296      dirName="On"
297      cyclebasename="datacycle"
298      indexStartHour="4"
299      timeShift="3"
300      timeDuration="30"
301      firstCycleId="0"
302      lastp1CycleId="${#cycleArray[@]}"
303      ;;
304    OFF)
305      $ECHO "process signal files for OFF phases"
306      ingain="-gain gain_${dateSelected}_${srclower}.fits"
307      freqfilter=""
308      nwinmean="5"
309      tspwin="5120"
310      dirName="Off"
311      cyclebasename="datacycle"
312      indexStartHour="8"
313      timeShift="3"
314      timeDuration="30"
315      firstCycleId="0"
316      lastp1CycleId="${#cycleArray[@]}"
317      ;;
318    MEANOFF)
319      $ECHO "process signal files for OFF phases (MEAN)"
320      ingain="-gain gain_${dateSelected}_${srclower}.fits"
321      freqfilter=""
322      nwinmean="25000"
323      tspwin="0"
324      dirName="Off"
325      cyclebasename="meancycle"
326      indexStartHour="8"
327      timeShift="3"
328      timeDuration="30"
329      firstCycleId="0"
330      lastp1CycleId="${#cycleArray[@]}"
331      ;;
332    MEANON)
333      $ECHO "process signal files for ON phases (MEAN)"
334      ingain="-gain gain_${dateSelected}_${srclower}.fits"
335      freqfilter=""
336      nwinmean="25000"
337      tspwin="0"
338      dirName="On"
339      cyclebasename="meancycle"
340      indexStartHour="4"
341      timeShift="3"
342      timeDuration="30"
343      firstCycleId="0"
344      lastp1CycleId="${#cycleArray[@]}"
345      ;;
346    CALIBON)
347      $ECHO "process signal files for CALIBRATION ON phases"
348      ingain="-gain gain_${dateSelected}_${srclower}.fits"
349      freqfilter="-freqfilter -"
350      nwinmean="1"
351      tspwin="1024"
352      dirName="On"
353      cyclebasename="calibcycle"
354      indexStartHour="2"
355      timeShift="-4"
356      timeDuration="14"
357      firstCycleId="0"
358      lastp1CycleId="${#cycleArray[@]}"
359      ;;
360    CALIBOFF)
361      $ECHO "process signal files for CALIBRATION OFF phases"
362      ingain="-gain gain_${dateSelected}_${srclower}.fits"
363      freqfilter="-freqfilter -"
364      nwinmean="1"
365      tspwin="1024"
366      dirName="Off"
367      cyclebasename="calibcycle"
368      indexStartHour="6"
369      timeShift="-4"
370      timeDuration="14"
371      firstCycleId="0"
372      lastp1CycleId="${#cycleArray[@]}"
373      ;;
374    *)
375    $ECHO "FATAL (${scriptName}): type of processing"
376    usage
377    exit 1
378    ;;
379esac
380#
381$ECHO "DEBUG (${scriptName}): END"
382#
383#
384#Compute time windows (Warning usage of GMT date)
385#
386$ECHO "DEBUG (${scriptName}): START compute time windows"
387OUT=./baotiming.$$
388${AWK} -v datesys=${DATE} -v type=${indexStartHour} -v shift=${timeShift} '
389BEGIN {FS=","}
390{
391  phaseStart=$type
392  cmd=sprintf("%s +%%s -d  \"%s\" ",datesys,phaseStart);
393  (cmd | getline phstart);
394  close(cmd);
395  newstart = phstart+shift;
396  cmd=sprintf("%s +%%T -d \"1970-01-01 %s sec GMT\"",datesys,newstart);
397  (cmd | getline newStart);
398  close(cmd);
399  print $1,newStart,phaseStart;
400}
401'  ${scaFileSummarized} > ${OUT}
402#
403$ECHO "DEBUG (${scriptName}): END"
404#
405$ECHO "DEBUG (${scriptName}): START cat ${OUT}"
406$CAT ${OUT}
407$ECHO "DEBUG (${scriptName}): END cat ${OUT}"
408#
409$ECHO "DEBUG (${scriptName}): START filling cycleArrayBis timeArray"
410#
411declare -a cycleArrayBis
412declare -a timeArray
413$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
414IFS=$DefaultIFS
415#select the selected cycles
416idtofind="0"
417while read v1 v2 v3
418do
419 if [ ${idtofind} -lt ${#cycleArray[*]} ]; then
420    if [ ${v1} = ${cycleArray[${idtofind}]} ]; then
421        cycleArrayBis[${#cycleArrayBis[*]}]=$v1
422        timeArray[${#timeArray[*]}]=$v2
423        idtofind=`expr ${idtofind} + 1`
424    fi
425 fi
426done < ${OUT}
427$ECHO "DEBUG (${scriptName}): number of elements ${#cycleArrayBis[*]} ${#timeArray[*]}"
428$ECHO "DEBUG (${scriptName}): END"
429
430
431#########################
432# specmfib exe
433#########################
434prg="/afs/in2p3.fr/throng/baoradio/Library/Progs/TAcq/Objs/specmfib"
435if [ ! -e $prg ]; then
436    $ECHO "FATAL (${scriptName}): $prg not found"
437    exit 1
438fi
439
440
441dirFiberfile=${iojobpath}/${localpath}
442
443curDir=`pwd`
444#
445$ECHO "DEBUG (${scriptName}): in principle $dirFiberfile == $curDir "
446#
447#force single channel
448forceSingle="-singlechan"
449#debug
450prtlevel="1"
451
452
453#Loop on cycles
454##
455$ECHO "DEBUG (${scriptName}): START loop on cycles Ids ${firstCycleId} to ${lastp1CycleId}"
456
457#write the script to be send to Batch
458#get gain file if necessary
459if [ ${typeofproc} != "GAIN" ]; then
460    OUT2=./igetStatus.$$
461    iget /baoradio/data/AmasNancay/${localpath}/gain_${dateSelected}_${srclower}.fits >${OUT2} 2>&1
462    igetStatus=`$GREP "^ERROR" ${OUT2}`
463    if [ "<${igetStatus}>" != "<>" ]; then
464        $ECHO "FATAL (${scriptName}): error while iget:"
465        $ECHO $igetStatus
466        exit 1
467    fi
468    $RM ${OUT2}
469fi   
470
471
472for ((i=${firstCycleId};i<${lastp1CycleId};i++)); do
473#
474
475####
476# Check daq status
477#JEC 1/10/11 Use generic baodaqstatus name
478####
479#get the daq current irod status
480#tag=`${DATE} +%F`
481#OUT1=${publicpath}/baodaqstatus-${tag}.txt
482OUT1=${publicpath}/baodaqstatus-current.txt
483
484if [ ! -e ${OUT1}  -o ! -r ${OUT1} ]; then
485    $ECHO "FATAL (${scriptName}): ${OUT1} has a problem"
486    exit 1
487#    $ECHO "We should bring the DAQ status more up to date, this take 1 or 2min..."
488#JEC 29/9/11 avoid finished stuff
489#    $RM -f ${tmppublicpath}/statusdaq.finished
490#    ${scriptpath}/statusdaq.sh > ${OUT1}
491#    while [ ! -f "${tmppublicpath}/statusdaq.finished" ]; do
492#       sleep 30
493#    done
494#    $RM ${tmppublicpath}/statusdaq.finished
495#protect against remove/rewriting   
496#    $CHMOD -v 444 ${OUT1}
497fi
498
499#time window selection
500    tmproc="-tmproc ${timeArray[${i}]},${timeDuration}"
501#cycle <number>
502    cycle="${cycleArray[${i}]}"
503    if [ "<${firstCycle}>" != "<>" -a "<${lastCycle}>" != "<>" ]; then
504        if [ ${cycle} -lt ${firstCycle} -o ${cycle} -gt ${lastCycle} ]; then
505            continue
506        fi
507    fi
508    $ECHO "DEBUG (${scriptName}): process cycle $cycle"
509
510#cycle <name><number>
511    cycle="${cyclebasename}${cycleArray[${i}]}"
512#signal files selection
513    infiles="${filesArray[${i}]}"
514#
515#Prepare BQS parameters: MEMORY, CPU...#
516    infilesArray=( `$ECHO $infiles | $AWK 'BEGIN {FS=","} {for(i=1;i<NF;i++)print $i;}'` )
517    IFS='
518    '
519    infilesArray=( $( printf "%s\n" "${infilesArray[@]}" | awk 'x[$0]++ == 0' ) )
520    IFS=$DefaultIFS
521    firstFile=${infilesArray[0]}
522    lastFile=${infilesArray[1]}
523#
524#
525#
526#get signals files
527    OUT3=./getsignals.$$
528#JEC 29/9/11 avoid finished stuff
529#    $RM -f ${tmppublicpath}/getsignalfiles.finished
530    ${scriptpath}/getsignalfiles.sh ${sourceRadio} ${dateSelected} ${firstFile}  ${lastFile} > ${OUT3} 2>&1
531#    while [ ! -f "${tmppublicpath}/getsignalfiles.finished" ]; do
532#       $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getsignalfiles.finished"
533#       date +%T
534#       sleep 30
535#    done
536#    $RM ${tmppublicpath}/getsignalfiles.finished
537
538    getSignals=`$GREP "^FATAL" ${OUT3}`
539    if [ "<$getSignals>" != "<>" ]; then
540        $ECHO "FATAL (${scriptName}): error while get signals"
541        $ECHO $getSignals
542        exit 1
543    fi
544    $RM ${OUT3}
545
546#location of the signal file
547    fibfile="${curDir} 1,2"
548#
549    outDir=${curDir}/${dirName}/${cycle}
550    $MKDIR -p ${outDir}
551
552#
553# where ami...
554#
555    $ECHO "We are here `pwd`"
556    $LS -lrth
557    $FIND . -name "igetStatus.*" | $XARGS -i  $CAT {}
558    $LS ./Fiber1 | $WC -l
559    $LS ./Fiber2 | $WC -l
560    $LS -lR ${outDir}
561
562#Note: here act=gain means only that specmfib will use median-like algorithms (not only for gains...)
563    $prg -act ${action} $forceSingle $freqfilter  -prt $prtlevel -out ${outDir} -nmean $nwinmean $ingain $tmproc -tspwin ${tspwin},0,0 -in $infiles ${fibfile}
564
565    $ECHO    ${outDir}
566    $LS -lrt ${outDir}
567
568#save results to Irods (use option -f ONLY to force override)
569    emptydir=`$FIND ${curDir}/${dirName} -maxdepth 1 -type f | $WC -l`
570    if [ "<${emptydir}>" != "<0>" ]; then
571      iput -v -K -r -f ${curDir}/${dirName} /baoradio/data/AmasNancay/${localpath}
572    else
573      $ECHO "DEBUG (${scriptName}): directory ${curDir}/${dirName} is empty"
574    fi
575#
576#clean the signal files which can at the end of the day fill completly the scratch directory of the batch work/AmasNancayer
577    $RM -rf ${iojobpath}/${localpath}/Fiber1
578    $RM -rf ${iojobpath}/${localpath}/Fiber2
579#    $RM -rf ${iojobpath}/${localpath}/${typeofproc} JEC 25/10/11
580    $RM -rf ${curDir}/${dirName}
581#
582#
583
584#end loop on cycles
585done
586#
587$ECHO "DEBUG (${scriptName}): END loop on cycles"
588
589exit 0
Note: See TracBrowser for help on using the repository browser.