source: BAORadio/AmasNancay/trunk/proc_rawonoff.sh @ 555

Last change on this file since 555 was 555, checked in by campagne, 13 years ago

action:mspec + minor typo (jec)
  • Property svn:executable set to *
File size: 16.1 KB
Line 
1#!/bin/sh -xvf
2#Basic tuning of system fuction used.
3AWK=/bin/awk
4SED=/bin/sed
5GREP=/bin/grep
6WC=/usr/bin/wc
7RM=/bin/rm
8CAT=/bin/cat
9TOUCH=/bin/touch
10DATE=/bin/date
11ECHO=/bin/echo
12LS=/bin/ls
13MKDIR=/bin/mkdir
14TR=/usr/bin/tr
15FIND=/usr/bin/find
16PRINTF=/usr/bin/printf
17XARGS=/usr/bin/xargs
18SORT=/bin/sort
19QSUB=qsub
20
21DefaultIFS=$' \t\n'
22IFS=$DefaultIFS
23
24
25scriptName="`basename $0`"
26$ECHO "Processing script ${scriptName} at `date`"
27######
28# Local functions
29######
30usage() {
31  echo "usage: $scriptName <source> <YYYYMMDD-directory> <type of process>"
32  echo "      -<source> as Abell85"
33  echo "      -<YYYYMMDD-directory> as 20110428"
34  echo "      -<type of process> as GAIN|CALIBON|CALIBOFF|ON|OFF"
35}
36######
37# argument test
38######
39
40#Process cmd line args: the -src option is mandatory (source name as Abell85)
41#action: gain|mspec (gain: gain-like doucble median filtering
42#                   mspec: mean+sigma wo filetring)
43action="gain"
44sourceRadio=
45dateSelected=
46typeofproc=
47#use -sim option to simulate processing (debug the script as if...)
48simulationMode=Off
49#
50#first and last cycle to process
51#
52firstCycle=
53lastCycle=
54while [ $# -gt 0 ]
55do
56  case "$1" in
57      -act)  action=$2;         shift;;
58      -src)  sourceRadio=$2;    shift;;
59      -date) dateSelected=$2;   shift;;
60      -type) typeofproc=$2;     shift;;
61      -fcycle) firstCycle=$2;   shift;;
62      -lcycle) lastCycle=$2;    shift;;
63      -sim)  simulationMode=On;;
64        -h)
65            echo >&2 \
66            "usage: $0 -src souce -date YYYYMMDD -type type [-fcycle firstCycle] [-lcycle lastCycle] [-sim to trig simulation mode]"
67            exit 1;;
68        *)  break;;     # terminate while loop
69    esac
70    shift
71done
72
73#Check input
74
75if [ "<${sourceRadio}>" = "<>" ]; then
76    $ECHO "FATAL: You have forgotten to select the source option (-src)"
77    exit 1
78fi
79
80if [ "<${dateSelected}>" = "<>" ]; then
81    $ECHO "FATAL: You have forgotten to select the date option (-date)"
82    exit 1
83fi
84
85if [ "<${typeofproc}>" = "<>" ]; then
86    $ECHO "FATAL: You have forgotten to select the type option (-type)"
87    exit 1
88fi
89
90
91
92srclower=`${ECHO} ${sourceRadio} | ${TR} "[:upper:]" "[:lower:]" `
93
94
95#Path to public backupable path
96publicpath="/afs/in2p3.fr/home/c/campagne/public"
97
98#temporary files to synchronize scripts
99tmppublicpath=${TMPPUBLICPATH}
100$LS -l ${tmppublicpath} > /dev/null
101
102
103#Path where the job will do temporary IO
104if [ ${ENVIRONMENT} = "INTERACTIVE" ]; then
105    iojobpath="/sps/baoradio/AmasNancay/JEC"
106    $MKDIR -p $iojobpath
107elif [ ${ENVIRONMENT} = "BATCH" ] ; then
108    iojobpath=${TMPBATCH}
109else
110    $ECHO "FATAL (${scriptName}): environment is ${ENVIRONMENT} not allowed"
111    touch ${tmppublicpath}/proc_script.finished
112    exit 1
113fi
114cd ${iojobpath}
115
116localpath="${sourceRadio}/${dateSelected}${srclower}"
117$MKDIR -p ./${localpath}
118cd ./${localpath}
119mainDirectory=`pwd`
120
121
122#Path of the utility scripts
123scriptpath="/afs/in2p3.fr/home/c/campagne/private/work/AmasNancay"
124
125
126#select file per cycle
127#$ECHO "Select files process..."
128#keep only lines with structure: cycle_number first,last,step
129#Get sca files
130$ECHO "DEBUG (${scriptName}): START call getscafiles.sh ${sourceRadio} ${dateSelected}"
131OUT1=./getScaStatus.$$
132#JEC 29/9/11 avoid finished stuff
133#$RM -f ${tmppublicpath}/getscafiles.finished
134${scriptpath}/getscafiles.sh ${sourceRadio} ${dateSelected} > ${OUT1} 2>&1
135#while [ ! -f "${tmppublicpath}/getscafiles.finished" ]; do
136#    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getscafiles.finished"
137#    date +%T
138#    sleep 30
139#done
140#$RM -f ${tmppublicpath}/getscafiles.finished
141$ECHO "DEBUG (${scriptName}): END"
142
143
144getScaStatus=`$GREP "^\(FATAL\|ERROR\)" ${OUT1}`
145
146if [ "<${getScaStatus}>" != "<>" ]; then
147  $ECHO "FATAL (${scriptName}): error to get sca file for $sourceRadio $dateSelected"
148  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
149  $CAT ${OUT1}
150  $ECHO "DEBUG (${scriptName}): END"
151#JEC 29/9/11 avoid finished stuff
152#  touch ${tmppublicpath}/proc_script.finished
153  exit 1
154fi
155
156
157scaFileSummarized=`$FIND . -name "sca*.sum" -print`
158if [ "<${scaFileSummarized}>" = "<>" ]; then
159  $ECHO "FATAL (${scriptName}): strange sca file summarized not available: $sourceRadio $dateSelected"
160  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
161  $CAT ${OUT1}
162  $ECHO "DEBUG (${scriptName}): END"
163#JEC 29/9/11 avoid finished stuff
164#  touch ${tmppublicpath}/proc_script.finished
165  exit 1
166fi
167$RM ${OUT1}
168
169scaFileSummarized=`basename $scaFileSummarized`
170
171#set Irods environment
172. /usr/local/shared/bin/irods_env.sh -noverbose
173#download header files to be used for signal files selction
174$MKDIR -p ./Fiber1
175dirFiberfile=`pwd`/Fiber1
176
177dirIrods="/baoradio/data/AmasNancay/${sourceRadio}/${dateSelected}${srclower}/Fiber1"
178
179listOfFiles=( `ils ${dirIrods} | $GREP -i "header" | $XARGS -i basename {} | $SORT -k1.7n` )
180IFS='
181'
182listOfFiles=( $( $PRINTF "%s\n" "${listOfFiles[@]}" | $AWK 'x[$0]++ == 0' ) )
183IFS=$DefaultIFS
184
185#
186$ECHO "DEBUG (${scriptName}): Download if necessary the headers"
187#
188for ifile in ${listOfFiles[@]}
189do
190#test if the headers have not yet been downloaded
191  if [ ! -e ${dirFiberfile}/${ifile} ]; then
192      OUT2=./getheaders.$$
193      iget -f -K ${dirIrods}/${ifile} ${dirFiberfile} >${OUT2} 2>&1
194      igetStatus=`$GREP "^ERROR" ${OUT2}`
195      if [ "<$igetStatus>" != "<>" ]; then
196          $ECHO "FATAL (${scriptName}): error while iget:"
197          $ECHO $igetStatus
198#JEC 29/9/11 avoid finished stuff
199#         touch ${tmppublicpath}/proc_script.finished
200          exit 1
201      fi
202      $RM -f $OUT2
203  fi
204done
205#
206$ECHO "DEBUG (${scriptName}): START call select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc}"
207#
208#Get list of file to process
209headerFileSelected=./selectSignal.$$
210#JEC 29/9/11 avoid finished stuff
211#$RM -f ${tmppublicpath}/select.finished
212${scriptpath}/select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc} | $GREP "^#[0-9]" | $SED "s/^#//" > $headerFileSelected
213#while [ ! -f "${tmppublicpath}/select.finished" ]; do
214#    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/select.finished"
215#    date +%T
216#    sleep 30
217#done
218#$RM ${tmppublicpath}/select.finished
219#
220$ECHO "DEBUG (${scriptName}): END"
221#
222$ECHO "DEBUG (${scriptName}): START filling cycleArray, filesArray"
223#
224declare -a cycleArray
225declare -a filesArray
226$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
227IFS=$DefaultIFS
228$ECHO "DEBUG (${scriptName}): START cat $headerFileSelected"
229$CAT $headerFileSelected
230$ECHO "DEBUG (${scriptName}): END cat $headerFileSelected"
231
232while read v1 v2
233do
234 cycleArray[${#cycleArray[*]}]=$v1
235 filesArray[${#filesArray[*]}]=$v2
236done < $headerFileSelected
237
238if [ ${#cycleArray[*]} -eq 0 -o ${#filesArray[*]} -eq 0 ]; then
239   $ECHO "FATAL (${scriptName}): no header file selected (${#cycleArray[*]}, ${#filesArray[*]})"
240#JEC 29/9/11 avoid finished stuff
241#   touch ${tmppublicpath}/proc_script.finished
242   exit 1
243else
244   $ECHO "DEBUG (${scriptName}): non zero elements found for cycleArray and filesArray" 
245fi
246
247#
248$ECHO "DEBUG (${scriptName}): number of elements <${#cycleArray[*]}> <${#filesArray[*]}>"
249$ECHO "DEBUG (${scriptName}): END"
250#
251# (timeShift)
252#Add 3sec to start ON / OFF
253#Substract 4sec to start DAB ON / OFF
254#
255#Take care: Cycle numbering starts at 1 in SCAFiles while Arrays index start at 0
256#
257#for GAIN the reference cycle takes the middle of the run
258#
259#nwinmean donne le nombre de fenetre tspwin utilise pour faire calculer la moyenne des medianes de chaque fenetre
260#This is the nimber of paquets of the time window which change according to processing type
261$ECHO "DEBUG (${scriptName}): START switch on $typeofproc"
262#
263case "$typeofproc" in
264    GAIN)
265      $ECHO "process signal files for GAIN phase"
266      ingain=""
267      freqfilter="-freqfilter -"
268      nwinmean="1"
269      tspwin="5120"
270      dirName="Off"
271      cyclebasename="gaincycle"
272      indexStartHour="6"
273      timeShift="-4"
274      timeDuration="14"
275      firstCycleId=`expr  ${#cycleArray[*]} / 2`
276      firstCycleId=`expr ${firstCycleId} - 1`
277      lastp1CycleId=`expr ${firstCycleId} + 1`
278      ;;
279    ON)
280      $ECHO "process signal files for ON phases"
281      ingain="-gain gain_${dateSelected}_${srclower}.fits"
282      freqfilter="-freqfilter 16"
283      nwinmean="5"
284      tspwin="5120"
285      dirName="On"
286      cyclebasename="datacycle"
287      indexStartHour="4"
288      timeShift="3"
289      timeDuration="30"
290      firstCycleId="0"
291      lastp1CycleId="${#cycleArray[@]}"
292      ;;
293    OFF)
294      $ECHO "process signal files for OFF phases"
295      ingain="-gain gain_${dateSelected}_${srclower}.fits"
296      freqfilter="-freqfilter 16"
297      nwinmean="5"
298      tspwin="5120"
299      dirName="Off"
300      cyclebasename="datacycle"
301      indexStartHour="8"
302      timeShift="3"
303      timeDuration="30"
304      firstCycleId="0"
305      lastp1CycleId="${#cycleArray[@]}"
306      ;;
307    MEANOFF)
308      $ECHO "process signal files for OFF phases (MEAN)"
309      ingain="-gain gain_${dateSelected}_${srclower}.fits"
310      freqfilter="-freqfilter 16"
311      nwinmean="5"
312      tspwin="5120"
313      dirName="Off"
314      cyclebasename="meancycle"
315      indexStartHour="8"
316      timeShift="3"
317      timeDuration="30"
318      firstCycleId="0"
319      lastp1CycleId="${#cycleArray[@]}"
320      ;;
321    MEANON)
322      $ECHO "process signal files for OFF phases (MEAN)"
323      ingain="-gain gain_${dateSelected}_${srclower}.fits"
324      freqfilter="-freqfilter 16"
325      nwinmean="5"
326      tspwin="5120"
327      dirName="On"
328      cyclebasename="meancycle"
329      indexStartHour="8"
330      timeShift="3"
331      timeDuration="30"
332      firstCycleId="0"
333      lastp1CycleId="${#cycleArray[@]}"
334      ;;
335    CALIBON)
336      $ECHO "process signal files for CALIBRATION ON phases"
337      ingain="-gain gain_${dateSelected}_${srclower}.fits"
338      freqfilter="-freqfilter 16"
339      nwinmean="1"
340      tspwin="1024"
341      dirName="On"
342      cyclebasename="calibcycle"
343      indexStartHour="2"
344      timeShift="-4"
345      timeDuration="14"
346      firstCycleId="0"
347      lastp1CycleId="${#cycleArray[@]}"
348      ;;
349    CALIBOFF)
350      $ECHO "process signal files for CALIBRATION OFF phases"
351      ingain="-gain gain_${dateSelected}_${srclower}.fits"
352      freqfilter="-freqfilter 16"
353      nwinmean="1"
354      tspwin="1024"
355      dirName="Off"
356      cyclebasename="calibcycle"
357      indexStartHour="6"
358      timeShift="-4"
359      timeDuration="14"
360      firstCycleId="0"
361      lastp1CycleId="${#cycleArray[@]}"
362      ;;
363    *)
364    $ECHO "FATAL (${scriptName}): type of processing"
365    usage
366#JEC 29/9/11 avoid finished stuff
367#    touch ${tmppublicpath}/proc_script.finished
368    exit 1
369    ;;
370esac
371#
372$ECHO "DEBUG (${scriptName}): END"
373#
374#
375#Compute time windows (Warning usage of GMT date)
376#
377$ECHO "DEBUG (${scriptName}): START compute time windows"
378OUT=./baotiming.$$
379${AWK} -v datesys=${DATE} -v type=${indexStartHour} -v shift=${timeShift} '
380BEGIN {FS=","}
381{
382  phaseStart=$type
383  cmd=sprintf("%s +%%s -d  \"%s\" ",datesys,phaseStart);
384  (cmd | getline phstart);
385  close(cmd);
386  newstart = phstart+shift;
387  cmd=sprintf("%s +%%T -d \"1970-01-01 %s sec GMT\"",datesys,newstart);
388  (cmd | getline newStart);
389  close(cmd);
390  print $1,newStart,phaseStart;
391}
392'  ${scaFileSummarized} > ${OUT}
393#
394$ECHO "DEBUG (${scriptName}): END"
395#
396$ECHO "DEBUG (${scriptName}): START cat ${OUT}"
397$CAT ${OUT}
398$ECHO "DEBUG (${scriptName}): END cat ${OUT}"
399#
400$ECHO "DEBUG (${scriptName}): START filling cylceArrayBis timeArray"
401#
402declare -a cycleArrayBis
403declare -a timeArray
404$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
405IFS=$DefaultIFS
406#select the selected cycles
407idtofind="0"
408while read v1 v2 v3
409do
410 if [ ${idtofind} -lt ${#cycleArray[*]} ]; then
411    if [ ${v1} = ${cycleArray[${idtofind}]} ]; then
412        cycleArrayBis[${#cycleArrayBis[*]}]=$v1
413        timeArray[${#timeArray[*]}]=$v2
414        idtofind=`expr ${idtofind} + 1`
415    fi
416 fi
417done < ${OUT}
418$ECHO "DEBUG (${scriptName}): number of elements ${#cycleArrayBis[*]} ${#timeArray[*]}"
419$ECHO "DEBUG (${scriptName}): END"
420
421
422#########################
423# specmfib exe
424#########################
425prg="/afs/in2p3.fr/throng/baoradio/Library/Progs/TAcq/Objs/specmfib"
426if [ ! -e $prg ]; then
427    $ECHO "FATAL (${scriptName}): $prg not found"
428#JEC 29/9/11 avoid finished stuff
429#    touch ${tmppublicpath}/proc_script.finished
430    exit 1
431fi
432
433
434dirFiberfile=${iojobpath}/${localpath}
435
436curDir=`pwd`
437#
438$ECHO "DEBUG (${scriptName}): in principle $dirFiberfile == $curDir "
439#
440#force single channel
441forceSingle="-singlechan"
442#debug
443prtlevel="1"
444
445
446#Loop on cycles
447##
448$ECHO "DEBUG (${scriptName}): START loop on cycles Ids ${firstCycleId} to ${lastp1CycleId}"
449
450#write the script to be send to Batch
451#get gain file if necessary
452if [ ${typeofproc} != "GAIN" ]; then
453    OUT2=./igetStatus.$$
454    iget /baoradio/data/AmasNancay/${localpath}/gain_${dateSelected}_${srclower}.fits >${OUT2} 2>&1
455    igetStatus=`$GREP "^ERROR" ${OUT2}`
456    if [ "<${igetStatus}>" != "<>" ]; then
457        $ECHO "FATAL (${scriptName}): error while iget:"
458        $ECHO $igetStatus
459#JEC 29/9/11 avoid finished stuff
460#       touch ${tmppublicpath}/${jobBatchName}.finished
461        exit 1
462    fi
463    $RM ${OUT2}
464fi   
465
466
467for ((i=${firstCycleId};i<${lastp1CycleId};i++)); do
468#
469
470####
471# Check daq status
472#JEC 1/10/11 Use generic baodaqstatus name
473####
474#get the daq current irod status
475#tag=`${DATE} +%F`
476#OUT1=${publicpath}/baodaqstatus-${tag}.txt
477OUT1=${publicpath}/baodaqstatus-current.txt
478
479if [ ! -e ${OUT1}  -o ! -r ${OUT1} ]; then
480    $ECHO "FATAL (${scriptName}): ${OUT1} has a problem"
481    exit 1
482#    $ECHO "We should bring the DAQ status more up to date, this take 1 or 2min..."
483#JEC 29/9/11 avoid finished stuff
484#    $RM -f ${tmppublicpath}/statusdaq.finished
485#    ${scriptpath}/statusdaq.sh > ${OUT1}
486#    while [ ! -f "${tmppublicpath}/statusdaq.finished" ]; do
487#       sleep 30
488#    done
489#    $RM ${tmppublicpath}/statusdaq.finished
490#protect against remove/rewriting   
491#    $CHMOD -v 444 ${OUT1}
492fi
493
494#time window selection
495    tmproc="-tmproc ${timeArray[${i}]},${timeDuration}"
496#cycle <number>
497    cycle="${cycleArray[${i}]}"
498    if [ "<${firstCycle}>" != "<>" -a "<${lastCycle}>" != "<>" ]; then
499        if [ ${cycle} -lt ${firstCycle} -o ${cycle} -gt ${lastCycle} ]; then
500            continue
501        fi
502    fi
503    $ECHO "DEBUG (${scriptName}): process cycle $cycle"
504
505#cycle <name><number>
506    cycle="${cyclebasename}${cycleArray[${i}]}"
507#signal files selection
508    infiles="${filesArray[${i}]}"
509#
510#Prepare BQS parameters: MEMORY, CPU...#
511    infilesArray=( `$ECHO $infiles | $AWK 'BEGIN {FS=","} {for(i=1;i<NF;i++)print $i;}'` )
512    IFS='
513    '
514    infilesArray=( $( printf "%s\n" "${infilesArray[@]}" | awk 'x[$0]++ == 0' ) )
515    IFS=$DefaultIFS
516    firstFile=${infilesArray[0]}
517    lastFile=${infilesArray[1]}
518#
519#
520#
521#get signals files
522    OUT3=./getsignals.$$
523#JEC 29/9/11 avoid finished stuff
524#    $RM -f ${tmppublicpath}/getsignalfiles.finished
525    ${scriptpath}/getsignalfiles.sh ${sourceRadio} ${dateSelected} ${firstFile}  ${lastFile} > ${OUT3} 2>&1
526#    while [ ! -f "${tmppublicpath}/getsignalfiles.finished" ]; do
527#       $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getsignalfiles.finished"
528#       date +%T
529#       sleep 30
530#    done
531#    $RM ${tmppublicpath}/getsignalfiles.finished
532
533    getSignals=`$GREP "^FATAL" ${OUT3}`
534    if [ "<$getSignals>" != "<>" ]; then
535        $ECHO "FATAL (${scriptName}): error while get signals"
536        $ECHO $getSignals
537#JEC 29/9/11 avoid finished stuff
538#       touch ${tmppublicpath}/${jobBatchName}.finished
539        exit 1
540    fi
541    $RM ${OUT3}
542
543#location of the signal file
544    fibfile="${curDir} 1,2"
545#
546    outDir=${curDir}/${dirName}/${cycle}
547    $MKDIR -p ${outDir}
548
549#
550# where ami...
551#
552    $ECHO "We are here `pwd`"
553    $LS -lrth
554    $FIND . -name "igetStatus.*" | $XARGS -i  $CAT {}
555    $LS ./Fiber1 | $WC -l
556    $LS ./Fiber2 | $WC -l
557    $LS -lR ${outDir}
558
559#Note: here act=gain means only that specmfib will use median-like algorithms (not only for gains...)
560    $prg -act gain $forceSingle $freqfilter  -prt $prtlevel -out ${outDir} -nmean $nwinmean $ingain $tmproc -tspwin ${tspwin},0,0 -in $infiles ${fibfile}
561
562    $ECHO    ${outDir}
563    $LS -lrt ${outDir}
564
565#save results to Irods (use option -f ONLY to force override)
566    iput -v -K -r ${curDir}/${dirName} /baoradio/data/AmasNancay/${localpath}
567#
568#clean the signal files which can at the end of the day fill completly the scratch directory of the batch work/AmasNancayer
569    $RM -rf ${iojobpath}/${localpath}/Fiber1
570    $RM -rf ${iojobpath}/${localpath}/Fiber2
571    $RM -rf ${iojobpath}/${localpath}/${typeofproc}
572
573#
574#
575
576#end loop on cycles
577done
578#
579$ECHO "DEBUG (${scriptName}): END loop on cycles"
580#
581#JEC 29/9/11 avoid finished stuff
582#touch ${tmppublicpath}/proc_script.finished
583exit 0
Note: See TracBrowser for help on using the repository browser.