source: BAORadio/AmasNancay/v4/proc_specmfib.sh @ 635

Last change on this file since 635 was 575, checked in by campagne, 13 years ago

introduce SCRIPTPATH environment variable (jec)
  • Property svn:executable set to *
File size: 16.3 KB
Line 
1#!/bin/sh -xvf
2#Basic tuning of system fuction used.
3AWK=/bin/awk
4SED=/bin/sed
5GREP=/bin/grep
6WC=/usr/bin/wc
7RM=/bin/rm
8CAT=/bin/cat
9TOUCH=/bin/touch
10DATE=/bin/date
11ECHO=/bin/echo
12LS=/bin/ls
13MKDIR=/bin/mkdir
14TR=/usr/bin/tr
15FIND=/usr/bin/find
16PRINTF=/usr/bin/printf
17XARGS=/usr/bin/xargs
18SORT=/bin/sort
19QSUB=qsub
20
21DefaultIFS=$' \t\n'
22IFS=$DefaultIFS
23
24
25scriptName="`basename $0`"
26$ECHO "Processing script ${scriptName} at `date`"
27######
28# Local functions
29######
30usage() {
31  echo "usage: $scriptName <source> <YYYYMMDD-directory> <type of process>"
32  echo "      -<source> as Abell85"
33  echo "      -<YYYYMMDD-directory> as 20110428"
34  echo "      -<type of process> as GAIN|CALIBON|CALIBOFF|ON|OFF"
35}
36######
37# argument test
38######
39
40#Process cmd line args: the -src option is mandatory (source name as Abell85)
41#action: gain|mspec (gain: gain-like doucble median filtering
42#                   mspec: mean+sigma wo filetring)
43action="gain"
44sourceRadio=
45dateSelected=
46typeofproc=
47#use -sim option to simulate processing (debug the script as if...)
48simulationMode=Off
49#
50#first and last cycle to process
51#
52firstCycle=
53lastCycle=
54while [ $# -gt 0 ]
55do
56  case "$1" in
57      -act)  action=$2;         shift;;
58      -src)  sourceRadio=$2;    shift;;
59      -date) dateSelected=$2;   shift;;
60      -type) typeofproc=$2;     shift;;
61      -fcycle) firstCycle=$2;   shift;;
62      -lcycle) lastCycle=$2;    shift;;
63      -sim)  simulationMode=On;;
64        -h)
65            echo >&2 \
66            "usage: $0 [-act <action> ] -src souce -date YYYYMMDD -type type [-fcycle firstCycle] [-lcycle lastCycle] [-sim to trig simulation mode]"
67            exit 1;;
68        *)  break;;     # terminate while loop
69    esac
70    shift
71done
72
73#Check input
74
75if [ "<${sourceRadio}>" = "<>" ]; then
76    $ECHO "FATAL: You have forgotten to select the source option (-src)"
77    exit 1
78fi
79
80if [ "<${dateSelected}>" = "<>" ]; then
81    $ECHO "FATAL: You have forgotten to select the date option (-date)"
82    exit 1
83fi
84
85if [ "<${typeofproc}>" = "<>" ]; then
86    $ECHO "FATAL: You have forgotten to select the type option (-type)"
87    exit 1
88
89fi
90#else
91#    offtype=`$ECHO ${typeofproc} | $GREP -i OFF | $WC -l`
92#    ontype=`$ECHO ${typeofproc} | $GREP -i ON | $WC -l`
93#    if [ "<${offtype}>" = "<1>" -a  "<${ontype}>" = "<0>" ]; then
94#       datatype="OFF"
95#    elif [ "<${offtype}>" = "<0>" -a  "<${ontype}>" = "<1>" ]; then
96#       datatype="ON"
97#    else
98#       $ECHO "FATAL: not supported: -type ${typeofproc}"
99#       exit 1
100#    fi
101#fi
102
103
104srclower=`${ECHO} ${sourceRadio} | ${TR} "[:upper:]" "[:lower:]" `
105
106
107#Path to public backupable path
108publicpath="/afs/in2p3.fr/home/c/campagne/public"
109
110#temporary files to synchronize scripts
111tmppublicpath=${TMPPUBLICPATH}
112$LS -l ${tmppublicpath} > /dev/null
113
114
115#Path where the job will do temporary IO
116if [ ${ENVIRONMENT} = "INTERACTIVE" ]; then
117    iojobpath="/sps/baoradio/AmasNancay/JEC"
118    $MKDIR -p $iojobpath
119elif [ ${ENVIRONMENT} = "BATCH" ] ; then
120    iojobpath=${TMPBATCH}
121else
122    $ECHO "FATAL (${scriptName}): environment is ${ENVIRONMENT} not allowed"
123    touch ${tmppublicpath}/proc_script.finished
124    exit 1
125fi
126cd ${iojobpath}
127
128localpath="${sourceRadio}/${dateSelected}${srclower}"
129$MKDIR -p ./${localpath}
130cd ./${localpath}
131mainDirectory=`pwd`
132
133
134#Path of the utility scripts
135scriptpath=${SCRIPTPATH}
136
137
138#select file per cycle
139#$ECHO "Select files process..."
140#keep only lines with structure: cycle_number first,last,step
141#Get sca files
142$ECHO "DEBUG (${scriptName}): START call getscafiles.sh ${sourceRadio} ${dateSelected}"
143OUT1=./getScaStatus.$$
144#JEC 29/9/11 avoid finished stuff
145#$RM -f ${tmppublicpath}/getscafiles.finished
146${scriptpath}/getscafiles.sh ${sourceRadio} ${dateSelected} > ${OUT1} 2>&1
147#while [ ! -f "${tmppublicpath}/getscafiles.finished" ]; do
148#    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getscafiles.finished"
149#    date +%T
150#    sleep 30
151#done
152#$RM -f ${tmppublicpath}/getscafiles.finished
153$ECHO "DEBUG (${scriptName}): END"
154
155
156getScaStatus=`$GREP "^\(FATAL\|ERROR\)" ${OUT1}`
157
158if [ "<${getScaStatus}>" != "<>" ]; then
159  $ECHO "FATAL (${scriptName}): error to get sca file for $sourceRadio $dateSelected"
160  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
161  $CAT ${OUT1}
162  $ECHO "DEBUG (${scriptName}): END"
163#JEC 29/9/11 avoid finished stuff
164#  touch ${tmppublicpath}/proc_script.finished
165  exit 1
166fi
167
168
169scaFileSummarized=`$FIND . -name "sca*.sum" -print`
170if [ "<${scaFileSummarized}>" = "<>" ]; then
171  $ECHO "FATAL (${scriptName}): strange sca file summarized not available: $sourceRadio $dateSelected"
172  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
173  $CAT ${OUT1}
174  $ECHO "DEBUG (${scriptName}): END"
175#JEC 29/9/11 avoid finished stuff
176#  touch ${tmppublicpath}/proc_script.finished
177  exit 1
178fi
179$RM ${OUT1}
180
181scaFileSummarized=`basename $scaFileSummarized`
182
183#set Irods environment
184. /usr/local/shared/bin/irods_env.sh -noverbose
185#download header files to be used for signal files selction
186$MKDIR -p ./Fiber1
187dirFiberfile=`pwd`/Fiber1
188
189dirIrods="/baoradio/data/AmasNancay/${sourceRadio}/${dateSelected}${srclower}/Fiber1"
190
191listOfFiles=( `ils ${dirIrods} | $GREP -i "header" | $XARGS -i basename {} | $SORT -k1.7n` )
192IFS='
193'
194listOfFiles=( $( $PRINTF "%s\n" "${listOfFiles[@]}" | $AWK 'x[$0]++ == 0' ) )
195IFS=$DefaultIFS
196
197#
198$ECHO "DEBUG (${scriptName}): Download if necessary the headers"
199#
200for ifile in ${listOfFiles[@]}
201do
202#test if the headers have not yet been downloaded
203  if [ ! -e ${dirFiberfile}/${ifile} ]; then
204      OUT2=./getheaders.$$
205      iget -f -K ${dirIrods}/${ifile} ${dirFiberfile} >${OUT2} 2>&1
206      igetStatus=`$GREP "^ERROR" ${OUT2}`
207      if [ "<$igetStatus>" != "<>" ]; then
208          $ECHO "FATAL (${scriptName}): error while iget:"
209          $ECHO $igetStatus
210#JEC 29/9/11 avoid finished stuff
211#         touch ${tmppublicpath}/proc_script.finished
212          exit 1
213      fi
214      $RM -f $OUT2
215  fi
216done
217#
218$ECHO "DEBUG (${scriptName}): START call select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc}"
219#
220#Get list of file to process
221headerFileSelected=./selectSignal.$$
222#JEC 29/9/11 avoid finished stuff
223#$RM -f ${tmppublicpath}/select.finished
224${scriptpath}/select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc} | $GREP "^#[0-9]" | $SED "s/^#//" > $headerFileSelected
225#while [ ! -f "${tmppublicpath}/select.finished" ]; do
226#    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/select.finished"
227#    date +%T
228#    sleep 30
229#done
230#$RM ${tmppublicpath}/select.finished
231#
232$ECHO "DEBUG (${scriptName}): END"
233#
234$ECHO "DEBUG (${scriptName}): START filling cycleArray, filesArray"
235#
236declare -a cycleArray
237declare -a filesArray
238$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
239IFS=$DefaultIFS
240$ECHO "DEBUG (${scriptName}): START cat $headerFileSelected"
241$CAT $headerFileSelected
242$ECHO "DEBUG (${scriptName}): END cat $headerFileSelected"
243
244while read v1 v2
245do
246 cycleArray[${#cycleArray[*]}]=$v1
247 filesArray[${#filesArray[*]}]=$v2
248done < $headerFileSelected
249
250if [ ${#cycleArray[*]} -eq 0 -o ${#filesArray[*]} -eq 0 ]; then
251   $ECHO "FATAL (${scriptName}): no header file selected (${#cycleArray[*]}, ${#filesArray[*]})"
252#JEC 29/9/11 avoid finished stuff
253#   touch ${tmppublicpath}/proc_script.finished
254   exit 1
255else
256   $ECHO "DEBUG (${scriptName}): non zero elements found for cycleArray and filesArray" 
257fi
258
259#
260$ECHO "DEBUG (${scriptName}): number of elements <${#cycleArray[*]}> <${#filesArray[*]}>"
261$ECHO "DEBUG (${scriptName}): END"
262#
263# (timeShift)
264#Add 3sec to start ON / OFF
265#Substract 4sec to start DAB ON / OFF
266#
267#Take care: Cycle numbering starts at 1 in SCAFiles while Arrays index start at 0
268#
269#for GAIN the reference cycle takes the middle of the run
270#
271#nwinmean donne le nombre de fenetre tspwin utilise pour faire calculer la moyenne des medianes de chaque fenetre
272#This is the nimber of paquets of the time window which change according to processing type
273$ECHO "DEBUG (${scriptName}): START switch on $typeofproc"
274#
275case "$typeofproc" in
276    GAIN)
277      $ECHO "process signal files for GAIN phase"
278      ingain=""
279      freqfilter="-freqfilter -"
280      nwinmean="1"
281      tspwin="5120"
282      dirName="Off"
283      cyclebasename="gaincycle"
284      indexStartHour="6"
285      timeShift="-4"
286      timeDuration="14"
287      firstCycleId=`expr  ${#cycleArray[*]} / 2`
288      firstCycleId=`expr ${firstCycleId} - 1`
289      lastp1CycleId=`expr ${firstCycleId} + 1`
290      ;;
291    ON)
292      $ECHO "process signal files for ON phases"
293      ingain="-gain gain_${dateSelected}_${srclower}.fits"
294      freqfilter=""
295      nwinmean="5"
296      tspwin="5120"
297      dirName="On"
298      cyclebasename="datacycle"
299      indexStartHour="4"
300      timeShift="3"
301      timeDuration="30"
302      firstCycleId="0"
303      lastp1CycleId="${#cycleArray[@]}"
304      ;;
305    OFF)
306      $ECHO "process signal files for OFF phases"
307      ingain="-gain gain_${dateSelected}_${srclower}.fits"
308      freqfilter=""
309      nwinmean="5"
310      tspwin="5120"
311      dirName="Off"
312      cyclebasename="datacycle"
313      indexStartHour="8"
314      timeShift="3"
315      timeDuration="30"
316      firstCycleId="0"
317      lastp1CycleId="${#cycleArray[@]}"
318      ;;
319    MEANOFF)
320      $ECHO "process signal files for OFF phases (MEAN)"
321      ingain="-gain gain_${dateSelected}_${srclower}.fits"
322      nwinmean="25000"
323      tspwin="1"
324      dirName="Off"
325      cyclebasename="meancycle"
326      indexStartHour="8"
327      timeShift="3"
328      timeDuration="30"
329      firstCycleId="0"
330      lastp1CycleId="${#cycleArray[@]}"
331      ;;
332    MEANON)
333      $ECHO "process signal files for OFF phases (MEAN)"
334      ingain="-gain gain_${dateSelected}_${srclower}.fits"
335      nwinmean="25000"
336      tspwin="1"
337      dirName="On"
338      cyclebasename="meancycle"
339      indexStartHour="8"
340      timeShift="3"
341      timeDuration="30"
342      firstCycleId="0"
343      lastp1CycleId="${#cycleArray[@]}"
344      ;;
345    CALIBON)
346      $ECHO "process signal files for CALIBRATION ON phases"
347      ingain="-gain gain_${dateSelected}_${srclower}.fits"
348      freqfilter=""
349      nwinmean="1"
350      tspwin="1024"
351      dirName="On"
352      cyclebasename="calibcycle"
353      indexStartHour="2"
354      timeShift="-4"
355      timeDuration="14"
356      firstCycleId="0"
357      lastp1CycleId="${#cycleArray[@]}"
358      ;;
359    CALIBOFF)
360      $ECHO "process signal files for CALIBRATION OFF phases"
361      ingain="-gain gain_${dateSelected}_${srclower}.fits"
362      freqfilter=""
363      nwinmean="1"
364      tspwin="1024"
365      dirName="Off"
366      cyclebasename="calibcycle"
367      indexStartHour="6"
368      timeShift="-4"
369      timeDuration="14"
370      firstCycleId="0"
371      lastp1CycleId="${#cycleArray[@]}"
372      ;;
373    *)
374    $ECHO "FATAL (${scriptName}): type of processing"
375    usage
376#JEC 29/9/11 avoid finished stuff
377#    touch ${tmppublicpath}/proc_script.finished
378    exit 1
379    ;;
380esac
381#
382$ECHO "DEBUG (${scriptName}): END"
383#
384#
385#Compute time windows (Warning usage of GMT date)
386#
387$ECHO "DEBUG (${scriptName}): START compute time windows"
388OUT=./baotiming.$$
389${AWK} -v datesys=${DATE} -v type=${indexStartHour} -v shift=${timeShift} '
390BEGIN {FS=","}
391{
392  phaseStart=$type
393  cmd=sprintf("%s +%%s -d  \"%s\" ",datesys,phaseStart);
394  (cmd | getline phstart);
395  close(cmd);
396  newstart = phstart+shift;
397  cmd=sprintf("%s +%%T -d \"1970-01-01 %s sec GMT\"",datesys,newstart);
398  (cmd | getline newStart);
399  close(cmd);
400  print $1,newStart,phaseStart;
401}
402'  ${scaFileSummarized} > ${OUT}
403#
404$ECHO "DEBUG (${scriptName}): END"
405#
406$ECHO "DEBUG (${scriptName}): START cat ${OUT}"
407$CAT ${OUT}
408$ECHO "DEBUG (${scriptName}): END cat ${OUT}"
409#
410$ECHO "DEBUG (${scriptName}): START filling cylceArrayBis timeArray"
411#
412declare -a cycleArrayBis
413declare -a timeArray
414$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
415IFS=$DefaultIFS
416#select the selected cycles
417idtofind="0"
418while read v1 v2 v3
419do
420 if [ ${idtofind} -lt ${#cycleArray[*]} ]; then
421    if [ ${v1} = ${cycleArray[${idtofind}]} ]; then
422        cycleArrayBis[${#cycleArrayBis[*]}]=$v1
423        timeArray[${#timeArray[*]}]=$v2
424        idtofind=`expr ${idtofind} + 1`
425    fi
426 fi
427done < ${OUT}
428$ECHO "DEBUG (${scriptName}): number of elements ${#cycleArrayBis[*]} ${#timeArray[*]}"
429$ECHO "DEBUG (${scriptName}): END"
430
431
432#########################
433# specmfib exe
434#########################
435prg="/afs/in2p3.fr/throng/baoradio/Library/Progs/TAcq/Objs/specmfib"
436if [ ! -e $prg ]; then
437    $ECHO "FATAL (${scriptName}): $prg not found"
438#JEC 29/9/11 avoid finished stuff
439#    touch ${tmppublicpath}/proc_script.finished
440    exit 1
441fi
442
443
444dirFiberfile=${iojobpath}/${localpath}
445
446curDir=`pwd`
447#
448$ECHO "DEBUG (${scriptName}): in principle $dirFiberfile == $curDir "
449#
450#force single channel
451forceSingle="-singlechan"
452#debug
453prtlevel="1"
454
455
456#Loop on cycles
457##
458$ECHO "DEBUG (${scriptName}): START loop on cycles Ids ${firstCycleId} to ${lastp1CycleId}"
459
460#write the script to be send to Batch
461#get gain file if necessary
462if [ ${typeofproc} != "GAIN" ]; then
463    OUT2=./igetStatus.$$
464    iget /baoradio/data/AmasNancay/${localpath}/gain_${dateSelected}_${srclower}.fits >${OUT2} 2>&1
465    igetStatus=`$GREP "^ERROR" ${OUT2}`
466    if [ "<${igetStatus}>" != "<>" ]; then
467        $ECHO "FATAL (${scriptName}): error while iget:"
468        $ECHO $igetStatus
469#JEC 29/9/11 avoid finished stuff
470#       touch ${tmppublicpath}/${jobBatchName}.finished
471        exit 1
472    fi
473    $RM ${OUT2}
474fi   
475
476
477for ((i=${firstCycleId};i<${lastp1CycleId};i++)); do
478#
479
480####
481# Check daq status
482#JEC 1/10/11 Use generic baodaqstatus name
483####
484#get the daq current irod status
485#tag=`${DATE} +%F`
486#OUT1=${publicpath}/baodaqstatus-${tag}.txt
487OUT1=${publicpath}/baodaqstatus-current.txt
488
489if [ ! -e ${OUT1}  -o ! -r ${OUT1} ]; then
490    $ECHO "FATAL (${scriptName}): ${OUT1} has a problem"
491    exit 1
492#    $ECHO "We should bring the DAQ status more up to date, this take 1 or 2min..."
493#JEC 29/9/11 avoid finished stuff
494#    $RM -f ${tmppublicpath}/statusdaq.finished
495#    ${scriptpath}/statusdaq.sh > ${OUT1}
496#    while [ ! -f "${tmppublicpath}/statusdaq.finished" ]; do
497#       sleep 30
498#    done
499#    $RM ${tmppublicpath}/statusdaq.finished
500#protect against remove/rewriting   
501#    $CHMOD -v 444 ${OUT1}
502fi
503
504#time window selection
505    tmproc="-tmproc ${timeArray[${i}]},${timeDuration}"
506#cycle <number>
507    cycle="${cycleArray[${i}]}"
508    if [ "<${firstCycle}>" != "<>" -a "<${lastCycle}>" != "<>" ]; then
509        if [ ${cycle} -lt ${firstCycle} -o ${cycle} -gt ${lastCycle} ]; then
510            continue
511        fi
512    fi
513    $ECHO "DEBUG (${scriptName}): process cycle $cycle"
514
515#cycle <name><number>
516    cycle="${cyclebasename}${cycleArray[${i}]}"
517#signal files selection
518    infiles="${filesArray[${i}]}"
519#
520#Prepare BQS parameters: MEMORY, CPU...#
521    infilesArray=( `$ECHO $infiles | $AWK 'BEGIN {FS=","} {for(i=1;i<NF;i++)print $i;}'` )
522    IFS='
523    '
524    infilesArray=( $( printf "%s\n" "${infilesArray[@]}" | awk 'x[$0]++ == 0' ) )
525    IFS=$DefaultIFS
526    firstFile=${infilesArray[0]}
527    lastFile=${infilesArray[1]}
528#
529#
530#
531#get signals files
532    OUT3=./getsignals.$$
533#JEC 29/9/11 avoid finished stuff
534#    $RM -f ${tmppublicpath}/getsignalfiles.finished
535    ${scriptpath}/getsignalfiles.sh ${sourceRadio} ${dateSelected} ${firstFile}  ${lastFile} > ${OUT3} 2>&1
536#    while [ ! -f "${tmppublicpath}/getsignalfiles.finished" ]; do
537#       $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getsignalfiles.finished"
538#       date +%T
539#       sleep 30
540#    done
541#    $RM ${tmppublicpath}/getsignalfiles.finished
542
543    getSignals=`$GREP "^FATAL" ${OUT3}`
544    if [ "<$getSignals>" != "<>" ]; then
545        $ECHO "FATAL (${scriptName}): error while get signals"
546        $ECHO $getSignals
547#JEC 29/9/11 avoid finished stuff
548#       touch ${tmppublicpath}/${jobBatchName}.finished
549        exit 1
550    fi
551    $RM ${OUT3}
552
553#location of the signal file
554    fibfile="${curDir} 1,2"
555#
556    outDir=${curDir}/${dirName}/${cycle}
557    $MKDIR -p ${outDir}
558
559#
560# where ami...
561#
562    $ECHO "We are here `pwd`"
563    $LS -lrth
564    $FIND . -name "igetStatus.*" | $XARGS -i  $CAT {}
565    $LS ./Fiber1 | $WC -l
566    $LS ./Fiber2 | $WC -l
567    $LS -lR ${outDir}
568
569#Note: here act=gain means only that specmfib will use median-like algorithms (not only for gains...)
570    $prg -act ${action} $forceSingle $freqfilter  -prt $prtlevel -out ${outDir} -nmean $nwinmean $ingain $tmproc -tspwin ${tspwin},0,0 -in $infiles ${fibfile}
571
572    $ECHO    ${outDir}
573    $LS -lrt ${outDir}
574
575#save results to Irods (use option -f ONLY to force override)
576    iput -v -K -r ${curDir}/${dirName} /baoradio/data/AmasNancay/${localpath}
577#
578#clean the signal files which can at the end of the day fill completly the scratch directory of the batch work/AmasNancayer
579    $RM -rf ${iojobpath}/${localpath}/Fiber1
580    $RM -rf ${iojobpath}/${localpath}/Fiber2
581    $RM -rf ${iojobpath}/${localpath}/${typeofproc}
582
583#
584#
585
586#end loop on cycles
587done
588#
589$ECHO "DEBUG (${scriptName}): END loop on cycles"
590#
591#JEC 29/9/11 avoid finished stuff
592#touch ${tmppublicpath}/proc_script.finished
593exit 0
Note: See TracBrowser for help on using the repository browser.