source: BAORadio/AmasNancay/trunk/proc_rawonoff.sh @ 556

Last change on this file since 556 was 556, checked in by campagne, 13 years ago

generalise the process to download the files for final analysis (jec)
  • Property svn:executable set to *
File size: 16.4 KB
Line 
1#!/bin/sh -xvf
2#Basic tuning of system fuction used.
3AWK=/bin/awk
4SED=/bin/sed
5GREP=/bin/grep
6WC=/usr/bin/wc
7RM=/bin/rm
8CAT=/bin/cat
9TOUCH=/bin/touch
10DATE=/bin/date
11ECHO=/bin/echo
12LS=/bin/ls
13MKDIR=/bin/mkdir
14TR=/usr/bin/tr
15FIND=/usr/bin/find
16PRINTF=/usr/bin/printf
17XARGS=/usr/bin/xargs
18SORT=/bin/sort
19QSUB=qsub
20
21DefaultIFS=$' \t\n'
22IFS=$DefaultIFS
23
24
25scriptName="`basename $0`"
26$ECHO "Processing script ${scriptName} at `date`"
27######
28# Local functions
29######
30usage() {
31  echo "usage: $scriptName <source> <YYYYMMDD-directory> <type of process>"
32  echo "      -<source> as Abell85"
33  echo "      -<YYYYMMDD-directory> as 20110428"
34  echo "      -<type of process> as GAIN|CALIBON|CALIBOFF|ON|OFF"
35}
36######
37# argument test
38######
39
40#Process cmd line args: the -src option is mandatory (source name as Abell85)
41#action: gain|mspec (gain: gain-like doucble median filtering
42#                   mspec: mean+sigma wo filetring)
43action="gain"
44sourceRadio=
45dateSelected=
46typeofproc=
47#use -sim option to simulate processing (debug the script as if...)
48simulationMode=Off
49#
50#first and last cycle to process
51#
52firstCycle=
53lastCycle=
54while [ $# -gt 0 ]
55do
56  case "$1" in
57      -act)  action=$2;         shift;;
58      -src)  sourceRadio=$2;    shift;;
59      -date) dateSelected=$2;   shift;;
60      -type) typeofproc=$2;     shift;;
61      -fcycle) firstCycle=$2;   shift;;
62      -lcycle) lastCycle=$2;    shift;;
63      -sim)  simulationMode=On;;
64        -h)
65            echo >&2 \
66            "usage: $0 -src souce -date YYYYMMDD -type type [-fcycle firstCycle] [-lcycle lastCycle] [-sim to trig simulation mode]"
67            exit 1;;
68        *)  break;;     # terminate while loop
69    esac
70    shift
71done
72
73#Check input
74
75if [ "<${sourceRadio}>" = "<>" ]; then
76    $ECHO "FATAL: You have forgotten to select the source option (-src)"
77    exit 1
78fi
79
80if [ "<${dateSelected}>" = "<>" ]; then
81    $ECHO "FATAL: You have forgotten to select the date option (-date)"
82    exit 1
83fi
84
85if [ "<${typeofproc}>" = "<>" ]; then
86    $ECHO "FATAL: You have forgotten to select the type option (-type)"
87    exit 1
88else
89    offtype=`$ECHO ${typeofproc} | $GREP -i OFF | $WC -l`
90    ontype=`$ECHO ${typeofproc} | $GREP -i ON | $WC -l`
91    if [ "<${offtype}>" = "<1>" -a  "<${ontype}>" = "<0>" ]; then
92        datatype="OFF"
93    elif [ "<${offtype}>" = "<0>" -a  "<${ontype}>" = "<1>" ]; then
94        datatype="ON"
95    else
96        $ECHO "FATAL: not supported: -type ${typeofproc}"
97        exit 1
98    fi
99fi
100
101
102srclower=`${ECHO} ${sourceRadio} | ${TR} "[:upper:]" "[:lower:]" `
103
104
105#Path to public backupable path
106publicpath="/afs/in2p3.fr/home/c/campagne/public"
107
108#temporary files to synchronize scripts
109tmppublicpath=${TMPPUBLICPATH}
110$LS -l ${tmppublicpath} > /dev/null
111
112
113#Path where the job will do temporary IO
114if [ ${ENVIRONMENT} = "INTERACTIVE" ]; then
115    iojobpath="/sps/baoradio/AmasNancay/JEC"
116    $MKDIR -p $iojobpath
117elif [ ${ENVIRONMENT} = "BATCH" ] ; then
118    iojobpath=${TMPBATCH}
119else
120    $ECHO "FATAL (${scriptName}): environment is ${ENVIRONMENT} not allowed"
121    touch ${tmppublicpath}/proc_script.finished
122    exit 1
123fi
124cd ${iojobpath}
125
126localpath="${sourceRadio}/${dateSelected}${srclower}"
127$MKDIR -p ./${localpath}
128cd ./${localpath}
129mainDirectory=`pwd`
130
131
132#Path of the utility scripts
133scriptpath="/afs/in2p3.fr/home/c/campagne/private/work/AmasNancay"
134
135
136#select file per cycle
137#$ECHO "Select files process..."
138#keep only lines with structure: cycle_number first,last,step
139#Get sca files
140$ECHO "DEBUG (${scriptName}): START call getscafiles.sh ${sourceRadio} ${dateSelected}"
141OUT1=./getScaStatus.$$
142#JEC 29/9/11 avoid finished stuff
143#$RM -f ${tmppublicpath}/getscafiles.finished
144${scriptpath}/getscafiles.sh ${sourceRadio} ${dateSelected} > ${OUT1} 2>&1
145#while [ ! -f "${tmppublicpath}/getscafiles.finished" ]; do
146#    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getscafiles.finished"
147#    date +%T
148#    sleep 30
149#done
150#$RM -f ${tmppublicpath}/getscafiles.finished
151$ECHO "DEBUG (${scriptName}): END"
152
153
154getScaStatus=`$GREP "^\(FATAL\|ERROR\)" ${OUT1}`
155
156if [ "<${getScaStatus}>" != "<>" ]; then
157  $ECHO "FATAL (${scriptName}): error to get sca file for $sourceRadio $dateSelected"
158  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
159  $CAT ${OUT1}
160  $ECHO "DEBUG (${scriptName}): END"
161#JEC 29/9/11 avoid finished stuff
162#  touch ${tmppublicpath}/proc_script.finished
163  exit 1
164fi
165
166
167scaFileSummarized=`$FIND . -name "sca*.sum" -print`
168if [ "<${scaFileSummarized}>" = "<>" ]; then
169  $ECHO "FATAL (${scriptName}): strange sca file summarized not available: $sourceRadio $dateSelected"
170  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
171  $CAT ${OUT1}
172  $ECHO "DEBUG (${scriptName}): END"
173#JEC 29/9/11 avoid finished stuff
174#  touch ${tmppublicpath}/proc_script.finished
175  exit 1
176fi
177$RM ${OUT1}
178
179scaFileSummarized=`basename $scaFileSummarized`
180
181#set Irods environment
182. /usr/local/shared/bin/irods_env.sh -noverbose
183#download header files to be used for signal files selction
184$MKDIR -p ./Fiber1
185dirFiberfile=`pwd`/Fiber1
186
187dirIrods="/baoradio/data/AmasNancay/${sourceRadio}/${dateSelected}${srclower}/Fiber1"
188
189listOfFiles=( `ils ${dirIrods} | $GREP -i "header" | $XARGS -i basename {} | $SORT -k1.7n` )
190IFS='
191'
192listOfFiles=( $( $PRINTF "%s\n" "${listOfFiles[@]}" | $AWK 'x[$0]++ == 0' ) )
193IFS=$DefaultIFS
194
195#
196$ECHO "DEBUG (${scriptName}): Download if necessary the headers"
197#
198for ifile in ${listOfFiles[@]}
199do
200#test if the headers have not yet been downloaded
201  if [ ! -e ${dirFiberfile}/${ifile} ]; then
202      OUT2=./getheaders.$$
203      iget -f -K ${dirIrods}/${ifile} ${dirFiberfile} >${OUT2} 2>&1
204      igetStatus=`$GREP "^ERROR" ${OUT2}`
205      if [ "<$igetStatus>" != "<>" ]; then
206          $ECHO "FATAL (${scriptName}): error while iget:"
207          $ECHO $igetStatus
208#JEC 29/9/11 avoid finished stuff
209#         touch ${tmppublicpath}/proc_script.finished
210          exit 1
211      fi
212      $RM -f $OUT2
213  fi
214done
215#
216$ECHO "DEBUG (${scriptName}): START call select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc}"
217#
218#Get list of file to process
219headerFileSelected=./selectSignal.$$
220#JEC 29/9/11 avoid finished stuff
221#$RM -f ${tmppublicpath}/select.finished
222${scriptpath}/select.sh ${scaFileSummarized} ${dirFiberfile} ${datatype} | $GREP "^#[0-9]" | $SED "s/^#//" > $headerFileSelected
223#while [ ! -f "${tmppublicpath}/select.finished" ]; do
224#    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/select.finished"
225#    date +%T
226#    sleep 30
227#done
228#$RM ${tmppublicpath}/select.finished
229#
230$ECHO "DEBUG (${scriptName}): END"
231#
232$ECHO "DEBUG (${scriptName}): START filling cycleArray, filesArray"
233#
234declare -a cycleArray
235declare -a filesArray
236$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
237IFS=$DefaultIFS
238$ECHO "DEBUG (${scriptName}): START cat $headerFileSelected"
239$CAT $headerFileSelected
240$ECHO "DEBUG (${scriptName}): END cat $headerFileSelected"
241
242while read v1 v2
243do
244 cycleArray[${#cycleArray[*]}]=$v1
245 filesArray[${#filesArray[*]}]=$v2
246done < $headerFileSelected
247
248if [ ${#cycleArray[*]} -eq 0 -o ${#filesArray[*]} -eq 0 ]; then
249   $ECHO "FATAL (${scriptName}): no header file selected (${#cycleArray[*]}, ${#filesArray[*]})"
250#JEC 29/9/11 avoid finished stuff
251#   touch ${tmppublicpath}/proc_script.finished
252   exit 1
253else
254   $ECHO "DEBUG (${scriptName}): non zero elements found for cycleArray and filesArray" 
255fi
256
257#
258$ECHO "DEBUG (${scriptName}): number of elements <${#cycleArray[*]}> <${#filesArray[*]}>"
259$ECHO "DEBUG (${scriptName}): END"
260#
261# (timeShift)
262#Add 3sec to start ON / OFF
263#Substract 4sec to start DAB ON / OFF
264#
265#Take care: Cycle numbering starts at 1 in SCAFiles while Arrays index start at 0
266#
267#for GAIN the reference cycle takes the middle of the run
268#
269#nwinmean donne le nombre de fenetre tspwin utilise pour faire calculer la moyenne des medianes de chaque fenetre
270#This is the nimber of paquets of the time window which change according to processing type
271$ECHO "DEBUG (${scriptName}): START switch on $typeofproc"
272#
273case "$typeofproc" in
274    GAIN)
275      $ECHO "process signal files for GAIN phase"
276      ingain=""
277      freqfilter="-freqfilter -"
278      nwinmean="1"
279      tspwin="5120"
280      dirName="Off"
281      cyclebasename="gaincycle"
282      indexStartHour="6"
283      timeShift="-4"
284      timeDuration="14"
285      firstCycleId=`expr  ${#cycleArray[*]} / 2`
286      firstCycleId=`expr ${firstCycleId} - 1`
287      lastp1CycleId=`expr ${firstCycleId} + 1`
288      ;;
289    ON)
290      $ECHO "process signal files for ON phases"
291      ingain="-gain gain_${dateSelected}_${srclower}.fits"
292      freqfilter="-freqfilter 16"
293      nwinmean="5"
294      tspwin="5120"
295      dirName="On"
296      cyclebasename="datacycle"
297      indexStartHour="4"
298      timeShift="3"
299      timeDuration="30"
300      firstCycleId="0"
301      lastp1CycleId="${#cycleArray[@]}"
302      ;;
303    OFF)
304      $ECHO "process signal files for OFF phases"
305      ingain="-gain gain_${dateSelected}_${srclower}.fits"
306      freqfilter="-freqfilter 16"
307      nwinmean="5"
308      tspwin="5120"
309      dirName="Off"
310      cyclebasename="datacycle"
311      indexStartHour="8"
312      timeShift="3"
313      timeDuration="30"
314      firstCycleId="0"
315      lastp1CycleId="${#cycleArray[@]}"
316      ;;
317    MEANOFF)
318      $ECHO "process signal files for OFF phases (MEAN)"
319      ingain="-gain gain_${dateSelected}_${srclower}.fits"
320      nwinmean="25000"
321      tspwin="1"
322      dirName="Off"
323      cyclebasename="meancycle"
324      indexStartHour="8"
325      timeShift="3"
326      timeDuration="30"
327      firstCycleId="0"
328      lastp1CycleId="${#cycleArray[@]}"
329      ;;
330    MEANON)
331      $ECHO "process signal files for OFF phases (MEAN)"
332      ingain="-gain gain_${dateSelected}_${srclower}.fits"
333      nwinmean="25000"
334      tspwin="1"
335      dirName="On"
336      cyclebasename="meancycle"
337      indexStartHour="8"
338      timeShift="3"
339      timeDuration="30"
340      firstCycleId="0"
341      lastp1CycleId="${#cycleArray[@]}"
342      ;;
343    CALIBON)
344      $ECHO "process signal files for CALIBRATION ON phases"
345      ingain="-gain gain_${dateSelected}_${srclower}.fits"
346      freqfilter="-freqfilter 16"
347      nwinmean="1"
348      tspwin="1024"
349      dirName="On"
350      cyclebasename="calibcycle"
351      indexStartHour="2"
352      timeShift="-4"
353      timeDuration="14"
354      firstCycleId="0"
355      lastp1CycleId="${#cycleArray[@]}"
356      ;;
357    CALIBOFF)
358      $ECHO "process signal files for CALIBRATION OFF phases"
359      ingain="-gain gain_${dateSelected}_${srclower}.fits"
360      freqfilter="-freqfilter 16"
361      nwinmean="1"
362      tspwin="1024"
363      dirName="Off"
364      cyclebasename="calibcycle"
365      indexStartHour="6"
366      timeShift="-4"
367      timeDuration="14"
368      firstCycleId="0"
369      lastp1CycleId="${#cycleArray[@]}"
370      ;;
371    *)
372    $ECHO "FATAL (${scriptName}): type of processing"
373    usage
374#JEC 29/9/11 avoid finished stuff
375#    touch ${tmppublicpath}/proc_script.finished
376    exit 1
377    ;;
378esac
379#
380$ECHO "DEBUG (${scriptName}): END"
381#
382#
383#Compute time windows (Warning usage of GMT date)
384#
385$ECHO "DEBUG (${scriptName}): START compute time windows"
386OUT=./baotiming.$$
387${AWK} -v datesys=${DATE} -v type=${indexStartHour} -v shift=${timeShift} '
388BEGIN {FS=","}
389{
390  phaseStart=$type
391  cmd=sprintf("%s +%%s -d  \"%s\" ",datesys,phaseStart);
392  (cmd | getline phstart);
393  close(cmd);
394  newstart = phstart+shift;
395  cmd=sprintf("%s +%%T -d \"1970-01-01 %s sec GMT\"",datesys,newstart);
396  (cmd | getline newStart);
397  close(cmd);
398  print $1,newStart,phaseStart;
399}
400'  ${scaFileSummarized} > ${OUT}
401#
402$ECHO "DEBUG (${scriptName}): END"
403#
404$ECHO "DEBUG (${scriptName}): START cat ${OUT}"
405$CAT ${OUT}
406$ECHO "DEBUG (${scriptName}): END cat ${OUT}"
407#
408$ECHO "DEBUG (${scriptName}): START filling cylceArrayBis timeArray"
409#
410declare -a cycleArrayBis
411declare -a timeArray
412$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
413IFS=$DefaultIFS
414#select the selected cycles
415idtofind="0"
416while read v1 v2 v3
417do
418 if [ ${idtofind} -lt ${#cycleArray[*]} ]; then
419    if [ ${v1} = ${cycleArray[${idtofind}]} ]; then
420        cycleArrayBis[${#cycleArrayBis[*]}]=$v1
421        timeArray[${#timeArray[*]}]=$v2
422        idtofind=`expr ${idtofind} + 1`
423    fi
424 fi
425done < ${OUT}
426$ECHO "DEBUG (${scriptName}): number of elements ${#cycleArrayBis[*]} ${#timeArray[*]}"
427$ECHO "DEBUG (${scriptName}): END"
428
429
430#########################
431# specmfib exe
432#########################
433prg="/afs/in2p3.fr/throng/baoradio/Library/Progs/TAcq/Objs/specmfib"
434if [ ! -e $prg ]; then
435    $ECHO "FATAL (${scriptName}): $prg not found"
436#JEC 29/9/11 avoid finished stuff
437#    touch ${tmppublicpath}/proc_script.finished
438    exit 1
439fi
440
441
442dirFiberfile=${iojobpath}/${localpath}
443
444curDir=`pwd`
445#
446$ECHO "DEBUG (${scriptName}): in principle $dirFiberfile == $curDir "
447#
448#force single channel
449forceSingle="-singlechan"
450#debug
451prtlevel="1"
452
453
454#Loop on cycles
455##
456$ECHO "DEBUG (${scriptName}): START loop on cycles Ids ${firstCycleId} to ${lastp1CycleId}"
457
458#write the script to be send to Batch
459#get gain file if necessary
460if [ ${typeofproc} != "GAIN" ]; then
461    OUT2=./igetStatus.$$
462    iget /baoradio/data/AmasNancay/${localpath}/gain_${dateSelected}_${srclower}.fits >${OUT2} 2>&1
463    igetStatus=`$GREP "^ERROR" ${OUT2}`
464    if [ "<${igetStatus}>" != "<>" ]; then
465        $ECHO "FATAL (${scriptName}): error while iget:"
466        $ECHO $igetStatus
467#JEC 29/9/11 avoid finished stuff
468#       touch ${tmppublicpath}/${jobBatchName}.finished
469        exit 1
470    fi
471    $RM ${OUT2}
472fi   
473
474
475for ((i=${firstCycleId};i<${lastp1CycleId};i++)); do
476#
477
478####
479# Check daq status
480#JEC 1/10/11 Use generic baodaqstatus name
481####
482#get the daq current irod status
483#tag=`${DATE} +%F`
484#OUT1=${publicpath}/baodaqstatus-${tag}.txt
485OUT1=${publicpath}/baodaqstatus-current.txt
486
487if [ ! -e ${OUT1}  -o ! -r ${OUT1} ]; then
488    $ECHO "FATAL (${scriptName}): ${OUT1} has a problem"
489    exit 1
490#    $ECHO "We should bring the DAQ status more up to date, this take 1 or 2min..."
491#JEC 29/9/11 avoid finished stuff
492#    $RM -f ${tmppublicpath}/statusdaq.finished
493#    ${scriptpath}/statusdaq.sh > ${OUT1}
494#    while [ ! -f "${tmppublicpath}/statusdaq.finished" ]; do
495#       sleep 30
496#    done
497#    $RM ${tmppublicpath}/statusdaq.finished
498#protect against remove/rewriting   
499#    $CHMOD -v 444 ${OUT1}
500fi
501
502#time window selection
503    tmproc="-tmproc ${timeArray[${i}]},${timeDuration}"
504#cycle <number>
505    cycle="${cycleArray[${i}]}"
506    if [ "<${firstCycle}>" != "<>" -a "<${lastCycle}>" != "<>" ]; then
507        if [ ${cycle} -lt ${firstCycle} -o ${cycle} -gt ${lastCycle} ]; then
508            continue
509        fi
510    fi
511    $ECHO "DEBUG (${scriptName}): process cycle $cycle"
512
513#cycle <name><number>
514    cycle="${cyclebasename}${cycleArray[${i}]}"
515#signal files selection
516    infiles="${filesArray[${i}]}"
517#
518#Prepare BQS parameters: MEMORY, CPU...#
519    infilesArray=( `$ECHO $infiles | $AWK 'BEGIN {FS=","} {for(i=1;i<NF;i++)print $i;}'` )
520    IFS='
521    '
522    infilesArray=( $( printf "%s\n" "${infilesArray[@]}" | awk 'x[$0]++ == 0' ) )
523    IFS=$DefaultIFS
524    firstFile=${infilesArray[0]}
525    lastFile=${infilesArray[1]}
526#
527#
528#
529#get signals files
530    OUT3=./getsignals.$$
531#JEC 29/9/11 avoid finished stuff
532#    $RM -f ${tmppublicpath}/getsignalfiles.finished
533    ${scriptpath}/getsignalfiles.sh ${sourceRadio} ${dateSelected} ${firstFile}  ${lastFile} > ${OUT3} 2>&1
534#    while [ ! -f "${tmppublicpath}/getsignalfiles.finished" ]; do
535#       $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getsignalfiles.finished"
536#       date +%T
537#       sleep 30
538#    done
539#    $RM ${tmppublicpath}/getsignalfiles.finished
540
541    getSignals=`$GREP "^FATAL" ${OUT3}`
542    if [ "<$getSignals>" != "<>" ]; then
543        $ECHO "FATAL (${scriptName}): error while get signals"
544        $ECHO $getSignals
545#JEC 29/9/11 avoid finished stuff
546#       touch ${tmppublicpath}/${jobBatchName}.finished
547        exit 1
548    fi
549    $RM ${OUT3}
550
551#location of the signal file
552    fibfile="${curDir} 1,2"
553#
554    outDir=${curDir}/${dirName}/${cycle}
555    $MKDIR -p ${outDir}
556
557#
558# where ami...
559#
560    $ECHO "We are here `pwd`"
561    $LS -lrth
562    $FIND . -name "igetStatus.*" | $XARGS -i  $CAT {}
563    $LS ./Fiber1 | $WC -l
564    $LS ./Fiber2 | $WC -l
565    $LS -lR ${outDir}
566
567#Note: here act=gain means only that specmfib will use median-like algorithms (not only for gains...)
568    $prg -act ${action} $forceSingle $freqfilter  -prt $prtlevel -out ${outDir} -nmean $nwinmean $ingain $tmproc -tspwin ${tspwin},0,0 -in $infiles ${fibfile}
569
570    $ECHO    ${outDir}
571    $LS -lrt ${outDir}
572
573#save results to Irods (use option -f ONLY to force override)
574    iput -v -K -r ${curDir}/${dirName} /baoradio/data/AmasNancay/${localpath}
575#
576#clean the signal files which can at the end of the day fill completly the scratch directory of the batch work/AmasNancayer
577    $RM -rf ${iojobpath}/${localpath}/Fiber1
578    $RM -rf ${iojobpath}/${localpath}/Fiber2
579    $RM -rf ${iojobpath}/${localpath}/${typeofproc}
580
581#
582#
583
584#end loop on cycles
585done
586#
587$ECHO "DEBUG (${scriptName}): END loop on cycles"
588#
589#JEC 29/9/11 avoid finished stuff
590#touch ${tmppublicpath}/proc_script.finished
591exit 0
Note: See TracBrowser for help on using the repository browser.