source: BAORadio/AmasNancay/proc_script.sh @ 520

Last change on this file since 520 was 520, checked in by campagne, 13 years ago

tune BATCH parameters (jec)
File size: 15.0 KB
Line 
1#!/bin/sh -xvf
2#Basic tuning of system fuction used.
3AWK=/bin/awk
4SED=/bin/sed
5GREP=/bin/grep
6WC=/usr/bin/wc
7RM=/bin/rm
8CAT=/bin/cat
9TOUCH=/bin/touch
10DATE=/bin/date
11ECHO=/bin/echo
12LS=/bin/ls
13MKDIR=/bin/mkdir
14TR=/usr/bin/tr
15FIND=/usr/bin/find
16PRINTF=/usr/bin/printf
17XARGS=/usr/bin/xargs
18SORT=/bin/sort
19QSUB=qsub
20
21DefaultIFS=$' \t\n'
22IFS=$DefaultIFS
23
24
25scriptName="`basename $0`"
26$ECHO "Processing script ${scriptName} at `date`"
27######
28# Local functions
29######
30usage() {
31  echo "usage: $scriptName <source> <YYYYMMDD-directory> <type of process>"
32  echo "      -<source> as Abell85"
33  echo "      -<YYYYMMDD-directory> as 20110428"
34  echo "      -<type of process> as GAIN|CALIBON|CALIBOFF|ON|OFF"
35}
36######
37# argument test
38######
39if [ ! $# = 3 ]; then
40  usage
41  touch proc_script.finished
42  exit 1
43fi
44sourceRadio=$1
45srclower=`${ECHO} ${sourceRadio} | ${TR} "[:upper:]" "[:lower:]" `
46dateDAQ=$2
47typeofproc=$3
48
49#Path to public backupable path
50publicpath="/afs/in2p3.fr/home/c/campagne/public"
51
52#temporary files to synchronize scripts
53tmppublicpath=${TMPPUBLICPATH}
54$LS -l ${tmppublicpath} > /dev/null
55
56
57#Path where the job will do temporary IO
58if [ ${ENVIRONMENT} = "INTERACTIVE" ]; then
59    iojobpath="/sps/baoradio/AmasNancay/JEC"
60    $MKDIR -p $iojobpath
61elif [ ${ENVIRONMENT} = "BATCH" ] ; then
62    iojobpath=${TMPBATCH}
63else
64    $ECHO "FATAL (${scriptName}): environment is ${ENVIRONMENT} not allowed"
65    touch ${tmppublicpath}/proc_script.finished
66    exit 1
67fi
68cd ${iojobpath}
69
70localpath="${sourceRadio}/${dateDAQ}${srclower}"
71$MKDIR -p ./${localpath}
72cd ./${localpath}
73mainDirectory=`pwd`
74
75
76#Path of the utility scripts
77scriptpath="/afs/in2p3.fr/home/c/campagne/private/work/AmasNancay"
78
79
80#select file per cycle
81#$ECHO "Select files process..."
82#keep only lines with structure: cycle_number first,last,step
83#Get sca files
84$ECHO "DEBUG (${scriptName}): START call getscafiles.sh ${sourceRadio} ${dateDAQ}"
85OUT1=./getScaStatus.$$
86$RM -f ${tmppublicpath}/getscafiles.finished
87${scriptpath}/getscafiles.sh ${sourceRadio} ${dateDAQ} > ${OUT1} 2>&1
88while [ ! -f "${tmppublicpath}/getscafiles.finished" ]; do
89    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getscafiles.finished"
90    date +%T
91    sleep 30
92done
93$RM ${tmppublicpath}/getscafiles.finished
94$ECHO "DEBUG (${scriptName}): END"
95
96
97getScaStatus=`$GREP "^\(FATAL\|ERROR\)" ${OUT1}`
98
99if [ "<${getScaStatus}>" != "<>" ]; then
100  $ECHO "FATAL (${scriptName}): error to get sca file for $sourceRadio $dateDAQ"
101  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
102  $CAT ${OUT1}
103  $ECHO "DEBUG (${scriptName}): END"
104  touch ${tmppublicpath}/proc_script.finished
105  exit 1
106fi
107
108
109scaFileSummarized=`$FIND . -name "sca*.sum" -print`
110if [ "<${scaFileSummarized}>" = "<>" ]; then
111  $ECHO "FATAL (${scriptName}): strange sca file summarized not available: $sourceRadio $dateDAQ"
112  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
113  $CAT ${OUT1}
114  $ECHO "DEBUG (${scriptName}): END"
115  touch ${tmppublicpath}/proc_script.finished
116  exit 1
117fi
118$RM ${OUT1}
119
120scaFileSummarized=`basename $scaFileSummarized`
121
122#set Irods environment
123. /usr/local/shared/bin/irods_env.sh -noverbose
124#download header files to be used for signal files selction
125$MKDIR -p ./Fiber1
126dirFiberfile=`pwd`/Fiber1
127
128dirIrods="/baoradio/data/AmasNancay/${sourceRadio}/${dateDAQ}${srclower}/Fiber1"
129
130listOfFiles=( `ils ${dirIrods} | $GREP -i "header" | $XARGS -i basename {} | $SORT -k1.7n` )
131IFS='
132'
133listOfFiles=( $( $PRINTF "%s\n" "${listOfFiles[@]}" | $AWK 'x[$0]++ == 0' ) )
134IFS=$DefaultIFS
135
136#
137$ECHO "DEBUG (${scriptName}): Download if necessary the headers"
138#
139for ifile in ${listOfFiles[@]}
140do
141#test if the headers have not yet been downloaded
142  if [ ! -e ${dirFiberfile}/${ifile} ]; then
143      OUT2=./getheaders.$$
144      iget -f -K ${dirIrods}/${ifile} ${dirFiberfile} >${OUT2} 2>&1
145      igetStatus=`$GREP "^ERROR" ${OUT2}`
146      if [ "<$igetStatus>" != "<>" ]; then
147          $ECHO "FATAL (${scriptName}): error while iget:"
148          $ECHO $igetStatus
149          touch ${tmppublicpath}/proc_script.finished
150          exit 1
151      fi
152      $RM -f $OUT2
153  fi
154done
155#
156$ECHO "DEBUG (${scriptName}): START call select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc}"
157#
158#Get list of file to process
159headerFileSelected=./selectSignal.$$
160$RM -f ${tmppublicpath}/select.finished
161${scriptpath}/select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc} | $GREP "^#[0-9]" | $SED "s/^#//" > $headerFileSelected
162while [ ! -f "${tmppublicpath}/select.finished" ]; do
163    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/select.finished"
164    date +%T
165    sleep 30
166done
167$RM ${tmppublicpath}/select.finished
168#
169$ECHO "DEBUG (${scriptName}): END"
170#
171$ECHO "DEBUG (${scriptName}): START filling cycleArray, filesArray"
172#
173declare -a cycleArray
174declare -a filesArray
175$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
176IFS=$DefaultIFS
177$ECHO "DEBUG (${scriptName}): START cat $headerFileSelected"
178$CAT $headerFileSelected
179$ECHO "DEBUG (${scriptName}): END cat $headerFileSelected"
180
181while read v1 v2
182do
183 cycleArray[${#cycleArray[*]}]=$v1
184 filesArray[${#filesArray[*]}]=$v2
185done < $headerFileSelected
186
187if [ ${#cycleArray[*]} -eq 0 -o ${#filesArray[*]} -eq 0 ]; then
188   $ECHO "FATAL (${scriptName}): no header file selected (${#cycleArray[*]}, ${#filesArray[*]})"
189   touch ${tmppublicpath}/proc_script.finished
190   exit 1
191else
192   $ECHO "DEBUG (${scriptName}): non zero elements found for cycleArray and filesArray" 
193fi
194
195#
196$ECHO "DEBUG (${scriptName}): number of elements <${#cycleArray[*]}> <${#filesArray[*]}>"
197$ECHO "DEBUG (${scriptName}): END"
198#
199# (timeShift)
200#Add 3sec to start ON / OFF
201#Substract 4sec to start DAB ON / OFF
202#
203#Take care: Cycle numbering starts at 1 in SCAFiles while Arrays index start at 0
204#
205#for GAIN the reference cycle takes the middle of the run
206#
207#nwinmean donne le nombre de fenetre tspwin utilise pour faire calculer la moyenne des medianes de chaque fenetre
208#This is the nimber of paquets of the time window which change according to processing type
209$ECHO "DEBUG (${scriptName}): START switch on $typeofproc"
210#
211case "$typeofproc" in
212    GAIN)
213      $ECHO "process signal files for GAIN phase"
214      ingain=""
215      freqfilter="-freqfilter -"
216      nwinmean="1"
217      tspwin="5120"
218      dirName="Off"
219      cyclebasename="gaincycle"
220      indexStartHour="6"
221      timeShift="-4"
222      timeDuration="14"
223      firstCycleId=`expr  ${#cycleArray[*]} / 2`
224      firstCycleId=`expr ${firstCycleId} - 1`
225      lastp1CycleId=`expr ${firstCycleId} + 1`
226      ;;
227    ON)
228      $ECHO "process signal files for ON phases"
229      ingain="-gain gain_${dateDAQ}_${srclower}.fits"
230      freqfilter="-freqfilter 16"
231      nwinmean="5"
232      tspwin="5120"
233      dirName="On"
234      cyclebasename="datacycle"
235      indexStartHour="4"
236      timeShift="3"
237      timeDuration="30"
238      firstCycleId="0"
239      lastp1CycleId="${#cycleArray[@]}"
240      ;;
241    OFF)
242      $ECHO "process signal files for OFF phases"
243      ingain="-gain gain_${dateDAQ}_${srclower}.fits"
244      freqfilter="-freqfilter 16"
245      nwinmean="5"
246      tspwin="5120"
247      dirName="Off"
248      cyclebasename="datacycle"
249      indexStartHour="8"
250      timeShift="3"
251      timeDuration="30"
252      firstCycleId="0"
253      lastp1CycleId="${#cycleArray[@]}"
254      ;;
255    CALIBON)
256      $ECHO "process signal files for CALIBRATION ON phases"
257      ingain="-gain gain_${dateDAQ}_${srclower}.fits"
258      freqfilter="-freqfilter 16"
259      nwinmean="1"
260      tspwin="1024"
261      dirName="On"
262      cyclebasename="calibcycle"
263      indexStartHour="2"
264      timeShift="-4"
265      timeDuration="14"
266      firstCycleId="0"
267      lastp1CycleId="${#cycleArray[@]}"
268      ;;
269    CALIBOFF)
270      $ECHO "process signal files for CALIBRATION OFF phases"
271      ingain="-gain gain_${dateDAQ}_${srclower}.fits"
272      freqfilter="-freqfilter 16"
273      nwinmean="1"
274      tspwin="1024"
275      dirName="Off"
276      cyclebasename="calibcycle"
277      indexStartHour="6"
278      timeShift="-4"
279      timeDuration="14"
280      firstCycleId="0"
281      lastp1CycleId="${#cycleArray[@]}"
282      ;;
283    *)
284    $ECHO "FATAL (${scriptName}): type of processing"
285    usage
286    touch ${tmppublicpath}/proc_script.finished
287    exit 1
288    ;;
289esac
290#
291$ECHO "DEBUG (${scriptName}): END"
292#
293#
294#Compute time windows (Warning usage of GMT date)
295#
296$ECHO "DEBUG (${scriptName}): START compute time windows"
297OUT=./baotiming.$$
298${AWK} -v datesys=${DATE} -v type=${indexStartHour} -v shift=${timeShift} '
299BEGIN {FS=","}
300{
301  phaseStart=$type
302  cmd=sprintf("%s +%%s -d  \"%s\" ",datesys,phaseStart);
303  (cmd | getline phstart);
304  close(cmd);
305  newstart = phstart+shift;
306  cmd=sprintf("%s +%%T -d \"1970-01-01 %s sec GMT\"",datesys,newstart);
307  (cmd | getline newStart);
308  close(cmd);
309  print $1,newStart,phaseStart;
310}
311'  ${scaFileSummarized} > ${OUT}
312#
313$ECHO "DEBUG (${scriptName}): END"
314#
315$ECHO "DEBUG (${scriptName}): START cat ${OUT}"
316$CAT ${OUT}
317$ECHO "DEBUG (${scriptName}): END cat ${OUT}"
318#
319$ECHO "DEBUG (${scriptName}): START filling cylceArrayBis timeArray"
320#
321declare -a cycleArrayBis
322declare -a timeArray
323$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
324IFS=$DefaultIFS
325#select the selected cycles
326idtofind="0"
327while read v1 v2 v3
328do
329 if [ ${idtofind} -lt ${#cycleArray[*]} ]; then
330    if [ ${v1} = ${cycleArray[${idtofind}]} ]; then
331        cycleArrayBis[${#cycleArrayBis[*]}]=$v1
332        timeArray[${#timeArray[*]}]=$v2
333        idtofind=`expr ${idtofind} + 1`
334    fi
335 fi
336done < ${OUT}
337$ECHO "DEBUG (${scriptName}): number of elements ${#cycleArrayBis[*]} ${#timeArray[*]}"
338$ECHO "DEBUG (${scriptName}): END"
339
340
341#########################
342# specmfib exe
343#########################
344prg="/afs/in2p3.fr/throng/baoradio/Library/Progs/TAcq/Objs/specmfib"
345if [ ! -e $prg ]; then
346    $ECHO "FATAL (${scriptName}): $prg not found"
347    touch ${tmppublicpath}/proc_script.finished
348    exit 1
349fi
350
351
352dirFiberfile=${iojobpath}/${localpath}
353
354curDir=`pwd`
355#
356$ECHO "DEBUG (${scriptName}): in principle $dirFiberfile == $curDir "
357#
358#force single channel
359forceSingle="-singlechan"
360#debug
361prtlevel="1"
362
363
364#Loop on cycles to write batch script and send to batch queue
365##
366$ECHO "DEBUG (${scriptName}): START loop on cycles ${firstCycleId} to ${lastp1CycleId}"
367
368for ((i=${firstCycleId};i<${lastp1CycleId};i++)); do
369#
370$ECHO "DEBUG (${scriptName}): process cycle $i"
371#time window selection
372    tmproc="-tmproc ${timeArray[${i}]},${timeDuration}"
373#cycle <name><number>
374    cycle="${cyclebasename}${cycleArray[${i}]}"
375#signal files selection
376    infiles="${filesArray[${i}]}"
377#
378#Prepare BQS parameters: MEMORY, CPU...#
379    infilesArray=( `$ECHO $infiles | $AWK 'BEGIN {FS=","} {for(i=1;i<NF;i++)print $i;}'` )
380    IFS='
381    '
382    infilesArray=( $( printf "%s\n" "${infilesArray[@]}" | awk 'x[$0]++ == 0' ) )
383    IFS=$DefaultIFS
384    firstFile=${infilesArray[0]}
385    lastFile=${infilesArray[1]}
386#compute Scratch Size MB (400MB per signal files), CPU Time
387    nFiles=`expr ${lastFile} - ${firstFile} + 1`
388# Fiber 1 & Fiber 2 => double the number of files
389    nFiles=`expr ${nFiles} \* 2`
390#add number of spectra generated (cf. (Nbrepaquets_per_files * nFiles /tspwin) * 100KB
391    nSpectra=`expr ${nFiles} \* 25600 /  ${tspwin}`
392#Guess formula to reserved the amount of space needed in scratch
393    scratchSize=`expr ${nFiles} \* 500 + ${nSpectra} \* 200 / 1000 + 5`
394# 30sec/signal_file * empirical_conversion_factor (=50)
395    cpu=`expr ${nFiles} \* 30 \* 50`
396#Virtual Memory (MB) Nb: at CCIN2P3 the reasonable max is 1GB
397    virtualmem=1000
398#Job and batch names
399    jobBatchName="${sourceRadio}-${dateDAQ}-${typeofproc}-cycle${cycleArray[${i}]}"
400    jobScriptName="${jobBatchName}.sh"
401    jobLogName="${jobBatchName}.log"
402#
403$ECHO "DEBUG (${scriptName}): start writing ${jobScriptName}"
404#
405#
406#write the script to be send to Batch
407#
408    $CAT > ${jobScriptName} <<EOF
409#!/bin/sh -xvf
410#    jobBatchName=$jobBatchName
411#    jobScriptName=$jobScriptName
412#    jobLogName=$jobLogName
413    scriptName=$jobScriptName
414
415    . /afs/in2p3.fr/home/throng/baoradio/setup.sh
416    . /usr/local/shared/bin/irods_env.sh -noverbose
417
418    iojobpath=\${TMPBATCH}
419    cd \${iojobpath}
420    localpath=${sourceRadio}/${dateDAQ}${srclower}
421    $MKDIR -p ./\${localpath}
422    cd ./\${localpath}
423
424    curDir=\`pwd\`
425#get gain file if necessary
426    if [ ${typeofproc} != "GAIN" ]; then
427        OUT2=./igetStatus.$$
428        iget /baoradio/data/AmasNancay/${localpath}/gain_${dateDAQ}_${srclower}.fits >\${OUT2} 2>&1
429        igetStatus=\`$GREP "^ERROR" \${OUT2}\`
430        if [ "<\${igetStatus}>" != "<>" ]; then
431            $ECHO "FATAL (\${scriptName}): error while iget:"
432            $ECHO \$igetStatus
433            touch ${tmppublicpath}/${jobBatchName}.finished
434            exit 1
435        fi
436        $RM \${OUT2}
437    fi   
438#get signals files
439    OUT3=./getsignals.$$
440    $RM -f ${tmppublicpath}/getsignalfiles.finished
441    ${scriptpath}/getsignalfiles.sh ${sourceRadio} ${dateDAQ} ${firstFile}  ${lastFile} > \${OUT3} 2>&1
442    while [ ! -f "${tmppublicpath}/getsignalfiles.finished" ]; do
443        $ECHO "INFO (\${scriptName}): waiting for ${tmppublicpath}/getsignalfiles.finished"
444        date +%T
445        sleep 30
446    done
447    $RM ${tmppublicpath}/getsignalfiles.finished
448
449    getSignals=\`$GREP "^FATAL" \${OUT3}\`
450    if [ "<\$getSignals>" != "<>" ]; then
451        $ECHO "FATAL (\${scriptName}): error while get signals"
452        $ECHO \$getSignals
453        touch ${tmppublicpath}/${jobBatchName}.finished
454        exit 1
455    fi
456    $RM \${OUT3}
457#location of the signal file
458    fibfile="\${curDir} 1,2"
459#
460    outDir=\${curDir}/${dirName}/${cycle}
461    $MKDIR -p \${outDir}
462
463#
464# where ami...
465#
466    $ECHO "We are here \`pwd\` to send the batch"
467    $LS -lrth
468    $FIND . -name "igetStatus.*" | $XARGS -i  $CAT {}
469    $LS ./Fiber1 | $WC -l
470    $LS ./Fiber2 | $WC -l
471    $LS -lR \${outDir}
472
473#Note: here act=gain means only that specmfib will use median-like algorithms (not only for gains...)
474    $prg -act gain $forceSingle $freqfilter  -prt $prtlevel -out \${outDir} -nmean $nwinmean $ingain $tmproc -tspwin ${tspwin},0,0 -in $infiles \${fibfile}
475
476    $ECHO    \${outDir}
477    $LS -lrt \${outDir}
478
479#save results to Irods (use option -f ONLY to force override)
480    iput -v -K -r \${curDir}/${dirName} /baoradio/data/AmasNancay/\${localpath}
481#
482#clean the signal files which can at the end of the day fill completly the scratch directory of the batch work/AmasNancayer
483    $RM -f \${iojobpath}/\${localpath}/Fiber1/signal*.fits
484    $RM -f \${iojobpath}/\${localpath}/Fiber2/signal*.fits
485    touch ${tmppublicpath}/${jobBatchName}.finished
486    exit 0
487EOF
488#
489#The EOF tag MUST BEGIN the line, this is the end of the script
490#
491    $ECHO "DEBUG (${scriptName}); START listing of script to be send to batch queue"
492    $CAT -v ${jobScriptName}
493    $ECHO "DEBUG (${scriptName}): END listing"
494#
495#send to BATCH
496#
497    $ECHO "DEBUG (${scriptName}): START send to batch listing"
498#
499    $RM -f ${tmppublicpath}/${jobBatchName}.finished
500
501    qsub -l u_sps_baoradio,u_irods,T=${cpu},M=${virtualmem}MB,scratch=${scratchSize}MB,platform=LINUX -eo -o ${tmppublicpath}/${jobLogName} -N ${jobBatchName}  ${jobScriptName}
502
503#    while [ ! -f "${tmppublicpath}/${jobBatchName}.finished" ]; do
504#       $ECHO "INFO (${scriptName}); waiting for ${tmppublicpath}/${jobBatchName}.finished"
505#       date +%T
506#       sleep 600
507#    done
508#    $RM ${tmppublicpath}/${jobBatchName}.finished
509    $ECHO "DEBUG (${scriptName}): end send to batch listing"
510#end loop on cycles
511done
512#
513$ECHO "DEBUG (${scriptName}): END loop on cycles"
514#
515touch ${tmppublicpath}/proc_script.finished
516exit 0
Note: See TracBrowser for help on using the repository browser.