source: BAORadio/AmasNancay/proc_script.sh @ 507

Last change on this file since 507 was 507, checked in by campagne, 13 years ago

New import
File size: 14.6 KB
Line 
1#!/bin/sh -xv
2#Basic tuning of system fuction used.
3AWK=/bin/awk
4SED=/bin/sed
5GREP=/bin/grep
6WC=/usr/bin/wc
7RM=/bin/rm
8CAT=/bin/cat
9TOUCH=/bin/touch
10DATE=/bin/date
11ECHO=/bin/echo
12LS=/bin/ls
13MKDIR=/bin/mkdir
14TR=/usr/bin/tr
15FIND=/usr/bin/find
16PRINTF=/usr/bin/printf
17XARGS=/usr/bin/xargs
18SORT=/bin/sort
19QSUB=qsub
20
21DefaultIFS=$' \t\n'
22IFS=$DefaultIFS
23
24
25scriptName="`basename $0`"
26$ECHO "Processing script ${scriptName} at `date`"
27######
28# Local functions
29######
30usage() {
31  echo "usage: $scriptName <source> <YYYYMMDD-directory> <type of process>"
32  echo "      -<source> as Abell85"
33  echo "      -<YYYYMMDD-directory> as 20110428"
34  echo "      -<type of process> as GAIN|CALIBON|CALIBOFF|ON|OFF"
35}
36######
37# argument test
38######
39if [ ! $# = 3 ]; then
40  usage
41  touch proc_script.finished
42  exit 1
43fi
44sourceRadio=$1
45srclower=`${ECHO} ${sourceRadio} | ${TR} "[:upper:]" "[:lower:]" `
46dateDAQ=$2
47typeofproc=$3
48
49#Path to public backupable path
50publicpath="/afs/in2p3.fr/home/c/campagne/public"
51
52#temporary files to synchronize scripts
53tmppublicpath="/sps/baoradio/AmasNancay"
54$LS -l ${tmppublicpath} > /dev/null
55
56
57#Path where the job will do temporary IO
58if [ ${ENVIRONMENT} = "INTERACTIVE" ]; then
59    iojobpath="/sps/baoradio/AmasNancay/JEC"
60    $MKDIR -p $iojobpath
61elif [ ${ENVIRONMENT} = "BATCH" ] ; then
62    iojobpath=${TMPBATCH}
63else
64    $ECHO "FATAL (${scriptName}): environment is ${ENVIRONMENT} not allowed"
65    touch ${tmppublicpath}/proc_script.finished
66    exit 1
67fi
68cd ${iojobpath}
69
70localpath="${sourceRadio}/${dateDAQ}${srclower}"
71$MKDIR -p ./${localpath}
72cd ./${localpath}
73mainDirectory=`pwd`
74
75
76#Path of the utility scripts
77scriptpath="/afs/in2p3.fr/home/c/campagne/private/work"
78
79
80#select file per cycle
81#$ECHO "Select files process..."
82#keep only lines with structure: cycle_number first,last,step
83#Get sca files
84$ECHO "DEBUG (${scriptName}): START call getscafiles.sh ${sourceRadio} ${dateDAQ}"
85OUT1=./getScaStatus.$$
86$RM -f ${tmppublicpath}/getscafiles.finished
87${scriptpath}/getscafiles.sh ${sourceRadio} ${dateDAQ} > ${OUT1} 2>&1
88while [ ! -f "${tmppublicpath}/getscafiles.finished" ]; do
89    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getscafiles.finished"
90    date +%T
91    sleep 30
92done
93$RM ${tmppublicpath}/getscafiles.finished
94$ECHO "DEBUG (${scriptName}): END"
95
96
97getScaStatus=`$GREP "^\(FATAL\|ERROR\)" ${OUT1}`
98
99if [ "<${getScaStatus}>" != "<>" ]; then
100  $ECHO "FATAL (${scriptName}): error to get sca file for $sourceRadio $dateDAQ"
101  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
102  $CAT ${OUT1}
103  $ECHO "DEBUG (${scriptName}): END"
104  touch ${tmppublicpath}/proc_script.finished
105  exit 1
106fi
107
108
109scaFileSummarized=`$FIND . -name "sca*.sum" -print`
110if [ "<${scaFileSummarized}>" = "<>" ]; then
111  $ECHO "FATAL (${scriptName}): strange sca file summarized not available: $sourceRadio $dateDAQ"
112  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
113  $CAT ${OUT1}
114  $ECHO "DEBUG (${scriptName}): END"
115  touch ${tmppublicpath}/proc_script.finished
116  exit 1
117fi
118$RM ${OUT1}
119
120scaFileSummarized=`basename $scaFileSummarized`
121
122#set Irods environment
123. /usr/local/shared/bin/irods_env.sh -noverbose
124#download header files to be used for signal files selction
125$MKDIR -p ./Fiber1
126dirFiberfile=`pwd`/Fiber1
127
128dirIrods="/baoradio/data/AmasNancay/${sourceRadio}/${dateDAQ}${srclower}/Fiber1"
129
130listOfFiles=( `ils ${dirIrods} | $GREP -i "header" | $XARGS -i basename {} | $SORT -k1.7n` )
131IFS='
132'
133listOfFiles=( $( $PRINTF "%s\n" "${listOfFiles[@]}" | $AWK 'x[$0]++ == 0' ) )
134IFS=$DefaultIFS
135
136#
137$ECHO "DEBUG (${scriptName}): Download if necessary the headers"
138#
139for ifile in ${listOfFiles[@]}
140do
141#test if the headers have not yet been downloaded
142  if [ ! -e ${dirFiberfile}/${ifile} ]; then
143      OUT2=./getheaders.$$
144      iget -f -K ${dirIrods}/${ifile} ${dirFiberfile} >${OUT2} 2>&1
145      igetStatus=`$GREP "^ERROR" ${OUT2}`
146      if [ "<$igetStatus>" != "<>" ]; then
147          $ECHO "FATAL (${scriptName}): error while iget:"
148          $ECHO $igetStatus
149          touch ${tmppublicpath}/proc_script.finished
150          exit 1
151      fi
152      $RM -f $OUT2
153  fi
154done
155#
156$ECHO "DEBUG (${scriptName}): START call select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc}"
157#
158#Get list of file to process
159headerFileSelected=./selectSignal.$$
160$RM -f ${tmppublicpath}/select.finished
161${scriptpath}/select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc} | $GREP "^#[0-9]" | $SED "s/^#//" > $headerFileSelected
162while [ ! -f "${tmppublicpath}/select.finished" ]; do
163    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/select.finished"
164    date +%T
165    sleep 30
166done
167$RM ${tmppublicpath}/select.finished
168#
169$ECHO "DEBUG (${scriptName}): END"
170#
171$ECHO "DEBUG (${scriptName}): START filling cycleArray, filesArray"
172#
173declare -a cycleArray
174declare -a filesArray
175$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
176IFS=$DefaultIFS
177$ECHO "DEBUG (${scriptName}): START cat $headerFileSelected"
178$CAT $headerFileSelected
179$ECHO "DEBUG (${scriptName}): END cat $headerFileSelected"
180
181while read v1 v2
182do
183 cycleArray[${#cycleArray[*]}]=$v1
184 filesArray[${#filesArray[*]}]=$v2
185done < $headerFileSelected
186
187if [ ${#cycleArray[*]} -eq 0 -o ${#filesArray[*]} -eq 0 ]; then
188   $ECHO "FATAL (${scriptName}): no header file selected (${#cycleArray[*]}, ${#filesArray[*]})"
189   touch ${tmppublicpath}/proc_script.finished
190   exit 1
191else
192   $ECHO "DEBUG (${scriptName}): non zero elements found for cycleArray and filesArray" 
193fi
194
195#
196$ECHO "DEBUG (${scriptName}): number of elements <${#cycleArray[*]}> <${#filesArray[*]}>"
197$ECHO "DEBUG (${scriptName}): END"
198#
199# (timeShift)
200#Add 3sec to start ON / OFF
201#Substract 4sec to start DAB ON / OFF
202#
203#Take care: Cycle numbering starts at 1 in SCAFiles while Arrays index start at 0
204#
205#for GAIN the reference cycle takes the middle of the run
206#
207$ECHO "DEBUG (${scriptName}): START switch on $typeofproc"
208#
209case "$typeofproc" in
210    GAIN)
211      $ECHO "process signal files for GAIN phase"
212      ingain=""
213      freqfilter="-freqfilter -"
214      tspwin="5120"
215      dirName="Off"
216      cyclebasename="gaincycle"
217      indexStartHour="6"
218      timeShift="-4"
219      timeDuration="14"
220      firstCycleId=`expr  ${#cycleArray[*]} / 2`
221      firstCycleId=`expr ${firstCycleId} - 1`
222      lastp1CycleId=`expr ${firstCycleId} + 1`
223      ;;
224    ON)
225      $ECHO "process signal files for ON phases"
226      ingain="-gain gain_${dateDAQ}_${srclower}.fits"
227      freqfilter="-freqfilter 16"
228      tspwin="25000"
229      dirName="On"
230      cyclebasename="datacycle"
231      indexStartHour="4"
232      timeShift="3"
233      timeDuration="30"
234      firstCycleId="0"
235      lastp1CycleId="${#cycleArray[@]}"
236      ;;
237    OFF)
238      $ECHO "process signal files for OFF phases"
239      ingain="-gain gain_${dateDAQ}_${srclower}.fits"
240      freqfilter="-freqfilter 16"
241      tspwin="25000"
242      dirName="Off"
243      cyclebasename="datacycle"
244      indexStartHour="8"
245      timeShift="3"
246      timeDuration="30"
247      firstCycleId="0"
248      lastp1CycleId="${#cycleArray[@]}"
249      ;;
250    CALIBON)
251      $ECHO "process signal files for CALIBRATION ON phases"
252      ingain="-gain gain_${dateDAQ}_${srclower}.fits"
253      freqfilter="-freqfilter 16"
254      tspwin="1024"
255      dirName="On"
256      cyclebasename="calibcycle"
257      indexStartHour="2"
258      timeShift="-4"
259      timeDuration="14"
260      firstCycleId="0"
261      lastp1CycleId="${#cycleArray[@]}"
262      ;;
263    CALIBOFF)
264      $ECHO "process signal files for CALIBRATION OFF phases"
265      ingain="-gain gain_${dateDAQ}_${srclower}.fits"
266      freqfilter="-freqfilter 16"
267      tspwin="1024"
268      dirName="Off"
269      cyclebasename="calibcycle"
270      indexStartHour="6"
271      timeShift="-4"
272      timeDuration="14"
273      firstCycleId="0"
274      lastp1CycleId="${#cycleArray[@]}"
275      ;;
276    *)
277    $ECHO "FATAL (${scriptName}): type of processing"
278    usage
279    touch ${tmppublicpath}/proc_script.finished
280    exit 1
281    ;;
282esac
283#
284$ECHO "DEBUG (${scriptName}): END"
285#
286#
287#Compute time windows (Warning usage of GMT date)
288#
289$ECHO "DEBUG (${scriptName}): START compute time windows"
290OUT=./baotiming.$$
291${AWK} -v datesys=${DATE} -v type=${indexStartHour} -v shift=${timeShift} '
292BEGIN {FS=","}
293{
294  phaseStart=$type
295  cmd=sprintf("%s +%%s -d  \"%s\" ",datesys,phaseStart);
296  (cmd | getline phstart);
297  close(cmd);
298  newstart = phstart+shift;
299  cmd=sprintf("%s +%%T -d \"1970-01-01 %s sec GMT\"",datesys,newstart);
300  (cmd | getline newStart);
301  close(cmd);
302  print $1,newStart,phaseStart;
303}
304'  ${scaFileSummarized} > ${OUT}
305#
306$ECHO "DEBUG (${scriptName}): END"
307#
308$ECHO "DEBUG (${scriptName}): START cat ${OUT}"
309$CAT ${OUT}
310$ECHO "DEBUG (${scriptName}): END cat ${OUT}"
311#
312$ECHO "DEBUG (${scriptName}): START filling cylceArrayBis timeArray"
313#
314declare -a cycleArrayBis
315declare -a timeArray
316$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
317IFS=$DefaultIFS
318#select the selected cycles
319idtofind="0"
320while read v1 v2 v3
321do
322 if [ ${idtofind} -lt ${#cycleArray[*]} ]; then
323    if [ ${v1} = ${cycleArray[${idtofind}]} ]; then
324        cycleArrayBis[${#cycleArrayBis[*]}]=$v1
325        timeArray[${#timeArray[*]}]=$v2
326        idtofind=`expr ${idtofind} + 1`
327    fi
328 fi
329done < ${OUT}
330$ECHO "DEBUG (${scriptName}): number of elements ${#cycleArrayBis[*]} ${#timeArray[*]}"
331$ECHO "DEBUG (${scriptName}): END"
332
333
334#########################
335# specmfib exe
336#########################
337prg="/afs/in2p3.fr/throng/baoradio/Library/Progs/TAcq/Objs/specmfib"
338if [ ! -e $prg ]; then
339    $ECHO "FATAL (${scriptName}): $prg not found"
340    touch ${tmppublicpath}/proc_script.finished
341    exit 1
342fi
343
344
345dirFiberfile=${iojobpath}/${localpath}
346
347curDir=`pwd`
348#
349$ECHO "DEBUG (${scriptName}): in principle $dirFiberfile == $curDir "
350#
351#force single channel
352forceSingle="-singlechan"
353#nwinmean donne le nombre de fenetre tspwin utilise pour faire calculer la moyenne des medianes de chaque fenetre
354#This is the nimber of paquets of the time window which change according to processing type
355nwinmean="1"
356#debug
357prtlevel="1"
358
359
360#Loop on cycles to write batch script and send to batch queue
361##
362$ECHO "DEBUG (${scriptName}): START loop on cycles ${firstCycleId} to ${lastp1CycleId}"
363
364for ((i=${firstCycleId};i<${lastp1CycleId};i++)); do
365#
366$ECHO "DEBUG (${scriptName}): process cycle $i"
367#time window selection
368    tmproc="-tmproc ${timeArray[${i}]},${timeDuration}"
369#cycle <name><number>
370    cycle="${cyclebasename}${cycleArray[${i}]}"
371#signal files selection
372    infiles="${filesArray[${i}]}"
373#
374#Prepare BQS parameters: MEMORY, CPU...#
375    infilesArray=( `$ECHO $infiles | $AWK 'BEGIN {FS=","} {for(i=1;i<NF;i++)print $i;}'` )
376    IFS='
377    '
378    infilesArray=( $( printf "%s\n" "${infilesArray[@]}" | awk 'x[$0]++ == 0' ) )
379    IFS=$DefaultIFS
380    firstFile=${infilesArray[0]}
381    lastFile=${infilesArray[1]}
382#compute Scratch Size MB (400MB per signal files), CPU Time
383    nFiles=`expr ${lastFile} - ${firstFile} + 1`
384# Fiber 1 & Fiber 2 => double the number of files
385    nFiles=`expr ${nFiles} \* 2`
386#add number of spectra generated (cf. (Nbrepaquets_per_files * nFiles /tspwin) * 100KB
387    nSpectra=`expr ${nFiles} \* 25600 /  ${tspwin}`
388#Guess formula to reserved the amount of space needed in scratch
389    scratchSize=`expr ${nFiles} \* 500 + ${nSpectra} \* 200 / 1000 + 5`
390# 25sec/signal_file * empirical_conversion_factor (=50)
391    cpu=`expr ${nFiles} \* 25 \* 50`
392#Virtual Memory (MB)
393    virtualmem=1000
394#Job and batch names
395    jobBatchName="${sourceRadio}-${dateDAQ}-${typeofproc}-cycle${cycleArray[${i}]}"
396    jobScriptName="${jobBatchName}.sh"
397    jobLogName="${jobBatchName}.log"
398#
399$ECHO "DEBUG (${scriptName}): start writing ${jobScriptName}"
400#
401#
402#write the script to be send to Batch
403#
404    $CAT > ${jobScriptName} <<EOF
405#!/bin/sh
406#    jobBatchName=$jobBatchName
407#    jobScriptName=$jobScriptName
408#    jobLogName=$jobLogName
409    scriptName=$jobScriptName
410
411    . /afs/in2p3.fr/home/throng/baoradio/setup.sh
412    . /usr/local/shared/bin/irods_env.sh -noverbose
413
414    iojobpath=\${TMPBATCH}
415    cd \${iojobpath}
416    localpath=${sourceRadio}/${dateDAQ}${srclower}
417    $MKDIR -p ./\${localpath}
418    cd ./\${localpath}
419
420    curDir=\`pwd\`
421#get gain file if necessary
422    if [ ${typeofproc} != "GAIN" ]; then
423        OUT2=./igetStatus.$$
424        iget /baoradio/data/AmasNancay/${localpath}/gain_${dateDAQ}_${srclower}.fits >\${OUT2} 2>&1
425        igetStatus=\`$GREP "^ERROR" \${OUT2}\`
426        if [ "<\${igetStatus}>" != "<>" ]; then
427            $ECHO "FATAL (\${scriptName}): error while iget:"
428            $ECHO \$igetStatus
429            touch ${tmppublicpath}/${jobBatchName}.finished
430            exit 1
431        fi
432        $RM \${OUT2}
433    fi   
434#get signals files
435    OUT3=./getsignals.$$
436    $RM -f ${tmppublicpath}/getsignalfiles.finished
437    ${scriptpath}/getsignalfiles.sh ${sourceRadio} ${dateDAQ} ${firstFile}  ${lastFile} > \${OUT3} 2>&1
438    while [ ! -f "${tmppublicpath}/getsignalfiles.finished" ]; do
439        $ECHO "INFO (\${scriptName}): waiting for ${tmppublicpath}/getsignalfiles.finished"
440        date +%T
441        sleep 30
442    done
443    $RM ${tmppublicpath}/getsignalfiles.finished
444
445    getSignals=\`$GREP "^FATAL" \${OUT3}\`
446    if [ "<\$getSignals>" != "<>" ]; then
447        $ECHO "FATAL (\${scriptName}): error while get signals"
448        $ECHO \$getSignals
449        touch ${tmppublicpath}/${jobBatchName}.finished
450        exit 1
451    fi
452    $RM \${OUT3}
453#location of the signal file
454    fibfile="\${curDir} 1,2"
455#
456    outDir=\${curDir}/${dirName}/${cycle}
457    $MKDIR -p \${outDir}
458#Note: here act=gain means only that specmfib will use median-like algorithms (not only for gains...)
459    $prg -act gain $forceSingle $freqfilter  -prt $prtlevel -out \${outDir} -nmean $nwinmean $ingain $tmproc -tspwin ${tspwin},0,0 -in $infiles \${fibfile}
460
461    $ECHO    \${outDir}
462    $LS -lrt \${outDir}/*
463
464#save results to Irods (use option -f ONLY to force override)
465    iput -v -K -r \${curDir}/${dirName} /baoradio/data/AmasNancay/\${localpath}
466#
467#clean the signal files which can at the end of the day fill completly the scratch directory of the batch worker
468    $RM -f \${iojobpath}/\${localpath}/Fiber1/signal*.fits
469    $RM -f \${iojobpath}/\${localpath}/Fiber2/signal*.fits
470    touch ${tmppublicpath}/${jobBatchName}.finished
471    exit 0
472EOF
473#
474#The EOF tag MUST BEGIN the line, this is the end of the script
475#
476    $ECHO "DEBUG (${scriptName}); START listing of script to be send to batch queue"
477    $CAT -v ${jobScriptName}
478    $ECHO "DEBUG (${scriptName}): END listing"
479#
480#send to BATCH
481#
482    $ECHO "DEBUG (${scriptName}): START send to batch listing"
483#
484    $RM -f ${tmppublicpath}/${jobBatchName}.finished
485
486    qsub -l u_sps_baoradio,u_irods,T=${cpu},M=${virtualmem}MB,scratch=${scratchSize}MB,platform=LINUX -eo -o ${tmppublicpath}/${jobLogName} -N ${jobBatchName}  ${jobScriptName}
487
488#    while [ ! -f "${tmppublicpath}/${jobBatchName}.finished" ]; do
489#       $ECHO "INFO (${scriptName}); waiting for ${tmppublicpath}/${jobBatchName}.finished"
490#       date +%T
491#       sleep 600
492#    done
493#    $RM ${tmppublicpath}/${jobBatchName}.finished
494    $ECHO "DEBUG (${scriptName}): end send to batch listing"
495#end loop on cycles
496done
497#
498$ECHO "DEBUG (${scriptName}): END loop on cycles"
499#
500touch ${tmppublicpath}/proc_script.finished
501exit 0
Note: See TracBrowser for help on using the repository browser.