source: BAORadio/AmasNancay/trunk/proc_script.sh @ 579

Last change on this file since 579 was 579, checked in by torrento, 13 years ago

version adaptee a grid engin (ana)
File size: 14.3 KB
Line 
1#!/bin/sh -xvf
2#Basic tuning of system fuction used.
3AWK=/bin/awk
4SED=/bin/sed
5GREP=/bin/grep
6WC=/usr/bin/wc
7RM=/bin/rm
8CAT=/bin/cat
9TOUCH=/bin/touch
10DATE=/bin/date
11ECHO=/bin/echo
12LS=/bin/ls
13MKDIR=/bin/mkdir
14TR=/usr/bin/tr
15FIND=/usr/bin/find
16PRINTF=/usr/bin/printf
17XARGS=/usr/bin/xargs
18SORT=/bin/sort
19QSUB=qsub
20
21DefaultIFS=$' \t\n'
22IFS=$DefaultIFS
23
24
25scriptName="`basename $0`"
26$ECHO "Processing script ${scriptName} at `date`"
27######
28# Local functions
29######
30usage() {
31  echo "usage: $scriptName <source> <YYYYMMDD-directory> <type of process>"
32  echo "      -<source> as Abell85"
33  echo "      -<YYYYMMDD-directory> as 20110428"
34  echo "      -<type of process> as GAIN|CALIBON|CALIBOFF|ON|OFF"
35}
36######
37# argument test
38######
39if [ ! $# = 3 ]; then
40  usage
41  exit 1
42fi
43sourceRadio=$1
44srclower=`${ECHO} ${sourceRadio} | ${TR} "[:upper:]" "[:lower:]" `
45dateDAQ=$2
46typeofproc=$3
47
48#Path to public backupable path
49publicpath="/afs/in2p3.fr/home/c/campagne/public"
50
51#temporary files to synchronize scripts
52tmppublicpath=${TMPPUBLICPATH}
53$LS -l ${tmppublicpath} > /dev/null
54
55
56#Path where the job will do temporary IO
57. ${SCRIPTPATH}/set_iojobpath.sh
58iojobpath=$(set_iojobpath)
59cd ${iojobpath}
60
61localpath="${sourceRadio}/${dateDAQ}${srclower}"
62$MKDIR -p ./${localpath}
63cd ./${localpath}
64mainDirectory=`pwd`
65
66
67#Path of the utility scripts
68scriptpath=${SCRIPTPATH}
69
70
71#select file per cycle
72#$ECHO "Select files process..."
73#keep only lines with structure: cycle_number first,last,step
74#Get sca files
75$ECHO "DEBUG (${scriptName}): START call getscafiles.sh ${sourceRadio} ${dateDAQ}"
76OUT1=./getScaStatus.$$
77$RM -f ${tmppublicpath}/getscafiles.finished
78${scriptpath}/getscafiles.sh ${sourceRadio} ${dateDAQ} > ${OUT1} 2>&1
79while [ ! -f "${tmppublicpath}/getscafiles.finished" ]; do
80    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getscafiles.finished"
81    date +%T
82    sleep 30
83done
84$RM ${tmppublicpath}/getscafiles.finished
85$ECHO "DEBUG (${scriptName}): END"
86
87
88getScaStatus=`$GREP "^\(FATAL\|ERROR\)" ${OUT1}`
89
90if [ "<${getScaStatus}>" != "<>" ]; then
91  $ECHO "FATAL (${scriptName}): error to get sca file for $sourceRadio $dateDAQ"
92  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
93  $CAT ${OUT1}
94  $ECHO "DEBUG (${scriptName}): END"
95  exit 1
96fi
97
98
99scaFileSummarized=`$FIND . -name "sca*.sum" -print`
100if [ "<${scaFileSummarized}>" = "<>" ]; then
101  $ECHO "FATAL (${scriptName}): strange sca file summarized not available: $sourceRadio $dateDAQ"
102  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
103  $CAT ${OUT1}
104  $ECHO "DEBUG (${scriptName}): END"
105  exit 1
106fi
107$RM ${OUT1}
108
109scaFileSummarized=`basename $scaFileSummarized`
110
111#set Irods environment
112. /usr/local/shared/bin/irods_env.sh -noverbose
113#download header files to be used for signal files selction
114$MKDIR -p ./Fiber1
115dirFiberfile=`pwd`/Fiber1
116
117dirIrods="/baoradio/data/AmasNancay/${sourceRadio}/${dateDAQ}${srclower}/Fiber1"
118
119listOfFiles=( `ils ${dirIrods} | $GREP -i "header" | $XARGS -i basename {} | $SORT -k1.7n` )
120IFS='
121'
122listOfFiles=( $( $PRINTF "%s\n" "${listOfFiles[@]}" | $AWK 'x[$0]++ == 0' ) )
123IFS=$DefaultIFS
124
125#
126$ECHO "DEBUG (${scriptName}): Download if necessary the headers"
127#
128for ifile in ${listOfFiles[@]}
129do
130#test if the headers have not yet been downloaded
131  if [ ! -e ${dirFiberfile}/${ifile} ]; then
132      OUT2=./getheaders.$$
133      iget -f -K ${dirIrods}/${ifile} ${dirFiberfile} >${OUT2} 2>&1
134      igetStatus=`$GREP "^ERROR" ${OUT2}`
135      if [ "<$igetStatus>" != "<>" ]; then
136          $ECHO "FATAL (${scriptName}): error while iget:"
137          $ECHO $igetStatus
138          exit 1
139      fi
140      $RM -f $OUT2
141  fi
142done
143#
144$ECHO "DEBUG (${scriptName}): START call select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc}"
145#
146#Get list of file to process
147headerFileSelected=./selectSignal.$$
148$RM -f ${tmppublicpath}/select.finished
149${scriptpath}/select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc} | $GREP "^#[0-9]" | $SED "s/^#//" > $headerFileSelected
150while [ ! -f "${tmppublicpath}/select.finished" ]; do
151    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/select.finished"
152    date +%T
153    sleep 30
154done
155$RM ${tmppublicpath}/select.finished
156#
157$ECHO "DEBUG (${scriptName}): END"
158#
159$ECHO "DEBUG (${scriptName}): START filling cycleArray, filesArray"
160#
161declare -a cycleArray
162declare -a filesArray
163$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
164IFS=$DefaultIFS
165$ECHO "DEBUG (${scriptName}): START cat $headerFileSelected"
166$CAT $headerFileSelected
167$ECHO "DEBUG (${scriptName}): END cat $headerFileSelected"
168
169while read v1 v2
170do
171 cycleArray[${#cycleArray[*]}]=$v1
172 filesArray[${#filesArray[*]}]=$v2
173done < $headerFileSelected
174
175if [ ${#cycleArray[*]} -eq 0 -o ${#filesArray[*]} -eq 0 ]; then
176   $ECHO "FATAL (${scriptName}): no header file selected (${#cycleArray[*]}, ${#filesArray[*]})"
177   exit 1
178else
179   $ECHO "DEBUG (${scriptName}): non zero elements found for cycleArray and filesArray" 
180fi
181
182#
183$ECHO "DEBUG (${scriptName}): number of elements <${#cycleArray[*]}> <${#filesArray[*]}>"
184$ECHO "DEBUG (${scriptName}): END"
185#
186# (timeShift)
187#Add 3sec to start ON / OFF
188#Substract 4sec to start DAB ON / OFF
189#
190#Take care: Cycle numbering starts at 1 in SCAFiles while Arrays index start at 0
191#
192#for GAIN the reference cycle takes the middle of the run
193#
194#nwinmean donne le nombre de fenetre tspwin utilise pour faire calculer la moyenne des medianes de chaque fenetre
195#This is the nimber of paquets of the time window which change according to processing type
196$ECHO "DEBUG (${scriptName}): START switch on $typeofproc"
197#
198case "$typeofproc" in
199    GAIN)
200      $ECHO "process signal files for GAIN phase"
201      ingain=""
202      freqfilter="-freqfilter -"
203      nwinmean="1"
204      tspwin="5120"
205      dirName="Off"
206      cyclebasename="gaincycle"
207      indexStartHour="6"
208      timeShift="-4"
209      timeDuration="14"
210      firstCycleId=`expr  ${#cycleArray[*]} / 2`
211      firstCycleId=`expr ${firstCycleId} - 1`
212      lastp1CycleId=`expr ${firstCycleId} + 1`
213      ;;
214    ON)
215      $ECHO "process signal files for ON phases"
216      ingain="-gain gain_${dateDAQ}_${srclower}.fits"
217      freqfilter="-freqfilter 16"
218      nwinmean="5"
219      tspwin="5120"
220      dirName="On"
221      cyclebasename="datacycle"
222      indexStartHour="4"
223      timeShift="3"
224      timeDuration="30"
225      firstCycleId="0"
226      lastp1CycleId="${#cycleArray[@]}"
227      ;;
228    OFF)
229      $ECHO "process signal files for OFF phases"
230      ingain="-gain gain_${dateDAQ}_${srclower}.fits"
231      freqfilter="-freqfilter 16"
232      nwinmean="5"
233      tspwin="5120"
234      dirName="Off"
235      cyclebasename="datacycle"
236      indexStartHour="8"
237      timeShift="3"
238      timeDuration="30"
239      firstCycleId="0"
240      lastp1CycleId="${#cycleArray[@]}"
241      ;;
242    CALIBON)
243      $ECHO "process signal files for CALIBRATION ON phases"
244      ingain="-gain gain_${dateDAQ}_${srclower}.fits"
245      freqfilter="-freqfilter 16"
246      nwinmean="1"
247      tspwin="1024"
248      dirName="On"
249      cyclebasename="calibcycle"
250      indexStartHour="2"
251      timeShift="-4"
252      timeDuration="14"
253      firstCycleId="0"
254      lastp1CycleId="${#cycleArray[@]}"
255      ;;
256    CALIBOFF)
257      $ECHO "process signal files for CALIBRATION OFF phases"
258      ingain="-gain gain_${dateDAQ}_${srclower}.fits"
259      freqfilter="-freqfilter 16"
260      nwinmean="1"
261      tspwin="1024"
262      dirName="Off"
263      cyclebasename="calibcycle"
264      indexStartHour="6"
265      timeShift="-4"
266      timeDuration="14"
267      firstCycleId="0"
268      lastp1CycleId="${#cycleArray[@]}"
269      ;;
270    *)
271    $ECHO "FATAL (${scriptName}): type of processing"
272    usage
273    exit 1
274    ;;
275esac
276#
277$ECHO "DEBUG (${scriptName}): END"
278#
279#
280#Compute time windows (Warning usage of GMT date)
281#
282$ECHO "DEBUG (${scriptName}): START compute time windows"
283OUT=./baotiming.$$
284${AWK} -v datesys=${DATE} -v type=${indexStartHour} -v shift=${timeShift} '
285BEGIN {FS=","}
286{
287  phaseStart=$type
288  cmd=sprintf("%s +%%s -d  \"%s\" ",datesys,phaseStart);
289  (cmd | getline phstart);
290  close(cmd);
291  newstart = phstart+shift;
292  cmd=sprintf("%s +%%T -d \"1970-01-01 %s sec GMT\"",datesys,newstart);
293  (cmd | getline newStart);
294  close(cmd);
295  print $1,newStart,phaseStart;
296}
297'  ${scaFileSummarized} > ${OUT}
298#
299$ECHO "DEBUG (${scriptName}): END"
300#
301$ECHO "DEBUG (${scriptName}): START cat ${OUT}"
302$CAT ${OUT}
303$ECHO "DEBUG (${scriptName}): END cat ${OUT}"
304#
305$ECHO "DEBUG (${scriptName}): START filling cylceArrayBis timeArray"
306#
307declare -a cycleArrayBis
308declare -a timeArray
309$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
310IFS=$DefaultIFS
311#select the selected cycles
312idtofind="0"
313while read v1 v2 v3
314do
315 if [ ${idtofind} -lt ${#cycleArray[*]} ]; then
316    if [ ${v1} = ${cycleArray[${idtofind}]} ]; then
317        cycleArrayBis[${#cycleArrayBis[*]}]=$v1
318        timeArray[${#timeArray[*]}]=$v2
319        idtofind=`expr ${idtofind} + 1`
320    fi
321 fi
322done < ${OUT}
323$ECHO "DEBUG (${scriptName}): number of elements ${#cycleArrayBis[*]} ${#timeArray[*]}"
324$ECHO "DEBUG (${scriptName}): END"
325
326
327#########################
328# specmfib exe
329#########################
330prg="/afs/in2p3.fr/throng/baoradio/Library/Progs/TAcq/Objs/specmfib"
331if [ ! -e $prg ]; then
332    $ECHO "FATAL (${scriptName}): $prg not found"
333    exit 1
334fi
335
336
337dirFiberfile=${iojobpath}/${localpath}
338
339curDir=`pwd`
340#
341$ECHO "DEBUG (${scriptName}): in principle $dirFiberfile == $curDir "
342#
343#force single channel
344forceSingle="-singlechan"
345#debug
346prtlevel="1"
347
348
349#Loop on cycles to write batch script and send to batch queue
350##
351$ECHO "DEBUG (${scriptName}): START loop on cycles ${firstCycleId} to ${lastp1CycleId}"
352
353for ((i=${firstCycleId};i<${lastp1CycleId};i++)); do
354#
355$ECHO "DEBUG (${scriptName}): process cycle $i"
356#time window selection
357    tmproc="-tmproc ${timeArray[${i}]},${timeDuration}"
358#cycle <name><number>
359    cycle="${cyclebasename}${cycleArray[${i}]}"
360#signal files selection
361    infiles="${filesArray[${i}]}"
362#
363#Prepare BQS parameters: MEMORY, CPU...#
364    infilesArray=( `$ECHO $infiles | $AWK 'BEGIN {FS=","} {for(i=1;i<NF;i++)print $i;}'` )
365    IFS='
366    '
367    infilesArray=( $( printf "%s\n" "${infilesArray[@]}" | awk 'x[$0]++ == 0' ) )
368    IFS=$DefaultIFS
369    firstFile=${infilesArray[0]}
370    lastFile=${infilesArray[1]}
371#compute Scratch Size MB (400MB per signal files), CPU Time
372    nFiles=`expr ${lastFile} - ${firstFile} + 1`
373# Fiber 1 & Fiber 2 => double the number of files
374    nFiles=`expr ${nFiles} \* 2`
375#add number of spectra generated (cf. (Nbrepaquets_per_files * nFiles /tspwin) * 100KB
376    nSpectra=`expr ${nFiles} \* 25600 /  ${tspwin}`
377#Guess formula to reserved the amount of space needed in scratch
378    scratchSize=`expr ${nFiles} \* 500 + ${nSpectra} \* 200 / 1000 + 5`
379# 60sec/signal_file * empirical_conversion_factor (=50)
380    cpu=`expr ${nFiles} \* 60 \* 50`
381#Virtual Memory (MB) Nb: at CCIN2P3 the reasonable max is 1GB
382    virtualmem=1000
383#Job and batch names
384    jobBatchName="${sourceRadio}-${dateDAQ}-${typeofproc}-cycle${cycleArray[${i}]}"
385    jobScriptName="${jobBatchName}.sh"
386    jobLogName="${jobBatchName}.log"
387#
388$ECHO "DEBUG (${scriptName}): start writing ${jobScriptName}"
389#
390#
391#write the script to be send to Batch
392#
393    $CAT > ${jobScriptName} <<EOF
394#!/bin/sh -xvf
395#    jobBatchName=$jobBatchName
396#    jobScriptName=$jobScriptName
397#    jobLogName=$jobLogName
398    scriptName=$jobScriptName
399
400    . /afs/in2p3.fr/home/throng/baoradio/setup.sh
401    . /usr/local/shared/bin/irods_env.sh -noverbose
402
403    iojobpath=\${TMPBATCH}
404    cd \${iojobpath}
405    localpath=${sourceRadio}/${dateDAQ}${srclower}
406    $MKDIR -p ./\${localpath}
407    cd ./\${localpath}
408
409    curDir=\`pwd\`
410#get gain file if necessary
411    if [ ${typeofproc} != "GAIN" ]; then
412        OUT2=./igetStatus.$$
413        iget /baoradio/data/AmasNancay/${localpath}/gain_${dateDAQ}_${srclower}.fits >\${OUT2} 2>&1
414        igetStatus=\`$GREP "^ERROR" \${OUT2}\`
415        if [ "<\${igetStatus}>" != "<>" ]; then
416            $ECHO "FATAL (\${scriptName}): error while iget:"
417            $ECHO \$igetStatus
418            exit 1
419        fi
420        $RM \${OUT2}
421    fi   
422#get signals files
423    OUT3=./getsignals.$$
424    $RM -f ${tmppublicpath}/getsignalfiles.finished
425    ${scriptpath}/getsignalfiles.sh ${sourceRadio} ${dateDAQ} ${firstFile}  ${lastFile} > \${OUT3} 2>&1
426    while [ ! -f "${tmppublicpath}/getsignalfiles.finished" ]; do
427        $ECHO "INFO (\${scriptName}): waiting for ${tmppublicpath}/getsignalfiles.finished"
428        date +%T
429        sleep 30
430    done
431    $RM ${tmppublicpath}/getsignalfiles.finished
432
433    getSignals=\`$GREP "^FATAL" \${OUT3}\`
434    if [ "<\$getSignals>" != "<>" ]; then
435        $ECHO "FATAL (\${scriptName}): error while get signals"
436        $ECHO \$getSignals
437        exit 1
438    fi
439    $RM \${OUT3}
440#location of the signal file
441    fibfile="\${curDir} 1,2"
442#
443    outDir=\${curDir}/${dirName}/${cycle}
444    $MKDIR -p \${outDir}
445
446#
447# where ami...
448#
449    $ECHO "We are here \`pwd\` to send the batch"
450    $LS -lrth
451    $FIND . -name "igetStatus.*" | $XARGS -i  $CAT {}
452    $LS ./Fiber1 | $WC -l
453    $LS ./Fiber2 | $WC -l
454    $LS -lR \${outDir}
455
456#Note: here act=gain means only that specmfib will use median-like algorithms (not only for gains...)
457    $prg -act gain $forceSingle $freqfilter  -prt $prtlevel -out \${outDir} -nmean $nwinmean $ingain $tmproc -tspwin ${tspwin},0,0 -in $infiles \${fibfile}
458
459    $ECHO    \${outDir}
460    $LS -lrt \${outDir}
461
462#save results to Irods (use option -f ONLY to force override)
463    iput -v -K -r \${curDir}/${dirName} /baoradio/data/AmasNancay/\${localpath}
464#
465#clean the signal files which can at the end of the day fill completly the scratch directory of the batch work/AmasNancayer
466    $RM -f \${iojobpath}/\${localpath}/Fiber1/signal*.fits
467    $RM -f \${iojobpath}/\${localpath}/Fiber2/signal*.fits
468    exit 0
469EOF
470#
471#The EOF tag MUST BEGIN the line, this is the end of the script
472#
473    $ECHO "DEBUG (${scriptName}); START listing of script to be send to batch queue"
474    $CAT -v ${jobScriptName}
475    $ECHO "DEBUG (${scriptName}): END listing"
476#
477#send to BATCH
478#
479    $ECHO "DEBUG (${scriptName}): START send to batch listing"
480#
481    $RM -f ${tmppublicpath}/${jobBatchName}.finished
482
483##26/9/11 JEC use option -h to trig manualy the job on the queue...
484    qsub -h -l u_sps_baoradio,u_irods,u_irods_baoradio,T=${cpu},M=${virtualmem}MB,scratch=${scratchSize}MB,platform=LINUX -eo -o ${tmppublicpath}/${jobLogName} -N ${jobBatchName}  ${jobScriptName}
485
486#    while [ ! -f "${tmppublicpath}/${jobBatchName}.finished" ]; do
487#       $ECHO "INFO (${scriptName}); waiting for ${tmppublicpath}/${jobBatchName}.finished"
488#       date +%T
489#       sleep 600
490#    done
491#    $RM ${tmppublicpath}/${jobBatchName}.finished
492    $ECHO "DEBUG (${scriptName}): end send to batch listing"
493#end loop on cycles
494done
495#
496$ECHO "DEBUG (${scriptName}): END loop on cycles"
497#
498exit 0
Note: See TracBrowser for help on using the repository browser.