source: BAORadio/AmasNancay/v2/proc_rawonoff.sh @ 641

Last change on this file since 641 was 535, checked in by campagne, 13 years ago

nouvelle analyse sans calibration (jec)
  • Property svn:executable set to *
File size: 14.0 KB
Line 
1#!/bin/sh -xvf
2#Basic tuning of system fuction used.
3AWK=/bin/awk
4SED=/bin/sed
5GREP=/bin/grep
6WC=/usr/bin/wc
7RM=/bin/rm
8CAT=/bin/cat
9TOUCH=/bin/touch
10DATE=/bin/date
11ECHO=/bin/echo
12LS=/bin/ls
13MKDIR=/bin/mkdir
14TR=/usr/bin/tr
15FIND=/usr/bin/find
16PRINTF=/usr/bin/printf
17XARGS=/usr/bin/xargs
18SORT=/bin/sort
19QSUB=qsub
20
21DefaultIFS=$' \t\n'
22IFS=$DefaultIFS
23
24
25scriptName="`basename $0`"
26$ECHO "Processing script ${scriptName} at `date`"
27######
28# Local functions
29######
30usage() {
31  echo "usage: $scriptName <source> <YYYYMMDD-directory> <type of process>"
32  echo "      -<source> as Abell85"
33  echo "      -<YYYYMMDD-directory> as 20110428"
34  echo "      -<type of process> as GAIN|CALIBON|CALIBOFF|ON|OFF"
35}
36######
37# argument test
38######
39
40#Process cmd line args: the -src option is mandatory (source name as Abell85)
41sourceRadio=
42dateSelected=
43typeofproc=
44#use -sim option to simulate processing (debug the script as if...)
45simulationMode=Off
46#
47#first and last cycle to process
48#
49firstCycle=
50lastCycle=
51while [ $# -gt 0 ]
52do
53  case "$1" in
54      -src)  sourceRadio=$2;    shift;;
55      -date) dateSelected=$2;     shift;;
56      -type) typeofproc=$2; shift;;
57      -fcycle) firstCycle=$2; shift;;
58      -lcycle) lastCycle=$2; shift;;
59      -sim)  simulationMode=On;;
60        -h)
61            echo >&2 \
62            "usage: $0 -src souce -date YYYYMMDD -type type [-fcycle firstCycle] [-lcycle lastCycle] [-sim to trig simulation mode]"
63            exit 1;;
64        *)  break;;     # terminate while loop
65    esac
66    shift
67done
68
69#Check input
70
71if [ "<${sourceRadio}>" = "<>" ]; then
72    $ECHO "FATAL: You have forgotten to select the source option (-src)"
73    exit 1
74fi
75
76if [ "<${dateSelected}>" = "<>" ]; then
77    $ECHO "FATAL: You have forgotten to select the date option (-date)"
78    exit 1
79fi
80
81if [ "<${typeofproc}>" = "<>" ]; then
82    $ECHO "FATAL: You have forgotten to select the type option (-type)"
83    exit 1
84fi
85
86
87
88srclower=`${ECHO} ${sourceRadio} | ${TR} "[:upper:]" "[:lower:]" `
89
90
91#Path to public backupable path
92publicpath="/afs/in2p3.fr/home/c/campagne/public"
93
94#temporary files to synchronize scripts
95tmppublicpath=${TMPPUBLICPATH}
96$LS -l ${tmppublicpath} > /dev/null
97
98
99#Path where the job will do temporary IO
100if [ ${ENVIRONMENT} = "INTERACTIVE" ]; then
101    iojobpath="/sps/baoradio/AmasNancay/JEC"
102    $MKDIR -p $iojobpath
103elif [ ${ENVIRONMENT} = "BATCH" ] ; then
104    iojobpath=${TMPBATCH}
105else
106    $ECHO "FATAL (${scriptName}): environment is ${ENVIRONMENT} not allowed"
107    touch ${tmppublicpath}/proc_script.finished
108    exit 1
109fi
110cd ${iojobpath}
111
112localpath="${sourceRadio}/${dateSelected}${srclower}"
113$MKDIR -p ./${localpath}
114cd ./${localpath}
115mainDirectory=`pwd`
116
117
118#Path of the utility scripts
119scriptpath="/afs/in2p3.fr/home/c/campagne/private/work/AmasNancay"
120
121
122#select file per cycle
123#$ECHO "Select files process..."
124#keep only lines with structure: cycle_number first,last,step
125#Get sca files
126$ECHO "DEBUG (${scriptName}): START call getscafiles.sh ${sourceRadio} ${dateSelected}"
127OUT1=./getScaStatus.$$
128$RM -f ${tmppublicpath}/getscafiles.finished
129${scriptpath}/getscafiles.sh ${sourceRadio} ${dateSelected} > ${OUT1} 2>&1
130while [ ! -f "${tmppublicpath}/getscafiles.finished" ]; do
131    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getscafiles.finished"
132    date +%T
133    sleep 30
134done
135$RM -f ${tmppublicpath}/getscafiles.finished
136$ECHO "DEBUG (${scriptName}): END"
137
138
139getScaStatus=`$GREP "^\(FATAL\|ERROR\)" ${OUT1}`
140
141if [ "<${getScaStatus}>" != "<>" ]; then
142  $ECHO "FATAL (${scriptName}): error to get sca file for $sourceRadio $dateSelected"
143  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
144  $CAT ${OUT1}
145  $ECHO "DEBUG (${scriptName}): END"
146  touch ${tmppublicpath}/proc_script.finished
147  exit 1
148fi
149
150
151scaFileSummarized=`$FIND . -name "sca*.sum" -print`
152if [ "<${scaFileSummarized}>" = "<>" ]; then
153  $ECHO "FATAL (${scriptName}): strange sca file summarized not available: $sourceRadio $dateSelected"
154  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
155  $CAT ${OUT1}
156  $ECHO "DEBUG (${scriptName}): END"
157  touch ${tmppublicpath}/proc_script.finished
158  exit 1
159fi
160$RM ${OUT1}
161
162scaFileSummarized=`basename $scaFileSummarized`
163
164#set Irods environment
165. /usr/local/shared/bin/irods_env.sh -noverbose
166#download header files to be used for signal files selction
167$MKDIR -p ./Fiber1
168dirFiberfile=`pwd`/Fiber1
169
170dirIrods="/baoradio/data/AmasNancay/${sourceRadio}/${dateSelected}${srclower}/Fiber1"
171
172listOfFiles=( `ils ${dirIrods} | $GREP -i "header" | $XARGS -i basename {} | $SORT -k1.7n` )
173IFS='
174'
175listOfFiles=( $( $PRINTF "%s\n" "${listOfFiles[@]}" | $AWK 'x[$0]++ == 0' ) )
176IFS=$DefaultIFS
177
178#
179$ECHO "DEBUG (${scriptName}): Download if necessary the headers"
180#
181for ifile in ${listOfFiles[@]}
182do
183#test if the headers have not yet been downloaded
184  if [ ! -e ${dirFiberfile}/${ifile} ]; then
185      OUT2=./getheaders.$$
186      iget -f -K ${dirIrods}/${ifile} ${dirFiberfile} >${OUT2} 2>&1
187      igetStatus=`$GREP "^ERROR" ${OUT2}`
188      if [ "<$igetStatus>" != "<>" ]; then
189          $ECHO "FATAL (${scriptName}): error while iget:"
190          $ECHO $igetStatus
191          touch ${tmppublicpath}/proc_script.finished
192          exit 1
193      fi
194      $RM -f $OUT2
195  fi
196done
197#
198$ECHO "DEBUG (${scriptName}): START call select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc}"
199#
200#Get list of file to process
201headerFileSelected=./selectSignal.$$
202$RM -f ${tmppublicpath}/select.finished
203${scriptpath}/select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc} | $GREP "^#[0-9]" | $SED "s/^#//" > $headerFileSelected
204while [ ! -f "${tmppublicpath}/select.finished" ]; do
205    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/select.finished"
206    date +%T
207    sleep 30
208done
209$RM ${tmppublicpath}/select.finished
210#
211$ECHO "DEBUG (${scriptName}): END"
212#
213$ECHO "DEBUG (${scriptName}): START filling cycleArray, filesArray"
214#
215declare -a cycleArray
216declare -a filesArray
217$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
218IFS=$DefaultIFS
219$ECHO "DEBUG (${scriptName}): START cat $headerFileSelected"
220$CAT $headerFileSelected
221$ECHO "DEBUG (${scriptName}): END cat $headerFileSelected"
222
223while read v1 v2
224do
225 cycleArray[${#cycleArray[*]}]=$v1
226 filesArray[${#filesArray[*]}]=$v2
227done < $headerFileSelected
228
229if [ ${#cycleArray[*]} -eq 0 -o ${#filesArray[*]} -eq 0 ]; then
230   $ECHO "FATAL (${scriptName}): no header file selected (${#cycleArray[*]}, ${#filesArray[*]})"
231   touch ${tmppublicpath}/proc_script.finished
232   exit 1
233else
234   $ECHO "DEBUG (${scriptName}): non zero elements found for cycleArray and filesArray" 
235fi
236
237#
238$ECHO "DEBUG (${scriptName}): number of elements <${#cycleArray[*]}> <${#filesArray[*]}>"
239$ECHO "DEBUG (${scriptName}): END"
240#
241# (timeShift)
242#Add 3sec to start ON / OFF
243#Substract 4sec to start DAB ON / OFF
244#
245#Take care: Cycle numbering starts at 1 in SCAFiles while Arrays index start at 0
246#
247#for GAIN the reference cycle takes the middle of the run
248#
249#nwinmean donne le nombre de fenetre tspwin utilise pour faire calculer la moyenne des medianes de chaque fenetre
250#This is the nimber of paquets of the time window which change according to processing type
251$ECHO "DEBUG (${scriptName}): START switch on $typeofproc"
252#
253case "$typeofproc" in
254    GAIN)
255      $ECHO "process signal files for GAIN phase"
256      ingain=""
257      freqfilter="-freqfilter -"
258      nwinmean="1"
259      tspwin="5120"
260      dirName="Off"
261      cyclebasename="gaincycle"
262      indexStartHour="6"
263      timeShift="-4"
264      timeDuration="14"
265      firstCycleId=`expr  ${#cycleArray[*]} / 2`
266      firstCycleId=`expr ${firstCycleId} - 1`
267      lastp1CycleId=`expr ${firstCycleId} + 1`
268      ;;
269    ON)
270      $ECHO "process signal files for ON phases"
271      ingain="-gain gain_${dateSelected}_${srclower}.fits"
272      freqfilter="-freqfilter 16"
273      nwinmean="5"
274      tspwin="5120"
275      dirName="On"
276      cyclebasename="datacycle"
277      indexStartHour="4"
278      timeShift="3"
279      timeDuration="30"
280      firstCycleId="0"
281      lastp1CycleId="${#cycleArray[@]}"
282      ;;
283    OFF)
284      $ECHO "process signal files for OFF phases"
285      ingain="-gain gain_${dateSelected}_${srclower}.fits"
286      freqfilter="-freqfilter 16"
287      nwinmean="5"
288      tspwin="5120"
289      dirName="Off"
290      cyclebasename="datacycle"
291      indexStartHour="8"
292      timeShift="3"
293      timeDuration="30"
294      firstCycleId="0"
295      lastp1CycleId="${#cycleArray[@]}"
296      ;;
297    CALIBON)
298      $ECHO "process signal files for CALIBRATION ON phases"
299      ingain="-gain gain_${dateSelected}_${srclower}.fits"
300      freqfilter="-freqfilter 16"
301      nwinmean="1"
302      tspwin="1024"
303      dirName="On"
304      cyclebasename="calibcycle"
305      indexStartHour="2"
306      timeShift="-4"
307      timeDuration="14"
308      firstCycleId="0"
309      lastp1CycleId="${#cycleArray[@]}"
310      ;;
311    CALIBOFF)
312      $ECHO "process signal files for CALIBRATION OFF phases"
313      ingain="-gain gain_${dateSelected}_${srclower}.fits"
314      freqfilter="-freqfilter 16"
315      nwinmean="1"
316      tspwin="1024"
317      dirName="Off"
318      cyclebasename="calibcycle"
319      indexStartHour="6"
320      timeShift="-4"
321      timeDuration="14"
322      firstCycleId="0"
323      lastp1CycleId="${#cycleArray[@]}"
324      ;;
325    *)
326    $ECHO "FATAL (${scriptName}): type of processing"
327    usage
328    touch ${tmppublicpath}/proc_script.finished
329    exit 1
330    ;;
331esac
332#
333$ECHO "DEBUG (${scriptName}): END"
334#
335#
336#Compute time windows (Warning usage of GMT date)
337#
338$ECHO "DEBUG (${scriptName}): START compute time windows"
339OUT=./baotiming.$$
340${AWK} -v datesys=${DATE} -v type=${indexStartHour} -v shift=${timeShift} '
341BEGIN {FS=","}
342{
343  phaseStart=$type
344  cmd=sprintf("%s +%%s -d  \"%s\" ",datesys,phaseStart);
345  (cmd | getline phstart);
346  close(cmd);
347  newstart = phstart+shift;
348  cmd=sprintf("%s +%%T -d \"1970-01-01 %s sec GMT\"",datesys,newstart);
349  (cmd | getline newStart);
350  close(cmd);
351  print $1,newStart,phaseStart;
352}
353'  ${scaFileSummarized} > ${OUT}
354#
355$ECHO "DEBUG (${scriptName}): END"
356#
357$ECHO "DEBUG (${scriptName}): START cat ${OUT}"
358$CAT ${OUT}
359$ECHO "DEBUG (${scriptName}): END cat ${OUT}"
360#
361$ECHO "DEBUG (${scriptName}): START filling cylceArrayBis timeArray"
362#
363declare -a cycleArrayBis
364declare -a timeArray
365$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
366IFS=$DefaultIFS
367#select the selected cycles
368idtofind="0"
369while read v1 v2 v3
370do
371 if [ ${idtofind} -lt ${#cycleArray[*]} ]; then
372    if [ ${v1} = ${cycleArray[${idtofind}]} ]; then
373        cycleArrayBis[${#cycleArrayBis[*]}]=$v1
374        timeArray[${#timeArray[*]}]=$v2
375        idtofind=`expr ${idtofind} + 1`
376    fi
377 fi
378done < ${OUT}
379$ECHO "DEBUG (${scriptName}): number of elements ${#cycleArrayBis[*]} ${#timeArray[*]}"
380$ECHO "DEBUG (${scriptName}): END"
381
382
383#########################
384# specmfib exe
385#########################
386prg="/afs/in2p3.fr/throng/baoradio/Library/Progs/TAcq/Objs/specmfib"
387if [ ! -e $prg ]; then
388    $ECHO "FATAL (${scriptName}): $prg not found"
389    touch ${tmppublicpath}/proc_script.finished
390    exit 1
391fi
392
393
394dirFiberfile=${iojobpath}/${localpath}
395
396curDir=`pwd`
397#
398$ECHO "DEBUG (${scriptName}): in principle $dirFiberfile == $curDir "
399#
400#force single channel
401forceSingle="-singlechan"
402#debug
403prtlevel="1"
404
405
406#Loop on cycles
407##
408$ECHO "DEBUG (${scriptName}): START loop on cycles Ids ${firstCycleId} to ${lastp1CycleId}"
409
410#write the script to be send to Batch
411#get gain file if necessary
412if [ ${typeofproc} != "GAIN" ]; then
413    OUT2=./igetStatus.$$
414    iget /baoradio/data/AmasNancay/${localpath}/gain_${dateSelected}_${srclower}.fits >${OUT2} 2>&1
415    igetStatus=`$GREP "^ERROR" ${OUT2}`
416    if [ "<${igetStatus}>" != "<>" ]; then
417        $ECHO "FATAL (${scriptName}): error while iget:"
418        $ECHO $igetStatus
419        touch ${tmppublicpath}/${jobBatchName}.finished
420        exit 1
421    fi
422    $RM ${OUT2}
423fi   
424
425
426for ((i=${firstCycleId};i<${lastp1CycleId};i++)); do
427#
428#time window selection
429    tmproc="-tmproc ${timeArray[${i}]},${timeDuration}"
430#cycle <number>
431    cycle="${cycleArray[${i}]}"
432    if [ "<${firstCycle}>" != "<>" -a "<${lastCycle}>" != "<>" ]; then
433        if [ ${cycle} -lt ${firstCycle} -o ${cycle} -gt ${lastCycle} ]; then
434            continue
435        fi
436    fi
437    $ECHO "DEBUG (${scriptName}): process cycle $cycle"
438
439#cycle <name><number>
440    cycle="${cyclebasename}${cycleArray[${i}]}"
441#signal files selection
442    infiles="${filesArray[${i}]}"
443#
444#Prepare BQS parameters: MEMORY, CPU...#
445    infilesArray=( `$ECHO $infiles | $AWK 'BEGIN {FS=","} {for(i=1;i<NF;i++)print $i;}'` )
446    IFS='
447    '
448    infilesArray=( $( printf "%s\n" "${infilesArray[@]}" | awk 'x[$0]++ == 0' ) )
449    IFS=$DefaultIFS
450    firstFile=${infilesArray[0]}
451    lastFile=${infilesArray[1]}
452#
453#
454#
455#get signals files
456    OUT3=./getsignals.$$
457    $RM -f ${tmppublicpath}/getsignalfiles.finished
458    ${scriptpath}/getsignalfiles.sh ${sourceRadio} ${dateSelected} ${firstFile}  ${lastFile} > ${OUT3} 2>&1
459    while [ ! -f "${tmppublicpath}/getsignalfiles.finished" ]; do
460        $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getsignalfiles.finished"
461        date +%T
462        sleep 30
463    done
464    $RM ${tmppublicpath}/getsignalfiles.finished
465
466    getSignals=`$GREP "^FATAL" ${OUT3}`
467    if [ "<$getSignals>" != "<>" ]; then
468        $ECHO "FATAL (${scriptName}): error while get signals"
469        $ECHO $getSignals
470        touch ${tmppublicpath}/${jobBatchName}.finished
471        exit 1
472    fi
473    $RM ${OUT3}
474
475#location of the signal file
476    fibfile="${curDir} 1,2"
477#
478    outDir=${curDir}/${dirName}/${cycle}
479    $MKDIR -p ${outDir}
480
481#
482# where ami...
483#
484    $ECHO "We are here `pwd`"
485    $LS -lrth
486    $FIND . -name "igetStatus.*" | $XARGS -i  $CAT {}
487    $LS ./Fiber1 | $WC -l
488    $LS ./Fiber2 | $WC -l
489    $LS -lR ${outDir}
490
491#Note: here act=gain means only that specmfib will use median-like algorithms (not only for gains...)
492    $prg -act gain $forceSingle $freqfilter  -prt $prtlevel -out ${outDir} -nmean $nwinmean $ingain $tmproc -tspwin ${tspwin},0,0 -in $infiles ${fibfile}
493
494    $ECHO    ${outDir}
495    $LS -lrt ${outDir}
496
497#save results to Irods (use option -f ONLY to force override)
498    iput -v -K -r ${curDir}/${dirName} /baoradio/data/AmasNancay/${localpath}
499#
500#clean the signal files which can at the end of the day fill completly the scratch directory of the batch work/AmasNancayer
501    $RM -rf ${iojobpath}/${localpath}/Fiber1
502    $RM -rf ${iojobpath}/${localpath}/Fiber2
503    $RM -rf ${iojobpath}/${localpath}/${typeofproc}
504
505#
506#
507
508#end loop on cycles
509done
510#
511$ECHO "DEBUG (${scriptName}): END loop on cycles"
512#
513touch ${tmppublicpath}/proc_script.finished
514exit 0
Note: See TracBrowser for help on using the repository browser.