Index: /BAORadio/AmasNancay/trunk/proc_specmfib.sh
===================================================================
--- /BAORadio/AmasNancay/trunk/proc_specmfib.sh	(revision 567)
+++ /BAORadio/AmasNancay/trunk/proc_specmfib.sh	(revision 567)
@@ -0,0 +1,591 @@
+#!/bin/sh -xvf
+#Basic tuning of system fuction used.
+AWK=/bin/awk
+SED=/bin/sed
+GREP=/bin/grep
+WC=/usr/bin/wc
+RM=/bin/rm
+CAT=/bin/cat
+TOUCH=/bin/touch
+DATE=/bin/date
+ECHO=/bin/echo
+LS=/bin/ls
+MKDIR=/bin/mkdir
+TR=/usr/bin/tr
+FIND=/usr/bin/find
+PRINTF=/usr/bin/printf
+XARGS=/usr/bin/xargs
+SORT=/bin/sort
+QSUB=qsub
+
+DefaultIFS=$' \t\n'
+IFS=$DefaultIFS
+
+
+scriptName="`basename $0`"
+$ECHO "Processing script ${scriptName} at `date`"
+######
+# Local functions
+######
+usage() {
+  echo "usage: $scriptName <source> <YYYYMMDD-directory> <type of process>"
+  echo "      -<source> as Abell85"
+  echo "      -<YYYYMMDD-directory> as 20110428"
+  echo "      -<type of process> as GAIN|CALIBON|CALIBOFF|ON|OFF"
+}
+######
+# argument test
+######
+
+#Process cmd line args: the -src option is mandatory (source name as Abell85)
+#action: gain|mspec (gain: gain-like doucble median filtering
+#                   mspec: mean+sigma wo filetring)
+action="gain"
+sourceRadio=
+dateSelected=
+typeofproc=
+#use -sim option to simulate processing (debug the script as if...)
+simulationMode=Off
+#
+#first and last cycle to process
+#
+firstCycle=
+lastCycle=
+while [ $# -gt 0 ]
+do
+  case "$1" in
+      -act)  action=$2;         shift;;
+      -src)  sourceRadio=$2;    shift;;
+      -date) dateSelected=$2;   shift;;
+      -type) typeofproc=$2;     shift;;
+      -fcycle) firstCycle=$2;   shift;;
+      -lcycle) lastCycle=$2;    shift;;
+      -sim)  simulationMode=On;;
+	-h)
+	    echo >&2 \
+	    "usage: $0 -src souce -date YYYYMMDD -type type [-fcycle firstCycle] [-lcycle lastCycle] [-sim to trig simulation mode]"
+	    exit 1;;
+	*)  break;;	# terminate while loop
+    esac
+    shift
+done
+
+#Check input
+
+if [ "<${sourceRadio}>" = "<>" ]; then
+    $ECHO "FATAL: You have forgotten to select the source option (-src)"
+    exit 1
+fi
+
+if [ "<${dateSelected}>" = "<>" ]; then
+    $ECHO "FATAL: You have forgotten to select the date option (-date)"
+    exit 1
+fi
+
+if [ "<${typeofproc}>" = "<>" ]; then
+    $ECHO "FATAL: You have forgotten to select the type option (-type)"
+    exit 1
+else
+    offtype=`$ECHO ${typeofproc} | $GREP -i OFF | $WC -l`
+    ontype=`$ECHO ${typeofproc} | $GREP -i ON | $WC -l`
+    if [ "<${offtype}>" = "<1>" -a  "<${ontype}>" = "<0>" ]; then
+	datatype="OFF"
+    elif [ "<${offtype}>" = "<0>" -a  "<${ontype}>" = "<1>" ]; then
+	datatype="ON"
+    else
+	$ECHO "FATAL: not supported: -type ${typeofproc}"
+	exit 1
+    fi
+fi
+
+
+srclower=`${ECHO} ${sourceRadio} | ${TR} "[:upper:]" "[:lower:]" `
+
+
+#Path to public backupable path
+publicpath="/afs/in2p3.fr/home/c/campagne/public"
+
+#temporary files to synchronize scripts
+tmppublicpath=${TMPPUBLICPATH}
+$LS -l ${tmppublicpath} > /dev/null
+
+
+#Path where the job will do temporary IO
+if [ ${ENVIRONMENT} = "INTERACTIVE" ]; then
+    iojobpath="/sps/baoradio/AmasNancay/JEC"
+    $MKDIR -p $iojobpath
+elif [ ${ENVIRONMENT} = "BATCH" ] ; then
+    iojobpath=${TMPBATCH}
+else
+    $ECHO "FATAL (${scriptName}): environment is ${ENVIRONMENT} not allowed"
+    touch ${tmppublicpath}/proc_script.finished
+    exit 1
+fi
+cd ${iojobpath}
+
+localpath="${sourceRadio}/${dateSelected}${srclower}"
+$MKDIR -p ./${localpath}
+cd ./${localpath}
+mainDirectory=`pwd`
+
+
+#Path of the utility scripts
+scriptpath="/afs/in2p3.fr/home/c/campagne/private/work/AmasNancay"
+
+
+#select file per cycle
+#$ECHO "Select files process..."
+#keep only lines with structure: cycle_number first,last,step
+#Get sca files
+$ECHO "DEBUG (${scriptName}): START call getscafiles.sh ${sourceRadio} ${dateSelected}"
+OUT1=./getScaStatus.$$
+#JEC 29/9/11 avoid finished stuff
+#$RM -f ${tmppublicpath}/getscafiles.finished
+${scriptpath}/getscafiles.sh ${sourceRadio} ${dateSelected} > ${OUT1} 2>&1
+#while [ ! -f "${tmppublicpath}/getscafiles.finished" ]; do
+#    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getscafiles.finished"
+#    date +%T
+#    sleep 30
+#done
+#$RM -f ${tmppublicpath}/getscafiles.finished
+$ECHO "DEBUG (${scriptName}): END"
+
+
+getScaStatus=`$GREP "^\(FATAL\|ERROR\)" ${OUT1}`
+
+if [ "<${getScaStatus}>" != "<>" ]; then
+  $ECHO "FATAL (${scriptName}): error to get sca file for $sourceRadio $dateSelected"
+  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
+  $CAT ${OUT1}
+  $ECHO "DEBUG (${scriptName}): END"
+#JEC 29/9/11 avoid finished stuff
+#  touch ${tmppublicpath}/proc_script.finished
+  exit 1
+fi
+
+
+scaFileSummarized=`$FIND . -name "sca*.sum" -print`
+if [ "<${scaFileSummarized}>" = "<>" ]; then
+  $ECHO "FATAL (${scriptName}): strange sca file summarized not available: $sourceRadio $dateSelected"
+  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
+  $CAT ${OUT1}
+  $ECHO "DEBUG (${scriptName}): END"
+#JEC 29/9/11 avoid finished stuff
+#  touch ${tmppublicpath}/proc_script.finished
+  exit 1
+fi
+$RM ${OUT1}
+
+scaFileSummarized=`basename $scaFileSummarized`
+
+#set Irods environment
+. /usr/local/shared/bin/irods_env.sh -noverbose
+#download header files to be used for signal files selction
+$MKDIR -p ./Fiber1
+dirFiberfile=`pwd`/Fiber1
+
+dirIrods="/baoradio/data/AmasNancay/${sourceRadio}/${dateSelected}${srclower}/Fiber1"
+
+listOfFiles=( `ils ${dirIrods} | $GREP -i "header" | $XARGS -i basename {} | $SORT -k1.7n` )
+IFS='
+'
+listOfFiles=( $( $PRINTF "%s\n" "${listOfFiles[@]}" | $AWK 'x[$0]++ == 0' ) )
+IFS=$DefaultIFS
+
+#
+$ECHO "DEBUG (${scriptName}): Download if necessary the headers"
+#
+for ifile in ${listOfFiles[@]}
+do
+#test if the headers have not yet been downloaded
+  if [ ! -e ${dirFiberfile}/${ifile} ]; then
+      OUT2=./getheaders.$$
+      iget -f -K ${dirIrods}/${ifile} ${dirFiberfile} >${OUT2} 2>&1
+      igetStatus=`$GREP "^ERROR" ${OUT2}`
+      if [ "<$igetStatus>" != "<>" ]; then
+	  $ECHO "FATAL (${scriptName}): error while iget:"
+	  $ECHO $igetStatus
+#JEC 29/9/11 avoid finished stuff
+#	  touch ${tmppublicpath}/proc_script.finished
+	  exit 1
+      fi
+      $RM -f $OUT2
+  fi
+done
+#
+$ECHO "DEBUG (${scriptName}): START call select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc}"
+#
+#Get list of file to process
+headerFileSelected=./selectSignal.$$
+#JEC 29/9/11 avoid finished stuff
+#$RM -f ${tmppublicpath}/select.finished
+${scriptpath}/select.sh ${scaFileSummarized} ${dirFiberfile} ${datatype} | $GREP "^#[0-9]" | $SED "s/^#//" > $headerFileSelected
+#while [ ! -f "${tmppublicpath}/select.finished" ]; do
+#    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/select.finished"
+#    date +%T
+#    sleep 30
+#done
+#$RM ${tmppublicpath}/select.finished
+#
+$ECHO "DEBUG (${scriptName}): END"
+#
+$ECHO "DEBUG (${scriptName}): START filling cycleArray, filesArray"
+#
+declare -a cycleArray
+declare -a filesArray
+$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
+IFS=$DefaultIFS
+$ECHO "DEBUG (${scriptName}): START cat $headerFileSelected"
+$CAT $headerFileSelected
+$ECHO "DEBUG (${scriptName}): END cat $headerFileSelected"
+
+while read v1 v2
+do
+ cycleArray[${#cycleArray[*]}]=$v1
+ filesArray[${#filesArray[*]}]=$v2
+done < $headerFileSelected
+
+if [ ${#cycleArray[*]} -eq 0 -o ${#filesArray[*]} -eq 0 ]; then
+   $ECHO "FATAL (${scriptName}): no header file selected (${#cycleArray[*]}, ${#filesArray[*]})"
+#JEC 29/9/11 avoid finished stuff
+#   touch ${tmppublicpath}/proc_script.finished
+   exit 1
+else
+   $ECHO "DEBUG (${scriptName}): non zero elements found for cycleArray and filesArray" 
+fi
+
+#
+$ECHO "DEBUG (${scriptName}): number of elements <${#cycleArray[*]}> <${#filesArray[*]}>"
+$ECHO "DEBUG (${scriptName}): END"
+#
+# (timeShift)
+#Add 3sec to start ON / OFF
+#Substract 4sec to start DAB ON / OFF
+#
+#Take care: Cycle numbering starts at 1 in SCAFiles while Arrays index start at 0
+#
+#for GAIN the reference cycle takes the middle of the run
+#
+#nwinmean donne le nombre de fenetre tspwin utilise pour faire calculer la moyenne des medianes de chaque fenetre
+#This is the nimber of paquets of the time window which change according to processing type
+$ECHO "DEBUG (${scriptName}): START switch on $typeofproc"
+#
+case "$typeofproc" in
+    GAIN)
+      $ECHO "process signal files for GAIN phase"
+      ingain=""
+      freqfilter="-freqfilter -"
+      nwinmean="1"
+      tspwin="5120"
+      dirName="Off"
+      cyclebasename="gaincycle"
+      indexStartHour="6"
+      timeShift="-4"
+      timeDuration="14"
+      firstCycleId=`expr  ${#cycleArray[*]} / 2`
+      firstCycleId=`expr ${firstCycleId} - 1`
+      lastp1CycleId=`expr ${firstCycleId} + 1`
+      ;;
+    ON)
+      $ECHO "process signal files for ON phases"
+      ingain="-gain gain_${dateSelected}_${srclower}.fits"
+      freqfilter=""
+      nwinmean="5"
+      tspwin="5120"
+      dirName="On"
+      cyclebasename="datacycle"
+      indexStartHour="4"
+      timeShift="3"
+      timeDuration="30"
+      firstCycleId="0"
+      lastp1CycleId="${#cycleArray[@]}"
+      ;;
+    OFF)
+      $ECHO "process signal files for OFF phases"
+      ingain="-gain gain_${dateSelected}_${srclower}.fits"
+      freqfilter=""
+      nwinmean="5"
+      tspwin="5120"
+      dirName="Off"
+      cyclebasename="datacycle"
+      indexStartHour="8"
+      timeShift="3"
+      timeDuration="30"
+      firstCycleId="0"
+      lastp1CycleId="${#cycleArray[@]}"
+      ;;
+    MEANOFF)
+      $ECHO "process signal files for OFF phases (MEAN)"
+      ingain="-gain gain_${dateSelected}_${srclower}.fits"
+      nwinmean="25000"
+      tspwin="1"
+      dirName="Off"
+      cyclebasename="meancycle"
+      indexStartHour="8"
+      timeShift="3"
+      timeDuration="30"
+      firstCycleId="0"
+      lastp1CycleId="${#cycleArray[@]}"
+      ;;
+    MEANON)
+      $ECHO "process signal files for OFF phases (MEAN)"
+      ingain="-gain gain_${dateSelected}_${srclower}.fits"
+      nwinmean="25000"
+      tspwin="1"
+      dirName="On"
+      cyclebasename="meancycle"
+      indexStartHour="8"
+      timeShift="3"
+      timeDuration="30"
+      firstCycleId="0"
+      lastp1CycleId="${#cycleArray[@]}"
+      ;;
+    CALIBON)
+      $ECHO "process signal files for CALIBRATION ON phases"
+      ingain="-gain gain_${dateSelected}_${srclower}.fits"
+      freqfilter=""
+      nwinmean="1"
+      tspwin="1024"
+      dirName="On"
+      cyclebasename="calibcycle"
+      indexStartHour="2"
+      timeShift="-4"
+      timeDuration="14"
+      firstCycleId="0"
+      lastp1CycleId="${#cycleArray[@]}"
+      ;;
+    CALIBOFF)
+      $ECHO "process signal files for CALIBRATION OFF phases"
+      ingain="-gain gain_${dateSelected}_${srclower}.fits"
+      freqfilter=""
+      nwinmean="1"
+      tspwin="1024"
+      dirName="Off"
+      cyclebasename="calibcycle"
+      indexStartHour="6"
+      timeShift="-4"
+      timeDuration="14"
+      firstCycleId="0"
+      lastp1CycleId="${#cycleArray[@]}"
+      ;;
+    *)
+    $ECHO "FATAL (${scriptName}): type of processing"
+    usage
+#JEC 29/9/11 avoid finished stuff
+#    touch ${tmppublicpath}/proc_script.finished
+    exit 1
+    ;;
+esac
+#
+$ECHO "DEBUG (${scriptName}): END"
+#
+#
+#Compute time windows (Warning usage of GMT date)
+#
+$ECHO "DEBUG (${scriptName}): START compute time windows"
+OUT=./baotiming.$$
+${AWK} -v datesys=${DATE} -v type=${indexStartHour} -v shift=${timeShift} '
+BEGIN {FS=","}
+{
+  phaseStart=$type
+  cmd=sprintf("%s +%%s -d  \"%s\" ",datesys,phaseStart);
+  (cmd | getline phstart);
+  close(cmd);
+  newstart = phstart+shift;
+  cmd=sprintf("%s +%%T -d \"1970-01-01 %s sec GMT\"",datesys,newstart);
+  (cmd | getline newStart);
+  close(cmd);
+  print $1,newStart,phaseStart;
+}
+'  ${scaFileSummarized} > ${OUT}
+#
+$ECHO "DEBUG (${scriptName}): END"
+#
+$ECHO "DEBUG (${scriptName}): START cat ${OUT}"
+$CAT ${OUT}
+$ECHO "DEBUG (${scriptName}): END cat ${OUT}"
+#
+$ECHO "DEBUG (${scriptName}): START filling cylceArrayBis timeArray"
+#
+declare -a cycleArrayBis
+declare -a timeArray
+$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
+IFS=$DefaultIFS
+#select the selected cycles
+idtofind="0"
+while read v1 v2 v3
+do
+ if [ ${idtofind} -lt ${#cycleArray[*]} ]; then
+    if [ ${v1} = ${cycleArray[${idtofind}]} ]; then
+	cycleArrayBis[${#cycleArrayBis[*]}]=$v1
+	timeArray[${#timeArray[*]}]=$v2
+	idtofind=`expr ${idtofind} + 1`
+    fi
+ fi
+done < ${OUT}
+$ECHO "DEBUG (${scriptName}): number of elements ${#cycleArrayBis[*]} ${#timeArray[*]}"
+$ECHO "DEBUG (${scriptName}): END"
+
+
+#########################
+# specmfib exe
+#########################
+prg="/afs/in2p3.fr/throng/baoradio/Library/Progs/TAcq/Objs/specmfib"
+if [ ! -e $prg ]; then
+    $ECHO "FATAL (${scriptName}): $prg not found"
+#JEC 29/9/11 avoid finished stuff
+#    touch ${tmppublicpath}/proc_script.finished
+    exit 1
+fi
+
+
+dirFiberfile=${iojobpath}/${localpath}
+
+curDir=`pwd`
+#
+$ECHO "DEBUG (${scriptName}): in principle $dirFiberfile == $curDir "
+#
+#force single channel
+forceSingle="-singlechan"
+#debug
+prtlevel="1"
+
+
+#Loop on cycles
+##
+$ECHO "DEBUG (${scriptName}): START loop on cycles Ids ${firstCycleId} to ${lastp1CycleId}"
+
+#write the script to be send to Batch
+#get gain file if necessary
+if [ ${typeofproc} != "GAIN" ]; then
+    OUT2=./igetStatus.$$
+    iget /baoradio/data/AmasNancay/${localpath}/gain_${dateSelected}_${srclower}.fits >${OUT2} 2>&1
+    igetStatus=`$GREP "^ERROR" ${OUT2}`
+    if [ "<${igetStatus}>" != "<>" ]; then
+	$ECHO "FATAL (${scriptName}): error while iget:"
+	$ECHO $igetStatus
+#JEC 29/9/11 avoid finished stuff
+#	touch ${tmppublicpath}/${jobBatchName}.finished
+	exit 1
+    fi
+    $RM ${OUT2}
+fi    
+
+
+for ((i=${firstCycleId};i<${lastp1CycleId};i++)); do
+#
+
+####
+# Check daq status
+#JEC 1/10/11 Use generic baodaqstatus name 
+####
+#get the daq current irod status
+#tag=`${DATE} +%F`
+#OUT1=${publicpath}/baodaqstatus-${tag}.txt
+OUT1=${publicpath}/baodaqstatus-current.txt
+
+if [ ! -e ${OUT1}  -o ! -r ${OUT1} ]; then
+    $ECHO "FATAL (${scriptName}): ${OUT1} has a problem"
+    exit 1
+#    $ECHO "We should bring the DAQ status more up to date, this take 1 or 2min..."
+#JEC 29/9/11 avoid finished stuff
+#    $RM -f ${tmppublicpath}/statusdaq.finished
+#    ${scriptpath}/statusdaq.sh > ${OUT1}
+#    while [ ! -f "${tmppublicpath}/statusdaq.finished" ]; do
+#	sleep 30
+#    done
+#    $RM ${tmppublicpath}/statusdaq.finished
+#protect against remove/rewriting    
+#    $CHMOD -v 444 ${OUT1} 
+fi
+
+#time window selection
+    tmproc="-tmproc ${timeArray[${i}]},${timeDuration}"
+#cycle <number>
+    cycle="${cycleArray[${i}]}"
+    if [ "<${firstCycle}>" != "<>" -a "<${lastCycle}>" != "<>" ]; then
+	if [ ${cycle} -lt ${firstCycle} -o ${cycle} -gt ${lastCycle} ]; then
+	    continue
+	fi
+    fi
+    $ECHO "DEBUG (${scriptName}): process cycle $cycle"
+
+#cycle <name><number>
+    cycle="${cyclebasename}${cycleArray[${i}]}"
+#signal files selection
+    infiles="${filesArray[${i}]}"
+#
+#Prepare BQS parameters: MEMORY, CPU...#
+    infilesArray=( `$ECHO $infiles | $AWK 'BEGIN {FS=","} {for(i=1;i<NF;i++)print $i;}'` )
+    IFS='
+    '
+    infilesArray=( $( printf "%s\n" "${infilesArray[@]}" | awk 'x[$0]++ == 0' ) )
+    IFS=$DefaultIFS
+    firstFile=${infilesArray[0]}
+    lastFile=${infilesArray[1]}
+#
+#
+#
+#get signals files
+    OUT3=./getsignals.$$
+#JEC 29/9/11 avoid finished stuff
+#    $RM -f ${tmppublicpath}/getsignalfiles.finished
+    ${scriptpath}/getsignalfiles.sh ${sourceRadio} ${dateSelected} ${firstFile}  ${lastFile} > ${OUT3} 2>&1
+#    while [ ! -f "${tmppublicpath}/getsignalfiles.finished" ]; do
+#	$ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getsignalfiles.finished"
+#	date +%T
+#	sleep 30
+#    done
+#    $RM ${tmppublicpath}/getsignalfiles.finished
+
+    getSignals=`$GREP "^FATAL" ${OUT3}`
+    if [ "<$getSignals>" != "<>" ]; then
+	$ECHO "FATAL (${scriptName}): error while get signals"
+	$ECHO $getSignals
+#JEC 29/9/11 avoid finished stuff
+#	touch ${tmppublicpath}/${jobBatchName}.finished
+	exit 1
+    fi
+    $RM ${OUT3}
+
+#location of the signal file
+    fibfile="${curDir} 1,2"
+#
+    outDir=${curDir}/${dirName}/${cycle}
+    $MKDIR -p ${outDir}
+
+#
+# where ami...
+#
+    $ECHO "We are here `pwd`"
+    $LS -lrth
+    $FIND . -name "igetStatus.*" | $XARGS -i  $CAT {}
+    $LS ./Fiber1 | $WC -l
+    $LS ./Fiber2 | $WC -l
+    $LS -lR ${outDir}
+
+#Note: here act=gain means only that specmfib will use median-like algorithms (not only for gains...)
+    $prg -act ${action} $forceSingle $freqfilter  -prt $prtlevel -out ${outDir} -nmean $nwinmean $ingain $tmproc -tspwin ${tspwin},0,0 -in $infiles ${fibfile}
+
+    $ECHO    ${outDir}
+    $LS -lrt ${outDir}
+
+#save results to Irods (use option -f ONLY to force override)
+    iput -v -K -r ${curDir}/${dirName} /baoradio/data/AmasNancay/${localpath}
+#
+#clean the signal files which can at the end of the day fill completly the scratch directory of the batch work/AmasNancayer
+    $RM -rf ${iojobpath}/${localpath}/Fiber1
+    $RM -rf ${iojobpath}/${localpath}/Fiber2
+    $RM -rf ${iojobpath}/${localpath}/${typeofproc}
+
+#
+#
+
+#end loop on cycles
+done
+#
+$ECHO "DEBUG (${scriptName}): END loop on cycles"
+#
+#JEC 29/9/11 avoid finished stuff
+#touch ${tmppublicpath}/proc_script.finished
+exit 0
