#!/bin/sh -xvf #Perform the ON-OFF analysis DATE=/bin/date GREP=/bin/grep AWK=/bin/awk ECHO=/bin/echo WC=/usr/bin/wc CAT=/bin/cat PRINTF=/usr/bin/printf FIND=/usr/bin/find MKDIR=/bin/mkdir XARGS=/usr/bin/xargs SORT=/bin/sort RM=/bin/rm TR=/usr/bin/tr CP=/bin/cp LS=/bin/ls CHMOD=/bin/chmod DefaultIFS=$' \t\n' IFS=$DefaultIFS #set Irods environment . /usr/local/shared/bin/irods_env.sh -noverbose scriptName="`basename $0`" $ECHO "Processing script ${scriptName} at `date`" #which source to analyse #Process cmd line args: the -src option is mandatory (source name as Abell85) sourceRadio= dateSelected= #use -sim option to simulate processing (debug the script as if...) simulationMode=Off while [ $# -gt 0 ] do case "$1" in -src) sourceRadio=$2; shift;; -date) dateSelected=$2; shift;; -sim) simulationMode=On;; -h) echo >&2 \ "usage: $0 -src souce -date YYYYMMDD [-freq freq in MHz (default ${freqBAOCalib})] [-sim to trig simulation mode]" exit 1;; *) break;; # terminate while loop esac shift done if [ ${simulationMode} = "On" ]; then $ECHO "INFO ${scriptName} running in SIMUL mode" fi if [ "<${sourceRadio}>" = "<>" ]; then $ECHO "FATAL: You have forgotten to select the source option (-src)" exit 1 fi case ${sourceRadio} in Abell85) $ECHO "INFO (${scriptName}): process ${sourceRadio}";; Abell2440) $ECHO "INFO (${scriptName}): process ${sourceRadio}";; Abell1205) $ECHO "INFO (${scriptName}): process ${sourceRadio}";; *) ECHO "FATAL (${scriptName}): process ${sourceRadio} not yet foreseen" exit 1;; esac srclower=`${ECHO} ${sourceRadio} | ${TR} "[:upper:]" "[:lower:]" ` #Path to public backupable path publicpath="/afs/in2p3.fr/home/c/campagne/public" #temporary files to synchronize scripts tmppublicpath=${TMPPUBLICPATH} #wakeup the NFS disk $LS -l ${tmppublicpath} > /dev/null #clean previous spurious files $RM -f ${tmppublicpath}/*.finished #Path where the job will do temporary IO . ${SCRIPTPATH}/isInteractive.sh isInteractive=$(isInteractive) if [ ${isInteractive} == "1" ]; then $ECHO "Usage in INTERACTIVE may be destructive... To be used with great care !!!!" exit 0; fi . ${SCRIPTPATH}/set_iojobpath.sh iojobpath=$(set_iojobpath) cd ${iojobpath} localpath="${sourceRadio}" $MKDIR -p ./${localpath} cd ./${localpath} #save the top directory topDir=`pwd` #Path of the utility scripts scriptpath=${SCRIPTPATH} #JEC 1/10/11 Use generic baodaqstatus name #get the daq current irod status #tag=`${DATE} +%F` #OUT1=${publicpath}/baodaqstatus-${tag}.txt OUT1=${publicpath}/baodaqstatus-current.txt if [ ! -e ${OUT1} -o ! -r ${OUT1} ]; then $ECHO "FATAL (${scriptName}): ${OUT1} has a problem" exit 1 fi $ECHO "You have selected sourceRadio = ${sourceRadio} [date = ${dateSelected}]" tableau=( `$GREP -i "${dateSelected}${srclower}" ${OUT1} | $AWK '( NF==4 ) { print $2 }' ` ) IFS=' ' tableau=( $( $PRINTF "%s\n" "${tableau[@]}" | $AWK 'x[$0]++ == 0' ) ) IFS=$DefaultIFS for i in ${tableau[@]} do # # start a "fresh" session cd ${topDir} $LS | $XARGS -i $RM -rf {} $ECHO "DEBUG: (${scriptName}) irods root dir $i" dateDAQ=`$ECHO ${i} | $AWK '{split($0,a,"/"); print a[6];}' | $AWK "{sub(/${srclower}/,\"\"); print}"` $ECHO "DEBUG: (${scriptName}): we are in directory `pwd` the topDir=${topDir}" $LS -lrt gainDone=`ils ${i} 2>&1 | $GREP -i "gain_.*\.fits" | $WC -l` if [ ! ${gainDone} -gt 0 ]; then $ECHO "FATAL (${scriptName}): gain file DOES NOT exist skip `basename $i` " continue fi ###### $ECHO ">>>>>>>>>>>>> ANALYSIS Part of ${sourceRadio} ${dateDAQ}" #go to the .../source/date-sourcelowercase directory #download Off/datacycle<> and On/datacycle<> directories $MKDIR -p ${topDir}/${dateDAQ}${srclower} cd ${topDir}/${dateDAQ}${srclower} aboveOnOffDir=`pwd` $ECHO "DEBUG (${scriptName}): we are in directory: <`pwd`>" # mode="Off" $ECHO "DEBUG (${scriptName}): START download the ${mode} data fits files" inFileDirectory="./${mode}" $MKDIR -p ${inFileDirectory} listOfDataCycle=( `ils ${i}/${mode} 2>&1 | $GREP -i "datacycle"|$AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.10n` ) IFS=' ' listOfDataCycle=( $( $PRINTF "%s\n" "${listOfDataCycle[@]}" | $AWK 'x[$0]++ == 0' ) ) IFS=$DefaultIFS firstCycleOff=`$ECHO ${listOfDataCycle[0]} | $AWK '{match($0,"[0-9]+",arr); print arr[0]}'` nCyclesOff=`expr ${#listOfDataCycle[*]} - 1` if [ ${nCyclesOff} -eq -1 ]; then continue fi lastCycleOff=`$ECHO ${listOfDataCycle[${nCyclesOff}]} | $AWK '{match($0,"[0-9]+",arr); print arr[0]}'` listOfDataCycle=( "${listOfDataCycle[@]/#/${i}/${mode}/}" ) for j in ${listOfDataCycle[@]} do irodsDownDir="${j}" OUT=./getInFits.$$ iget -r -f -K ${irodsDownDir} ${inFileDirectory} > ${OUT} 2>&1 igetStatus=`$GREP "^ERROR" ${OUT}` if [ "<$igetStatus>" != "<>" ]; then $ECHO "FATAL (${scriptName}): error while iget fits files:" $ECHO $igetStatus $ECHO " ==> skip this run $sourceRadio $dateDAQ" continue fi $RM -f ${OUT} done $ECHO "DEBUG (${scriptName}): END download the ${mode} data fits files" # mode="On" $ECHO "DEBUG (${scriptName}): START download the ${mode} data fits files" inFileDirectory="./${mode}" $MKDIR -p ${inFileDirectory} listOfDataCycle=( `ils ${i}/${mode} 2>&1 | $GREP -i "datacycle" |$AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.10n` ) IFS=' ' listOfDataCycle=( $( $PRINTF "%s\n" "${listOfDataCycle[@]}" | $AWK 'x[$0]++ == 0' ) ) IFS=$DefaultIFS firstCycleOn=`$ECHO ${listOfDataCycle[0]} | $AWK '{match($0,"[0-9]+",arr); print arr[0]}'` nCyclesOn=`expr ${#listOfDataCycle[*]} - 1` if [ ${nCyclesOn} -eq -1 ]; then continue fi lastCycleOn=`$ECHO ${listOfDataCycle[${nCyclesOn}]} | $AWK '{match($0,"[0-9]+",arr); print arr[0]}'` listOfDataCycle=( "${listOfDataCycle[@]/#/${i}/${mode}/}" ) # #Check if there is the same number of cycles in Off and On mode # #first cycle = max(firstCycleOn, firstCycleOff) #last cycle = min(lastCycleOn,lastCycleOff) if [ ${firstCycleOn} -eq ${firstCycleOff} -a ${lastCycleOn} -eq ${lastCycleOff} ]; then firstCycle=${firstCycleOff} lastCycle=${lastCycleOff} else $ECHO "INFO (${scriptName}): missmatch between Off and On mode" $ECHO "${firstCycleOn} != ${firstCycleOff} OR ${lastCycleOn} != ${lastCycleOff}" firstCycle=${firstCycleOff} if [ ${firstCycleOn} -gt ${firstCycle} ]; then firstCycle=${firstCycleOn} fi lastCycle=${lastCycleOff} if [ ${lastCycleOn} -lt ${lastCycle} ]; then lastCycle=${lastCycleOn} fi fi $ECHO "INFO (${scriptName}): use cycles [${firstCycle}, ${lastCycle}]" for j in ${listOfDataCycle[@]} do irodsDownDir="${j}" OUT=./getInFits.$$ iget -r -f -K ${irodsDownDir} ${inFileDirectory} > ${OUT} 2>&1 igetStatus=`$GREP "^ERROR" ${OUT}` if [ "<$igetStatus>" != "<>" ]; then $ECHO "FATAL (${scriptName}): error while iget fits files:" $ECHO $igetStatus $ECHO " ==> skip this run $sourceRadio $dateDAQ" continue fi $RM -f ${OUT} done $ECHO "DEBUG (${scriptName}): END download the ${mode} data fits files" #Get sca file $ECHO "DEBUG (${scriptName}): START call getscafiles.sh ${sourceRadio} ${dateDAQ}" OUT1=./getScaStatus.$$ $RM -f ${tmppublicpath}/getscafiles.finished ${scriptpath}/getscafiles.sh ${sourceRadio} ${dateDAQ} > ${OUT1} 2>&1 while [ ! -f "${tmppublicpath}/getscafiles.finished" ]; do $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getscafiles.finished" date +%T sleep 30 done $RM ${tmppublicpath}/getscafiles.finished $ECHO "DEBUG (${scriptName}): END" # getScaStatus=`$GREP "^\(FATAL\|ERROR\)" ${OUT1}` # if [ "<${getScaStatus}>" != "<>" ]; then $ECHO "FATAL (${scriptName}): error to get sca file for $sourceRadio $dateDAQ" $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}" $CAT ${OUT1} $ECHO "DEBUG (${scriptName}): END" continue fi $RM -f ${OUT1} scaFile=`$FIND . -name "sca*.sum.trans" | $XARGS -i basename {}` #prepare job submission cd ${aboveOnOffDir} #the default spectra name is medfiltmtx, otherwise use -specname logFile="analyse_RAWDATA_${sourceRadio}_${dateDAQ}.log" $ECHO "DEBUG (${scriptName}): execute the analysis... should not take long. `date`" if [ ${simulationMode} = "On" ]; then $ECHO "SIMUL: ${scriptpath}/Objs/analyse -act rawOnOff -inPath ${iojobpath} -source ${sourceRadio} -date ${dateDAQ} -sca ${scaFile} -specdir datacycle -numcycle ${firstCycle},${lastCycle} -debug 1 >& ${logFile}" cat > ${logFile} <& ${logFile} fi $ECHO "DEBUG (${scriptName}): analysis finished `date`. Look for errors before saving to Irods" rcstatus=`$GREP -i "Ok rawOnOff finished" ${logFile} | $WC -l` if [ ${rcstatus} -eq 0 ]; then $ECHO "INFO (${scriptName}): analysis problem for ${sourceRadio} ${dateDAQ}" $ECHO " START logfile" $CAT ${logFile} $ECHO " END logFile" #clean and leave cd ${aboveOnOffDir} $LS | $XARGS -i $RM -rf {} continue fi #Save into Irods $ECHO "DEBUG (${scriptName}): START save output files" $LS ${aboveOnOffDir} #use -f option for iput ONLY to force override if [ ${simulationMode} = "On" ]; then $ECHO "SIMUL: here we look for output files and put them in Irods" else $LS -lrth $FIND . -name "dataRaw_*" -print | $XARGS -i iput -f -v -K {} ${i} $FIND . -name "diffOnOffRaw_*" -print | $XARGS -i iput -f -v -K {} ${i} fi #save analysis logfile $LS -l ${tmppublicpath} > /dev/null $CP ${logFile} ${tmppublicpath} $ECHO "DEBUG (${scriptName}): END save output files" #clean up to avoid scratch SIZE EXCEED LIMIT cd ${aboveOnOffDir} $LS | $XARGS -i $RM -rf {} done exit 0