source: BAORadio/AmasNancay/trunk/anacalibmaker.sh @ 572

#!/bin/sh -xvf
#do calibration
DATE=/bin/date
GREP=/bin/grep
AWK=/bin/awk
ECHO=/bin/echo
WC=/usr/bin/wc
CAT=/bin/cat
PRINTF=/usr/bin/printf
FIND=/usr/bin/find
MKDIR=/bin/mkdir
XARGS=/usr/bin/xargs
SORT=/bin/sort
RM=/bin/rm
TR=/usr/bin/tr
CP=/bin/cp
LS=/bin/ls
CHMOD=/bin/chmod

DefaultIFS=$' \t\n'
IFS=$DefaultIFS

#set Irods environment    
. /usr/local/shared/bin/irods_env.sh -noverbose

scriptName="`basename $0`"
$ECHO "Processing script ${scriptName} at `date`"
#which source to analyse

#Process cmd line args: the -src option is mandatory (source name as Abell85)
sourceRadio=
dateSelected=
freqBAOCalib=1346
bwBAOCalib=6.25
#use -sim option to simulate processing (debug the script as if...)
simulationMode=Off
while [ $# -gt 0 ]
  do
  case "$1" in
      -src)  sourceRadio=$2; shift;;
      -date) dateSelected=$2;     shift;;
      -freq) freqBAOCalib=$2; shift;;
      -bwd)  bwBAOCalib=$2; shift;;
      -sim)  simulationMode=On;;
      -h)
      echo >&2 \
          "usage: $0 -src souce -date YYYYMMDD [-freq freq in MHz (default ${freqBAOCalib})] [-bwd bandwidth in MHz (default ${bwBAOCalib})]"
      exit 1;;
      *)  break;;       # terminate while loop
  esac
  shift
done

if [ ${simulationMode} = "On" ]; then
    $ECHO "INFO ${scriptName} running in SIMUL mode"
fi

if [ "<${sourceRadio}>" = "<>" ]; then
    $ECHO "FATAL: You have forgotten to select the source option (-src)"
    exit 1
fi

case ${sourceRadio} in
    Abell85) $ECHO "INFO (${scriptName}): process ${sourceRadio}";;
    Abell2440) $ECHO "INFO (${scriptName}): process ${sourceRadio}";;
    Abell1205) $ECHO "INFO (${scriptName}): process ${sourceRadio}";;
    *) ECHO "FATAL (${scriptName}): process ${sourceRadio} not yet foreseen"
    exit 1;;
esac

srclower=`${ECHO} ${sourceRadio} | ${TR} "[:upper:]" "[:lower:]" `

#Path to public backupable path
publicpath="/afs/in2p3.fr/home/c/campagne/public"


#temporary files to synchronize scripts
tmppublicpath=${TMPPUBLICPATH}
#wakeup the NFS disk
$LS -l ${tmppublicpath} > /dev/null
#clean previous spurious files
$RM -f ${tmppublicpath}/*.finished

#Path where the job will do temporary IO
if [ ${ENVIRONMENT} = "INTERACTIVE" ]; then
    iojobpath="/sps/baoradio/AmasNancay/JEC"
    $MKDIR -p $iojobpath
    $ECHO "Usage in INTERACTIVE may be destructive... To be used with great care !!!!"
    exit 0;
elif [ ${ENVIRONMENT} = "BATCH" ] ; then
    iojobpath=${TMPBATCH}
else
    $ECHO "FATAL (${scriptName}): environment is ${ENVIRONMENT} not allowed"
    exit 1
fi
cd ${iojobpath}

localpath="${sourceRadio}"
$MKDIR -p ./${localpath}
cd ./${localpath}

#save the top directory
topDir=`pwd`

#Path of the utility scripts
scriptpath="/afs/in2p3.fr/home/c/campagne/private/work/AmasNancay"

#JEC 1/10/11 Use generic baodaqstatus name 
OUT1=${publicpath}/baodaqstatus-current.txt
#
#get the daq current irod status
#tag=`${DATE} +%F`
#OUT1=${publicpath}/baodaqstatus-${tag}.txt
#
if [ ! -e ${OUT1}  -o ! -r ${OUT1} ]; then
    $ECHO "FATAL (${scriptName}): ${OUT1} has a problem"
    exit 1
fi


#Nb: if we want to implement a loop on sources we have to take care of srclower...

$ECHO "You have selected sourceRadio = $sourceRadio[date = ${dateSelected}]"
tableau=( `$GREP -i "${dateSelected}${srclower}" ${OUT1} | $AWK '( NF==4 ) { print $2 }' ` )
IFS='
' 
tableau=( $( $PRINTF "%s\n" "${tableau[@]}" | $AWK 'x[$0]++ == 0' ) )
IFS=$DefaultIFS


for i in ${tableau[@]}
  do
#
# start a "fresh" session  
  cd ${topDir}
  $LS | $XARGS -i $RM -rf {}

  
  $ECHO "DEBUG: (${scriptName}) irods root dir $i"
  dateDAQ=`$ECHO ${i} | $AWK '{split($0,a,"/"); print a[6];}' | $AWK "{sub(/${srclower}/,\"\"); print}"`

  $ECHO "DEBUG: (${scriptName}): we are in directory `pwd` the topDir=${topDir}"
  $LS -lrt


#  alreadyDone=`ils ${i} 2>&1 | $GREP -i "calib_.*\.txt" | $WC -l`
#  if [ ${alreadyDone} -gt 0 ]; then
#      $ECHO "INFO (${scriptName}): calibration coef files already exist"
#      continue
#  fi

  gainDone=`ils ${i} 2>&1 | $GREP -i "gain_.*\.fits" | $WC -l`
  if [ ! ${gainDone} -gt 0 ]; then
      $ECHO "FATAL (${scriptName}): gain file DOES NOT exist skip `basename $i` "
      continue
  fi

######
  $ECHO ">>>>>>>>>>>>> ANALYSIS Part of ${sourceRadio} ${dateDAQ}"
#go to the .../source/date-sourcelowercase directory
#download Off/calibcycle<> and On/calibcycle<> directories
  $MKDIR -p ${topDir}/${dateDAQ}${srclower}
  cd ${topDir}/${dateDAQ}${srclower}
  aboveOnOffDir=`pwd`
  $ECHO "DEBUG (${scriptName}): we are in directory: <`pwd`>"
#
  mode="Off"
  $ECHO "DEBUG (${scriptName}): START download the ${mode} calib fits files"
  inFileDirectory="./${mode}"
  $MKDIR -p ${inFileDirectory}
  listOfCalibCycle=( `ils ${i}/${mode} 2>&1 | $GREP -i "calibcycle"|$AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.11n` )
  IFS='
'
  listOfCalibCycle=( $( $PRINTF "%s\n" "${listOfCalibCycle[@]}" | $AWK 'x[$0]++ == 0' ) )
  IFS=$DefaultIFS

  firstCycleOff=`$ECHO ${listOfCalibCycle[0]} | $AWK '{match($0,"[0-9]+",arr); print arr[0]}'`
  nCyclesOff=`expr ${#listOfCalibCycle[*]} - 1`

  lastCycleOff=`$ECHO ${listOfCalibCycle[${nCyclesOff}]} | $AWK '{match($0,"[0-9]+",arr); print arr[0]}'`

  listOfCalibCycle=( "${listOfCalibCycle[@]/#/${i}/${mode}/}" )

  
  for j in ${listOfCalibCycle[@]}
    do
    irodsDownDir="${j}"
    OUT=./getInFits.$$
    iget -r -f -K ${irodsDownDir} ${inFileDirectory} > ${OUT} 2>&1
    igetStatus=`$GREP "^ERROR" ${OUT}`
    if [ "<$igetStatus>" != "<>" ]; then
        $ECHO "FATAL (${scriptName}): error while iget fits files:"
        $ECHO $igetStatus
        $ECHO " ==> skip this run $sourceRadio $dateDAQ"
        continue
    fi
    $RM -f ${OUT}
  done
  $ECHO "DEBUG (${scriptName}): END download the ${mode} calib fits files"
#
  mode="On"
  $ECHO "DEBUG (${scriptName}): START download the ${mode} calib fits files"
  inFileDirectory="./${mode}"
  $MKDIR -p ${inFileDirectory}

  listOfCalibCycle=( `ils ${i}/${mode} 2>&1 | $GREP -i "calibcycle" |$AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.11n` )
  IFS='
  '
  listOfCalibCycle=( $( $PRINTF "%s\n" "${listOfCalibCycle[@]}" | $AWK 'x[$0]++ == 0' ) )
  IFS=$DefaultIFS

  firstCycleOn=`$ECHO ${listOfCalibCycle[0]} | $AWK '{match($0,"[0-9]+",arr); print arr[0]}'`
  nCyclesOn=`expr ${#listOfCalibCycle[*]} - 1`

  lastCycleOn=`$ECHO ${listOfCalibCycle[${nCyclesOn}]} | $AWK '{match($0,"[0-9]+",arr); print arr[0]}'`

  listOfCalibCycle=( "${listOfCalibCycle[@]/#/${i}/${mode}/}" )


#
#Check if there is the same number of cycles in Off and On mode
#      
#first cycle = max(firstCycleOn, firstCycleOff)
#last  cycle = min(lastCycleOn,lastCycleOff)

  if [ ${firstCycleOn} -eq ${firstCycleOff} -a ${lastCycleOn} -eq ${lastCycleOff} ]; then
      firstCycle=${firstCycleOff}
      lastCycle=${lastCycleOff}
  else
      $ECHO "INFO (${scriptName}): missmatch between Off and On mode"
      $ECHO "${firstCycleOn} != ${firstCycleOff} OR ${lastCycleOn} != ${lastCycleOff}"
      firstCycle=${firstCycleOff}
      if [ ${firstCycleOn} -gt ${firstCycle} ]; then
          firstCycle=${firstCycleOn}
      fi
      lastCycle=${lastCycleOff}
      if [ ${lastCycleOn} -lt ${lastCycle} ]; then
          lastCycle=${lastCycleOn}
      fi
  fi
  $ECHO "INFO (${scriptName}): use cycles [${firstCycle}, ${lastCycle}]"
  

  for j in ${listOfCalibCycle[@]}
    do
    irodsDownDir="${j}"
    OUT=./getInFits.$$
    iget -r -f -K ${irodsDownDir} ${inFileDirectory} > ${OUT} 2>&1
    igetStatus=`$GREP "^ERROR" ${OUT}`
    if [ "<$igetStatus>" != "<>" ]; then
        $ECHO "FATAL (${scriptName}): error while iget fits files:"
        $ECHO $igetStatus
        $ECHO " ==> skip this run $sourceRadio $dateDAQ"
        continue
    fi
    $RM -f ${OUT}
  done
  $ECHO "DEBUG (${scriptName}): END download the ${mode} calib fits files"
#Get sca file
  $ECHO "DEBUG (${scriptName}): START call getscafiles.sh ${sourceRadio} ${dateDAQ}"
  OUT1=./getScaStatus.$$
  $RM -f ${tmppublicpath}/getscafiles.finished
  ${scriptpath}/getscafiles.sh ${sourceRadio} ${dateDAQ} > ${OUT1} 2>&1
  while [ ! -f "${tmppublicpath}/getscafiles.finished" ]; do
      $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getscafiles.finished"
      date +%T
      sleep 30
  done
  $RM ${tmppublicpath}/getscafiles.finished
  $ECHO "DEBUG (${scriptName}): END"
#
  getScaStatus=`$GREP "^\(FATAL\|ERROR\)" ${OUT1}`
#       
  if [ "<${getScaStatus}>" != "<>" ]; then
      $ECHO "FATAL (${scriptName}): error to get sca file for $sourceRadio $dateDAQ"
      $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
      $CAT ${OUT1}
      $ECHO "DEBUG (${scriptName}): END"
      continue
  fi
  $RM -f ${OUT1}

  scaFile=`$FIND . -name "sca*.sum.trans" | $XARGS -i basename {}`
  
#prepare job submission
  cd ${aboveOnOffDir}
#the default spectra name is medfiltmtx, otherwise use -specname <a_string>

  logFile="analyse_CALIB_${sourceRadio}_${dateDAQ}.log"
  $ECHO "DEBUG (${scriptName}): execute the analysis... should not take long. `date`"
  if [ ${simulationMode} = "On" ]; then
      $ECHO "SIMUL: ${scriptpath}/Objs/analyse -act calib -inPath ${iojobpath}  -source  ${sourceRadio} -date ${dateDAQ} -sca ${scaFile} -specdir calibcycle -numcycle ${firstCycle},${lastCycle} -calibopt mean  -debuglev 1 -freqBAOCalib ${freqBAOCalib} -bwBAOCalib ${bwBAOCalib} >& ${logFile}"
      cat > ${logFile} <<EOF
      Ok calibration finished
EOF
#WARNING: the EOF must start at 1st column overwise the rest of the script is part of the file cat...
  else
      ${scriptpath}/Objs/analyse -act calib -inPath ${iojobpath}  -source  ${sourceRadio} -date ${dateDAQ} -sca ${scaFile} -specdir calibcycle  -numcycle ${firstCycle},${lastCycle} -calibopt mean  -debuglev 1 -freqBAOCalib ${freqBAOCalib} -bwBAOCalib ${bwBAOCalib} >& ${logFile}
  fi
#
  $ECHO "DEBUG (${scriptName}): analysis finished `date`. Look for errors before saving to Irods"
  rcstatus=`$GREP -i "Ok calibration finished" ${logFile} | $WC -l`
  if [ ${rcstatus} -eq 0 ]; then
      $ECHO "INFO (${scriptName}): analysis problem for ${sourceRadio} ${dateDAQ}"
      $ECHO "                      START logfile"
      $CAT ${logFile}
      $ECHO "                      END   logFile"
#clean and leave
      cd ${aboveOnOffDir}
      $LS | $XARGS -i $RM -rf {}
      continue
  fi
#Save into Irods
  $ECHO "DEBUG (${scriptName}): START save output files"
  $LS ${aboveOnOffDir}
#use -f option for iput ONLY to force override
  if [ ${simulationMode} = "On" ]; then
      $ECHO "SIMUL: here we look for calib file and put them in Irods"
  else
      $LS -lrth
      $FIND . -name "calib_*" -print | $XARGS -i  iput -f  -v -K {} ${i}
  fi
#save analysis logfile
  $LS -l ${tmppublicpath} > /dev/null
  $CP ${logFile} ${tmppublicpath}
  $ECHO "DEBUG (${scriptName}): END save output files"
#clean up to avoid scratch SIZE EXCEED LIMIT
  cd ${aboveOnOffDir}
  $LS | $XARGS -i $RM -rf {}

done

exit 0