source: BAORadio/AmasNancay/trunk/anadriftmaker.sh@ 673

#!/bin/sh -xvf
#do calibration
DATE=/bin/date
GREP=/bin/grep
AWK=/bin/awk
ECHO=/bin/echo
WC=/usr/bin/wc
CAT=/bin/cat
PRINTF=/usr/bin/printf
FIND=/usr/bin/find
MKDIR=/bin/mkdir
XARGS=/usr/bin/xargs
SORT=/bin/sort
RM=/bin/rm
TR=/usr/bin/tr
CP=/bin/cp
LS=/bin/ls
CHMOD=/bin/chmod

DefaultIFS=$' \t\n'
IFS=$DefaultIFS

#set Irods environment    
. /usr/local/shared/bin/irods_env.sh -noverbose

scriptName="`basename $0`"
$ECHO "Processing script ${scriptName} at `date`"
#which source to analyse

#Process cmd line args: the -src option is mandatory (source name as Abell85)
sourceRadio=
dateSelected=
#use -sim option to simulate processing (debug the script as if...)
simulationMode=Off
while [ $# -gt 0 ]
  do
  case "$1" in
      -src)  sourceRadio=$2; shift;;
      -date) dateSelected=$2;     shift;;
      -sim)  simulationMode=On;;
      -h)
      echo >&2 \
          "usage: $0 -src souce -date YYYYMMDD [-freq freq in MHz (default ${freqBAOCalib})] [-bwd bandwidth in MHz (default ${bwBAOCalib})]"
      exit 1;;
      *)  break;;       # terminate while loop
  esac
  shift
done

if [ ${simulationMode} = "On" ]; then
    $ECHO "INFO ${scriptName} running in SIMUL mode"
fi

if [ "<${sourceRadio}>" = "<>" ]; then
    $ECHO "FATAL: You have forgotten to select the source option (-src)"
    exit 1
fi

#TODO in principe the naming convention is <SOURCE>DR but NGC4383 is an exception
case ${sourceRadio} in
    NGC4383DR) $ECHO "INFO (${scriptName}): process ${sourceRadio}";;
    3C273DR) $ECHO "INFO (${scriptName}): process ${sourceRadio}";;
    3C161DR) $ECHO "INFO (${scriptName}): process ${sourceRadio}";;
    3C161BDR) $ECHO "INFO (${scriptName}): process ${sourceRadio}";;
    *) ECHO "FATAL (${scriptName}): process ${sourceRadio} not yet foreseen"
    exit 1;;
esac

srclower=`${ECHO} ${sourceRadio} | ${TR} "[:upper:]" "[:lower:]" `

#Path to public backupable path
publicpath="/afs/in2p3.fr/home/c/campagne/public"


#temporary files to synchronize scripts
tmppublicpath=${TMPPUBLICPATH}
#wakeup the NFS disk
$LS -l ${tmppublicpath} > /dev/null
#clean previous spurious files
$RM -f ${tmppublicpath}/*.finished

#Path where the job will do temporary IO
. ${SCRIPTPATH}/isInteractive.sh
isInteractive=$(isInteractive)
if [ ${isInteractive} == "1" ]; then
  $ECHO "Usage in INTERACTIVE may be destructive... To be used with great care !!!!"
   exit 0;
fi

. ${SCRIPTPATH}/set_iojobpath.sh
iojobpath=$(set_iojobpath)
cd ${iojobpath}

localpath="${sourceRadio}"
$MKDIR -p ./${localpath}
cd ./${localpath}

#save the top directory
topDir=`pwd`

#Path of the utility scripts
scriptpath=${SCRIPTPATH}

#Use generic baodaqstatus name 
OUT1=${publicpath}/baodaqstatus-current.txt
if [ ! -e ${OUT1}  -o ! -r ${OUT1} ]; then
    $ECHO "FATAL (${scriptName}): ${OUT1} has a problem"
    exit 1
fi


#Nb: if we want to implement a loop on sources we have to take care of srclower...

$ECHO "You have selected sourceRadio = $sourceRadio[date = ${dateSelected}]"
#JEC 14/12/11 One should take care of DR extension for Drift Scan
#tableau=( `$GREP -i "${dateSelected}${srclower}" ${OUT1} | $AWK '( NF==4 ) { print $2 }' ` )
tableau=( `$GREP -i "${sourceRadio}/${dateSelected}${srclower}" ${OUT1} | $AWK '( NF==4 ) { print $2 }' ` )
IFS='
' 
tableau=( $( $PRINTF "%s\n" "${tableau[@]}" | $AWK 'x[$0]++ == 0' ) )
IFS=$DefaultIFS


for i in ${tableau[@]}
  do
#
# start a "fresh" session  
  cd ${topDir}
  $LS | $XARGS -i $RM -rf {}

  
  $ECHO "DEBUG: (${scriptName}) irods root dir $i"
  dateDAQ=`$ECHO ${i} | $AWK '{split($0,a,"/"); print a[6];}' | $AWK "{sub(/${srclower}/,\"\"); print}"`

  $ECHO "DEBUG: (${scriptName}): we are in directory `pwd` the topDir=${topDir}"
  $LS -lrt


######
  $ECHO ">>>>>>>>>>>>> ANALYSIS Part of ${sourceRadio} ${dateDAQ}"
#go to the .../source/date-sourcelowercase directory
#download Off/calibcycle<> and On/calibcycle<> directories
  $MKDIR -p ${topDir}/${dateDAQ}${srclower}
  cd ${topDir}/${dateDAQ}${srclower}
  aboveOnOffDir=`pwd`
  $ECHO "DEBUG (${scriptName}): we are in directory: <`pwd`>"
#
  mode="On"
  $ECHO "DEBUG (${scriptName}): START download the ${mode} fits files"
  inFileDirectory="./${mode}"
  $MKDIR -p ${inFileDirectory}

  listOfDataCycle=( `ils ${i}/${mode} 2>&1 | $GREP -i "datacycle" |$AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.10n` )
  IFS='
  '
  listOfDataCycle=( $( $PRINTF "%s\n" "${listOfDataCycle[@]}" | $AWK 'x[$0]++ == 0' ) )
  IFS=$DefaultIFS

  firstCycle=`$ECHO ${listOfDataCycle[0]} | $AWK '{match($0,"[0-9]+",arr); print arr[0]}'`
  nCycles=`expr ${#listOfDataCycle[*]} - 1`

  lastCycle=`$ECHO ${listOfDataCycle[${nCycles}]} | $AWK '{match($0,"[0-9]+",arr); print arr[0]}'`

  listOfDataCycle=( "${listOfDataCycle[@]/#/${i}/${mode}/}" )

  $ECHO "INFO (${scriptName}): use cycles [${firstCycle}, ${lastCycle}]"
  

  for j in ${listOfDataCycle[@]}
    do
    irodsDownDir="${j}"
    OUT=./getInFits.$$
    iget -r -f -K ${irodsDownDir} ${inFileDirectory} > ${OUT} 2>&1
    igetStatus=`$GREP "^ERROR" ${OUT}`
    if [ "<$igetStatus>" != "<>" ]; then
        $ECHO "FATAL (${scriptName}): error while iget fits files:"
        $ECHO $igetStatus
        $ECHO " ==> skip this run $sourceRadio $dateDAQ"
        continue
    fi
    $RM -f ${OUT}
  done
  $ECHO "DEBUG (${scriptName}): END download the ${mode} fits files"
#Get sca file
  $ECHO "DEBUG (${scriptName}): START call getscafiles.sh ${sourceRadio} ${dateDAQ}"
  OUT1=./getScaStatus.$$
#  $RM -f ${tmppublicpath}/getscafiles.finished
  ${scriptpath}/getscafiles.sh ${sourceRadio} ${dateDAQ} > ${OUT1} 2>&1
#  $RM ${tmppublicpath}/getscafiles.finished
  $ECHO "DEBUG (${scriptName}): END"
#
  getScaStatus=`$GREP "^\(FATAL\|ERROR\)" ${OUT1}`
#       
  if [ "<${getScaStatus}>" != "<>" ]; then
      $ECHO "FATAL (${scriptName}): error to get sca file for $sourceRadio $dateDAQ"
      $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
      $CAT ${OUT1}
      $ECHO "DEBUG (${scriptName}): END"
      continue
  fi
  $RM -f ${OUT1}

  scaFile=`$FIND . -name "sca*.sum.trans" | $XARGS -i basename {}`
  
#prepare job submission
  cd ${aboveOnOffDir}
#the default spectra name is medfiltmtx, otherwise use -specname <a_string>

  logFile="analyse_DRIFT_${sourceRadio}_${dateDAQ}.log"
  $ECHO "DEBUG (${scriptName}): execute the analysis... should not take long. `date`"
  if [ ${simulationMode} = "On" ]; then
      $ECHO "SIMUL: ${scriptpath}/Objs/analyse -act driftScanImg -inPath ${iojobpath}  -source  ${sourceRadio} -date ${dateDAQ} -sca ${scaFile} -specdir datacycle -numcycle ${firstCycle},${lastCycle} -debug 1 >& ${logFile}"
      cat > ${logFile} <<EOF
      Ok calibration finished
EOF
#WARNING: the EOF must start at 1st column overwise the rest of the script is part of the file cat...
  else
      ${scriptpath}/Objs/analyse -act driftScanImg -inPath ${iojobpath}  -source  ${sourceRadio} -date ${dateDAQ} -sca ${scaFile} -specdir datacycle -numcycle ${firstCycle},${lastCycle} -debug 1 >& ${logFile}
  fi
#
  $ECHO "DEBUG (${scriptName}): analysis finished `date`. Look for errors before saving to Irods"
  rcstatus=`$GREP -i "Ok drift finished" ${logFile} | $WC -l`
  if [ ${rcstatus} -eq 0 ]; then
      $ECHO "INFO (${scriptName}): analysis problem for ${sourceRadio} ${dateDAQ}"
      $ECHO "                      START logfile"
      $CAT ${logFile}
      $ECHO "                      END   logFile"
#clean and leave
      cd ${aboveOnOffDir}
      $LS | $XARGS -i $RM -rf {}
      continue
  fi
#Save into Irods
  $ECHO "DEBUG (${scriptName}): START save output files"
  $LS ${aboveOnOffDir}
#use -f option for iput ONLY to force override
  if [ ${simulationMode} = "On" ]; then
      $ECHO "SIMUL: here we look for calib file and put them in Irods"
  else
      $LS -lrth
      $FIND . -name "img_*" -print | $XARGS -i  iput -f  -v -K {} ${i}
  fi
#save analysis logfile
  $LS -l ${tmppublicpath} > /dev/null
  $CP ${logFile} ${tmppublicpath}
  $ECHO "DEBUG (${scriptName}): END save output files"
#clean up to avoid scratch SIZE EXCEED LIMIT
  cd ${aboveOnOffDir}
  $LS | $XARGS -i $RM -rf {}

done

exit 0