#!/bin/sh -xvf
#Perform the ON-OFF analysis
DATE=/bin/date
GREP=/bin/grep
AWK=/bin/awk
ECHO=/bin/echo
WC=/usr/bin/wc
CAT=/bin/cat
PRINTF=/usr/bin/printf
FIND=/usr/bin/find
MKDIR=/bin/mkdir
XARGS=/usr/bin/xargs
SORT=/bin/sort
RM=/bin/rm
TR=/usr/bin/tr
CP=/bin/cp
LS=/bin/ls
CHMOD=/bin/chmod

DefaultIFS=$' \t\n'
IFS=$DefaultIFS

#set Irods environment    
. /usr/local/shared/bin/irods_env.sh -noverbose

scriptName="`basename $0`"
$ECHO "Processing script ${scriptName} at `date`"
#which source to analyse

#Process cmd line args: the -src option is mandatory (source name as Abell85)
sourceRadio=
dateSelected=
freqBAOCalib=1346
#use -sim option to simulate processing (debug the script as if...)
simulationMode=Off
while [ $# -gt 0 ]
do
  case "$1" in
      -src)  sourceRadio=$2;    shift;;
      -date) dateSelected=$2;     shift;;
      -freq) freqBAOCalib=$2;   shift;;
      -sim)  simulationMode=On;;
	-h)
	    echo >&2 \
	    "usage: $0 -src souce -date YYYYMMDD [-freq freq in MHz (default ${freqBAOCalib})] [-sim to trig simulation mode]"
	    exit 1;;
	*)  break;;	# terminate while loop
    esac
    shift
done

if [ ${simulationMode} = "On" ]; then
    $ECHO "INFO ${scriptName} running in SIMUL mode"
fi

if [ "<${sourceRadio}>" = "<>" ]; then
    $ECHO "FATAL: You have forgotten to select the source option (-src)"
    exit 1
fi

case ${sourceRadio} in
    Abell85) $ECHO "INFO (${scriptName}): process ${sourceRadio}";;
    Abell2440) $ECHO "INFO (${scriptName}): process ${sourceRadio}";;
    Abell1205) $ECHO "INFO (${scriptName}): process ${sourceRadio}";;
    *) ECHO "FATAL (${scriptName}): process ${sourceRadio} not yet foreseen"
    exit 1;;
esac

srclower=`${ECHO} ${sourceRadio} | ${TR} "[:upper:]" "[:lower:]" `

#Path to public backupable path
publicpath="/afs/in2p3.fr/home/c/campagne/public"


#temporary files to synchronize scripts
tmppublicpath=${TMPPUBLICPATH}
#wakeup the NFS disk
$LS -l ${tmppublicpath} > /dev/null
#clean previous spurious files
$RM -f ${tmppublicpath}/*.finished

#Path where the job will do temporary IO
if [ ${ENVIRONMENT} = "INTERACTIVE" ]; then
    iojobpath="/sps/baoradio/AmasNancay/JEC"
    $MKDIR -p $iojobpath
elif [ ${ENVIRONMENT} = "BATCH" ] ; then
    iojobpath=${TMPBATCH}
else
    $ECHO "FATAL (${scriptName}): environment is ${ENVIRONMENT} not allowed"
    exit 1
fi
cd ${iojobpath}

localpath="${sourceRadio}"
$MKDIR -p ./${localpath}
cd ./${localpath}

#save the top directory
topDir=`pwd`

#Path of the utility scripts
scriptpath="/afs/in2p3.fr/home/c/campagne/private/work/AmasNancay"

#get the daq current irod status
tag=`${DATE} +%F`
OUT1=${publicpath}/baodaqstatus-${tag}.txt

if [ ! -e ${OUT1}  -o ! -r ${OUT1} ]; then
    $ECHO "We should bring the DAQ status more up to date, this take 1 or 2sec..."
    $RM -f ${tmppublicpath}/statusdaq.finished
    ${scriptpath}/statusdaq.sh > ${OUT1}
    while [ ! -f "${tmppublicpath}/statusdaq.finished" ]; do
	sleep 30
    done
    $RM ${tmppublicpath}/statusdaq.finished
#protect against remove/rewriting    
    $CHMOD -v 444 ${OUT1} 
fi


#Nb: if we want to implement a loop on sources we have to take care of srclower...

$ECHO "You have selected sourceRadio = ${sourceRadio} [date = ${dateSelected}]"
tableau=( `$GREP -i "${dateSelected}${srclower}" ${OUT1} | $AWK '( NF==4 ) { print $2 }' ` )
IFS='
' 
tableau=( $( $PRINTF "%s\n" "${tableau[@]}" | $AWK 'x[$0]++ == 0' ) )
IFS=$DefaultIFS


for i in ${tableau[@]}
  do
#
# start a "fresh" session  
  cd ${topDir}
  $LS | $XARGS -i $RM -rf {}
  
  $ECHO "DEBUG: (${scriptName}) irods root dir $i"
  dateDAQ=`$ECHO ${i} | $AWK '{split($0,a,"/"); print a[6];}' | $AWK "{sub(/${srclower}/,\"\"); print}"`

  $ECHO "DEBUG: (${scriptName}): we are in directory `pwd` the topDir=${topDir}"
  $LS -lrt


#Look if the date has changed since beginning of the job!
  curtag=`${DATE} +%F`
  if [ "<${curtag}>" != "<${tag}>" ]; then
      tag=${curtag}
#it is necessary to update the daq status as the day has changed during the processing... (batch can take a long time...)
      OUT1=${publicpath}/baodaqstatus-${curtag}.txt
      $ECHO "We should bring the DAQ status more up to date, this take 1 or 2sec..."
      $RM -f ${tmppublicpath}/statusdaq.finished
      ${scriptpath}/statusdaq.sh > ${OUT1}
      while [ ! -f "${tmppublicpath}/statusdaq.finished" ]; do
	  sleep 30
      done
      $RM ${tmppublicpath}/statusdaq.finished
#protect against remove/rewriting    
      $CHMOD -v 444 ${OUT1} 
  fi
#
# look if the gain & calibration files exist
#
  gainDone=`ils ${i} 2>&1 | $GREP -i "gain_.*\.fits" | $WC -l`
  if [ ! ${gainDone} -gt 0 ]; then
      $ECHO "FATAL (${scriptName}): gain file DOES NOT exist skip `basename $i` "
      continue
  fi

  calibDone=`ils ${i} 2>&1 | $GREP -i "calib_.*\.txt" | $WC -l`
  if [ ! ${calibDone} -gt 0 ]; then
      $ECHO "FATAL (${scriptName}): calibration files DO NOT exist skip `basename $i` "
      continue
  fi
#
  somethingTodo=1
  input4dataOff=`ils ${i}/Off 2>&1 | $GREP -i "datacycle" | $AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.10n`  
  input4dataOn=`ils ${i}/On 2>&1 | $GREP -i "datacycle" | $AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.10n`  
#
  if [ "<${input4dataOff}>" != "<>" -a "<${input4dataOn}>" != "<>"  ]; then
#look at the content of the datacycle directory as may be the previous batch job has crashed
      tableauOff=( `$ECHO ${input4dataOff}` )
      IFS='
'
      tableauOff=( $( $PRINTF "%s\n" "${tableauOff[@]}" | $AWK 'x[$0]++ == 0' ) )
      IFS=$DefaultIFS
#
      nTotFilesOff=0
      for j in ${tableauOff[@]}
	do
	nFiles=`ils ${i}/Off/${j} | $GREP -i "medfiltmtx" | $WC -l`
	nTotFilesOff=`expr ${nTotFilesOff} + ${nFiles}`
      done
#
      tableauOn=( `$ECHO ${input4dataOn}` )
      IFS='
      '
      tableauOn=( $( $PRINTF "%s\n" "${tableauOn[@]}" | $AWK 'x[$0]++ == 0' ) )
      IFS=$DefaultIFS
#
      nTotFilesOn=0
      for j in ${tableauOn[@]}
	do
	nFiles=`ils ${i}/On/${j} | $GREP -i "medfiltmtx" | $WC -l`
	nTotFilesOn=`expr ${nTotFilesOn} + ${nFiles}`
      done
      diffOnOff=`expr ${nTotFilesOff} - ${nTotFilesOn}`
      if [ ${diffOnOff} -lt 0 ]; then
	  zero=0
	  diffOnOff=`expr ${zero} - ${diffOnOff}` 
      fi
#allow files differences
      if [  ${nTotFilesOff} -gt 0 -a ${nTotFilesOn} -gt 0  ]; then
	  somethingTodo=0
	  $ECHO "DEBUG (${scriptName}): $sourceRadio $dateDAQ has <${nTotFilesOff}> Off-files and <${nTotFilesOn}> On-files but OK"
      else
	  $ECHO "INFO (${scriptName}): $sourceRadio $dateDAQ has <${nTotFilesOff}> Off-files and <${nTotFilesOn}> On-files => one should rebuild has something is wrong"
      fi
# 
  fi 
  if [ ${somethingTodo} -eq 1 ]; then
#
      $ECHO ">>>>>>>>>>>>>>>> Do $sourceRadio $dateDAQ ANALOFF"
      $RM -f ${tmppublicpath}/proc_script.finished
      if [ ${simulationMode} = "On" ]; then
	  $ECHO "SIMUL: ${scriptpath}/proc_script.sh $sourceRadio $dateDAQ OFF"
      else
	  ${scriptpath}/proc_script.sh $sourceRadio $dateDAQ OFF
	  while [ ! -e "${tmppublicpath}/proc_script.finished"  ]; do
	      $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/proc_script.finished"
	      date +%T
	      sleep 60
	  done
      fi
      $ECHO "INFO (${scriptName}): proc_script has finished"
      $ECHO ">>>>>>>>>>>> Done $sourceRadio $dateDAQ OFF"
      $ECHO ">>>>>>>>>>>>>>>> Do $sourceRadio $dateDAQ ON"
      $RM ${tmppublicpath}/proc_script.finished
      
      if [ ${simulationMode} = "On" ]; then
	  $ECHO "SIMUL: ${scriptpath}/proc_script.sh $sourceRadio $dateDAQ ON"
      else
	  ${scriptpath}/proc_script.sh $sourceRadio $dateDAQ ON
	  while [ ! -e "${tmppublicpath}/proc_script.finished"  ]; do
	      $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/proc_script.finished"
	      date +%T
	      sleep 60
	  done
      fi
      $ECHO "INFO (${scriptName}): proc_script has finished"
      $RM ${tmppublicpath}/proc_script.finished
#Todo see if there is errors in the log file before copy
      $ECHO ">>>>>>>>>>>> Done $sourceRadio $dateDAQ ON"
######
  else
######
      $ECHO ">>>>>>>>>>>>> ANALYSIS Part of ${sourceRadio} ${dateDAQ}"
      #go to the .../source/date-sourcelowercase directory
#download Off/datacycle<> and On/datacycle<> directories
      $MKDIR -p ${topDir}/${dateDAQ}${srclower}
      cd ${topDir}/${dateDAQ}${srclower}
      aboveOnOffDir=`pwd`
      $ECHO "DEBUG (${scriptName}): we are in directory: <`pwd`>"
#
      mode="Off"
      $ECHO "DEBUG (${scriptName}): START download the ${mode} data fits files"
      inFileDirectory="./${mode}"
      $MKDIR -p ${inFileDirectory}
      listOfDataCycle=( `ils ${i}/${mode} 2>&1 | $GREP -i "datacycle"|$AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.10n` )
      IFS='
'
      listOfDataCycle=( $( $PRINTF "%s\n" "${listOfDataCycle[@]}" | $AWK 'x[$0]++ == 0' ) )
      IFS=$DefaultIFS

      firstCycleOff=`$ECHO ${listOfDataCycle[0]} | $AWK '{match($0,"[0-9]+",arr); print arr[0]}'`
      nCyclesOff=`expr ${#listOfDataCycle[*]} - 1`

      lastCycleOff=`$ECHO ${listOfDataCycle[${nCyclesOff}]} | $AWK '{match($0,"[0-9]+",arr); print arr[0]}'`

      listOfDataCycle=( "${listOfDataCycle[@]/#/${i}/${mode}/}" )

      
      for j in ${listOfDataCycle[@]}
	do
	irodsDownDir="${j}"
	OUT=./getInFits.$$
	iget -r -f -K ${irodsDownDir} ${inFileDirectory} > ${OUT} 2>&1
	igetStatus=`$GREP "^ERROR" ${OUT}`
	if [ "<$igetStatus>" != "<>" ]; then
	    $ECHO "FATAL (${scriptName}): error while iget fits files:"
	    $ECHO $igetStatus
	    $ECHO " ==> skip this run $sourceRadio $dateDAQ"
	    continue
	fi
	$RM -f ${OUT}
      done
      $ECHO "DEBUG (${scriptName}): END download the ${mode} data fits files"
#
      mode="On"
      $ECHO "DEBUG (${scriptName}): START download the ${mode} data fits files"
      inFileDirectory="./${mode}"
      $MKDIR -p ${inFileDirectory}

      listOfDataCycle=( `ils ${i}/${mode} 2>&1 | $GREP -i "datacycle" |$AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.10n` )
      IFS='
'
      listOfDataCycle=( $( $PRINTF "%s\n" "${listOfDataCycle[@]}" | $AWK 'x[$0]++ == 0' ) )
      IFS=$DefaultIFS

      firstCycleOn=`$ECHO ${listOfDataCycle[0]} | $AWK '{match($0,"[0-9]+",arr); print arr[0]}'`
      nCyclesOn=`expr ${#listOfDataCycle[*]} - 1`

      lastCycleOn=`$ECHO ${listOfDataCycle[${nCyclesOn}]} | $AWK '{match($0,"[0-9]+",arr); print arr[0]}'`

      listOfDataCycle=( "${listOfDataCycle[@]/#/${i}/${mode}/}" )


#
#Check if there is the same number of cycles in Off and On mode
#      
#first cycle = max(firstCycleOn, firstCycleOff)
#last  cycle = min(lastCycleOn,lastCycleOff)

      if [ ${firstCycleOn} -eq ${firstCycleOff} -a ${lastCycleOn} -eq ${lastCycleOff} ]; then
	  firstCycle=${firstCycleOff}
	  lastCycle=${lastCycleOff}
      else
	  $ECHO "INFO (${scriptName}): missmatch between Off and On mode"
	  $ECHO "${firstCycleOn} != ${firstCycleOff} OR ${lastCycleOn} != ${lastCycleOff}"
	  firstCycle=${firstCycleOff}
	  if [ ${firstCycleOn} -gt ${firstCycle} ]; then
	      firstCycle=${firstCycleOn}
	  fi
	  lastCycle=${lastCycleOff}
	  if [ ${lastCycleOn} -lt ${lastCycle} ]; then
	      lastCycle=${lastCycleOn}
	  fi
      fi
      $ECHO "INFO (${scriptName}): use cycles [${firstCycle}, ${lastCycle}]"
      

      for j in ${listOfDataCycle[@]}
	do
	irodsDownDir="${j}"
	OUT=./getInFits.$$
	iget -r -f -K ${irodsDownDir} ${inFileDirectory} > ${OUT} 2>&1
	igetStatus=`$GREP "^ERROR" ${OUT}`
	if [ "<$igetStatus>" != "<>" ]; then
	    $ECHO "FATAL (${scriptName}): error while iget fits files:"
	    $ECHO $igetStatus
	    $ECHO " ==> skip this run $sourceRadio $dateDAQ"
	    continue
	fi
	$RM -f ${OUT}
      done
      $ECHO "DEBUG (${scriptName}): END download the ${mode} data fits files"
#Get sca file
      $ECHO "DEBUG (${scriptName}): START call getscafiles.sh ${sourceRadio} ${dateDAQ}"
      OUT1=./getScaStatus.$$
      $RM -f ${tmppublicpath}/getscafiles.finished
      ${scriptpath}/getscafiles.sh ${sourceRadio} ${dateDAQ} > ${OUT1} 2>&1
      while [ ! -f "${tmppublicpath}/getscafiles.finished" ]; do
	  $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getscafiles.finished"
	  date +%T
	  sleep 30
      done
      $RM ${tmppublicpath}/getscafiles.finished
      $ECHO "DEBUG (${scriptName}): END"
#
      getScaStatus=`$GREP "^\(FATAL\|ERROR\)" ${OUT1}`
#	
      if [ "<${getScaStatus}>" != "<>" ]; then
	  $ECHO "FATAL (${scriptName}): error to get sca file for $sourceRadio $dateDAQ"
	  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
	  $CAT ${OUT1}
	  $ECHO "DEBUG (${scriptName}): END"
	  continue
      fi
      $RM -f ${OUT1}

      scaFile=`$FIND . -name "sca*.sum.trans" | $XARGS -i basename {}`


#Get the calibration files
      OUT2=./calibFiles.$$
      ils ${i} | ${GREP} -i "calib.*\.txt" > ${OUT2}
      listeOfCalib=( `$CAT ${OUT2}` )
      IFS='
'
      listeOfCalib=( $( $PRINTF "%s\n" "${listeOfCalib[@]}" | $AWK 'x[$0]++ == 0' ) )
      IFS=$DefaultIFS
      $RM $OUT2
      for fcal in ${listeOfCalib[@]}
      do
	OUT3=./getCalib.$$
	iget -f -K -v ${i}/${fcal} ${aboveOnOffDir}> ${OUT3} 2>&1
	igetStatus=`$GREP "^ERROR" ${OUT3}`
	if [ "<$igetStatus>" != "<>" ]; then
	    $ECHO "FATAL (${scriptName}): error while iget calib files:"
	    $ECHO $igetStatus
	    exit 1
	fi
	$RM -f ${OUT3}
      done
      
#prepare job submission
      cd ${aboveOnOffDir}
#the default spectra name is medfiltmtx, otherwise use -specname <a_string>

      logFile="analyse_DATA_${sourceRadio}_${dateDAQ}.log"
      $ECHO "DEBUG (${scriptName}): execute the analysis... should not take long. `date`"
      if [ ${simulationMode} = "On" ]; then
	  $ECHO "SIMUL: ${scriptpath}/Objs/analyse -act dataOnOff -inPath ${iojobpath}  -source  ${sourceRadio} -date ${dateDAQ} -sca ${scaFile} -specdir datacycle -numcycle ${firstCycle},${lastCycle} -debuglev 1 -freqBAOCalib ${freqBAOCalib} >& ${logFile}"
	  cat > ${logFile} <<EOF
	  Ok calibration finished
EOF
      else
	  ${scriptpath}/Objs/analyse -act dataOnOff -inPath ${iojobpath}  -source  ${sourceRadio} -date ${dateDAQ} -sca ${scaFile} -specdir datacycle -numcycle ${firstCycle},${lastCycle} -debuglev 1 -freqBAOCalib ${freqBAOCalib} >& ${logFile}
      fi
      $ECHO "DEBUG (${scriptName}): analysis finished `date`. Look for errors before saving to Irods"
      rcstatus=`$GREP -i "Ok dataOnOff finished" ${logFile} | $WC -l`
      if [ ${rcstatus} -eq 0 ]; then
	  $ECHO "INFO (${scriptName}): analysis problem for ${sourceRadio} ${dateDAQ}"
	  $ECHO "                      START logfile"
	  $CAT ${logFile}
	  $ECHO "                      END   logFile"
#clean and leave
	  cd ${aboveOnOffDir}
	  $LS | $XARGS -i $RM -rf {}
	  continue
      fi
#Save into Irods
      $ECHO "DEBUG (${scriptName}): START save output files"
      $LS ${aboveOnOffDir}
#use -f option for iput ONLY to force override
      if [ ${simulationMode} = "On" ]; then
	  $ECHO "SIMUL: here we look for calib file and put them in Irods"
      else
	  $LS -lrth
	  $FIND . -name "dataRaw_*" -print | $XARGS -i  iput  -v -K {} ${i}
	  $FIND . -name "dataBAOCalib_*" -print | $XARGS -i  iput  -v -K {} ${i}
	  $FIND . -name "dataBAORTCalib_*" -print | $XARGS -i  iput  -v -K {} ${i}
	  $FIND . -name "diffOnOff_*" -print | $XARGS -i  iput  -v -K {} ${i}
      fi
#save analysis logfile
      $LS -l ${tmppublicpath} > /dev/null
      $CP ${logFile} ${tmppublicpath}
      $ECHO "DEBUG (${scriptName}): END save output files"
#clean up to avoid scratch SIZE EXCEED LIMIT
      cd ${aboveOnOffDir}
      $LS | $XARGS -i $RM -rf {}
  fi
done

exit 0