source: BAORadio/AmasNancay/rawonoffmaker.sh @ 535

#!/bin/sh
#Perform the ON-OFF analysis
DATE=/bin/date
GREP=/bin/grep
AWK=/bin/awk
ECHO=/bin/echo
WC=/usr/bin/wc
CAT=/bin/cat
PRINTF=/usr/bin/printf
FIND=/usr/bin/find
MKDIR=/bin/mkdir
XARGS=/usr/bin/xargs
SORT=/bin/sort
RM=/bin/rm
TR=/usr/bin/tr
CP=/bin/cp
LS=/bin/ls
CHMOD=/bin/chmod

DefaultIFS=$' \t\n'
IFS=$DefaultIFS

#set Irods environment    
. /usr/local/shared/bin/irods_env.sh -noverbose

scriptName="`basename $0`"
$ECHO "Processing script ${scriptName} at `date`"
#which source to analyse

#Process cmd line args: the -src option is mandatory (source name as Abell85)
sourceRadio=
dateSelected=
#use -sim option to simulate processing (debug the script as if...)
simulationMode=Off
while [ $# -gt 0 ]
do
  case "$1" in
      -src)  sourceRadio=$2;    shift;;
      -date) dateSelected=$2;     shift;;
      -sim)  simulationMode=On;;
        -h)
            echo >&2 \
            "usage: $0 -src souce -date YYYYMMDD [-freq freq in MHz (default ${freqBAOCalib})] [-sim to trig simulation mode]"
            exit 1;;
        *)  break;;     # terminate while loop
    esac
    shift
done

if [ ${simulationMode} = "On" ]; then
    $ECHO "INFO ${scriptName} running in SIMUL mode"
fi

if [ "<${sourceRadio}>" = "<>" ]; then
    $ECHO "FATAL: You have forgotten to select the source option (-src)"
    exit 1
fi

case ${sourceRadio} in
    Abell85) $ECHO "INFO (${scriptName}): process ${sourceRadio}";;
    Abell2440) $ECHO "INFO (${scriptName}): process ${sourceRadio}";;
    Abell1205) $ECHO "INFO (${scriptName}): process ${sourceRadio}";;
    *) ECHO "FATAL (${scriptName}): process ${sourceRadio} not yet foreseen"
    exit 1;;
esac

srclower=`${ECHO} ${sourceRadio} | ${TR} "[:upper:]" "[:lower:]" `

#Path to public backupable path
publicpath="/afs/in2p3.fr/home/c/campagne/public"


#temporary files to synchronize scripts
tmppublicpath=${TMPPUBLICPATH}
#wakeup the NFS disk
$LS -l ${tmppublicpath} > /dev/null
#clean previous spurious files
$RM -f ${tmppublicpath}/*.finished

#Path where the job will do temporary IO
if [ ${ENVIRONMENT} = "INTERACTIVE" ]; then
    iojobpath="/sps/baoradio/AmasNancay/JEC"
    $MKDIR -p $iojobpath
elif [ ${ENVIRONMENT} = "BATCH" ] ; then
    iojobpath=${TMPBATCH}
else
    $ECHO "FATAL (${scriptName}): environment is ${ENVIRONMENT} not allowed"
    exit 1
fi
cd ${iojobpath}

localpath="${sourceRadio}"
$MKDIR -p ./${localpath}
cd ./${localpath}

#save the top directory
topDir=`pwd`

#Path of the utility scripts
scriptpath="/afs/in2p3.fr/home/c/campagne/private/work/AmasNancay"

#get the daq current irod status
tag=`${DATE} +%F`
OUT1=${publicpath}/baodaqstatus-${tag}.txt

if [ ! -e ${OUT1}  -o ! -r ${OUT1} ]; then
    $ECHO "We should bring the DAQ status more up to date, this take 1 or 2sec..."
    $RM -f ${tmppublicpath}/statusdaq.finished
    ${scriptpath}/statusdaq.sh > ${OUT1}
    while [ ! -f "${tmppublicpath}/statusdaq.finished" ]; do
        sleep 30
    done
    $RM ${tmppublicpath}/statusdaq.finished
#protect against remove/rewriting    
    $CHMOD -v 444 ${OUT1} 
fi


#Nb: if we want to implement a loop on sources we have to take care of srclower...

$ECHO "You have selected sourceRadio = ${sourceRadio} [date = ${dateSelected}]"
tableau=( `$GREP -i "${dateSelected}${srclower}" ${OUT1} | $AWK '( NF==4 ) { print $2 }' ` )
IFS='
' 
tableau=( $( $PRINTF "%s\n" "${tableau[@]}" | $AWK 'x[$0]++ == 0' ) )
IFS=$DefaultIFS


for i in ${tableau[@]}
  do
#
# start a "fresh" session  
  cd ${topDir}
  $LS | $XARGS -i $RM -rf {}
  
  $ECHO "DEBUG: (${scriptName}) irods root dir $i"
  dateDAQ=`$ECHO ${i} | $AWK '{split($0,a,"/"); print a[6];}' | $AWK "{sub(/${srclower}/,\"\"); print}"`

  $ECHO "DEBUG: (${scriptName}): we are in directory `pwd` the topDir=${topDir}"
  $LS -lrt


#Look if the date has changed since beginning of the job!
  curtag=`${DATE} +%F`
  if [ "<${curtag}>" != "<${tag}>" ]; then
      tag=${curtag}
#it is necessary to update the daq status as the day has changed during the processing... (batch can take a long time...)
      OUT1=${publicpath}/baodaqstatus-${curtag}.txt
      $ECHO "We should bring the DAQ status more up to date, this take 1 or 2sec..."
      $RM -f ${tmppublicpath}/statusdaq.finished
      ${scriptpath}/statusdaq.sh > ${OUT1}
      while [ ! -f "${tmppublicpath}/statusdaq.finished" ]; do
          sleep 30
      done
      $RM ${tmppublicpath}/statusdaq.finished
#protect against remove/rewriting    
      $CHMOD -v 444 ${OUT1} 
  fi
#
# look if the gain & calibration files exist
#
  gainDone=`ils ${i} 2>&1 | $GREP -i "gain_.*\.fits" | $WC -l`
  if [ ! ${gainDone} -gt 0 ]; then
      $ECHO "FATAL (${scriptName}): gain file DOES NOT exist skip `basename $i` "
      continue
  fi

#
  somethingTodo=1
  input4dataOff=`ils ${i}/Off 2>&1 | $GREP -i "datacycle" | $AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.10n`  
  input4dataOn=`ils ${i}/On 2>&1 | $GREP -i "datacycle" | $AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.10n`  
#
  if [ "<${input4dataOff}>" != "<>" -a "<${input4dataOn}>" != "<>"  ]; then
#look at the content of the datacycle directory as may be the previous batch job has crashed
      tableauOff=( `$ECHO ${input4dataOff}` )
      IFS='
'
      tableauOff=( $( $PRINTF "%s\n" "${tableauOff[@]}" | $AWK 'x[$0]++ == 0' ) )
      IFS=$DefaultIFS
#
      nTotFilesOff=0
      for j in ${tableauOff[@]}
        do
        nFiles=`ils ${i}/Off/${j} | $GREP -i "medfiltmtx" | $WC -l`
        nTotFilesOff=`expr ${nTotFilesOff} + ${nFiles}`
      done
#
      tableauOn=( `$ECHO ${input4dataOn}` )
      IFS='
      '
      tableauOn=( $( $PRINTF "%s\n" "${tableauOn[@]}" | $AWK 'x[$0]++ == 0' ) )
      IFS=$DefaultIFS
#
      nTotFilesOn=0
      for j in ${tableauOn[@]}
        do
        nFiles=`ils ${i}/On/${j} | $GREP -i "medfiltmtx" | $WC -l`
        nTotFilesOn=`expr ${nTotFilesOn} + ${nFiles}`
      done
      diffOnOff=`expr ${nTotFilesOff} - ${nTotFilesOn}`
      if [ ${diffOnOff} -lt 0 ]; then
          zero=0
          diffOnOff=`expr ${zero} - ${diffOnOff}` 
      fi
#allow files differences but non zero
      if [  ${nTotFilesOff} -gt 0 -a ${nTotFilesOn} -gt 0  ]; then
          somethingTodo=0
          $ECHO "DEBUG (${scriptName}): $sourceRadio $dateDAQ has <${nTotFilesOff}> Off-files and <${nTotFilesOn}> On-files but OK"
      else
          $ECHO "INFO (${scriptName}): $sourceRadio $dateDAQ has <${nTotFilesOff}> Off-files and <${nTotFilesOn}> On-files => one should rebuild has something is wrong"
      fi
#
#look at consecutive cycles

      rmDirectory=0

      cycleIdOff=`$ECHO $input4dataOff | sed 's/datacycle//g'`
      tableauOff=( `$ECHO ${cycleIdOff}` )
      IFS='
'
      tableauOff=( $( $PRINTF "%s\n" "${tableauOff[@]}" | $AWK 'x[$0]++ == 0' ) )
      IFS=$DefaultIFS
      
      lastIdOff=`expr ${#tableauOff[*]} - 1`
      ncycleOff=`expr ${tableauOff[${lastIdOff}]} - ${tableauOff[0]} + 1`
      ncycleOffVerif=${#tableauOff[*]}
      if [ "<${ncycleOff}>" != "<${ncycleOffVerif}>"  ]; then
          $ECHO "$sourceRadio $dateDAQ: There is a lack of OFF cycles in the sequence: ${tableauOff[@]}"
          rmDirectory=1
          somethingTodo=1
      else
          $ECHO "$sourceRadio $dateDAQ: Ok at least consecutive OFF cycles: ${tableauOff[@]}"
      fi

      cycleIdOn=`$ECHO $input4dataOn | sed 's/datacycle//g'`
      tableauOn=( `$ECHO ${cycleIdOn}` )
      IFS='
'
      tableauOn=( $( $PRINTF "%s\n" "${tableauOn[@]}" | $AWK 'x[$0]++ == 0' ) )
      IFS=$DefaultIFS
      
      lastIdOn=`expr ${#tableauOn[*]} - 1`
      ncycleOn=`expr ${tableauOn[${lastIdOn}]} - ${tableauOn[0]} + 1`
      ncycleOnVerif=${#tableauOn[*]}
      if [ "<${ncycleOn}>" != "<${ncycleOnVerif}>"  ]; then
          $ECHO "$sourceRadio $dateDAQ: There is a lack of ON cycles in the sequence: ${tableauOn[@]}"
          rmDirectory=1
          somethingTodo=1
      else
          $ECHO "$sourceRadio $dateDAQ: Ok at least consecutive ON cycles:  ${tableauOn[@]}"
      fi

      if [ "<${cycleIdOff}>" != "<${cycleIdOn}>" ]; then
          $ECHO "$sourceRadio $dateDAQ: cycles seq. missmatch ON and OFF"
          rmDirectory=1
          somethingTodo=1         
      fi
# 
  fi 
########
  if [ ${somethingTodo} -eq 1 ]; then
#
      $ECHO ">>>>>>>>>>>>>>>> Do $sourceRadio $dateDAQ ANALOFF"

      if [ ${rmDirectory} -eq 1 ]; then
#delete datacycle Off
          dirToRmOff=`ils ${i}/Off 2>&1 | $GREP -i "datacycle" | $AWK 'BEGIN{FS="C- "}{print $2}'`  
          tableauOff=( `$ECHO ${dirToRmOff}` )
          IFS='
'
          tableauOff=( $( $PRINTF "%s\n" "${tableauOff[@]}" | $AWK 'x[$0]++ == 0' ) )
          IFS=$DefaultIFS
          for j in ${tableauOff[@]}
            do
            irm -f -r -v ${j}
          done
#delete datacycle On
          dirToRmOn=`ils ${i}/On 2>&1 | $GREP -i "datacycle" | $AWK 'BEGIN{FS="C- "}{print $2}'`  
          tableauOn=( `$ECHO ${dirToRmOn}` )
          IFS='
'
          tableauOn=( $( $PRINTF "%s\n" "${tableauOn[@]}" | $AWK 'x[$0]++ == 0' ) )
          IFS=$DefaultIFS
          for j in ${tableauOn[@]}
            do
            irm -f -r -v ${j}
          done
      fi
          
      $RM -f ${tmppublicpath}/proc_script.finished
      if [ ${simulationMode} = "On" ]; then
          $ECHO "SIMUL: ${scriptpath}/proc_script.sh $sourceRadio $dateDAQ OFF"
      else
          ${scriptpath}/proc_script.sh $sourceRadio $dateDAQ OFF
          while [ ! -e "${tmppublicpath}/proc_script.finished"  ]; do
              $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/proc_script.finished"
              date +%T
              sleep 60
          done
      fi
      $ECHO "INFO (${scriptName}): proc_script has finished"
      $ECHO ">>>>>>>>>>>> Done $sourceRadio $dateDAQ OFF"
      $ECHO ">>>>>>>>>>>>>>>> Do $sourceRadio $dateDAQ ON"
      $RM ${tmppublicpath}/proc_script.finished
      
      if [ ${simulationMode} = "On" ]; then
          $ECHO "SIMUL: ${scriptpath}/proc_script.sh $sourceRadio $dateDAQ ON"
      else
          ${scriptpath}/proc_script.sh $sourceRadio $dateDAQ ON
          while [ ! -e "${tmppublicpath}/proc_script.finished"  ]; do
              $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/proc_script.finished"
              date +%T
              sleep 60
          done
      fi
      $ECHO "INFO (${scriptName}): proc_script has finished"
      $RM ${tmppublicpath}/proc_script.finished
#Todo see if there is errors in the log file before copy
      $ECHO ">>>>>>>>>>>> Done $sourceRadio $dateDAQ ON"
  fi
done

exit 0