source: BAORadio/AmasNancay/trunk/gainmaker.sh @ 575

#!/bin/sh -xvf
#download SCA file, proceed to introspection, and upload results
DATE=/bin/date
GREP=/bin/grep
AWK=/bin/awk
ECHO=/bin/echo
WC=/usr/bin/wc
CAT=/bin/cat
PRINTF=/usr/bin/printf
FIND=/usr/bin/find
MKDIR=/bin/mkdir
XARGS=/usr/bin/xargs
SORT=/bin/sort
RM=/bin/rm
TR=/usr/bin/tr
CP=/bin/cp
LS=/bin/ls
CHMOD=/bin/chmod

DefaultIFS=$' \t\n'
IFS=$DefaultIFS

#set Irods environment    
. /usr/local/shared/bin/irods_env.sh -noverbose

scriptName="`basename $0`"
#$ECHO "Processing script ${scriptName} at `date`"

#$ECHO "SHOULD BE RETUNED FOR THE inPath cmd line of analyse.cc (Sorry...): 21/7/11 by JEC"
#exit 1

#Process cmd line args: the -src option is mandatory (source name as Abell85)
sourceRadio=
#use -sim option to simulate processing (debug the script as if...)
simulationMode=Off
while [ $# -gt 0 ]
do
  case "$1" in
      -src)  sourceRadio=$2; shift;;
      -sim) simulationMode=On;;
        -h)
            echo >&2 \
            "usage: $0 -src souce [-sim]"
            exit 1;;
        *)  break;;     # terminate while loop
    esac
    shift
done

if [ ${simulationMode} = "On" ]; then
    $ECHO "INFO ${scriptName} running in SIMUL mode"
fi


if [ "<${sourceRadio}>" = "<>" ]; then
    $ECHO "FATAL: You have forgotten to select the source option (-src)"
    exit 1
fi

case ${sourceRadio} in
    Abell85) $ECHO "INFO (${scriptName}): process ${sourceRadio}";;
    Abell2440) $ECHO "INFO (${scriptName}): process ${sourceRadio}";;
    Abell1205) $ECHO "INFO (${scriptName}): process ${sourceRadio}";;
    *) ECHO "FATAL (${scriptName}): process ${sourceRadio} not yet foreseen"
    exit 1;;
esac


srclower=`${ECHO} ${sourceRadio} | ${TR} "[:upper:]" "[:lower:]" `

#Path to public backupable path
publicpath="/afs/in2p3.fr/home/c/campagne/public"


#temporary files to synchronize scripts
tmppublicpath=${TMPPUBLICPATH}
$LS -l ${tmppublicpath} > /dev/null


#Path where the job will do temporary IO
if [ ${ENVIRONMENT} = "INTERACTIVE" ]; then
    iojobpath="/sps/baoradio/AmasNancay/JEC"
    $MKDIR -p $iojobpath
    $ECHO "Usage in INTERACTIVE may be destructive... To be used with great care !!!!"
    exit 0;   
elif [ ${ENVIRONMENT} = "BATCH" ] ; then
    iojobpath=${TMPBATCH}
else
    $ECHO "FATAL (${scriptName}): environment is ${ENVIRONMENT} not allowed"
    exit 1
fi
cd ${iojobpath}

localpath="${sourceRadio}"
$MKDIR -p ./${localpath}
cd ./${localpath}

#save the top directory
topDir=`pwd`

#Path of the utility scripts
scriptpath=${SCRIPTPATH}


#JEC 1/10/11 Use generic baodaqstatus name 
#get the daq current irod status
#tag=`${DATE} +%F`
#OUT1=${publicpath}/baodaqstatus-${tag}.txt
OUT1=${publicpath}/baodaqstatus-current.txt

if [ ! -e ${OUT1}  -o ! -r ${OUT1} ]; then
    $ECHO "FATAL (${scriptName}): ${OUT1} has a problem"
    exit 1
#    $RM -f ${tmppublicpath}/statusdaq.finished
#    ${scriptpath}/statusdaq.sh > ${OUT1}
#    while [ ! -f "${tmppublicpath}/statusdaq.finished" ]; do
#       sleep 30
#    done
#    $RM ${tmppublicpath}/statusdaq.finished
#protect against remove/rewriting    
#    $CHMOD -v 444 ${OUT1} 
#
fi

#
#We forbib to perform the gain for all the sources in one shot!
#
#if [ ! "<${sourceRadio}>" = "All" ]; then
$ECHO "You have selected sourceRadio = $sourceRadio"
tableau=( `$GREP -i ${sourceRadio} ${OUT1} | $AWK '( NF==4 ) { print $2 }' ` )
#else
#    tableau=( `$CAT ${OUT1} | $AWK '( NF==4 ) { print $2 }'`  )
#fi
IFS='
'
tableau=( $( $PRINTF "%s\n" "${tableau[@]}" | $AWK 'x[$0]++ == 0' ) )
IFS=$DefaultIFS


for i in ${tableau[@]}
  do
  $ECHO "DEBUG (${scriptName}): irods root dir $i"
  dateDAQ=`$ECHO ${i} | $AWK '{split($0,a,"/"); print a[6];}' | $AWK "{sub(/${srclower}/,\"\"); print}"`
#JEC 1/10/11 Use generic baodaqstatus name 
#Look if the date has changed since beginning of the job!
#  curtag=`${DATE} +%F`
#  if [ "<${curtag}>" != "<${tag}>" ]; then
#      tag=${curtag}
#it is necessary to update the daq status as the day has changed during the processing... (batch can take a long time...)
#      OUT1=${publicpath}/baodaqstatus-${curtag}.txt
#      $ECHO "We should bring the DAQ status more up to date, this take 1 or 2sec..."
#      $RM -f ${tmppublicpath}/statusdaq.finished
#      ${scriptpath}/statusdaq.sh > ${OUT1}
#      while [ ! -f "${tmppublicpath}/statusdaq.finished" ]; do
#         sleep 30
#      done
#      $RM ${tmppublicpath}/statusdaq.finished
#protect against remove/rewriting    
#      $CHMOD -v 444 ${OUT1} 
#  fi
#
# look if the fits file of the gains already exists
  alreadyDone=`ils ${i} 2>&1 | $GREP -i "gain_.*\.fits" | $WC -l`
  if [ ${alreadyDone} -gt 0 ]; then
      $ECHO "INFO (${scriptName}): Fits file for gains already exists"
      continue
  fi
# Something should be done...
#look if the files for the input to gain computations have been done
  somethingTodo=1
  input4gainOk1=`ils ${i}/Off 2>&1 | $GREP -i "^ERROR" | $WC -l` 
  input4gainOk2=`ils ${i}/Off 2>&1 | $GREP -i "gaincycle"` 
  if [ $input4gainOk1 -eq 0 -a "<${input4gainOk2}>" != "<>"  ]; then
      gaincycle=`$ECHO ${input4gainOk2} | $AWK '($1 == "C-"){print $2}'| $XARGS -i  basename {}`
      input4gainOk3=`ils ${i}/Off/${gaincycle} 2>&1 | $GREP -i "medfiltmtx" | $WC -l`
      if [ $input4gainOk3 -gt 0 ]; then
          somethingTodo=0
      fi
  fi
  if [ $somethingTodo -eq 1 ]; then
#
      $ECHO ">>>>>>>>>>>>>>>> Do proc_script $sourceRadio $dateDAQ GAIN"
      $RM -f ${tmppublicpath}/proc_script.finished
      if [ ${simulationMode} = "On" ]; then
          $ECHO "SIMUL: ${scriptpath}/proc_script.sh $sourceRadio $dateDAQ GAIN"
      else      
          ${scriptpath}/proc_script.sh $sourceRadio $dateDAQ GAIN
          while [ ! -e "${tmppublicpath}/proc_script.finished"  ]; do
              $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/proc_script.finished"
              date +%T
              sleep 60
          done
      fi
      $ECHO "INFO (${scriptName}): proc_script has finished"
      $RM ${tmppublicpath}/proc_script.finished
#Todo see if there is errors in the log file before copy
      $ECHO ">>>>>>>>>>>>>>>> Done proc_script $sourceRadio $dateDAQ GAIN"
######
  fi
######
  $ECHO ">>>>>>>>>>>>> ANALYSIS Part of ${sourceRadio} ${dateDAQ}"
#go to the .../source/date-sourcelowercase directory
  $MKDIR -p ${topDir}/${dateDAQ}${srclower}
  cd ${topDir}/${dateDAQ}${srclower}
  aboveOnOffDir=`pwd`
  $ECHO "DEBUG (${scriptName}): we are in directory: <`pwd`>"
# So the input files are there, so the fits file can be determined
  mode="Off"
  spectraGenDirName=`$ECHO ${gaincycle} | $AWK '{match($0,"[a-zA-Z]+",arr); print arr[0]}'`
  cycleNumber=`$ECHO ${gaincycle} | $AWK '{match($0,"[0-9]+",arr); print arr[0]}'`
#care: the gaincycle variable contains the cycle number
  $ECHO "DEBUG (${scriptName}): START download the fits files"
  inFileDirectory="./${mode}"
  $MKDIR -p ${inFileDirectory}
  irodsDownDir="${i}/${mode}/${gaincycle}"
  OUT=./getInFits.$$
  iget -r -f -K ${irodsDownDir} ${inFileDirectory} > ${OUT} 2>&1
  $LS ${inFileDirectory}/${gaincycle}
  igetStatus=`$GREP "^ERROR" ${OUT}`
  if [ "<$igetStatus>" != "<>" ]; then
      $ECHO "FATAL (${scriptName}): error while iget fits files:"
      $ECHO $igetStatus
      continue
  fi
  $RM -f ${OUT}
  $ECHO "DEBUG (${scriptName}): END download the fits files"
#Get sca files
  $ECHO "DEBUG (${scriptName}): START call getscafiles.sh ${sourceRadio} ${dateDAQ}"
  OUT1=./getScaStatus.$$
  $RM -f ${tmppublicpath}/getscafiles.finished
  ${scriptpath}/getscafiles.sh ${sourceRadio} ${dateDAQ} > ${OUT1} 2>&1
  while [ ! -f "${tmppublicpath}/getscafiles.finished" ]; do
      $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getscafiles.finished"
      date +%T
      sleep 30
  done
  $RM ${tmppublicpath}/getscafiles.finished
  $ECHO "DEBUG (${scriptName}): END"
#
  getScaStatus=`$GREP "^\(FATAL\|ERROR\)" ${OUT1}`
#       
  if [ "<${getScaStatus}>" != "<>" ]; then
      $ECHO "FATAL (${scriptName}): error to get sca file for $sourceRadio $dateDAQ"
      $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
      $CAT ${OUT1}
      $ECHO "DEBUG (${scriptName}): END"
      continue
  fi
  $RM -f ${OUT1}
  $ECHO "DEBUG (${scriptName}): where am i `pwd`"
  $LS -lrt
  
  scaFile=`$FIND . -name "sca*.sum.trans" | $XARGS -i basename {}`
#       
#the default spectra name is medfiltmtx, otherwise use -specname <a_string>
  logFile="analyse_GAIN_${sourceRadio}_${dateDAQ}.log"
  $ECHO "DEBUG (${scriptName}): execute the analysis... should not take long. `date`"
  if [ ${simulationMode} = "On" ]; then
      $ECHO "SIMUL: ${scriptpath}/Objs/analyse -act gain  -inPath ${iojobpath} -source  ${sourceRadio} -date ${dateDAQ} -sca ${scaFile} -specdir ${spectraGenDirName} -mode ${mode} -numcycle ${cycleNumber},${cycleNumber} -debug 1 >& ${logFile}"
      cat > ${logFile} <<EOF
      Ok gain finished
EOF

  else
      ${scriptpath}/Objs/analyse -act gain  -inPath ${iojobpath} -source  ${sourceRadio} -date ${dateDAQ} -sca ${scaFile} -specdir ${spectraGenDirName} -mode ${mode} -numcycle ${cycleNumber},${cycleNumber} -debuglev 1 >& ${logFile}
  fi
#
  $ECHO "DEBUG (${scriptName}): analysis finished `date`. Look for errors before saving to Irods"
  rcstatus=`$GREP -i "OK gain finished" ${logFile} | $WC -l`
  if [ ${rcstatus} -eq 0 ]; then
      $ECHO "INFO (${scriptName}): analysis problem for ${sourceRadio} ${dateDAQ}"
      $ECHO "                      START logfile"
      $CAT ${logFile}
      $ECHO "                      END   logFile"
#clean and leave
      cd ${aboveOnOffDir}
      $RM -rf ./*
      continue
  fi
#Save into Irods
  $ECHO "DEBUG (${scriptName}): START save output files"
  $LS ${aboveOnOffDir}
#use -f option of iput ONLY to force override
  if [ ${simulationMode} = "On" ]; then
      $ECHO "SIMUL: here we look for gain file and put them in Irods"
  else
      $FIND . -name "gain_*" -print | $XARGS -i  iput  -v -K {} ${i}
  fi
#save analysis logfile
  $LS -l ${tmppublicpath} > /dev/null
  $CP ${logFile} ${tmppublicpath}
  $ECHO "DEBUG (${scriptName}): END save output files"
#clean
  cd ${aboveOnOffDir}
  $RM -rf ./*
done

exit 0