#!/bin/sh -xvf
#Basic tuning of system fuction used.
AWK=/bin/awk
SED=/bin/sed
GREP=/bin/grep
WC=/usr/bin/wc
RM=/bin/rm
CAT=/bin/cat
TOUCH=/bin/touch
DATE=/bin/date
ECHO=/bin/echo
LS=/bin/ls
MKDIR=/bin/mkdir
TR=/usr/bin/tr
FIND=/usr/bin/find
PRINTF=/usr/bin/printf
XARGS=/usr/bin/xargs
SORT=/bin/sort
QSUB=qsub

DefaultIFS=$' \t\n'
IFS=$DefaultIFS


scriptName="`basename $0`"
$ECHO "Processing script ${scriptName} at `date`"
######
# Local functions
######
usage() {
  echo "usage: $scriptName <source> <YYYYMMDD-directory> <type of process>"
  echo "      -<source> as Abell85"
  echo "      -<YYYYMMDD-directory> as 20110428"
  echo "      -<type of process> as GAIN|CALIBON|CALIBOFF|ON|OFF"
}
######
# argument test
######
if [ ! $# = 3 ]; then
  usage
  touch proc_script.finished
  exit 1
fi
sourceRadio=$1
srclower=`${ECHO} ${sourceRadio} | ${TR} "[:upper:]" "[:lower:]" `
dateDAQ=$2
typeofproc=$3

#Path to public backupable path
publicpath="/afs/in2p3.fr/home/c/campagne/public"

#temporary files to synchronize scripts
tmppublicpath=${TMPPUBLICPATH}
$LS -l ${tmppublicpath} > /dev/null


#Path where the job will do temporary IO
if [ ${ENVIRONMENT} = "INTERACTIVE" ]; then
    iojobpath="/sps/baoradio/AmasNancay/JEC"
    $MKDIR -p $iojobpath
elif [ ${ENVIRONMENT} = "BATCH" ] ; then
    iojobpath=${TMPBATCH}
else
    $ECHO "FATAL (${scriptName}): environment is ${ENVIRONMENT} not allowed"
    touch ${tmppublicpath}/proc_script.finished
    exit 1
fi
cd ${iojobpath}

localpath="${sourceRadio}/${dateDAQ}${srclower}"
$MKDIR -p ./${localpath}
cd ./${localpath}
mainDirectory=`pwd`


#Path of the utility scripts
scriptpath="/afs/in2p3.fr/home/c/campagne/private/work/AmasNancay"


#select file per cycle
#$ECHO "Select files process..."
#keep only lines with structure: cycle_number first,last,step
#Get sca files
$ECHO "DEBUG (${scriptName}): START call getscafiles.sh ${sourceRadio} ${dateDAQ}"
OUT1=./getScaStatus.$$
$RM -f ${tmppublicpath}/getscafiles.finished
${scriptpath}/getscafiles.sh ${sourceRadio} ${dateDAQ} > ${OUT1} 2>&1
while [ ! -f "${tmppublicpath}/getscafiles.finished" ]; do
    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getscafiles.finished"
    date +%T
    sleep 30
done
$RM ${tmppublicpath}/getscafiles.finished
$ECHO "DEBUG (${scriptName}): END"


getScaStatus=`$GREP "^\(FATAL\|ERROR\)" ${OUT1}`

if [ "<${getScaStatus}>" != "<>" ]; then
  $ECHO "FATAL (${scriptName}): error to get sca file for $sourceRadio $dateDAQ"
  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
  $CAT ${OUT1}
  $ECHO "DEBUG (${scriptName}): END"
  touch ${tmppublicpath}/proc_script.finished
  exit 1
fi


scaFileSummarized=`$FIND . -name "sca*.sum" -print`
if [ "<${scaFileSummarized}>" = "<>" ]; then
  $ECHO "FATAL (${scriptName}): strange sca file summarized not available: $sourceRadio $dateDAQ"
  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
  $CAT ${OUT1}
  $ECHO "DEBUG (${scriptName}): END"
  touch ${tmppublicpath}/proc_script.finished
  exit 1
fi
$RM ${OUT1}

scaFileSummarized=`basename $scaFileSummarized`

#set Irods environment
. /usr/local/shared/bin/irods_env.sh -noverbose
#download header files to be used for signal files selction
$MKDIR -p ./Fiber1
dirFiberfile=`pwd`/Fiber1

dirIrods="/baoradio/data/AmasNancay/${sourceRadio}/${dateDAQ}${srclower}/Fiber1"

listOfFiles=( `ils ${dirIrods} | $GREP -i "header" | $XARGS -i basename {} | $SORT -k1.7n` )
IFS='
'
listOfFiles=( $( $PRINTF "%s\n" "${listOfFiles[@]}" | $AWK 'x[$0]++ == 0' ) )
IFS=$DefaultIFS

#
$ECHO "DEBUG (${scriptName}): Download if necessary the headers"
#
for ifile in ${listOfFiles[@]}
do
#test if the headers have not yet been downloaded
  if [ ! -e ${dirFiberfile}/${ifile} ]; then
      OUT2=./getheaders.$$
      iget -f -K ${dirIrods}/${ifile} ${dirFiberfile} >${OUT2} 2>&1
      igetStatus=`$GREP "^ERROR" ${OUT2}`
      if [ "<$igetStatus>" != "<>" ]; then
	  $ECHO "FATAL (${scriptName}): error while iget:"
	  $ECHO $igetStatus
	  touch ${tmppublicpath}/proc_script.finished
	  exit 1
      fi
      $RM -f $OUT2
  fi
done
#
$ECHO "DEBUG (${scriptName}): START call select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc}"
#
#Get list of file to process
headerFileSelected=./selectSignal.$$
$RM -f ${tmppublicpath}/select.finished
${scriptpath}/select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc} | $GREP "^#[0-9]" | $SED "s/^#//" > $headerFileSelected
while [ ! -f "${tmppublicpath}/select.finished" ]; do
    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/select.finished"
    date +%T
    sleep 30
done
$RM ${tmppublicpath}/select.finished
#
$ECHO "DEBUG (${scriptName}): END"
#
$ECHO "DEBUG (${scriptName}): START filling cycleArray, filesArray"
#
declare -a cycleArray
declare -a filesArray
$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
IFS=$DefaultIFS
$ECHO "DEBUG (${scriptName}): START cat $headerFileSelected"
$CAT $headerFileSelected
$ECHO "DEBUG (${scriptName}): END cat $headerFileSelected"

while read v1 v2
do
 cycleArray[${#cycleArray[*]}]=$v1
 filesArray[${#filesArray[*]}]=$v2
done < $headerFileSelected

if [ ${#cycleArray[*]} -eq 0 -o ${#filesArray[*]} -eq 0 ]; then
   $ECHO "FATAL (${scriptName}): no header file selected (${#cycleArray[*]}, ${#filesArray[*]})"
   touch ${tmppublicpath}/proc_script.finished
   exit 1
else
   $ECHO "DEBUG (${scriptName}): non zero elements found for cycleArray and filesArray" 
fi

#
$ECHO "DEBUG (${scriptName}): number of elements <${#cycleArray[*]}> <${#filesArray[*]}>"
$ECHO "DEBUG (${scriptName}): END"
#
# (timeShift)
#Add 3sec to start ON / OFF
#Substract 4sec to start DAB ON / OFF
#
#Take care: Cycle numbering starts at 1 in SCAFiles while Arrays index start at 0
#
#for GAIN the reference cycle takes the middle of the run
#
#nwinmean donne le nombre de fenetre tspwin utilise pour faire calculer la moyenne des medianes de chaque fenetre
#This is the nimber of paquets of the time window which change according to processing type
$ECHO "DEBUG (${scriptName}): START switch on $typeofproc"
#
case "$typeofproc" in
    GAIN)
      $ECHO "process signal files for GAIN phase"
      ingain=""
      freqfilter="-freqfilter -"
      nwinmean="1"
      tspwin="5120"
      dirName="Off"
      cyclebasename="gaincycle"
      indexStartHour="6"
      timeShift="-4"
      timeDuration="14"
      firstCycleId=`expr  ${#cycleArray[*]} / 2`
      firstCycleId=`expr ${firstCycleId} - 1`
      lastp1CycleId=`expr ${firstCycleId} + 1`
      ;;
    ON)
      $ECHO "process signal files for ON phases"
      ingain="-gain gain_${dateDAQ}_${srclower}.fits"
      freqfilter="-freqfilter 16"
      nwinmean="5"
      tspwin="5120"
      dirName="On"
      cyclebasename="datacycle"
      indexStartHour="4"
      timeShift="3"
      timeDuration="30"
      firstCycleId="0"
      lastp1CycleId="${#cycleArray[@]}"
      ;;
    OFF)
      $ECHO "process signal files for OFF phases"
      ingain="-gain gain_${dateDAQ}_${srclower}.fits"
      freqfilter="-freqfilter 16"
      nwinmean="5"
      tspwin="5120"
      dirName="Off"
      cyclebasename="datacycle"
      indexStartHour="8"
      timeShift="3"
      timeDuration="30"
      firstCycleId="0"
      lastp1CycleId="${#cycleArray[@]}"
      ;;
    CALIBON)
      $ECHO "process signal files for CALIBRATION ON phases"
      ingain="-gain gain_${dateDAQ}_${srclower}.fits"
      freqfilter="-freqfilter 16"
      nwinmean="1"
      tspwin="1024"
      dirName="On"
      cyclebasename="calibcycle"
      indexStartHour="2"
      timeShift="-4"
      timeDuration="14"
      firstCycleId="0"
      lastp1CycleId="${#cycleArray[@]}"
      ;;
    CALIBOFF)
      $ECHO "process signal files for CALIBRATION OFF phases"
      ingain="-gain gain_${dateDAQ}_${srclower}.fits"
      freqfilter="-freqfilter 16"
      nwinmean="1"
      tspwin="1024"
      dirName="Off"
      cyclebasename="calibcycle"
      indexStartHour="6"
      timeShift="-4"
      timeDuration="14"
      firstCycleId="0"
      lastp1CycleId="${#cycleArray[@]}"
      ;;
    *)
    $ECHO "FATAL (${scriptName}): type of processing"
    usage
    touch ${tmppublicpath}/proc_script.finished
    exit 1
    ;;
esac
#
$ECHO "DEBUG (${scriptName}): END"
#
#
#Compute time windows (Warning usage of GMT date)
#
$ECHO "DEBUG (${scriptName}): START compute time windows"
OUT=./baotiming.$$
${AWK} -v datesys=${DATE} -v type=${indexStartHour} -v shift=${timeShift} '
BEGIN {FS=","}
{
  phaseStart=$type
  cmd=sprintf("%s +%%s -d  \"%s\" ",datesys,phaseStart);
  (cmd | getline phstart);
  close(cmd);
  newstart = phstart+shift;
  cmd=sprintf("%s +%%T -d \"1970-01-01 %s sec GMT\"",datesys,newstart);
  (cmd | getline newStart);
  close(cmd);
  print $1,newStart,phaseStart;
}
'  ${scaFileSummarized} > ${OUT}
#
$ECHO "DEBUG (${scriptName}): END"
#
$ECHO "DEBUG (${scriptName}): START cat ${OUT}"
$CAT ${OUT}
$ECHO "DEBUG (${scriptName}): END cat ${OUT}"
#
$ECHO "DEBUG (${scriptName}): START filling cylceArrayBis timeArray"
#
declare -a cycleArrayBis
declare -a timeArray
$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
IFS=$DefaultIFS
#select the selected cycles
idtofind="0"
while read v1 v2 v3
do
 if [ ${idtofind} -lt ${#cycleArray[*]} ]; then
    if [ ${v1} = ${cycleArray[${idtofind}]} ]; then
	cycleArrayBis[${#cycleArrayBis[*]}]=$v1
	timeArray[${#timeArray[*]}]=$v2
	idtofind=`expr ${idtofind} + 1`
    fi
 fi
done < ${OUT}
$ECHO "DEBUG (${scriptName}): number of elements ${#cycleArrayBis[*]} ${#timeArray[*]}"
$ECHO "DEBUG (${scriptName}): END"


#########################
# specmfib exe
#########################
prg="/afs/in2p3.fr/throng/baoradio/Library/Progs/TAcq/Objs/specmfib"
if [ ! -e $prg ]; then
    $ECHO "FATAL (${scriptName}): $prg not found"
    touch ${tmppublicpath}/proc_script.finished
    exit 1
fi


dirFiberfile=${iojobpath}/${localpath}

curDir=`pwd`
#
$ECHO "DEBUG (${scriptName}): in principle $dirFiberfile == $curDir "
#
#force single channel
forceSingle="-singlechan"
#debug
prtlevel="1"


#Loop on cycles to write batch script and send to batch queue
##
$ECHO "DEBUG (${scriptName}): START loop on cycles ${firstCycleId} to ${lastp1CycleId}"

for ((i=${firstCycleId};i<${lastp1CycleId};i++)); do
#
$ECHO "DEBUG (${scriptName}): process cycle $i"
#time window selection
    tmproc="-tmproc ${timeArray[${i}]},${timeDuration}"
#cycle <name><number>
    cycle="${cyclebasename}${cycleArray[${i}]}"
#signal files selection
    infiles="${filesArray[${i}]}"
#
#Prepare BQS parameters: MEMORY, CPU...#
    infilesArray=( `$ECHO $infiles | $AWK 'BEGIN {FS=","} {for(i=1;i<NF;i++)print $i;}'` )
    IFS='
    '
    infilesArray=( $( printf "%s\n" "${infilesArray[@]}" | awk 'x[$0]++ == 0' ) )
    IFS=$DefaultIFS
    firstFile=${infilesArray[0]}
    lastFile=${infilesArray[1]}
#compute Scratch Size MB (400MB per signal files), CPU Time
    nFiles=`expr ${lastFile} - ${firstFile} + 1`
# Fiber 1 & Fiber 2 => double the number of files
    nFiles=`expr ${nFiles} \* 2`
#add number of spectra generated (cf. (Nbrepaquets_per_files * nFiles /tspwin) * 100KB
    nSpectra=`expr ${nFiles} \* 25600 /  ${tspwin}`
#Guess formula to reserved the amount of space needed in scratch
    scratchSize=`expr ${nFiles} \* 500 + ${nSpectra} \* 200 / 1000 + 5`
# 30sec/signal_file * empirical_conversion_factor (=50)
    cpu=`expr ${nFiles} \* 30 \* 50`
#Virtual Memory (MB) Nb: at CCIN2P3 the reasonable max is 1GB
    virtualmem=1000
#Job and batch names
    jobBatchName="${sourceRadio}-${dateDAQ}-${typeofproc}-cycle${cycleArray[${i}]}"
    jobScriptName="${jobBatchName}.sh"
    jobLogName="${jobBatchName}.log"
#
$ECHO "DEBUG (${scriptName}): start writing ${jobScriptName}"
#
#
#write the script to be send to Batch
#
    $CAT > ${jobScriptName} <<EOF
#!/bin/sh -xvf
#    jobBatchName=$jobBatchName
#    jobScriptName=$jobScriptName
#    jobLogName=$jobLogName
    scriptName=$jobScriptName

    . /afs/in2p3.fr/home/throng/baoradio/setup.sh
    . /usr/local/shared/bin/irods_env.sh -noverbose

    iojobpath=\${TMPBATCH}
    cd \${iojobpath}
    localpath=${sourceRadio}/${dateDAQ}${srclower}
    $MKDIR -p ./\${localpath}
    cd ./\${localpath}

    curDir=\`pwd\`
#get gain file if necessary
    if [ ${typeofproc} != "GAIN" ]; then
	OUT2=./igetStatus.$$
	iget /baoradio/data/AmasNancay/${localpath}/gain_${dateDAQ}_${srclower}.fits >\${OUT2} 2>&1
	igetStatus=\`$GREP "^ERROR" \${OUT2}\`
	if [ "<\${igetStatus}>" != "<>" ]; then
	    $ECHO "FATAL (\${scriptName}): error while iget:"
	    $ECHO \$igetStatus
	    touch ${tmppublicpath}/${jobBatchName}.finished
	    exit 1
	fi
        $RM \${OUT2}
    fi    
#get signals files
    OUT3=./getsignals.$$
    $RM -f ${tmppublicpath}/getsignalfiles.finished
    ${scriptpath}/getsignalfiles.sh ${sourceRadio} ${dateDAQ} ${firstFile}  ${lastFile} > \${OUT3} 2>&1
    while [ ! -f "${tmppublicpath}/getsignalfiles.finished" ]; do
	$ECHO "INFO (\${scriptName}): waiting for ${tmppublicpath}/getsignalfiles.finished"
	date +%T
	sleep 30
    done
    $RM ${tmppublicpath}/getsignalfiles.finished

    getSignals=\`$GREP "^FATAL" \${OUT3}\`
    if [ "<\$getSignals>" != "<>" ]; then
	$ECHO "FATAL (\${scriptName}): error while get signals"
	$ECHO \$getSignals
	touch ${tmppublicpath}/${jobBatchName}.finished
	exit 1
    fi
    $RM \${OUT3}
#location of the signal file
    fibfile="\${curDir} 1,2"
#
    outDir=\${curDir}/${dirName}/${cycle}
    $MKDIR -p \${outDir}

#
# where ami...
#
    $ECHO "We are here \`pwd\` to send the batch"
    $LS -lrth
    $FIND . -name "igetStatus.*" | $XARGS -i  $CAT {}
    $LS ./Fiber1 | $WC -l
    $LS ./Fiber2 | $WC -l
    $LS -lR \${outDir}

#Note: here act=gain means only that specmfib will use median-like algorithms (not only for gains...)
    $prg -act gain $forceSingle $freqfilter  -prt $prtlevel -out \${outDir} -nmean $nwinmean $ingain $tmproc -tspwin ${tspwin},0,0 -in $infiles \${fibfile}

    $ECHO    \${outDir}
    $LS -lrt \${outDir}

#save results to Irods (use option -f ONLY to force override)
    iput -v -K -r \${curDir}/${dirName} /baoradio/data/AmasNancay/\${localpath}
#
#clean the signal files which can at the end of the day fill completly the scratch directory of the batch work/AmasNancayer
    $RM -f \${iojobpath}/\${localpath}/Fiber1/signal*.fits
    $RM -f \${iojobpath}/\${localpath}/Fiber2/signal*.fits
    touch ${tmppublicpath}/${jobBatchName}.finished
    exit 0
EOF
#
#The EOF tag MUST BEGIN the line, this is the end of the script
#
    $ECHO "DEBUG (${scriptName}); START listing of script to be send to batch queue"
    $CAT -v ${jobScriptName}
    $ECHO "DEBUG (${scriptName}): END listing"
#
#send to BATCH
#
    $ECHO "DEBUG (${scriptName}): START send to batch listing"
#
    $RM -f ${tmppublicpath}/${jobBatchName}.finished

    qsub -l u_sps_baoradio,u_irods,T=${cpu},M=${virtualmem}MB,scratch=${scratchSize}MB,platform=LINUX -eo -o ${tmppublicpath}/${jobLogName} -N ${jobBatchName}  ${jobScriptName}

#    while [ ! -f "${tmppublicpath}/${jobBatchName}.finished" ]; do
#	$ECHO "INFO (${scriptName}); waiting for ${tmppublicpath}/${jobBatchName}.finished"
#	date +%T
#	sleep 600
#    done
#    $RM ${tmppublicpath}/${jobBatchName}.finished
    $ECHO "DEBUG (${scriptName}): end send to batch listing"
#end loop on cycles
done
#
$ECHO "DEBUG (${scriptName}): END loop on cycles"
#
touch ${tmppublicpath}/proc_script.finished
exit 0