#!/bin/sh -xvf
#Basic tuning of system fuction used.
AWK=/bin/awk
SED=/bin/sed
GREP=/bin/grep
WC=/usr/bin/wc
RM=/bin/rm
CAT=/bin/cat
TOUCH=/bin/touch
DATE=/bin/date
ECHO=/bin/echo
LS=/bin/ls
MKDIR=/bin/mkdir
TR=/usr/bin/tr
FIND=/usr/bin/find
PRINTF=/usr/bin/printf
XARGS=/usr/bin/xargs
SORT=/bin/sort
QSUB=qsub

DefaultIFS=$' \t\n'
IFS=$DefaultIFS


scriptName="`basename $0`"
$ECHO "Processing script ${scriptName} at `date`"
######
# Local functions
######
usage() {
  echo "usage: $scriptName <source> <YYYYMMDD-directory> <type of process>"
  echo "      -<source> as Abell85"
  echo "      -<YYYYMMDD-directory> as 20110428"
  echo "      -<type of process> as GAIN|CALIBON|CALIBOFF|ON|OFF"
}
######
# argument test
######

#Process cmd line args: the -src option is mandatory (source name as Abell85)
#action: gain|mspec (gain: gain-like doucble median filtering
#                   mspec: mean+sigma wo filetring)
action="gain"
sourceRadio=
dateSelected=
typeofproc=
#use -sim option to simulate processing (debug the script as if...)
simulationMode=Off
#
#first and last cycle to process
#
firstCycle=
lastCycle=
while [ $# -gt 0 ]
do
  case "$1" in
      -act)  action=$2;         shift;;
      -src)  sourceRadio=$2;    shift;;
      -date) dateSelected=$2;   shift;;
      -type) typeofproc=$2;     shift;;
      -fcycle) firstCycle=$2;   shift;;
      -lcycle) lastCycle=$2;    shift;;
      -sim)  simulationMode=On;;
	-h)
	    echo >&2 \
	    "usage: $0 -src souce -date YYYYMMDD -type type [-fcycle firstCycle] [-lcycle lastCycle] [-sim to trig simulation mode]"
	    exit 1;;
	*)  break;;	# terminate while loop
    esac
    shift
done

#Check input

if [ "<${sourceRadio}>" = "<>" ]; then
    $ECHO "FATAL: You have forgotten to select the source option (-src)"
    exit 1
fi

if [ "<${dateSelected}>" = "<>" ]; then
    $ECHO "FATAL: You have forgotten to select the date option (-date)"
    exit 1
fi

if [ "<${typeofproc}>" = "<>" ]; then
    $ECHO "FATAL: You have forgotten to select the type option (-type)"
    exit 1
else
    offtype=`$ECHO ${typeofproc} | $GREP -i OFF | $WC -l`
    ontype=`$ECHO ${typeofproc} | $GREP -i ON | $WC -l`
    if [ "<${offtype}>" = "<1>" -a  "<${ontype}>" = "<0>" ]; then
	datatype="OFF"
    elif [ "<${offtype}>" = "<0>" -a  "<${ontype}>" = "<1>" ]; then
	datatype="ON"
    else
	$ECHO "FATAL: not supported: -type ${typeofproc}"
	exit 1
    fi
fi


srclower=`${ECHO} ${sourceRadio} | ${TR} "[:upper:]" "[:lower:]" `


#Path to public backupable path
publicpath="/afs/in2p3.fr/home/c/campagne/public"

#temporary files to synchronize scripts
tmppublicpath=${TMPPUBLICPATH}
$LS -l ${tmppublicpath} > /dev/null


#Path where the job will do temporary IO
if [ ${ENVIRONMENT} = "INTERACTIVE" ]; then
    iojobpath="/sps/baoradio/AmasNancay/JEC"
    $MKDIR -p $iojobpath
elif [ ${ENVIRONMENT} = "BATCH" ] ; then
    iojobpath=${TMPBATCH}
else
    $ECHO "FATAL (${scriptName}): environment is ${ENVIRONMENT} not allowed"
    touch ${tmppublicpath}/proc_script.finished
    exit 1
fi
cd ${iojobpath}

localpath="${sourceRadio}/${dateSelected}${srclower}"
$MKDIR -p ./${localpath}
cd ./${localpath}
mainDirectory=`pwd`


#Path of the utility scripts
scriptpath="/afs/in2p3.fr/home/c/campagne/private/work/AmasNancay"


#select file per cycle
#$ECHO "Select files process..."
#keep only lines with structure: cycle_number first,last,step
#Get sca files
$ECHO "DEBUG (${scriptName}): START call getscafiles.sh ${sourceRadio} ${dateSelected}"
OUT1=./getScaStatus.$$
#JEC 29/9/11 avoid finished stuff
#$RM -f ${tmppublicpath}/getscafiles.finished
${scriptpath}/getscafiles.sh ${sourceRadio} ${dateSelected} > ${OUT1} 2>&1
#while [ ! -f "${tmppublicpath}/getscafiles.finished" ]; do
#    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getscafiles.finished"
#    date +%T
#    sleep 30
#done
#$RM -f ${tmppublicpath}/getscafiles.finished
$ECHO "DEBUG (${scriptName}): END"


getScaStatus=`$GREP "^\(FATAL\|ERROR\)" ${OUT1}`

if [ "<${getScaStatus}>" != "<>" ]; then
  $ECHO "FATAL (${scriptName}): error to get sca file for $sourceRadio $dateSelected"
  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
  $CAT ${OUT1}
  $ECHO "DEBUG (${scriptName}): END"
#JEC 29/9/11 avoid finished stuff
#  touch ${tmppublicpath}/proc_script.finished
  exit 1
fi


scaFileSummarized=`$FIND . -name "sca*.sum" -print`
if [ "<${scaFileSummarized}>" = "<>" ]; then
  $ECHO "FATAL (${scriptName}): strange sca file summarized not available: $sourceRadio $dateSelected"
  $ECHO "DEBUG (${scriptName}): START with cat ${OUT1}"
  $CAT ${OUT1}
  $ECHO "DEBUG (${scriptName}): END"
#JEC 29/9/11 avoid finished stuff
#  touch ${tmppublicpath}/proc_script.finished
  exit 1
fi
$RM ${OUT1}

scaFileSummarized=`basename $scaFileSummarized`

#set Irods environment
. /usr/local/shared/bin/irods_env.sh -noverbose
#download header files to be used for signal files selction
$MKDIR -p ./Fiber1
dirFiberfile=`pwd`/Fiber1

dirIrods="/baoradio/data/AmasNancay/${sourceRadio}/${dateSelected}${srclower}/Fiber1"

listOfFiles=( `ils ${dirIrods} | $GREP -i "header" | $XARGS -i basename {} | $SORT -k1.7n` )
IFS='
'
listOfFiles=( $( $PRINTF "%s\n" "${listOfFiles[@]}" | $AWK 'x[$0]++ == 0' ) )
IFS=$DefaultIFS

#
$ECHO "DEBUG (${scriptName}): Download if necessary the headers"
#
for ifile in ${listOfFiles[@]}
do
#test if the headers have not yet been downloaded
  if [ ! -e ${dirFiberfile}/${ifile} ]; then
      OUT2=./getheaders.$$
      iget -f -K ${dirIrods}/${ifile} ${dirFiberfile} >${OUT2} 2>&1
      igetStatus=`$GREP "^ERROR" ${OUT2}`
      if [ "<$igetStatus>" != "<>" ]; then
	  $ECHO "FATAL (${scriptName}): error while iget:"
	  $ECHO $igetStatus
#JEC 29/9/11 avoid finished stuff
#	  touch ${tmppublicpath}/proc_script.finished
	  exit 1
      fi
      $RM -f $OUT2
  fi
done
#
$ECHO "DEBUG (${scriptName}): START call select.sh ${scaFileSummarized} ${dirFiberfile} ${typeofproc}"
#
#Get list of file to process
headerFileSelected=./selectSignal.$$
#JEC 29/9/11 avoid finished stuff
#$RM -f ${tmppublicpath}/select.finished
${scriptpath}/select.sh ${scaFileSummarized} ${dirFiberfile} ${datatype} | $GREP "^#[0-9]" | $SED "s/^#//" > $headerFileSelected
#while [ ! -f "${tmppublicpath}/select.finished" ]; do
#    $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/select.finished"
#    date +%T
#    sleep 30
#done
#$RM ${tmppublicpath}/select.finished
#
$ECHO "DEBUG (${scriptName}): END"
#
$ECHO "DEBUG (${scriptName}): START filling cycleArray, filesArray"
#
declare -a cycleArray
declare -a filesArray
$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
IFS=$DefaultIFS
$ECHO "DEBUG (${scriptName}): START cat $headerFileSelected"
$CAT $headerFileSelected
$ECHO "DEBUG (${scriptName}): END cat $headerFileSelected"

while read v1 v2
do
 cycleArray[${#cycleArray[*]}]=$v1
 filesArray[${#filesArray[*]}]=$v2
done < $headerFileSelected

if [ ${#cycleArray[*]} -eq 0 -o ${#filesArray[*]} -eq 0 ]; then
   $ECHO "FATAL (${scriptName}): no header file selected (${#cycleArray[*]}, ${#filesArray[*]})"
#JEC 29/9/11 avoid finished stuff
#   touch ${tmppublicpath}/proc_script.finished
   exit 1
else
   $ECHO "DEBUG (${scriptName}): non zero elements found for cycleArray and filesArray" 
fi

#
$ECHO "DEBUG (${scriptName}): number of elements <${#cycleArray[*]}> <${#filesArray[*]}>"
$ECHO "DEBUG (${scriptName}): END"
#
# (timeShift)
#Add 3sec to start ON / OFF
#Substract 4sec to start DAB ON / OFF
#
#Take care: Cycle numbering starts at 1 in SCAFiles while Arrays index start at 0
#
#for GAIN the reference cycle takes the middle of the run
#
#nwinmean donne le nombre de fenetre tspwin utilise pour faire calculer la moyenne des medianes de chaque fenetre
#This is the nimber of paquets of the time window which change according to processing type
$ECHO "DEBUG (${scriptName}): START switch on $typeofproc"
#
case "$typeofproc" in
    GAIN)
      $ECHO "process signal files for GAIN phase"
      ingain=""
      freqfilter="-freqfilter -"
      nwinmean="1"
      tspwin="5120"
      dirName="Off"
      cyclebasename="gaincycle"
      indexStartHour="6"
      timeShift="-4"
      timeDuration="14"
      firstCycleId=`expr  ${#cycleArray[*]} / 2`
      firstCycleId=`expr ${firstCycleId} - 1`
      lastp1CycleId=`expr ${firstCycleId} + 1`
      ;;
    ON)
      $ECHO "process signal files for ON phases"
      ingain="-gain gain_${dateSelected}_${srclower}.fits"
      freqfilter=""
      nwinmean="5"
      tspwin="5120"
      dirName="On"
      cyclebasename="datacycle"
      indexStartHour="4"
      timeShift="3"
      timeDuration="30"
      firstCycleId="0"
      lastp1CycleId="${#cycleArray[@]}"
      ;;
    OFF)
      $ECHO "process signal files for OFF phases"
      ingain="-gain gain_${dateSelected}_${srclower}.fits"
      freqfilter=""
      nwinmean="5"
      tspwin="5120"
      dirName="Off"
      cyclebasename="datacycle"
      indexStartHour="8"
      timeShift="3"
      timeDuration="30"
      firstCycleId="0"
      lastp1CycleId="${#cycleArray[@]}"
      ;;
    MEANOFF)
      $ECHO "process signal files for OFF phases (MEAN)"
      ingain="-gain gain_${dateSelected}_${srclower}.fits"
      nwinmean="25000"
      tspwin="1"
      dirName="Off"
      cyclebasename="meancycle"
      indexStartHour="8"
      timeShift="3"
      timeDuration="30"
      firstCycleId="0"
      lastp1CycleId="${#cycleArray[@]}"
      ;;
    MEANON)
      $ECHO "process signal files for OFF phases (MEAN)"
      ingain="-gain gain_${dateSelected}_${srclower}.fits"
      nwinmean="25000"
      tspwin="1"
      dirName="On"
      cyclebasename="meancycle"
      indexStartHour="8"
      timeShift="3"
      timeDuration="30"
      firstCycleId="0"
      lastp1CycleId="${#cycleArray[@]}"
      ;;
    CALIBON)
      $ECHO "process signal files for CALIBRATION ON phases"
      ingain="-gain gain_${dateSelected}_${srclower}.fits"
      freqfilter=""
      nwinmean="1"
      tspwin="1024"
      dirName="On"
      cyclebasename="calibcycle"
      indexStartHour="2"
      timeShift="-4"
      timeDuration="14"
      firstCycleId="0"
      lastp1CycleId="${#cycleArray[@]}"
      ;;
    CALIBOFF)
      $ECHO "process signal files for CALIBRATION OFF phases"
      ingain="-gain gain_${dateSelected}_${srclower}.fits"
      freqfilter=""
      nwinmean="1"
      tspwin="1024"
      dirName="Off"
      cyclebasename="calibcycle"
      indexStartHour="6"
      timeShift="-4"
      timeDuration="14"
      firstCycleId="0"
      lastp1CycleId="${#cycleArray[@]}"
      ;;
    *)
    $ECHO "FATAL (${scriptName}): type of processing"
    usage
#JEC 29/9/11 avoid finished stuff
#    touch ${tmppublicpath}/proc_script.finished
    exit 1
    ;;
esac
#
$ECHO "DEBUG (${scriptName}): END"
#
#
#Compute time windows (Warning usage of GMT date)
#
$ECHO "DEBUG (${scriptName}): START compute time windows"
OUT=./baotiming.$$
${AWK} -v datesys=${DATE} -v type=${indexStartHour} -v shift=${timeShift} '
BEGIN {FS=","}
{
  phaseStart=$type
  cmd=sprintf("%s +%%s -d  \"%s\" ",datesys,phaseStart);
  (cmd | getline phstart);
  close(cmd);
  newstart = phstart+shift;
  cmd=sprintf("%s +%%T -d \"1970-01-01 %s sec GMT\"",datesys,newstart);
  (cmd | getline newStart);
  close(cmd);
  print $1,newStart,phaseStart;
}
'  ${scaFileSummarized} > ${OUT}
#
$ECHO "DEBUG (${scriptName}): END"
#
$ECHO "DEBUG (${scriptName}): START cat ${OUT}"
$CAT ${OUT}
$ECHO "DEBUG (${scriptName}): END cat ${OUT}"
#
$ECHO "DEBUG (${scriptName}): START filling cylceArrayBis timeArray"
#
declare -a cycleArrayBis
declare -a timeArray
$ECHO "DEBUG (${scriptName}): IFS value between<>: <$IFS>"
IFS=$DefaultIFS
#select the selected cycles
idtofind="0"
while read v1 v2 v3
do
 if [ ${idtofind} -lt ${#cycleArray[*]} ]; then
    if [ ${v1} = ${cycleArray[${idtofind}]} ]; then
	cycleArrayBis[${#cycleArrayBis[*]}]=$v1
	timeArray[${#timeArray[*]}]=$v2
	idtofind=`expr ${idtofind} + 1`
    fi
 fi
done < ${OUT}
$ECHO "DEBUG (${scriptName}): number of elements ${#cycleArrayBis[*]} ${#timeArray[*]}"
$ECHO "DEBUG (${scriptName}): END"


#########################
# specmfib exe
#########################
prg="/afs/in2p3.fr/throng/baoradio/Library/Progs/TAcq/Objs/specmfib"
if [ ! -e $prg ]; then
    $ECHO "FATAL (${scriptName}): $prg not found"
#JEC 29/9/11 avoid finished stuff
#    touch ${tmppublicpath}/proc_script.finished
    exit 1
fi


dirFiberfile=${iojobpath}/${localpath}

curDir=`pwd`
#
$ECHO "DEBUG (${scriptName}): in principle $dirFiberfile == $curDir "
#
#force single channel
forceSingle="-singlechan"
#debug
prtlevel="1"


#Loop on cycles
##
$ECHO "DEBUG (${scriptName}): START loop on cycles Ids ${firstCycleId} to ${lastp1CycleId}"

#write the script to be send to Batch
#get gain file if necessary
if [ ${typeofproc} != "GAIN" ]; then
    OUT2=./igetStatus.$$
    iget /baoradio/data/AmasNancay/${localpath}/gain_${dateSelected}_${srclower}.fits >${OUT2} 2>&1
    igetStatus=`$GREP "^ERROR" ${OUT2}`
    if [ "<${igetStatus}>" != "<>" ]; then
	$ECHO "FATAL (${scriptName}): error while iget:"
	$ECHO $igetStatus
#JEC 29/9/11 avoid finished stuff
#	touch ${tmppublicpath}/${jobBatchName}.finished
	exit 1
    fi
    $RM ${OUT2}
fi    


for ((i=${firstCycleId};i<${lastp1CycleId};i++)); do
#

####
# Check daq status
#JEC 1/10/11 Use generic baodaqstatus name 
####
#get the daq current irod status
#tag=`${DATE} +%F`
#OUT1=${publicpath}/baodaqstatus-${tag}.txt
OUT1=${publicpath}/baodaqstatus-current.txt

if [ ! -e ${OUT1}  -o ! -r ${OUT1} ]; then
    $ECHO "FATAL (${scriptName}): ${OUT1} has a problem"
    exit 1
#    $ECHO "We should bring the DAQ status more up to date, this take 1 or 2min..."
#JEC 29/9/11 avoid finished stuff
#    $RM -f ${tmppublicpath}/statusdaq.finished
#    ${scriptpath}/statusdaq.sh > ${OUT1}
#    while [ ! -f "${tmppublicpath}/statusdaq.finished" ]; do
#	sleep 30
#    done
#    $RM ${tmppublicpath}/statusdaq.finished
#protect against remove/rewriting    
#    $CHMOD -v 444 ${OUT1} 
fi

#time window selection
    tmproc="-tmproc ${timeArray[${i}]},${timeDuration}"
#cycle <number>
    cycle="${cycleArray[${i}]}"
    if [ "<${firstCycle}>" != "<>" -a "<${lastCycle}>" != "<>" ]; then
	if [ ${cycle} -lt ${firstCycle} -o ${cycle} -gt ${lastCycle} ]; then
	    continue
	fi
    fi
    $ECHO "DEBUG (${scriptName}): process cycle $cycle"

#cycle <name><number>
    cycle="${cyclebasename}${cycleArray[${i}]}"
#signal files selection
    infiles="${filesArray[${i}]}"
#
#Prepare BQS parameters: MEMORY, CPU...#
    infilesArray=( `$ECHO $infiles | $AWK 'BEGIN {FS=","} {for(i=1;i<NF;i++)print $i;}'` )
    IFS='
    '
    infilesArray=( $( printf "%s\n" "${infilesArray[@]}" | awk 'x[$0]++ == 0' ) )
    IFS=$DefaultIFS
    firstFile=${infilesArray[0]}
    lastFile=${infilesArray[1]}
#
#
#
#get signals files
    OUT3=./getsignals.$$
#JEC 29/9/11 avoid finished stuff
#    $RM -f ${tmppublicpath}/getsignalfiles.finished
    ${scriptpath}/getsignalfiles.sh ${sourceRadio} ${dateSelected} ${firstFile}  ${lastFile} > ${OUT3} 2>&1
#    while [ ! -f "${tmppublicpath}/getsignalfiles.finished" ]; do
#	$ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/getsignalfiles.finished"
#	date +%T
#	sleep 30
#    done
#    $RM ${tmppublicpath}/getsignalfiles.finished

    getSignals=`$GREP "^FATAL" ${OUT3}`
    if [ "<$getSignals>" != "<>" ]; then
	$ECHO "FATAL (${scriptName}): error while get signals"
	$ECHO $getSignals
#JEC 29/9/11 avoid finished stuff
#	touch ${tmppublicpath}/${jobBatchName}.finished
	exit 1
    fi
    $RM ${OUT3}

#location of the signal file
    fibfile="${curDir} 1,2"
#
    outDir=${curDir}/${dirName}/${cycle}
    $MKDIR -p ${outDir}

#
# where ami...
#
    $ECHO "We are here `pwd`"
    $LS -lrth
    $FIND . -name "igetStatus.*" | $XARGS -i  $CAT {}
    $LS ./Fiber1 | $WC -l
    $LS ./Fiber2 | $WC -l
    $LS -lR ${outDir}

#Note: here act=gain means only that specmfib will use median-like algorithms (not only for gains...)
    $prg -act ${action} $forceSingle $freqfilter  -prt $prtlevel -out ${outDir} -nmean $nwinmean $ingain $tmproc -tspwin ${tspwin},0,0 -in $infiles ${fibfile}

    $ECHO    ${outDir}
    $LS -lrt ${outDir}

#save results to Irods (use option -f ONLY to force override)
    iput -v -K -r ${curDir}/${dirName} /baoradio/data/AmasNancay/${localpath}
#
#clean the signal files which can at the end of the day fill completly the scratch directory of the batch work/AmasNancayer
    $RM -rf ${iojobpath}/${localpath}/Fiber1
    $RM -rf ${iojobpath}/${localpath}/Fiber2
    $RM -rf ${iojobpath}/${localpath}/${typeofproc}

#
#

#end loop on cycles
done
#
$ECHO "DEBUG (${scriptName}): END loop on cycles"
#
#JEC 29/9/11 avoid finished stuff
#touch ${tmppublicpath}/proc_script.finished
exit 0