Index: /BAORadio/AmasNancay/rawonoffmaker.sh
===================================================================
--- /BAORadio/AmasNancay/rawonoffmaker.sh	(revision 530)
+++ /BAORadio/AmasNancay/rawonoffmaker.sh	(revision 530)
@@ -0,0 +1,323 @@
+#!/bin/sh
+#Perform the ON-OFF analysis
+DATE=/bin/date
+GREP=/bin/grep
+AWK=/bin/awk
+ECHO=/bin/echo
+WC=/usr/bin/wc
+CAT=/bin/cat
+PRINTF=/usr/bin/printf
+FIND=/usr/bin/find
+MKDIR=/bin/mkdir
+XARGS=/usr/bin/xargs
+SORT=/bin/sort
+RM=/bin/rm
+TR=/usr/bin/tr
+CP=/bin/cp
+LS=/bin/ls
+CHMOD=/bin/chmod
+
+DefaultIFS=$' \t\n'
+IFS=$DefaultIFS
+
+#set Irods environment    
+. /usr/local/shared/bin/irods_env.sh -noverbose
+
+scriptName="`basename $0`"
+$ECHO "Processing script ${scriptName} at `date`"
+#which source to analyse
+
+#Process cmd line args: the -src option is mandatory (source name as Abell85)
+sourceRadio=
+dateSelected=
+#use -sim option to simulate processing (debug the script as if...)
+simulationMode=Off
+while [ $# -gt 0 ]
+do
+  case "$1" in
+      -src)  sourceRadio=$2;    shift;;
+      -date) dateSelected=$2;     shift;;
+      -sim)  simulationMode=On;;
+	-h)
+	    echo >&2 \
+	    "usage: $0 -src souce -date YYYYMMDD [-freq freq in MHz (default ${freqBAOCalib})] [-sim to trig simulation mode]"
+	    exit 1;;
+	*)  break;;	# terminate while loop
+    esac
+    shift
+done
+
+if [ ${simulationMode} = "On" ]; then
+    $ECHO "INFO ${scriptName} running in SIMUL mode"
+fi
+
+if [ "<${sourceRadio}>" = "<>" ]; then
+    $ECHO "FATAL: You have forgotten to select the source option (-src)"
+    exit 1
+fi
+
+case ${sourceRadio} in
+    Abell85) $ECHO "INFO (${scriptName}): process ${sourceRadio}";;
+    Abell2440) $ECHO "INFO (${scriptName}): process ${sourceRadio}";;
+    Abell1205) $ECHO "INFO (${scriptName}): process ${sourceRadio}";;
+    *) ECHO "FATAL (${scriptName}): process ${sourceRadio} not yet foreseen"
+    exit 1;;
+esac
+
+srclower=`${ECHO} ${sourceRadio} | ${TR} "[:upper:]" "[:lower:]" `
+
+#Path to public backupable path
+publicpath="/afs/in2p3.fr/home/c/campagne/public"
+
+
+#temporary files to synchronize scripts
+tmppublicpath=${TMPPUBLICPATH}
+#wakeup the NFS disk
+$LS -l ${tmppublicpath} > /dev/null
+#clean previous spurious files
+$RM -f ${tmppublicpath}/*.finished
+
+#Path where the job will do temporary IO
+if [ ${ENVIRONMENT} = "INTERACTIVE" ]; then
+    iojobpath="/sps/baoradio/AmasNancay/JEC"
+    $MKDIR -p $iojobpath
+elif [ ${ENVIRONMENT} = "BATCH" ] ; then
+    iojobpath=${TMPBATCH}
+else
+    $ECHO "FATAL (${scriptName}): environment is ${ENVIRONMENT} not allowed"
+    exit 1
+fi
+cd ${iojobpath}
+
+localpath="${sourceRadio}"
+$MKDIR -p ./${localpath}
+cd ./${localpath}
+
+#save the top directory
+topDir=`pwd`
+
+#Path of the utility scripts
+scriptpath="/afs/in2p3.fr/home/c/campagne/private/work/AmasNancay"
+
+#get the daq current irod status
+tag=`${DATE} +%F`
+OUT1=${publicpath}/baodaqstatus-${tag}.txt
+
+if [ ! -e ${OUT1}  -o ! -r ${OUT1} ]; then
+    $ECHO "We should bring the DAQ status more up to date, this take 1 or 2sec..."
+    $RM -f ${tmppublicpath}/statusdaq.finished
+    ${scriptpath}/statusdaq.sh > ${OUT1}
+    while [ ! -f "${tmppublicpath}/statusdaq.finished" ]; do
+	sleep 30
+    done
+    $RM ${tmppublicpath}/statusdaq.finished
+#protect against remove/rewriting    
+    $CHMOD -v 444 ${OUT1} 
+fi
+
+
+#Nb: if we want to implement a loop on sources we have to take care of srclower...
+
+$ECHO "You have selected sourceRadio = ${sourceRadio} [date = ${dateSelected}]"
+tableau=( `$GREP -i "${dateSelected}${srclower}" ${OUT1} | $AWK '( NF==4 ) { print $2 }' ` )
+IFS='
+' 
+tableau=( $( $PRINTF "%s\n" "${tableau[@]}" | $AWK 'x[$0]++ == 0' ) )
+IFS=$DefaultIFS
+
+
+for i in ${tableau[@]}
+  do
+#
+# start a "fresh" session  
+  cd ${topDir}
+  $LS | $XARGS -i $RM -rf {}
+  
+  $ECHO "DEBUG: (${scriptName}) irods root dir $i"
+  dateDAQ=`$ECHO ${i} | $AWK '{split($0,a,"/"); print a[6];}' | $AWK "{sub(/${srclower}/,\"\"); print}"`
+
+  $ECHO "DEBUG: (${scriptName}): we are in directory `pwd` the topDir=${topDir}"
+  $LS -lrt
+
+
+#Look if the date has changed since beginning of the job!
+  curtag=`${DATE} +%F`
+  if [ "<${curtag}>" != "<${tag}>" ]; then
+      tag=${curtag}
+#it is necessary to update the daq status as the day has changed during the processing... (batch can take a long time...)
+      OUT1=${publicpath}/baodaqstatus-${curtag}.txt
+      $ECHO "We should bring the DAQ status more up to date, this take 1 or 2sec..."
+      $RM -f ${tmppublicpath}/statusdaq.finished
+      ${scriptpath}/statusdaq.sh > ${OUT1}
+      while [ ! -f "${tmppublicpath}/statusdaq.finished" ]; do
+	  sleep 30
+      done
+      $RM ${tmppublicpath}/statusdaq.finished
+#protect against remove/rewriting    
+      $CHMOD -v 444 ${OUT1} 
+  fi
+#
+# look if the gain & calibration files exist
+#
+  gainDone=`ils ${i} 2>&1 | $GREP -i "gain_.*\.fits" | $WC -l`
+  if [ ! ${gainDone} -gt 0 ]; then
+      $ECHO "FATAL (${scriptName}): gain file DOES NOT exist skip `basename $i` "
+      continue
+  fi
+
+#
+  somethingTodo=1
+  input4dataOff=`ils ${i}/Off 2>&1 | $GREP -i "datacycle" | $AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.10n`  
+  input4dataOn=`ils ${i}/On 2>&1 | $GREP -i "datacycle" | $AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.10n`  
+#
+  if [ "<${input4dataOff}>" != "<>" -a "<${input4dataOn}>" != "<>"  ]; then
+#look at the content of the datacycle directory as may be the previous batch job has crashed
+      tableauOff=( `$ECHO ${input4dataOff}` )
+      IFS='
+'
+      tableauOff=( $( $PRINTF "%s\n" "${tableauOff[@]}" | $AWK 'x[$0]++ == 0' ) )
+      IFS=$DefaultIFS
+#
+      nTotFilesOff=0
+      for j in ${tableauOff[@]}
+	do
+	nFiles=`ils ${i}/Off/${j} | $GREP -i "medfiltmtx" | $WC -l`
+	nTotFilesOff=`expr ${nTotFilesOff} + ${nFiles}`
+      done
+#
+      tableauOn=( `$ECHO ${input4dataOn}` )
+      IFS='
+      '
+      tableauOn=( $( $PRINTF "%s\n" "${tableauOn[@]}" | $AWK 'x[$0]++ == 0' ) )
+      IFS=$DefaultIFS
+#
+      nTotFilesOn=0
+      for j in ${tableauOn[@]}
+	do
+	nFiles=`ils ${i}/On/${j} | $GREP -i "medfiltmtx" | $WC -l`
+	nTotFilesOn=`expr ${nTotFilesOn} + ${nFiles}`
+      done
+      diffOnOff=`expr ${nTotFilesOff} - ${nTotFilesOn}`
+      if [ ${diffOnOff} -lt 0 ]; then
+	  zero=0
+	  diffOnOff=`expr ${zero} - ${diffOnOff}` 
+      fi
+#allow files differences but non zero
+      if [  ${nTotFilesOff} -gt 0 -a ${nTotFilesOn} -gt 0  ]; then
+	  somethingTodo=0
+	  $ECHO "DEBUG (${scriptName}): $sourceRadio $dateDAQ has <${nTotFilesOff}> Off-files and <${nTotFilesOn}> On-files but OK"
+      else
+	  $ECHO "INFO (${scriptName}): $sourceRadio $dateDAQ has <${nTotFilesOff}> Off-files and <${nTotFilesOn}> On-files => one should rebuild has something is wrong"
+      fi
+#
+#look at consecutive cycles
+
+      rmDirectory=0
+
+      cycleIdOff=`$ECHO $input4dataOff | sed 's/datacycle//g'`
+      tableauOff=( `$ECHO ${cycleIdOff}` )
+      IFS='
+'
+      tableauOff=( $( $PRINTF "%s\n" "${tableauOff[@]}" | $AWK 'x[$0]++ == 0' ) )
+      IFS=$DefaultIFS
+      
+      lastIdOff=`expr ${#tableauOff[*]} - 1`
+      ncycleOff=`expr ${tableauOff[${lastIdOff}]} - ${tableauOff[0]} + 1`
+      ncycleOffVerif=${#tableauOff[*]}
+      if [ "<${ncycleOff}>" != "<${ncycleOffVerif}>"  ]; then
+	  $ECHO "$sourceRadio $dateDAQ: There is a lack of OFF cycles in the sequence: ${tableauOff[@]}"
+	  rmDirectory=1
+	  somethingTodo=1
+      else
+	  $ECHO "$sourceRadio $dateDAQ: Ok at least consecutive OFF cycles: ${tableauOff[@]}"
+      fi
+
+      cycleIdOn=`$ECHO $input4dataOn | sed 's/datacycle//g'`
+      tableauOn=( `$ECHO ${cycleIdOn}` )
+      IFS='
+'
+      tableauOn=( $( $PRINTF "%s\n" "${tableauOn[@]}" | $AWK 'x[$0]++ == 0' ) )
+      IFS=$DefaultIFS
+      
+      lastIdOn=`expr ${#tableauOn[*]} - 1`
+      ncycleOn=`expr ${tableauOn[${lastIdOn}]} - ${tableauOn[0]} + 1`
+      ncycleOnVerif=${#tableauOn[*]}
+      if [ "<${ncycleOn}>" != "<${ncycleOnVerif}>"  ]; then
+	  $ECHO "$sourceRadio $dateDAQ: There is a lack of ON cycles in the sequence: ${tableauOn[@]}"
+	  rmDirectory=1
+	  somethingTodo=1
+      else
+	  $ECHO "$sourceRadio $dateDAQ: Ok at least consecutive ON cycles:  ${tableauOn[@]}"
+      fi
+
+      if [ "<${cycleIdOff}>" != "<${cycleIdOn}>" ]; then
+	  $ECHO "$sourceRadio $dateDAQ: cycles seq. missmatch ON and OFF"
+	  rmDirectory=1
+	  somethingTodo=1	  
+      fi
+# 
+  fi 
+########
+  if [ ${somethingTodo} -eq 1 ]; then
+#
+      $ECHO ">>>>>>>>>>>>>>>> Do $sourceRadio $dateDAQ ANALOFF"
+
+      if [ ${rmDirectory} -eq 1 ]; then
+#delete datacycle Off
+	  dirToRmOff=`ils ${i}/Off 2>&1 | $GREP -i "datacycle" | $AWK 'BEGIN{FS="C- "}{print $2}'`  
+	  tableauOff=( `$ECHO ${dirToRmOff}` )
+	  IFS='
+'
+	  tableauOff=( $( $PRINTF "%s\n" "${tableauOff[@]}" | $AWK 'x[$0]++ == 0' ) )
+	  IFS=$DefaultIFS
+	  for j in ${tableauOff[@]}
+	    do
+	    irm -f -r -v ${j}
+	  done
+#delete datacycle On
+	  dirToRmOn=`ils ${i}/On 2>&1 | $GREP -i "datacycle" | $AWK 'BEGIN{FS="C- "}{print $2}'`  
+	  tableauOn=( `$ECHO ${dirToRmOn}` )
+	  IFS='
+'
+	  tableauOn=( $( $PRINTF "%s\n" "${tableauOn[@]}" | $AWK 'x[$0]++ == 0' ) )
+	  IFS=$DefaultIFS
+	  for j in ${tableauOn[@]}
+	    do
+	    irm -f -r -v ${j}
+	  done
+      fi
+	  
+      $RM -f ${tmppublicpath}/proc_script.finished
+      if [ ${simulationMode} = "On" ]; then
+	  $ECHO "SIMUL: ${scriptpath}/proc_script.sh $sourceRadio $dateDAQ OFF"
+      else
+	  ${scriptpath}/proc_script.sh $sourceRadio $dateDAQ OFF
+	  while [ ! -e "${tmppublicpath}/proc_script.finished"  ]; do
+	      $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/proc_script.finished"
+	      date +%T
+	      sleep 60
+	  done
+      fi
+      $ECHO "INFO (${scriptName}): proc_script has finished"
+      $ECHO ">>>>>>>>>>>> Done $sourceRadio $dateDAQ OFF"
+      $ECHO ">>>>>>>>>>>>>>>> Do $sourceRadio $dateDAQ ON"
+      $RM ${tmppublicpath}/proc_script.finished
+      
+      if [ ${simulationMode} = "On" ]; then
+	  $ECHO "SIMUL: ${scriptpath}/proc_script.sh $sourceRadio $dateDAQ ON"
+      else
+	  ${scriptpath}/proc_script.sh $sourceRadio $dateDAQ ON
+	  while [ ! -e "${tmppublicpath}/proc_script.finished"  ]; do
+	      $ECHO "INFO (${scriptName}): waiting for ${tmppublicpath}/proc_script.finished"
+	      date +%T
+	      sleep 60
+	  done
+      fi
+      $ECHO "INFO (${scriptName}): proc_script has finished"
+      $RM ${tmppublicpath}/proc_script.finished
+#Todo see if there is errors in the log file before copy
+      $ECHO ">>>>>>>>>>>> Done $sourceRadio $dateDAQ ON"
+  fi
+done
+
+exit 0
Index: /BAORadio/AmasNancay/submit2bqs-anarawonoffmaker.sh
===================================================================
--- /BAORadio/AmasNancay/submit2bqs-anarawonoffmaker.sh	(revision 530)
+++ /BAORadio/AmasNancay/submit2bqs-anarawonoffmaker.sh	(revision 530)
@@ -0,0 +1,42 @@
+#!/bin/sh -xv
+
+scriptName="`basename $0`"
+echo "Processing script ${scriptName} at `date`"
+
+#Process cmd line args: the -src option is mandatory (source name as Abell85)
+sourceRadio=
+dateSelected=
+dateJob=
+#use -sim option to simulate processing (debug the script as if...)
+simulationMode=""
+while [ $# -gt 0 ]
+do
+  case "$1" in
+      -src)  sourceRadio="$2"; shift;;
+      -date) dateSelected="-date $2"; dateJob="$2";     shift;;
+      -sim) 
+      simulationMode="-sim On"
+      $ECHO "INFO ${scriptName} running in SIMUL mode";;
+	-h)
+	echo >&2 \
+	    "usage: $0 -src souce -date YYYYMMDD [-freq freq in MHz (default ${freqBAOCalib})] [-sim to trig simulation mode]"
+	exit 1;;
+	*)  break;;	# terminate while loop
+    esac
+    shift
+done
+
+
+if [ "<${sourceRadio}>" = "<>" ]; then
+    $ECHO "FATAL: You have forgotten to select the source option (-src)"
+    exit 1
+fi
+
+
+jobBatchName="rawonoff-${sourceRadio}${dateJob}"
+
+jobLogName="${jobBatchName}.log.$$"
+
+qsub -l u_sps_baoradio,u_irods,T=120000,M=1000MB,scratch=1000MB,platform=LINUX -eo -o $jobLogName -N $jobBatchName -mb -me -mu campagne@lal.in2p3.fr <<EOF
+/afs/in2p3.fr/home/c/campagne/private/work/AmasNancay/anarawonoffmaker.sh  -src ${sourceRadio} ${dateSelected} ${simulationMode}
+EOF
