Index: /BAORadio/AmasNancay/analysis-scriptsAndprg-description.txt
===================================================================
--- /BAORadio/AmasNancay/analysis-scriptsAndprg-description.txt	(revision 518)
+++ /BAORadio/AmasNancay/analysis-scriptsAndprg-description.txt	(revision 519)
@@ -1,4 +1,4 @@
 Title: Status of the programs and scripts used for Amas@Nancay analysis at CCIN2P3
-Revision: 7/9/11
+Revision: 14/9/11
 Author: J.E Campagne 22/7/11
 ----------------------------
@@ -19,5 +19,5 @@
 		. include RT calibration
 	- comments
-		. ON-OFF part only validate for a single Abell85 run and not in batch mode
+		. ON-OFF part only validate for a single Abell85 run and not in batch mode (preliminary stage)
 		. GAIN part should be revalidated according to inPath argument
 		
@@ -81,4 +81,10 @@
 		- TO MIGRATE TO NEW BATCH 
 
+o submit2qs-onoffmaker.sh
+--------------------------
+	- send the onoffmaker.sh script to BQS system
+	- Todo list
+		- TO MIGRATE TO NEW BATCH 
+
 		
 C) High level scripts 
@@ -109,12 +115,22 @@
 ---------------
 	- perfom
-		1) the extraction of signalXYZ.fits files 
-			concerned by the OFF & ON calibration time window of all cycles concerned
-		2) . the analysis of the above fits files to determine the 
-			calibration constantes of each channel: produce calib_* files (monitoring +  calibration files)
-			. put the results into Irods
+		1) several x-checks to see at which stage of the analysis process we are
+		2) trig the proc_script.sh script if necessary to create the spectra fits files via specmfib.cc prg 
+		3) . prepare the analysis of the above fits files
+		   . put calib_* files (monitoring +  calibration files) into Irods
 	- use 
 		- statusdaq.sh, proc_script.sh, getscafiles.sh
 		- analysis.cc	
+
+o onoffmaker.sh (preliminary)
+----------------------------
+	- perform
+		1) several x-checks to see at which stage of the analysis process we are
+		2) trig the proc_script.sh script if necessary to create the spectra fits files via specmfib.cc prg 
+		3) prepare the analysis of the above fits files
+		4) put dataRaw_*        (raw spectra), 
+		       dataBAOCalib_*   (after BAO calibration), 
+		       dataBAORTCalib_* (after BAO & RT calibration), 
+		       diffOnOff_*      (ON-OFF spectra) 
 
 D) Lower level scripts
Index: /BAORadio/AmasNancay/onoffmaker.sh
===================================================================
--- /BAORadio/AmasNancay/onoffmaker.sh	(revision 518)
+++ /BAORadio/AmasNancay/onoffmaker.sh	(revision 519)
@@ -175,6 +175,6 @@
 #
   somethingTodo=1
-  input4dataOff=`ils ${i}/Off 2>&1 | $GREP -i "datacycle" | $AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.11n`  
-  input4dataOn=`ils ${i}/On 2>&1 | $GREP -i "datacycle" | $AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.11n`  
+  input4dataOff=`ils ${i}/Off 2>&1 | $GREP -i "datacycle" | $AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.10n`  
+  input4dataOn=`ils ${i}/On 2>&1 | $GREP -i "datacycle" | $AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.10n`  
 #
   if [ "<${input4dataOff}>" != "<>" -a "<${input4dataOn}>" != "<>"  ]; then
@@ -267,5 +267,5 @@
       inFileDirectory="./${mode}"
       $MKDIR -p ${inFileDirectory}
-      listOfDataCycle=( `ils ${i}/${mode} 2>&1 | $GREP -i "datacycle"|$AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.11n` )
+      listOfDataCycle=( `ils ${i}/${mode} 2>&1 | $GREP -i "datacycle"|$AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.10n` )
       IFS='
 '
@@ -302,5 +302,5 @@
       $MKDIR -p ${inFileDirectory}
 
-      listOfDataCycle=( `ils ${i}/${mode} 2>&1 | $GREP -i "datacycle" |$AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.11n` )
+      listOfDataCycle=( `ils ${i}/${mode} 2>&1 | $GREP -i "datacycle" |$AWK 'BEGIN{FS="C- "}{print $2}' | $XARGS -i basename {} | $SORT -k1.10n` )
       IFS='
 '
@@ -380,4 +380,16 @@
 
       scaFile=`$FIND . -name "sca*.sum.trans" | $XARGS -i basename {}`
+
+
+#Get the calibration files
+      OUT=./getCalib.$$
+      iget -f -K -v ${i}/calib_*.txt ${aboveOnOffDir}> ${OUT} 2>&1
+      igetStatus=`$GREP "^ERROR" ${OUT}`
+      if [ "<$igetStatus>" != "<>" ]; then
+	  $ECHO "FATAL (${scriptName}): error while iget calib files:"
+	  $ECHO $igetStatus
+	  exit 1
+      fi
+      $RM -f ${OUT}
       
 #prepare job submission
Index: /BAORadio/AmasNancay/proc_script.sh
===================================================================
--- /BAORadio/AmasNancay/proc_script.sh	(revision 518)
+++ /BAORadio/AmasNancay/proc_script.sh	(revision 519)
@@ -205,4 +205,6 @@
 #for GAIN the reference cycle takes the middle of the run
 #
+#nwinmean donne le nombre de fenetre tspwin utilise pour faire calculer la moyenne des medianes de chaque fenetre
+#This is the nimber of paquets of the time window which change according to processing type
 $ECHO "DEBUG (${scriptName}): START switch on $typeofproc"
 #
@@ -212,4 +214,5 @@
       ingain=""
       freqfilter="-freqfilter -"
+      nwinmean="1"
       tspwin="5120"
       dirName="Off"
@@ -226,5 +229,6 @@
       ingain="-gain gain_${dateDAQ}_${srclower}.fits"
       freqfilter="-freqfilter 16"
-      tspwin="25000"
+      nwinmean="5"
+      tspwin="5120"
       dirName="On"
       cyclebasename="datacycle"
@@ -239,5 +243,6 @@
       ingain="-gain gain_${dateDAQ}_${srclower}.fits"
       freqfilter="-freqfilter 16"
-      tspwin="25000"
+      nwinmean="5"
+      tspwin="5120"
       dirName="Off"
       cyclebasename="datacycle"
@@ -252,4 +257,5 @@
       ingain="-gain gain_${dateDAQ}_${srclower}.fits"
       freqfilter="-freqfilter 16"
+      nwinmean="1"
       tspwin="1024"
       dirName="On"
@@ -265,4 +271,5 @@
       ingain="-gain gain_${dateDAQ}_${srclower}.fits"
       freqfilter="-freqfilter 16"
+      nwinmean="1"
       tspwin="1024"
       dirName="Off"
@@ -351,7 +358,4 @@
 #force single channel
 forceSingle="-singlechan"
-#nwinmean donne le nombre de fenetre tspwin utilise pour faire calculer la moyenne des medianes de chaque fenetre
-#This is the nimber of paquets of the time window which change according to processing type
-nwinmean="1"
 #debug
 prtlevel="1"
@@ -390,5 +394,5 @@
 # 25sec/signal_file * empirical_conversion_factor (=50)
     cpu=`expr ${nFiles} \* 25 \* 50`
-#Virtual Memory (MB)
+#Virtual Memory (MB) Nb: at CCIN2P3 the reasonable max is 1GB
     virtualmem=1000
 #Job and batch names
@@ -403,5 +407,5 @@
 #
     $CAT > ${jobScriptName} <<EOF
-#!/bin/sh
+#!/bin/sh -xvf
 #    jobBatchName=$jobBatchName
 #    jobScriptName=$jobScriptName
@@ -456,9 +460,20 @@
     outDir=\${curDir}/${dirName}/${cycle}
     $MKDIR -p \${outDir}
+
+#
+# where ami...
+#
+    $ECHO "We are here \`pwd\` to send the batch"
+    $LS -lrth
+    $FIND . -name "igetStatus.*" | $XARGS -i  $CAT {}
+    $LS ./Fiber1 | $WC -l
+    $LS ./Fiber2 | $WC -l
+    $LS -lR \${outDir}
+
 #Note: here act=gain means only that specmfib will use median-like algorithms (not only for gains...)
     $prg -act gain $forceSingle $freqfilter  -prt $prtlevel -out \${outDir} -nmean $nwinmean $ingain $tmproc -tspwin ${tspwin},0,0 -in $infiles \${fibfile}
 
     $ECHO    \${outDir}
-    $LS -lrt \${outDir}/*
+    $LS -lrt \${outDir}
 
 #save results to Irods (use option -f ONLY to force override)
Index: /BAORadio/AmasNancay/statusonoff.sh
===================================================================
--- /BAORadio/AmasNancay/statusonoff.sh	(revision 519)
+++ /BAORadio/AmasNancay/statusonoff.sh	(revision 519)
@@ -0,0 +1,137 @@
+#!/bin/sh
+#Status de l'etat de la production des fichiers INPUT de calibrations
+GREP=/bin/grep
+AWK=/bin/awk
+ECHO=/bin/echo
+MKDIR=/bin/mkdir
+CAT=/bin/cat
+PRINTF=/usr/bin/printf
+RM=/bin/rm
+SORT=/bin/sort
+XARGS=/usr/bin/xargs
+WC=/usr/bin/wc
+DefaultIFS=$' \t\n'
+IFS=$DefaultIFS
+
+# Irods environment settings
+. /usr/local/shared/bin/irods_env.sh -noverbose
+
+scriptName="`basename $0`"
+$ECHO "Processing script ${scriptName} at `date`"
+
+#Path to public backupable path
+publicpath="/afs/in2p3.fr/home/c/campagne/public"
+
+#Path where the job will do temporary IO
+if [ ${ENVIRONMENT} == "INTERACTIVE" ]; then
+    iojobpath="/sps/baoradio/AmasNancay/JEC"
+    $MKDIR -p $iojobpath
+elif [ ${ENVIRONMENT} == "BATCH" ] ; then
+    iojobpath=${TMPBATCH}
+else
+    $ECHO "FATAL (${scriptName}): environment is ${ENVIRONMENT} not allowed"
+    exit 1
+fi
+cd ${iojobpath}
+
+
+OUT1=./baoils.$$
+ils /baoradio/data/AmasNancay/ > ${OUT1}
+OUT2=./baoawk1.$$
+$AWK '($1=="C-") {print $2}' ${OUT1} > ${OUT2}
+$RM $OUT1
+#Sequence 
+listeSources=( `$CAT $OUT2 | $AWK '{print $1}'` )
+IFS='
+'
+listeSources=( $( $PRINTF "%s\n" "${listeSources[@]}" | $AWK 'x[$0]++ == 0' ) )
+IFS=$DefaultIFS
+$RM $OUT2
+
+for i in ${listeSources[@]}
+do
+#nb: $i contains the full path
+  $ECHO "source ${i}"
+  sourceName=`basename ${i}`
+  listeDates=( `ils $i | $AWK '($1=="C-"){print $2}'`)
+  IFS='
+'
+  listeDates=( $( $PRINTF "%s\n" "${listeDates[@]}" | $AWK 'x[$0]++ == 0' ) )
+  IFS=$DefaultIFS
+
+  for j in ${listeDates[@]}
+  do
+#nb: $j contains the full path
+#  $ECHO "run ${j}"
+#
+#Off directory
+#
+    dateField=`basename ${j}`
+    sourceDate="${sourceName}-${dateField}"
+#look if final data processing files exist
+    onOffOk=`ils ${j} | $GREP -i "data|diff" | $WC -l`
+    offDirExist=`ils ${j} | $GREP -i "Off" | $WC -l`
+    if [ ${offDirExist} != 0 ]; then
+	listeCyclesOff=( `ils ${j}/Off | $GREP -i "datacycle" | $AWK '($1=="C-"){print $2}' | $XARGS -i basename {} | $SORT -k1.11n` )
+	IFS='
+  '
+	listeCyclesOff=( $( $PRINTF "%s\n" "${listeCyclesOff[@]}" | $AWK 'x[$0]++ == 0' ) )
+	IFS=$DefaultIFS
+	
+	if [ ${#listeCyclesOff[*]} != 0 ]; then    
+           for c in ${listeCyclesOff[@]}
+	   do
+#nb: $c has the format of datacycle<number>
+	     nFiles=`ils ${j}/Off/${c} | $GREP -i "medfiltmtx" | $WC -l`
+	     if [ ${nFiles} != 0 ]; then
+		 $ECHO "${sourceDate}/Off/${c}: ${nFiles} spectra (${onOffOk} ana)"
+	     else
+		 $ECHO "${sourceDate}/Off/${c}: ON-OFF failed"
+	     fi
+#end of loop on cycles 
+	   done
+	else
+	    $ECHO "${sourceDate} : No ON-OFF done for Off Diectory"
+	fi
+#end of Off exist
+    else
+	$ECHO "${sourceDate} : No Off Diectory"
+    fi
+#
+#On directory
+#
+    dateField=`basename ${j}`
+    sourceDate="${sourceName}-${dateField}"
+#look if final calibration files exist
+    onOffOk=`ils ${j} | $GREP -i "data|diff" | $WC -l`
+    onDirExist=`ils ${j} | $GREP -i "On" | $WC -l`
+    if [ ${onDirExist} != 0 ]; then
+	listeCyclesOn=( `ils ${j}/On | $GREP -i "datacycle" | $AWK '($1=="C-"){print $2}' | $XARGS -i basename {} | $SORT -k1.11n` )
+	IFS='
+  '
+	listeCyclesOn=( $( $PRINTF "%s\n" "${listeCyclesOn[@]}" | $AWK 'x[$0]++ == 0' ) )
+	IFS=$DefaultIFS
+	
+	if [ ${#listeCyclesOn[*]} != 0 ]; then    
+           for c in ${listeCyclesOn[@]}
+	   do
+#nb: $c has the format of datacycle<number>
+	     nFiles=`ils ${j}/On/${c} | $GREP -i "medfiltmtx" | $WC -l`
+	     if [ ${nFiles} != 0 ]; then
+		 $ECHO "${sourceDate}/On/${c}: ${nFiles} spectra (${onOffOk} ana)"
+	     else
+		 $ECHO "${sourceDate}/On/${c}: ON-OFF failed"
+	     fi
+#end of loop on cycles 
+	   done
+	else
+	    $ECHO "${sourceDate} : No ON-OFF done for On Diectory"
+	fi
+#end of On exist
+    else
+	$ECHO "${sourceDate} : No On Diectory"
+    fi
+#end of loop on dates
+  done 
+#end of loop on sources
+done
