source: PSPA/madxPSPA/testing/MadBuildAndTest.py @ 478

#!/usr/bin/python
from Notify import notify
from MadTestExceptions import MadException
import MadTrigTest

import threading
import time, datetime
import os
import sys
import re
import traceback
import optparse

global debugMode

threadList = []

rootDir = "/afs/cern.ch/user/n/nougaret/scratch1/mad-automation"

class MadBuildAndTestException(Exception):
    def __init__(self,value):
        self.value = value
    def __str__(self):
        return repr(self.value)

class updateTokensThread(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)
        self.killed = False
        threadList.append(self)
        self.fileTokens = open('MadBuildAndTest.trace','w')
    def setDebugMode(self,mode):
        self.debugMode = mode
    def run(self):
        global debugMode
        
        origin = datetime.datetime.now() # return string
        duration = 0 # init
        keytabDir = '/extra/home/nougaret/MadTestScriptAuth'
        keytabFile = keytabDir + '/' + 'mykeytab'
        outfile = rootDir+'/kinitReply.txt'

        initialCmd = '/usr/sue/bin/kinit -k -t ' + keytabFile + ' nougaret' \
                     + ' > '+outfile

        while True:
            now = datetime.datetime.now()            
            try:
            
                os.system('rm '+outfile)
                os.system(initialCmd) # forget kinit -R: only worked 5+1 days
                # by default unless kinit -r 10days could be used initially
                # which is not the case with an acron => instead rely on
                # keytable
                f = open(outfile,'r')
                msg = ''
                lines = f.readlines()
                for line in lines:
                    msg = msg + line + '\n'
                f.close()
                os.system('rm '+outfile)

                # must provide path to cope with reduced acron environment
                os.system('/usr/kerberos/bin/klist > ' + outfile)             
                f = open(outfile,'r')             
                lines = f.readlines()               
                msg2 = ''
                for line in lines:
                    msg2 = msg2 + line + '\n'
                f.close()              
                os.system('rm '+outfile)

                # refresh AFS ticket
                # must provide path to cope with reduced acron environment
                os.system('/usr/bin/aklog')

                if debugMode:
                    notify('jean-luc','Kerberos / AFS ticket','ran for ' \
                           +str((now-origin).days)+' days\n\n'\
                           + 'kinit reply:\n\n'+ msg+'\n\nklist output:\n\n'+\
                           msg2)
                
            except:
                traceback.print_tb(self.fileTokens)
                notify('jean-luc','Automatic message (PROBLEM)',\
                       'failed to refresh tokens!')
            for sec in range(1,21600+1): # sleep 6 hours
            #for sec in range(1,7200+1): # sleep 2 hours               
                time.sleep(1) # 1 sec
                if self.killed == True:
                    return
            
    def kill(self):
        self.killed = True # work-around: not yet able to kill Python threads
        close(self.fileTokens)

os.chdir(sys.path[0]) # change to Python script's directory
# ???
currentDir= os.getcwd()

# temporary: for the Intel compiler, lines split after the 80th character
# unless we set up a specific environment variable to set the line length
# note that this variable will be transmitted to the child processes
# launched through os.sytem() calls.
# all 'madx < input > output' calls where madx is compiled with the ifort
# Intel compiler will take FORT_FMT_RECL into account.
os.putenv('FORT_FMT_RECL','256') # lines split after 256 chars instead of
# the default 80

reportFile = open(rootDir+"/MadBuildAndTest_Report.txt","w")
now = time.ctime()
reportFile.write("MadBuildAndTest.py report from "+now+"\n")

# option parsing
try:
#if True:
    parser = optparse.OptionParser()
    parser.add_option("-a","--automatic", action="store_true", \
                      dest = "automatic", default=False, \
                      help="wake-up once a day and check for trigger tag in the CVS")
    parser.add_option("-m","--manual", \
                      action="store_true", dest="manual", \
                      help="start straight away, looking at the specified tag")
    parser.add_option("-t","--tag", \
                      action="store", type="string", dest="specified_ver", \
                      help="specify tag madX-XX_YY_ZZ")
    parser.add_option("-g","--debug", action="store_true", dest="debug",\
                      default=False,\
                      help="no commit and file written. no message sent.")
    parser.add_option("-s","--silent", action="store_true", dest="silent",\
                      default=False,\
                      help="no e-mail sent to module keepers")

    # 4 december 2009
    parser.add_option("-d","--dev",action="store_true",help="build and test also with the 'Makefile_develop' Lahey Fortran compiler")
    parser.add_option("-n","--nag",action="store_true",help="build and test also with the 'Makefile_nag' NAG f95 compiler")
    
    (options,args) = parser.parse_args()

    additionalCompilers = ''
    if options.nag:
        additionalCompilers += ' --nag '
    if options.dev:
        additionalCompilers += ' --dev '

    # check if all the command line arguments could be processed as options

    if len(args)>0:
        raise MadBuildAndTestException("arguments not understood as valid options")

    # check it all selected options are compatible
    if options.automatic and options.manual:
        raise MadBuildAndTestException("options -a and -m are mutually exclusive")

    global debugMode
    debugMode = options.debug

    if options.manual:
        # is it a well formed release tag?
        runTest = 'run-test'
        pattern = r'^madX-\d_\d{2}_\d{2}$'
        m = re.match(pattern,options.specified_ver)
        if m:
            # extract the version from the CVS at the specified tag in a local mirror under ./MadCvsExtract
            try:
                # delete any pre-existing directory
                #os.chdir(rootDir)
                # 23 june 2009: all the following can be removed as it's MadBuildPy.pl which creates ./MadCvsExtract
                #os.system('rm -rf ./temporary_MadCvsExtract')
                #os.mkdir('./temporary_MadCvsExtract')
                #os.chdir('./temporary_MadCvsExtract')
                #command = 'cvs -d :gserver:isscvs.cern.ch/local/reps/madx checkout -r ' + options.specified_ver + ' madX'
                #notify('jean-luc','CVS extract command','under "'+os.getcwd()+'", issue: '+command)
                #os.system(command)
                
                release = options.specified_ver

                # did the checkout succeed?
                # 23 june 2009: actually this CVS extract is useless, as MadBuilPy.pl carries-out its on extract
                # => such check-out could be removed eventually. but for test purpose, it still should work !!!

                # must go back to rootDir
                #os.chdir(rootDir)
                #os.system('rm -rf ./MadCvsExtract') # clean-up. directory will be recreated by MadBuildPy.pl
                
            except:
                print("failed to extract the specified release from the CVS ("+options.specified_ver+")")
                notify('jean-luc','failed to extract '+options.specified_ver,'failed to extract specified release from CVS')
            pass
        else:
            raise MadBuildAndTestException("incorrect version specified")
    else:
        # retreive the release number from the CVS
        # (idem as MadTrigTest.pl)
        trigTest = MadTrigTest.MadTestTrigger()
        trigTest.setDebugMode(options.debug) # if false, won't tag the CVS with 'test-XX_YY_ZZ'
        trigTest.run(rootDir)
        runTest = trigTest.getWhatToDo()
        release = trigTest.getRelease() # the tag with which to extract the CVS
        os.chdir(rootDir) # changed in between
     
#else:    
except:
    print("failed to parse the command line. exit.")
    (type,value,tb) = sys.exc_info()
    traceback.print_tb(tb)
    sys.exit()

th = updateTokensThread()
if options.debug:
    th.setDebugMode(True)
else:
    th.setDebugMode(False)
th.start()

try:
    if runTest == 'do-nothing':
        reportFile.write("No new release detected => no need to run "+\
                         "the test-suite\n")
        global debugMode
        if debugMode:
            notify('jean-luc','no new release detected','last release = '+release)
        
        #reportFile.close()
    elif not runTest == 'run-test':
        reportFile.write("don't know what to do!\n")
        #reportFile.close()
    else:
        reportFile.write("now about to release "+release+"\n")
        if options.manual:
            notify('jean-luc','manual release','new release = '+release)
        else:
            notify('admin','new release detected','new release = '+release)       

        # leave the try statement (go to finally if any)
        #th.kill()
        #sys.exit()

        # no work report for the time being
        #os.chdir(rootDir)
        #os.system("./MadWorkReport.pl")

        reportFile.write("entering MadBuild.pl, with release-tag "+\
                     release+"\n")
        os.chdir(rootDir)
        os.system("rm -rf ./MadSvnExtract")
        try:
            os.system("rm -rf ./tempfile")
        except:
            pass
        # 4 decembre 2009: invoke MadBuild.py instead of MadBuildPy.pl
        # os.system("./MadBuildPy.pl "+ release + " > ./tempfile") # MadBuildPy.pl instead of MadBuild.pl
        try:
            os.system("./MadBuild.py "+additionalCompilers + " --release " + release + "> ./tempfile")
            notify('jean-luc','here','managed to launch '+"./MadBuild.py "+additionalCompilers+  " --release " + release + "> ./tempfile")           
        except:
            notify('jean-luc','here','failed to launch '+"./MadBuild.py "+additionalCompilers+ release + "> ./tempfile")
            raise("forward exception")
        tempfile = open('./tempfile','r')
        templines = tempfile.readlines()
        if templines[0]=='False': # the return status indicating a compilation failure
            if options.manual:
                notify('jean-luc','stop test','because compilation did not succeed.')
            else:
                notify('admin','stop test','because compilation did not succeed.')
            # shall we kill the thread as well?
            th.kill()
            sys.exit()
        
        reportFile.write("MadBuildPy.pl completed\n")

        # no MadTest for the time-being
        reportFile.write("entering MadTest.pl\n")
        os.chdir(rootDir)

        # for the time being, invoke debug mode for target cororbit only
        #os.system("./MadTest.pl ./MadCvsExtract/madX")
        # WARNING: MadTestPy.pl should replace MadTest.pl to avoid sending
        # e-mail - instead, notification should take place within Python.
        if options.debug: # or global debugMode
            notify('jean-luc','Start test','invoke ./MadTest.py '+additionalCompilers)
        # os.system("./MadTestPy.pl ./MadCvsExtract/madX debug=c6t")
        if options.silent:
            # December 2nd, 2009: invoke the latest MadTest.py
            #os.system("./MadTestPy.pl ./MadCvsExtract/madX silent") # no e-mail
            os.system("./MadTest.py "+additionalCompilers+" --silent") # assumes the path to the MadCvsExtract/madX is known ...
            # currently the '--silent' option is recognized but not taken into account by MadTest.py
        else:
            # December 2nd, 2009: invoke the latest MadTest.py
            #os.system("./MadTestPy.pl ./MadCvsExtract/madX")
            os.system("./MadTest.py "+additionalCompilers) # assumes the path  to the MadCvsExtract/madX is known by MadTest.py
        reportFile.write("MadTestPy.py completed\n")

        if options.manual:
            notify('jean-luc','Completed test','MadTest.py completed')
        else:
            notify('admin','Completed test','MadTest.py completed')
            
        # final debug test to check we still have access to AFS
        # make sure still the script still has access to AFS
        # by listing the examples directory
        directory =  '/afs/cern.ch/user/n/nougaret/scratch1/'+\
                    'mad-automation/TESTING/'
        command = 'ls '+directory+ ' > '+directory+'theFile'
        # apparently os.system can't handle relative path names!!!!!!!????
        #print command
        os.system(command)
        time.sleep(5)
        afsDebugFile = open(directory+'theFile','r')
        lines = afsDebugFile.readlines()
        msg =''
        for line in lines:
            msg = msg + line + '\n'
        if debugMode:
            notify('jean-luc','Very Last Message',\
                   'Contents of '+directory+'\n'+\
               msg)              
        os.system('rm '+directory+'theFile')
        # end of final test


        #   reportFile.close()

except:
    reportFile.write("exception caught when trying Python scripts in:\n"+\
                     currentDir+"\n")
    traceback.print_exc(file=reportFile)

    notify('jean-luc','Failure',\
           "exception caught when trying the successive Python scripts in:\n"+\
           currentDir)

th.kill()
reportFile.close()