1 | #!/usr/bin/python |
---|
2 | from Notify import notify |
---|
3 | from MadTestExceptions import MadException |
---|
4 | import MadTrigTest |
---|
5 | |
---|
6 | import threading |
---|
7 | import time, datetime |
---|
8 | import os |
---|
9 | import sys |
---|
10 | import re |
---|
11 | import traceback |
---|
12 | import optparse |
---|
13 | |
---|
14 | global debugMode |
---|
15 | |
---|
16 | threadList = [] |
---|
17 | |
---|
18 | rootDir = "/afs/cern.ch/user/n/nougaret/scratch1/mad-automation" |
---|
19 | |
---|
20 | class MadBuildAndTestException(Exception): |
---|
21 | def __init__(self,value): |
---|
22 | self.value = value |
---|
23 | def __str__(self): |
---|
24 | return repr(self.value) |
---|
25 | |
---|
26 | class updateTokensThread(threading.Thread): |
---|
27 | def __init__(self): |
---|
28 | threading.Thread.__init__(self) |
---|
29 | self.killed = False |
---|
30 | threadList.append(self) |
---|
31 | self.fileTokens = open('MadBuildAndTest.trace','w') |
---|
32 | def setDebugMode(self,mode): |
---|
33 | self.debugMode = mode |
---|
34 | def run(self): |
---|
35 | global debugMode |
---|
36 | |
---|
37 | origin = datetime.datetime.now() # return string |
---|
38 | duration = 0 # init |
---|
39 | keytabDir = '/extra/home/nougaret/MadTestScriptAuth' |
---|
40 | keytabFile = keytabDir + '/' + 'mykeytab' |
---|
41 | outfile = rootDir+'/kinitReply.txt' |
---|
42 | |
---|
43 | initialCmd = '/usr/sue/bin/kinit -k -t ' + keytabFile + ' nougaret' \ |
---|
44 | + ' > '+outfile |
---|
45 | |
---|
46 | while True: |
---|
47 | now = datetime.datetime.now() |
---|
48 | try: |
---|
49 | |
---|
50 | os.system('rm '+outfile) |
---|
51 | os.system(initialCmd) # forget kinit -R: only worked 5+1 days |
---|
52 | # by default unless kinit -r 10days could be used initially |
---|
53 | # which is not the case with an acron => instead rely on |
---|
54 | # keytable |
---|
55 | f = open(outfile,'r') |
---|
56 | msg = '' |
---|
57 | lines = f.readlines() |
---|
58 | for line in lines: |
---|
59 | msg = msg + line + '\n' |
---|
60 | f.close() |
---|
61 | os.system('rm '+outfile) |
---|
62 | |
---|
63 | # must provide path to cope with reduced acron environment |
---|
64 | os.system('/usr/kerberos/bin/klist > ' + outfile) |
---|
65 | f = open(outfile,'r') |
---|
66 | lines = f.readlines() |
---|
67 | msg2 = '' |
---|
68 | for line in lines: |
---|
69 | msg2 = msg2 + line + '\n' |
---|
70 | f.close() |
---|
71 | os.system('rm '+outfile) |
---|
72 | |
---|
73 | # refresh AFS ticket |
---|
74 | # must provide path to cope with reduced acron environment |
---|
75 | os.system('/usr/bin/aklog') |
---|
76 | |
---|
77 | if debugMode: |
---|
78 | notify('jean-luc','Kerberos / AFS ticket','ran for ' \ |
---|
79 | +str((now-origin).days)+' days\n\n'\ |
---|
80 | + 'kinit reply:\n\n'+ msg+'\n\nklist output:\n\n'+\ |
---|
81 | msg2) |
---|
82 | |
---|
83 | except: |
---|
84 | traceback.print_tb(self.fileTokens) |
---|
85 | notify('jean-luc','Automatic message (PROBLEM)',\ |
---|
86 | 'failed to refresh tokens!') |
---|
87 | for sec in range(1,21600+1): # sleep 6 hours |
---|
88 | #for sec in range(1,7200+1): # sleep 2 hours |
---|
89 | time.sleep(1) # 1 sec |
---|
90 | if self.killed == True: |
---|
91 | return |
---|
92 | |
---|
93 | def kill(self): |
---|
94 | self.killed = True # work-around: not yet able to kill Python threads |
---|
95 | close(self.fileTokens) |
---|
96 | |
---|
97 | os.chdir(sys.path[0]) # change to Python script's directory |
---|
98 | # ??? |
---|
99 | currentDir= os.getcwd() |
---|
100 | |
---|
101 | # temporary: for the Intel compiler, lines split after the 80th character |
---|
102 | # unless we set up a specific environment variable to set the line length |
---|
103 | # note that this variable will be transmitted to the child processes |
---|
104 | # launched through os.sytem() calls. |
---|
105 | # all 'madx < input > output' calls where madx is compiled with the ifort |
---|
106 | # Intel compiler will take FORT_FMT_RECL into account. |
---|
107 | os.putenv('FORT_FMT_RECL','256') # lines split after 256 chars instead of |
---|
108 | # the default 80 |
---|
109 | |
---|
110 | reportFile = open(rootDir+"/MadBuildAndTest_Report.txt","w") |
---|
111 | now = time.ctime() |
---|
112 | reportFile.write("MadBuildAndTest.py report from "+now+"\n") |
---|
113 | |
---|
114 | # option parsing |
---|
115 | try: |
---|
116 | #if True: |
---|
117 | parser = optparse.OptionParser() |
---|
118 | parser.add_option("-a","--automatic", action="store_true", \ |
---|
119 | dest = "automatic", default=False, \ |
---|
120 | help="wake-up once a day and check for trigger tag in the CVS") |
---|
121 | parser.add_option("-m","--manual", \ |
---|
122 | action="store_true", dest="manual", \ |
---|
123 | help="start straight away, looking at the specified tag") |
---|
124 | parser.add_option("-t","--tag", \ |
---|
125 | action="store", type="string", dest="specified_ver", \ |
---|
126 | help="specify tag madX-XX_YY_ZZ") |
---|
127 | parser.add_option("-g","--debug", action="store_true", dest="debug",\ |
---|
128 | default=False,\ |
---|
129 | help="no commit and file written. no message sent.") |
---|
130 | parser.add_option("-s","--silent", action="store_true", dest="silent",\ |
---|
131 | default=False,\ |
---|
132 | help="no e-mail sent to module keepers") |
---|
133 | |
---|
134 | # 4 december 2009 |
---|
135 | parser.add_option("-d","--dev",action="store_true",help="build and test also with the 'Makefile_develop' Lahey Fortran compiler") |
---|
136 | parser.add_option("-n","--nag",action="store_true",help="build and test also with the 'Makefile_nag' NAG f95 compiler") |
---|
137 | |
---|
138 | (options,args) = parser.parse_args() |
---|
139 | |
---|
140 | additionalCompilers = '' |
---|
141 | if options.nag: |
---|
142 | additionalCompilers += ' --nag ' |
---|
143 | if options.dev: |
---|
144 | additionalCompilers += ' --dev ' |
---|
145 | |
---|
146 | # check if all the command line arguments could be processed as options |
---|
147 | |
---|
148 | if len(args)>0: |
---|
149 | raise MadBuildAndTestException("arguments not understood as valid options") |
---|
150 | |
---|
151 | # check it all selected options are compatible |
---|
152 | if options.automatic and options.manual: |
---|
153 | raise MadBuildAndTestException("options -a and -m are mutually exclusive") |
---|
154 | |
---|
155 | global debugMode |
---|
156 | debugMode = options.debug |
---|
157 | |
---|
158 | if options.manual: |
---|
159 | # is it a well formed release tag? |
---|
160 | runTest = 'run-test' |
---|
161 | pattern = r'^madX-\d_\d{2}_\d{2}$' |
---|
162 | m = re.match(pattern,options.specified_ver) |
---|
163 | if m: |
---|
164 | # extract the version from the CVS at the specified tag in a local mirror under ./MadCvsExtract |
---|
165 | try: |
---|
166 | # delete any pre-existing directory |
---|
167 | #os.chdir(rootDir) |
---|
168 | # 23 june 2009: all the following can be removed as it's MadBuildPy.pl which creates ./MadCvsExtract |
---|
169 | #os.system('rm -rf ./temporary_MadCvsExtract') |
---|
170 | #os.mkdir('./temporary_MadCvsExtract') |
---|
171 | #os.chdir('./temporary_MadCvsExtract') |
---|
172 | #command = 'cvs -d :gserver:isscvs.cern.ch/local/reps/madx checkout -r ' + options.specified_ver + ' madX' |
---|
173 | #notify('jean-luc','CVS extract command','under "'+os.getcwd()+'", issue: '+command) |
---|
174 | #os.system(command) |
---|
175 | |
---|
176 | release = options.specified_ver |
---|
177 | |
---|
178 | # did the checkout succeed? |
---|
179 | # 23 june 2009: actually this CVS extract is useless, as MadBuilPy.pl carries-out its on extract |
---|
180 | # => such check-out could be removed eventually. but for test purpose, it still should work !!! |
---|
181 | |
---|
182 | # must go back to rootDir |
---|
183 | #os.chdir(rootDir) |
---|
184 | #os.system('rm -rf ./MadCvsExtract') # clean-up. directory will be recreated by MadBuildPy.pl |
---|
185 | |
---|
186 | except: |
---|
187 | print("failed to extract the specified release from the CVS ("+options.specified_ver+")") |
---|
188 | notify('jean-luc','failed to extract '+options.specified_ver,'failed to extract specified release from CVS') |
---|
189 | pass |
---|
190 | else: |
---|
191 | raise MadBuildAndTestException("incorrect version specified") |
---|
192 | else: |
---|
193 | # retreive the release number from the CVS |
---|
194 | # (idem as MadTrigTest.pl) |
---|
195 | trigTest = MadTrigTest.MadTestTrigger() |
---|
196 | trigTest.setDebugMode(options.debug) # if false, won't tag the CVS with 'test-XX_YY_ZZ' |
---|
197 | trigTest.run(rootDir) |
---|
198 | runTest = trigTest.getWhatToDo() |
---|
199 | release = trigTest.getRelease() # the tag with which to extract the CVS |
---|
200 | os.chdir(rootDir) # changed in between |
---|
201 | |
---|
202 | #else: |
---|
203 | except: |
---|
204 | print("failed to parse the command line. exit.") |
---|
205 | (type,value,tb) = sys.exc_info() |
---|
206 | traceback.print_tb(tb) |
---|
207 | sys.exit() |
---|
208 | |
---|
209 | th = updateTokensThread() |
---|
210 | if options.debug: |
---|
211 | th.setDebugMode(True) |
---|
212 | else: |
---|
213 | th.setDebugMode(False) |
---|
214 | th.start() |
---|
215 | |
---|
216 | try: |
---|
217 | if runTest == 'do-nothing': |
---|
218 | reportFile.write("No new release detected => no need to run "+\ |
---|
219 | "the test-suite\n") |
---|
220 | global debugMode |
---|
221 | if debugMode: |
---|
222 | notify('jean-luc','no new release detected','last release = '+release) |
---|
223 | |
---|
224 | #reportFile.close() |
---|
225 | elif not runTest == 'run-test': |
---|
226 | reportFile.write("don't know what to do!\n") |
---|
227 | #reportFile.close() |
---|
228 | else: |
---|
229 | reportFile.write("now about to release "+release+"\n") |
---|
230 | if options.manual: |
---|
231 | notify('jean-luc','manual release','new release = '+release) |
---|
232 | else: |
---|
233 | notify('admin','new release detected','new release = '+release) |
---|
234 | |
---|
235 | # leave the try statement (go to finally if any) |
---|
236 | #th.kill() |
---|
237 | #sys.exit() |
---|
238 | |
---|
239 | # no work report for the time being |
---|
240 | #os.chdir(rootDir) |
---|
241 | #os.system("./MadWorkReport.pl") |
---|
242 | |
---|
243 | reportFile.write("entering MadBuild.pl, with release-tag "+\ |
---|
244 | release+"\n") |
---|
245 | os.chdir(rootDir) |
---|
246 | os.system("rm -rf ./MadSvnExtract") |
---|
247 | try: |
---|
248 | os.system("rm -rf ./tempfile") |
---|
249 | except: |
---|
250 | pass |
---|
251 | # 4 decembre 2009: invoke MadBuild.py instead of MadBuildPy.pl |
---|
252 | # os.system("./MadBuildPy.pl "+ release + " > ./tempfile") # MadBuildPy.pl instead of MadBuild.pl |
---|
253 | try: |
---|
254 | os.system("./MadBuild.py "+additionalCompilers + " --release " + release + "> ./tempfile") |
---|
255 | notify('jean-luc','here','managed to launch '+"./MadBuild.py "+additionalCompilers+ " --release " + release + "> ./tempfile") |
---|
256 | except: |
---|
257 | notify('jean-luc','here','failed to launch '+"./MadBuild.py "+additionalCompilers+ release + "> ./tempfile") |
---|
258 | raise("forward exception") |
---|
259 | tempfile = open('./tempfile','r') |
---|
260 | templines = tempfile.readlines() |
---|
261 | if templines[0]=='False': # the return status indicating a compilation failure |
---|
262 | if options.manual: |
---|
263 | notify('jean-luc','stop test','because compilation did not succeed.') |
---|
264 | else: |
---|
265 | notify('admin','stop test','because compilation did not succeed.') |
---|
266 | # shall we kill the thread as well? |
---|
267 | th.kill() |
---|
268 | sys.exit() |
---|
269 | |
---|
270 | reportFile.write("MadBuildPy.pl completed\n") |
---|
271 | |
---|
272 | # no MadTest for the time-being |
---|
273 | reportFile.write("entering MadTest.pl\n") |
---|
274 | os.chdir(rootDir) |
---|
275 | |
---|
276 | # for the time being, invoke debug mode for target cororbit only |
---|
277 | #os.system("./MadTest.pl ./MadCvsExtract/madX") |
---|
278 | # WARNING: MadTestPy.pl should replace MadTest.pl to avoid sending |
---|
279 | # e-mail - instead, notification should take place within Python. |
---|
280 | if options.debug: # or global debugMode |
---|
281 | notify('jean-luc','Start test','invoke ./MadTest.py '+additionalCompilers) |
---|
282 | # os.system("./MadTestPy.pl ./MadCvsExtract/madX debug=c6t") |
---|
283 | if options.silent: |
---|
284 | # December 2nd, 2009: invoke the latest MadTest.py |
---|
285 | #os.system("./MadTestPy.pl ./MadCvsExtract/madX silent") # no e-mail |
---|
286 | os.system("./MadTest.py "+additionalCompilers+" --silent") # assumes the path to the MadCvsExtract/madX is known ... |
---|
287 | # currently the '--silent' option is recognized but not taken into account by MadTest.py |
---|
288 | else: |
---|
289 | # December 2nd, 2009: invoke the latest MadTest.py |
---|
290 | #os.system("./MadTestPy.pl ./MadCvsExtract/madX") |
---|
291 | os.system("./MadTest.py "+additionalCompilers) # assumes the path to the MadCvsExtract/madX is known by MadTest.py |
---|
292 | reportFile.write("MadTestPy.py completed\n") |
---|
293 | |
---|
294 | if options.manual: |
---|
295 | notify('jean-luc','Completed test','MadTest.py completed') |
---|
296 | else: |
---|
297 | notify('admin','Completed test','MadTest.py completed') |
---|
298 | |
---|
299 | # final debug test to check we still have access to AFS |
---|
300 | # make sure still the script still has access to AFS |
---|
301 | # by listing the examples directory |
---|
302 | directory = '/afs/cern.ch/user/n/nougaret/scratch1/'+\ |
---|
303 | 'mad-automation/TESTING/' |
---|
304 | command = 'ls '+directory+ ' > '+directory+'theFile' |
---|
305 | # apparently os.system can't handle relative path names!!!!!!!???? |
---|
306 | #print command |
---|
307 | os.system(command) |
---|
308 | time.sleep(5) |
---|
309 | afsDebugFile = open(directory+'theFile','r') |
---|
310 | lines = afsDebugFile.readlines() |
---|
311 | msg ='' |
---|
312 | for line in lines: |
---|
313 | msg = msg + line + '\n' |
---|
314 | if debugMode: |
---|
315 | notify('jean-luc','Very Last Message',\ |
---|
316 | 'Contents of '+directory+'\n'+\ |
---|
317 | msg) |
---|
318 | os.system('rm '+directory+'theFile') |
---|
319 | # end of final test |
---|
320 | |
---|
321 | |
---|
322 | # reportFile.close() |
---|
323 | |
---|
324 | except: |
---|
325 | reportFile.write("exception caught when trying Python scripts in:\n"+\ |
---|
326 | currentDir+"\n") |
---|
327 | traceback.print_exc(file=reportFile) |
---|
328 | |
---|
329 | notify('jean-luc','Failure',\ |
---|
330 | "exception caught when trying the successive Python scripts in:\n"+\ |
---|
331 | currentDir) |
---|
332 | |
---|
333 | th.kill() |
---|
334 | reportFile.close() |
---|