1 | #!/usr/bin/python |
---|
2 | |
---|
3 | # currently works with following invocation: |
---|
4 | # MadTest.py -k -m Makefile -t aperture |
---|
5 | # MadTest.py -k -m Makefile -t ptc_twiss -c example1.madx |
---|
6 | # MadTest.py -k -m Makefile -t ptc_twiss |
---|
7 | import optparse |
---|
8 | import os |
---|
9 | import shutil |
---|
10 | import re |
---|
11 | import Notify |
---|
12 | import time |
---|
13 | import datetime |
---|
14 | |
---|
15 | # on 30 november 2009, experienced strange problem: |
---|
16 | # pathname of files contained paths with /afs/cern/.../si/slap/share.... |
---|
17 | def checkname(name): |
---|
18 | if name.find('/si/slap/share/')>-1: |
---|
19 | raise('found name:'+name) |
---|
20 | |
---|
21 | class State: # the state of a test |
---|
22 | pass |
---|
23 | |
---|
24 | extracting = State() |
---|
25 | init = State() |
---|
26 | incomplete = State() # missing resources |
---|
27 | ready = State() # all resources present |
---|
28 | running = State() |
---|
29 | aborted = State() |
---|
30 | completed = State() |
---|
31 | |
---|
32 | rootDir = "/afs/cern.ch/user/n/nougaret/scratch1/mad-automation/madX-examples/REF" |
---|
33 | |
---|
34 | topDir = "/afs/cern.ch/user/n/nougaret/scratch1/mad-automation" |
---|
35 | testingDir = topDir + "/TESTING" |
---|
36 | repositoryDir = topDir + "/madX-examples" |
---|
37 | madxDir = topDir + "/MadSvnExtract/madX" |
---|
38 | |
---|
39 | htmlRootDir = "/afs/cern.ch/user/n/nougaret/www/mad" |
---|
40 | mainHtmlPage = htmlRootDir + "/test.htm" |
---|
41 | |
---|
42 | makefiles = ['Makefile','Makefile_develop','Makefile_nag'] |
---|
43 | |
---|
44 | # all files with absolute path |
---|
45 | |
---|
46 | class Repository: # wrapper on CVS or SVN repository for examples |
---|
47 | def __init__(self): |
---|
48 | pass |
---|
49 | def checkout(self): |
---|
50 | # clean-up the directory in which the examples's repository is to be extracted |
---|
51 | try: |
---|
52 | shutil.rmtree(repositoryDir) |
---|
53 | except: |
---|
54 | pass # directory absent |
---|
55 | currentDir = os.getcwd() |
---|
56 | os.mkdir(repositoryDir) |
---|
57 | os.chdir(topDir) |
---|
58 | #checkoutCommand = "cvs -d :gserver:isscvs.cern.ch:/local/reps/madx-examples checkout madX-examples"; |
---|
59 | if options.singleTarget: |
---|
60 | checkoutCommand = "svn co svn+ssh://svn.cern.ch/reps/madx-examples/trunk/madX-examples"\ |
---|
61 | +"/REF/"+options.singleTarget+" "+repositoryDir+'/REF/'+options.singleTarget |
---|
62 | else: |
---|
63 | checkoutCommand = "svn co svn+ssh://svn.cern.ch/reps/madx-examples/trunk/madX-examples" + " " + repositoryDir |
---|
64 | os.system(checkoutCommand) |
---|
65 | os.chdir(currentDir) |
---|
66 | |
---|
67 | class Resource: |
---|
68 | def __init__(self,name,source,destinations): # the name is not unique, as this is within scope of a testcase |
---|
69 | checkname(source) |
---|
70 | # for a given resource, there are 3 destinations, i.e. one for each Makefile |
---|
71 | self.name = name |
---|
72 | self.source = source # the expanded filename of the file from CVS |
---|
73 | # check the resource indeed exists. If not this a temp file meant to be created at runtime |
---|
74 | if not os.path.exists(source): |
---|
75 | self.type = 'runtime' |
---|
76 | if options.verbose: |
---|
77 | print("WARNING resource file "+source+" is probably meant to be created at runtime") |
---|
78 | else: |
---|
79 | self.type = 'static' |
---|
80 | self.destinations = destinations # the expanded filename of the file in local work directory |
---|
81 | if options.verbose: |
---|
82 | print("create resource '"+name+"' retreived from '"+source+"'") |
---|
83 | |
---|
84 | def lookForDependantResources(test,filename,lvl=0,leafDir=0): |
---|
85 | # returns a list of filenames with absolute path |
---|
86 | |
---|
87 | resources = [] # list to be returned |
---|
88 | |
---|
89 | if lvl==0: |
---|
90 | source = filename |
---|
91 | name = source[source.rfind('/')+1:] |
---|
92 | parts = source.split('/') |
---|
93 | leafDir = parts[-2] |
---|
94 | if options.verbose: |
---|
95 | print("leafDir="+leafDir) |
---|
96 | # the leaf directory that contain the example and that should be renamed by test.testcaseDir |
---|
97 | destinations = [] |
---|
98 | for m in makefiles: |
---|
99 | destination = source.replace(rootDir,testingDir+'/'+m) # replace prefix between source and destination |
---|
100 | match = re.match(r'^test_\d+(_.+)?$',test.testcaseDir) |
---|
101 | if not match: |
---|
102 | raise("pattern should always match") |
---|
103 | if match.lastindex == 1: # testcaseDir like test_1_LHC |
---|
104 | destination = destination.replace('/'+leafDir+'/','/'+test.testcaseDir+'/') # testcaseDir like test_1_LHC |
---|
105 | else: |
---|
106 | destination = destination.replace('/'+leafDir+'/','/'+leafDir+'/'+test.testcaseDir+'/') # testcaseDir like test_1 |
---|
107 | if options.verbose: |
---|
108 | print("destination="+destination+" for source="+source+", leafDir="+leafDir) |
---|
109 | destinations.append(destination) |
---|
110 | resource = Resource(name,source,destinations) # primary resource is the MAD-X input file itself |
---|
111 | resources.append(resource) |
---|
112 | |
---|
113 | try: |
---|
114 | f = open(filename,'r') |
---|
115 | # regex to input a file (not anchored as an instruction may precede) |
---|
116 | commonPatterns = [re.compile(r'readmytable,?[\s\t]*file[\s\t]*=[\s\t]*[\"\']?([\w\.\_\-\d/]+)[\"\']?[\s\t]*'),\ |
---|
117 | re.compile(r'call,?[\s\t]*file[\s\t]*=[\s\t]*[\"\']?([\w\.\_\-\d/]+)[\"\']?[\s\t]*;'),\ |
---|
118 | re.compile(r'readtable,?[\s\t]*file[\s\t]*=[\s\t]*[\"\']?([\w\.\_\-\d/]+)[\"\']?[\s\t]*;')] |
---|
119 | |
---|
120 | # another - rare - instruction that calls a file from another (in sxf/sxfread.madx) |
---|
121 | #commonPatterns.append(\ |
---|
122 | # re.compile(r'sxfread[\s\t]*,?[\s\t]*file[\s\t]*=[\s\t]*[\"\']?([\w\._\-\d\/]+)[\"\']?[\s\t]*;')\ |
---|
123 | # ) |
---|
124 | |
---|
125 | commonPatterns.append(re.compile(r'sxfread, file = "(fv9\.sxf)";')) |
---|
126 | |
---|
127 | # another - exceptional - instruction that calls a file from another (in aperture/lhccell.madx) |
---|
128 | commonPatterns.append(re.compile(r'aperture,offsetelem=\"([\w\.]+)\".+')) |
---|
129 | |
---|
130 | # another - rare - instruction that calls a script available in the input directory |
---|
131 | # along the other input files. Such script must be copied locally along the other files. |
---|
132 | # This is for instance the case for the read.magnet.errors perl-script in twiss/test_5/ or foot |
---|
133 | scriptInvocationPattern = [re.compile(r'[Ss][Yy][Ss][Tt][Ee][Mm][\s\t]*,?'+\ |
---|
134 | '[\s\t]*[\"\']([\w\.\_\-\d]+)[\s\t]*'+\ |
---|
135 | '<?[\s\t]*([\w\.\_\-\d\/]*)[\s\t]*>?(.*)[\"\']')] |
---|
136 | |
---|
137 | |
---|
138 | for line in f.readlines(): |
---|
139 | #print("line="+line) |
---|
140 | line = line.rstrip('\n') |
---|
141 | if len(line.lstrip())>=1 and line.lstrip()[0]=='!': # if line is a comment discard it |
---|
142 | continue |
---|
143 | k = line.find('=') |
---|
144 | if not k == -1: |
---|
145 | # put to lower case everything in front of the equal sign to simply patterns |
---|
146 | line = line[0:k].lower() +"="+ line[k+1:] |
---|
147 | for i,p in enumerate(commonPatterns + scriptInvocationPattern): |
---|
148 | if i == len(commonPatterns + scriptInvocationPattern)-1: |
---|
149 | scriptInvocation = True |
---|
150 | else: |
---|
151 | scriptInvocation = False # the most usual case, as in call, file... |
---|
152 | m = p.search(line) |
---|
153 | if m: |
---|
154 | |
---|
155 | if scriptInvocation: |
---|
156 | command = m.group(1) |
---|
157 | if command == 'perl' or command == 'python' or command == 'gnuplot': |
---|
158 | source = m.group(2) |
---|
159 | elif not (command == 'mkdir' or command == 'ls' or command == 'cat' or command == 'rm' or command == 'grep' or \ |
---|
160 | command == 'echo' or command == 'ln' or command == 'cp'): |
---|
161 | source = m.group(1) # source is a script |
---|
162 | else: |
---|
163 | continue # the for-loop |
---|
164 | else: |
---|
165 | source = m.group(1) # the general case, as in call, file... |
---|
166 | |
---|
167 | if not source[0] == '/': # relative path to be converted to absolute |
---|
168 | dirPrefix = filename[0:filename.rfind('/')] |
---|
169 | source = dirPrefix+'/'+source |
---|
170 | source = os.path.normpath(source) # normalize the pathname |
---|
171 | else: |
---|
172 | pass |
---|
173 | |
---|
174 | # the short name of the resource is obtained by removing the complete path prefix |
---|
175 | name = source[source.rfind('/')+1:] |
---|
176 | # for each source, there are three destinations as there are 3 different makefiles |
---|
177 | destinations = [] |
---|
178 | for m in makefiles: |
---|
179 | destination = source.replace(rootDir,testingDir+'/'+m) # replace prefix between source and destination |
---|
180 | match = re.match(r'^test_\d+(_.+)?$',test.testcaseDir) |
---|
181 | if not match: |
---|
182 | raise("pattern should always match") |
---|
183 | if match.lastindex == 1: # testcaseDir like test_1_LHC |
---|
184 | destination = destination.replace('/'+leafDir+'/','/'+test.testcaseDir+'/') # testcaseDir like test_1_LHC |
---|
185 | else: |
---|
186 | destination = destination.replace('/'+leafDir+'/','/'+leafDir+'/'+test.testcaseDir+'/') # testcaseDir like test_1 |
---|
187 | destinations.append(destination) |
---|
188 | |
---|
189 | resource = Resource(name,source,destinations) |
---|
190 | resources.append(resource) |
---|
191 | |
---|
192 | # now dig further |
---|
193 | additionalResources = Resource.lookForDependantResources(test,source,lvl+1,\ |
---|
194 | leafDir) |
---|
195 | for a in additionalResources: |
---|
196 | resources.append(a) |
---|
197 | |
---|
198 | f.close() |
---|
199 | except: # failed to open the file: this may be a temporary file to be created at run time, any way one should |
---|
200 | # mark the resource as missing |
---|
201 | if options.verbose: |
---|
202 | print("caught exception - probably reference to a runtime resource not available at the start of the script") |
---|
203 | test.missing_resources.append(filename) |
---|
204 | if test.state == init: |
---|
205 | test.state = incomplete; |
---|
206 | |
---|
207 | return resources |
---|
208 | lookForDependantResources = staticmethod(lookForDependantResources) |
---|
209 | |
---|
210 | class Target: # a named "module" holding on or several tests |
---|
211 | targets = [] |
---|
212 | targetsDict = {} # for fast access (but random order) |
---|
213 | def __init__(self,name): |
---|
214 | self.name = name |
---|
215 | self.tests = [] |
---|
216 | Target.targets.append(self) |
---|
217 | Target.targetsDict[name] = self |
---|
218 | if options.verbose: |
---|
219 | print("create target "+name) |
---|
220 | def registerTest(targetname,test): |
---|
221 | registered = False # default |
---|
222 | for t in Target.targets: |
---|
223 | if t.name == targetname: |
---|
224 | registered = True |
---|
225 | if not registered: |
---|
226 | target = Target(targetname) |
---|
227 | target.tests.append(test) |
---|
228 | else: |
---|
229 | for t in Target.targets: |
---|
230 | if t.name == targetname: |
---|
231 | t.tests.append(test) |
---|
232 | registerTest = staticmethod(registerTest) |
---|
233 | |
---|
234 | class Output: |
---|
235 | def __init__(self,name,outcome,pageLink): |
---|
236 | self.name = name |
---|
237 | self.outcome = outcome |
---|
238 | self.weblink = pageLink |
---|
239 | |
---|
240 | class Test: # a test case |
---|
241 | tests = [] |
---|
242 | extracting = False # true when tests are being currently extracted from the CVS |
---|
243 | def __init__(self,name,program,input,output,subdirectory): |
---|
244 | |
---|
245 | self.name = name # the target |
---|
246 | self.program = program |
---|
247 | self.input = input # the main input file |
---|
248 | self.output = output # the main output file |
---|
249 | self.resources = [] # secondary inputs |
---|
250 | self.outputs = [] # secondary outputs |
---|
251 | self.missing_resources = [] |
---|
252 | self.state = init # initial state |
---|
253 | |
---|
254 | # following information only relevant when --dev or --nag option |
---|
255 | self.dev_tag = "undefined" |
---|
256 | self.nag_tag = "undefined" |
---|
257 | |
---|
258 | Target.registerTest(name, self) |
---|
259 | |
---|
260 | if subdirectory == 0 or subdirectory == '': |
---|
261 | self.testcaseDir = "test_"+str(len(Target.targetsDict[name].tests)) |
---|
262 | else: |
---|
263 | self.testcaseDir = "test_"+str(len(Target.targetsDict[name].tests))+"_"+subdirectory |
---|
264 | |
---|
265 | if not subdirectory == 0: |
---|
266 | self.subdirectory = subdirectory |
---|
267 | else: |
---|
268 | self.subdirectory = '' |
---|
269 | |
---|
270 | Test.tests.append(self) # irrespective of whether the tests has access to all resources it requires |
---|
271 | |
---|
272 | |
---|
273 | if self.collectResources(): |
---|
274 | if options.verbose: |
---|
275 | print("create test '"+name+"' with program '"+program+"', input '"+input+"', and output '"+output+"'") |
---|
276 | |
---|
277 | else: |
---|
278 | if options.verbose: |
---|
279 | print("fail to create test '"+name+"' with program = '"+program+"', input = '"+\ |
---|
280 | input+"', and output = '"+output+", subdir='",subdirectory) |
---|
281 | self.state = incomplete |
---|
282 | # issue a warning stating that the file has incomplete resources |
---|
283 | |
---|
284 | |
---|
285 | def collectResources(self): |
---|
286 | for entry in os.walk(rootDir,topdown=True): # walk file and directory structure under root |
---|
287 | foundDirectory = False |
---|
288 | |
---|
289 | if not self.subdirectory == '': |
---|
290 | if rootDir+'/'+self.name+'/'+self.subdirectory == entry[0]: |
---|
291 | foundDirectory = True |
---|
292 | else: |
---|
293 | if rootDir+'/'+self.name == entry[0]: |
---|
294 | foundDirectory = True |
---|
295 | |
---|
296 | if foundDirectory: |
---|
297 | if self.input in entry[2]: # got the primary input madx file |
---|
298 | # now check that all included files are also available (resources) |
---|
299 | self.resources = Resource.lookForDependantResources(self,entry[0]+'/'+self.input) # provide expanded filename |
---|
300 | return True # found the input madx file |
---|
301 | return False |
---|
302 | |
---|
303 | def copyResourcesToTestingDir(self): |
---|
304 | for r in self.resources: |
---|
305 | if r.type == 'runtime': |
---|
306 | # this resource is meant to be created at run time => do not attempt to copy as it does not exists yet |
---|
307 | continue |
---|
308 | if options.verbose: |
---|
309 | print("now to copy '"+r.source+"'"), |
---|
310 | for d in r.destinations: # a single resource has several destinations, i.e. one per Makefile |
---|
311 | destinationDir = d[:d.rfind('/')] |
---|
312 | |
---|
313 | if options.verbose: |
---|
314 | print("to: '"+destinationDir+"'") |
---|
315 | if not os.path.exists(destinationDir): # output directory does not exist => create it before copying file |
---|
316 | os.makedirs(destinationDir) # also create all intermediate-level directories to contain the leaf directory |
---|
317 | |
---|
318 | try: # try to copy file as well as permissions from source to destination |
---|
319 | # this is required for executable scripts called by an input madx file |
---|
320 | shutil.copyfile(r.source,d) |
---|
321 | shutil.copymode(r.source,d) # will fail for yet unidentified reasons |
---|
322 | except: |
---|
323 | # 14 april 2010 |
---|
324 | if r.source[:r.source.rfind('/')]==destinationDir: |
---|
325 | print 'WARNING do not copy '+r.source+' to its own directory '+destinationDir |
---|
326 | else: |
---|
327 | shutil.copyfile(r.source,d) |
---|
328 | # only copy the file - forget about permissions |
---|
329 | |
---|
330 | |
---|
331 | def run(self): |
---|
332 | currentDir = os.getcwd() |
---|
333 | for i,m in enumerate(makefiles): |
---|
334 | # retreive the name of the directory in which the madx input is located (first resource) |
---|
335 | script = self.resources[0].destinations[i] |
---|
336 | scriptDir = script[:script.rfind('/')] |
---|
337 | self.topDir = scriptDir # for future reuse when comparing the output with the reference |
---|
338 | os.chdir(scriptDir) |
---|
339 | if m == 'Makefile': |
---|
340 | command = madxDir+'/madx_'+m+' <'+self.input +'>'+self.output |
---|
341 | else: # Makefile_develop or Makefile_nag |
---|
342 | stderrfile = './stderr_redirected' |
---|
343 | command = '('+madxDir+'/madx_'+m+' <'+self.input +'>'+self.output + ')' + ">& " + stderrfile |
---|
344 | |
---|
345 | if options.verbose: |
---|
346 | print("now to execute "+command+" under "+scriptDir) |
---|
347 | |
---|
348 | startTime = (datetime.datetime.now()).ctime() |
---|
349 | os.system(command) |
---|
350 | endTime = (datetime.datetime.now()).ctime() |
---|
351 | |
---|
352 | # check if the program finished normally |
---|
353 | finishedNormally = False # default |
---|
354 | if os.path.exists(self.output): # output exists |
---|
355 | f = open(self.output,'r') |
---|
356 | finishedNormallyPattern = re.compile(r'\+[\s\t]+MAD-X[\s\t]+([\d\.]+)[\s\t]+finished normally[\s\t]+\+') |
---|
357 | for l in f.readlines(): |
---|
358 | if finishedNormallyPattern.search(l): |
---|
359 | finishedNormally = True |
---|
360 | # print("FINISHED NORMALLY") |
---|
361 | f.close() |
---|
362 | |
---|
363 | # now create two subdirectories, input and output to store the outcome |
---|
364 | os.mkdir('./input') |
---|
365 | os.mkdir('./output') |
---|
366 | class IOType: |
---|
367 | pass |
---|
368 | input = IOType() |
---|
369 | output = IOType() |
---|
370 | |
---|
371 | for f in os.listdir(scriptDir): |
---|
372 | if os.path.isdir(f): |
---|
373 | continue |
---|
374 | type = output # default |
---|
375 | for r in self.resources: |
---|
376 | # print("compare "+scriptDir+"/"+f+" with "+r.destinations[i]) |
---|
377 | |
---|
378 | if scriptDir+"/"+f == r.destinations[i]: # this is an (input) resource file |
---|
379 | type = input |
---|
380 | if type == input: |
---|
381 | shutil.move(f,'./input') |
---|
382 | else: |
---|
383 | shutil.move(f,'./output') |
---|
384 | |
---|
385 | whichTag = 'standard' # refers to the standard Makefile |
---|
386 | |
---|
387 | if m == 'Makefile_develop' or m == 'Makefile_nag': |
---|
388 | # then create a web page to store the contents of stderr |
---|
389 | if os.path.exists('./output/stderr_redirected'): |
---|
390 | ferror = open('./output/stderr_redirected','r') |
---|
391 | lines = ferror.readlines() |
---|
392 | if len(lines)==0 and finishedNormally: |
---|
393 | # stderr is empty and the program finished normally |
---|
394 | stderrReturnStatus = 'success' |
---|
395 | if options.verbose: |
---|
396 | print("stderr_redirected is empty") |
---|
397 | if m == 'Makefile_develop': |
---|
398 | whichTag = 'dev' # needed here? |
---|
399 | self.dev_tag = 'success' |
---|
400 | if m == 'Makefile_nag': |
---|
401 | whichTag = 'nag' # needed here? |
---|
402 | self.nag_tag = 'success' |
---|
403 | else: |
---|
404 | # distinguish between an ERROR and WARNING stderrReturnStatus |
---|
405 | # WARNING when the program finished normally, as seen in the madx output file, but with stderr |
---|
406 | # ERROR when |
---|
407 | if finishedNormally: |
---|
408 | stderrReturnStatus = 'warning' |
---|
409 | else: |
---|
410 | stderrReturnStatus = 'failure' |
---|
411 | |
---|
412 | if m == 'Makefile_develop': |
---|
413 | whichTag = 'dev' |
---|
414 | self.dev_tag = stderrReturnStatus |
---|
415 | if m == 'Makefile_nag': |
---|
416 | whichTag = 'nag' |
---|
417 | self.nag_tag = stderrReturnStatus |
---|
418 | htmlFile = htmlRootDir+"/details/"+"Error_"+whichTag+"_"+self.name+"_"+self.testcaseDir+".htm" |
---|
419 | if whichTag == 'nag': |
---|
420 | self.nag_link = "./details/"+"Error_"+whichTag+"_"+self.name+"_"+self.testcaseDir+".htm" |
---|
421 | elif whichTag == 'dev': |
---|
422 | self.dev_link = "./details/"+"Error_"+whichTag+"_"+self.name+"_"+self.testcaseDir+".htm" |
---|
423 | else: |
---|
424 | raise("should never reach this point") |
---|
425 | |
---|
426 | errorPage = ErrorWebPage(htmlFile,lines,stderrReturnStatus,startTime,endTime) |
---|
427 | #errorPage = ErrorWebPage("/user/nougaret/MAD-X/madX/testing/bidonErrorPage.html",lines) |
---|
428 | errorPage.output() |
---|
429 | ferror.close() |
---|
430 | else: |
---|
431 | if options.verbose: |
---|
432 | print("error: expected to find redirected stderr") |
---|
433 | |
---|
434 | os.chdir(currentDir) # back to the initial work directory |
---|
435 | |
---|
436 | def compareOutputWithReference(self): |
---|
437 | # only for the main Makefile |
---|
438 | # must pick-up all files under the /output directory and compare them with the reference |
---|
439 | outputDir = self.topDir+'/output' |
---|
440 | if options.verbose: |
---|
441 | print "outputDir="+outputDir |
---|
442 | files = os.listdir(outputDir) |
---|
443 | |
---|
444 | for fname in files: # the short name of the file without its path prefix |
---|
445 | try: |
---|
446 | os.remove('./tempfile') |
---|
447 | except: |
---|
448 | pass |
---|
449 | |
---|
450 | createdFile = outputDir + '/' + fname # fname with adequate prefix |
---|
451 | referenceFile = self.resources[0].source[:self.resources[0].source.rfind('/')] +'/'+ fname # fname with adequate prefix -> take the madx input host directory in the CVS and add the filename |
---|
452 | |
---|
453 | # make sure the reference file exists, otherwise we shall omit to look for differences |
---|
454 | if os.path.exists(referenceFile): |
---|
455 | |
---|
456 | #print("name of createdFile="+createdFile) |
---|
457 | #print("name of referenceFile="+referenceFile) |
---|
458 | |
---|
459 | # specific case when the HTML file name is of the form XX.map or XX.map.htm |
---|
460 | # webserver will fail to display the HTML although one can open it from the webfolder... |
---|
461 | # to overcome this limitation, we need to juggle with the HTML file name |
---|
462 | sublinkname = fname.replace('.map','.maAap') |
---|
463 | |
---|
464 | htmlFile = "./temp.html" # output HTML file, to be delivered to the web site... |
---|
465 | weblink = "./DiffResult_" + self.name + "_" + self.testcaseDir + "_" + sublinkname + ".htm" # again test.name stands for the target |
---|
466 | htmlFile = htmlRootDir+"/details/"+weblink |
---|
467 | |
---|
468 | os.system('./MadDiff.pl '+createdFile+' '+referenceFile+' ' +htmlFile+' > ./tempfile') |
---|
469 | |
---|
470 | tf = open("./tempfile","r") |
---|
471 | outcome = tf.readlines()[0] # single line |
---|
472 | |
---|
473 | if outcome == 'failure': |
---|
474 | pass |
---|
475 | #print("this is a failure") |
---|
476 | elif outcome == 'warning': |
---|
477 | pass |
---|
478 | #print("this is a warning") |
---|
479 | elif outcome == 'quasi-success': |
---|
480 | pass |
---|
481 | #print("this is a quasi-sucess") |
---|
482 | elif outcome == 'success': |
---|
483 | pass |
---|
484 | #print("this is a success") |
---|
485 | else: |
---|
486 | raise("should never reach this point, outcome = "+outcome) |
---|
487 | |
---|
488 | tf.close() |
---|
489 | |
---|
490 | os.remove('./tempfile') |
---|
491 | |
---|
492 | # store the short name of the output file, together with the outcome of the comparison |
---|
493 | |
---|
494 | # skip .ps and .eps files |
---|
495 | if fname[-3:]=='.ps' or fname[-4:]=='.eps': |
---|
496 | pass # skip |
---|
497 | else: |
---|
498 | |
---|
499 | out = Output(fname,outcome,weblink) |
---|
500 | self.outputs.append(out) |
---|
501 | |
---|
502 | # store output file information in the test object for subsequent reuse |
---|
503 | # by the web page output |
---|
504 | |
---|
505 | |
---|
506 | else: |
---|
507 | outcome = 'omit' |
---|
508 | |
---|
509 | |
---|
510 | |
---|
511 | |
---|
512 | |
---|
513 | class Tester: |
---|
514 | |
---|
515 | |
---|
516 | |
---|
517 | def __init__(self): |
---|
518 | pass |
---|
519 | |
---|
520 | def run(self): |
---|
521 | |
---|
522 | page = WebPage(mainHtmlPage) |
---|
523 | |
---|
524 | # first extract examples from the repository |
---|
525 | if not options.keep_data: |
---|
526 | rep = Repository() |
---|
527 | Test.extracting = True |
---|
528 | page.output() |
---|
529 | rep.checkout() |
---|
530 | Test.extracting = False |
---|
531 | |
---|
532 | testinfoPattern = re.compile(r'^[\s\t]*(.+)[\s\t]*<[\s\t]*(.+)[\s\t]*>[\s\t]*([\w\.\d\-\_]+)[\s\t]*,?[\s\t]*'+\ |
---|
533 | '(subdirectory=)?[\s\t]*(.*)[\s\t]*$') |
---|
534 | # e.g. ./madx < touschek.lhcinjection.madx > touschek.lhcinjection.out, subdirectory=LHC_injection |
---|
535 | # or ./madx < lep.madx > lep.out |
---|
536 | |
---|
537 | # process TestScenario.xml |
---|
538 | os.system('xsltproc --stringparam what list_targets ProcessScenario.xsl'+\ |
---|
539 | ' TestScenario.xml > ./outfile') |
---|
540 | f = open('./outfile','r') |
---|
541 | targets = f.readlines() |
---|
542 | for i,target in enumerate(targets): |
---|
543 | targets[i] = target.strip('\n') |
---|
544 | f.close() |
---|
545 | os.remove('./outfile') |
---|
546 | |
---|
547 | if options.singleTarget: |
---|
548 | if not options.singleTarget in targets: |
---|
549 | raise("specified target '"+options.singleTarget+"' does not exists in " + str(targets)) |
---|
550 | |
---|
551 | for target in targets: |
---|
552 | |
---|
553 | #target = target.rstrip('\n') |
---|
554 | |
---|
555 | if options.omit: |
---|
556 | if target == options.omit: |
---|
557 | if options.verbose: |
---|
558 | print("skip "+options.omit) |
---|
559 | continue |
---|
560 | |
---|
561 | if options.singleTarget and not options.singleTarget == target: |
---|
562 | continue # skip this test |
---|
563 | |
---|
564 | # extract detailed information about each test |
---|
565 | os.system('xsltproc --stringparam what list_tests'+\ |
---|
566 | ' --stringparam target '+target+\ |
---|
567 | ' ProcessScenario.xsl TestScenario.xml > ./outfile') |
---|
568 | f = open('./outfile','r') |
---|
569 | testinfos = f.readlines() |
---|
570 | f.close() |
---|
571 | os.remove('./outfile') |
---|
572 | if options.singleCase: |
---|
573 | recognizedSingleCase = False # default, must be set to True if valid |
---|
574 | for testinfo in testinfos: |
---|
575 | testinfo = testinfo.rstrip('\n') |
---|
576 | |
---|
577 | m = testinfoPattern.match(testinfo) |
---|
578 | if m: |
---|
579 | program = m.group(1).rstrip() |
---|
580 | input = m.group(2).rstrip() |
---|
581 | output = m.group(3).rstrip() |
---|
582 | if m.lastindex == 5: |
---|
583 | subdirectory = m.group(5) |
---|
584 | else: |
---|
585 | subdirectory = 0 |
---|
586 | |
---|
587 | if options.singleCase and not options.singleCase == input: |
---|
588 | continue |
---|
589 | elif options.singleCase and options.singleCase == input: |
---|
590 | recognizedSingleCase = True |
---|
591 | |
---|
592 | test = Test(target,program,input,output,subdirectory) |
---|
593 | |
---|
594 | else: |
---|
595 | if options.verbose: |
---|
596 | print("failed to parse line "+testinfo) |
---|
597 | |
---|
598 | if options.singleCase and not recognizedSingleCase: |
---|
599 | raise("specified single case '"+options.singleCase+"' does not exist.") |
---|
600 | |
---|
601 | try: |
---|
602 | shutil.rmtree(testingDir) |
---|
603 | except: |
---|
604 | pass # directory absent |
---|
605 | |
---|
606 | page.output() # refresh the web page |
---|
607 | |
---|
608 | # now populate testingDir with all sources and associated resources |
---|
609 | for t in Test.tests: |
---|
610 | t.copyResourcesToTestingDir() |
---|
611 | |
---|
612 | |
---|
613 | # now run the tests |
---|
614 | for t in Test.tests: |
---|
615 | |
---|
616 | t.state = running |
---|
617 | page.output() # to mark the current test as running |
---|
618 | t.run() |
---|
619 | t.compareOutputWithReference() |
---|
620 | t.state = completed # or aborted? |
---|
621 | page.output() # refresh the web page |
---|
622 | |
---|
623 | # notify module keepers if required |
---|
624 | if options.mail: |
---|
625 | Notify.notify("jean-luc","test completion","test completed.") # for the time-being |
---|
626 | |
---|
627 | page.output() # refresh the web page for the last time |
---|
628 | |
---|
629 | |
---|
630 | class WebPage: |
---|
631 | |
---|
632 | def __init__(self,name): |
---|
633 | self.name = name |
---|
634 | |
---|
635 | def header(self): |
---|
636 | self.contents += '<DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2//EN">\n' |
---|
637 | self.contents += '<html>\n' |
---|
638 | self.contents += '<head>\n' |
---|
639 | # remove the auto refresh and rely on the user to refresh on demand |
---|
640 | #self.contents += '<meta http-equiv="refresh" content="5" >' # page reloads itself every 5 seconds |
---|
641 | # this tag should only be present while the test is running, and be absent when completed. |
---|
642 | self.contents += '<title>MAD testing main page</title>\n' |
---|
643 | self.contents += '<link rel=stylesheet href="./MadTestWebStyle.css" type="text/css">\n' |
---|
644 | self.contents += '</head>\n' |
---|
645 | |
---|
646 | def body(self): |
---|
647 | self.contents += '<body>\n' |
---|
648 | when = (datetime.datetime.now()).ctime() |
---|
649 | self.contents += when |
---|
650 | self.contents += '<table>\n' |
---|
651 | if Test.extracting: |
---|
652 | self.contents += '<tr><td width="80%"><center><b>Script extracting the repository since '+when+' ...</center></td><td width="20%"><center><img src="repository.gif" width="134" height="139"></b></center></td></tr>\n' # image showing running job |
---|
653 | else: |
---|
654 | for target in Target.targets: |
---|
655 | self.contents += '<tr class="test_target"><td colspan="2"><div align="center"><strong>'+target.name+'</strong></div></td></tr>\n' |
---|
656 | for i,t in enumerate(target.tests): |
---|
657 | if not options.dev: |
---|
658 | devClass = "test_case" # for the time being |
---|
659 | devLink = 'dev' # actually no link |
---|
660 | else: |
---|
661 | pass # will be either failure, warning or success |
---|
662 | devClass = t.dev_tag |
---|
663 | if not devClass == 'undefined': |
---|
664 | devLink = '<a href="'+t.dev_link+'">dev</a>' |
---|
665 | else: |
---|
666 | devLink = 'dev' # actually no link |
---|
667 | if not options.nag: |
---|
668 | nagClass = "test_case" # for the time being |
---|
669 | nagLink = 'nag' # actually no link |
---|
670 | else: |
---|
671 | pass # will be either failure, warning or success |
---|
672 | nagClass = t.nag_tag |
---|
673 | if not nagClass == 'undefined': |
---|
674 | nagLink = '<a href="./details/Error_nag_aperture_test_1.htm">nag</a>' |
---|
675 | nagLink = '<a href="'+t.nag_link+'">nag</a>' |
---|
676 | else: |
---|
677 | nagLink = 'nag' # actually no link |
---|
678 | |
---|
679 | self.contents += '<tr class="test_case"><td width=\"80%\">'+t.testcaseDir+\ |
---|
680 | ': '+t.program +'<'+t.input+'>'+t.output+\ |
---|
681 | '</td><td width=\"20%\"><table width=\"100%\" style=\"text-align: center\"><tr>' +\ |
---|
682 | '<td class="'+devClass+'">'+devLink+'</td>'+\ |
---|
683 | '<td class="'+nagClass+'">'+nagLink+'</td>'+\ |
---|
684 | '</tr></table></td></tr>\n'; |
---|
685 | if t.state == init or t.state == incomplete: |
---|
686 | for r in t.resources: |
---|
687 | self.contents += '<tr><td>'+r.name+'</td></tr>\n' |
---|
688 | elif t.state == running: |
---|
689 | when = (datetime.datetime.now()).ctime() |
---|
690 | self.contents += '<tr><td width="80%"><center><b>Script running this test since '+\ |
---|
691 | when+' ...</center></td><td width="20%"><center><img src="underConstruction.gif" width="134" height="139"></b></center>'+\ |
---|
692 | '</td></tr>\n' # image showing running job |
---|
693 | elif t.state == completed or t.state == aborted: |
---|
694 | for o in t.outputs: |
---|
695 | if not o.outcome == 'omit': |
---|
696 | self.contents += '<tr class="'+o.outcome+'"><td width=\"80%\">'+o.name+\ |
---|
697 | '<td width="30%"><a href="./details/'+o.weblink+'">'+o.outcome+'</a></td></tr>\n' |
---|
698 | else: |
---|
699 | self.contents += '<tr class="'+o.outcome+'"><td width=\"80%\">'+o.name+\ |
---|
700 | '<td width="30%">no file for reference</td></tr>\n' |
---|
701 | else: |
---|
702 | raise("should never reach this point") |
---|
703 | |
---|
704 | |
---|
705 | self.contents += '<table>\n' |
---|
706 | self.contents += '<body>\n' |
---|
707 | def footer(self): |
---|
708 | self.contents += '</html>\n' |
---|
709 | |
---|
710 | def output(self): |
---|
711 | self.contents = "" |
---|
712 | self.header() |
---|
713 | self.body() |
---|
714 | self.footer() |
---|
715 | f = open(self.name,'w') |
---|
716 | for c in self.contents: |
---|
717 | f.write(c) |
---|
718 | f.close() |
---|
719 | |
---|
720 | def display(self): |
---|
721 | os.system('firefox ' + self.name+ '&') |
---|
722 | |
---|
723 | class ErrorWebPage(WebPage): |
---|
724 | def __init__(self,name,stderr_lines,stderrReturnStatus,startTime,endTime): |
---|
725 | self.name = name |
---|
726 | self.stderr_lines = stderr_lines |
---|
727 | self.stderr_return_status = stderrReturnStatus |
---|
728 | self.startTime = startTime |
---|
729 | self.endTime = endTime |
---|
730 | |
---|
731 | def header(self): |
---|
732 | self.contents += '<DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2//EN">\n' |
---|
733 | self.contents += '<html>\n' |
---|
734 | self.contents += '<head>\n' |
---|
735 | # remove the auto refresh and rely on the user to refresh on demand |
---|
736 | #self.contents += '<meta http-equiv="refresh" content="5" >' # page reloads itself every 5 seconds |
---|
737 | # this tag should only be present while the test is running, and be absent when completed. |
---|
738 | self.contents += '<title>MAD testing main page</title>\n' |
---|
739 | self.contents += '<link rel=stylesheet href="../MadTestWebStyle.css" type="text/css">\n' |
---|
740 | self.contents += '</head>\n' |
---|
741 | def body(self): |
---|
742 | self.contents += '<body>\n' |
---|
743 | self.contents += '<p>Test started '+self.startTime+', ended '+self.endTime+'</p>\n' |
---|
744 | self.contents += '<table width="75%" border="0">\n' |
---|
745 | # top coloured banner |
---|
746 | self.contents += '<tr class='+self.stderr_return_status+'><td width="80%">Contents of stderr</td><td width="20%">'+\ |
---|
747 | self.stderr_return_status+"</td></tr>\n" |
---|
748 | for l in self.stderr_lines: |
---|
749 | self.contents += '<tr><td colspan="2">'+l.rstrip("\n")+'</td></tr>\n' |
---|
750 | self.contents += "</table>\n" |
---|
751 | self.contents += '</body>\n' |
---|
752 | |
---|
753 | if __name__ == "__main__": |
---|
754 | usage = "%prog [options]" |
---|
755 | parser = optparse.OptionParser(usage) |
---|
756 | parser.add_option("--verbose","-v",help="display messages useful for debug", action="store_true") |
---|
757 | parser.add_option("--target","-t",help="select a specific target to run the test (e.g.ptc_twiss)",dest="singleTarget") |
---|
758 | parser.add_option("--case","-c",help="select a specific test-case to run the test, in combination with --target (e.g. example1.madx)",\ |
---|
759 | dest="singleCase") |
---|
760 | parser.add_option("--keep_data","-k",help="keep old data without extracting repository",action="store_true") |
---|
761 | parser.add_option("--quiet","-q",help="does not produce a web page",action="store_true") |
---|
762 | parser.add_option("--mail","-m",help="notify module keepers be e-mail",action="store_true") |
---|
763 | parser.add_option("--omit","-o",help="omit particular target causing trouble, for debugging",dest='omit') |
---|
764 | |
---|
765 | parser.add_option("--dev","-d",help="compiles and runs with Makefile_develop",action="store_true") |
---|
766 | parser.add_option("--nag","-n",help="compiles and runs with Makefile_nag",action="store_true") |
---|
767 | |
---|
768 | parser.add_option("--silent","-s",help="no e-mail (currently has no effect)",action="store_true") # currently has no effect |
---|
769 | |
---|
770 | (options, args) = parser.parse_args() |
---|
771 | |
---|
772 | if options.singleCase and not options.singleTarget: |
---|
773 | print("option --case assumes option --target is selected as well") |
---|
774 | |
---|
775 | # for the time-being do not bother about the --dev and --nag options |
---|
776 | makefiles = ['Makefile'] |
---|
777 | |
---|
778 | if options.dev: |
---|
779 | makefiles.append('Makefile_develop') |
---|
780 | |
---|
781 | if options.nag: |
---|
782 | makefiles.append('Makefile_nag') |
---|
783 | |
---|
784 | tester = Tester() |
---|
785 | tester.run() |
---|
786 | if options.verbose: |
---|
787 | print("program completed.") |
---|
788 | |
---|