Changeset 513
- Timestamp:
- Jun 27, 2009, 9:58:12 PM (15 years ago)
- Location:
- CMT/HEAD
- Files:
-
- 3 edited
Legend:
- Unmodified
- Added
- Removed
-
CMT/HEAD/ChangeLog
r512 r513 1 2009-06-27 <rybkin@lal.in2p3.fr> 402 2 3 * mgr/cmt_svn_checkout.py: Introduce the version using the Subversion CLI 4 (rather than Subversion Python bindings). Introduce the CMTHEADVERSION 5 environment variable to hold the string that is written into the 6 version.cmt file on "cmt checkout" of the package's trunk when it is 7 different from any existing tag. The value may contain the templates 8 <package>, <PACKAGE>, <revision>, where <revision> is replaced with 9 last_changed_rev of the trunk. Use "cmt -q config" and always return 0 from 10 function generate of CmtContext to avoid error messages even when the 11 function actually succeeds. Treat URL, OFFSET, PATH as (Subversion scheme) 12 URLs, introduce function urljoin (using "the last wins algorithm") to 13 construct the checkout URL from them 14 * mgr/requirements: For Linux, set macro svn_checkout_command to use the 15 Python in the PATH to execute mgr/cmt_svn_checkout.py 16 1 17 2009-06-18 <rybkin@lal.in2p3.fr> 401 2 18 -
CMT/HEAD/mgr/cmt_svn_checkout.py
r512 r513 23 23 """ 24 24 25 __version__ = '0. 2.0'26 __date__ = 'Thu Jun 182009'25 __version__ = '0.3.0' 26 __date__ = 'Thu Jun 27 2009' 27 27 __author__ = 'Grigory Rybkin' 28 28 … … 33 33 import os.path 34 34 import urlparse 35 # for Python 2.3 and older 36 if sys.version_info[0] == 2 and sys.version_info[1] < 4: 37 for p in ('svn', 'svn+ssh'): 38 if p not in urlparse.uses_netloc: 39 urlparse.uses_netloc.append(p) 35 40 import tempfile 41 import commands 42 import re 36 43 37 44 self = 'cmt_svn_checkout.py' 38 try:39 from svn import client, core40 except ImportError, e:41 print >>sys.stderr, '%s: cannot import Subversion Python bindings: %s' \42 % (self, str(e))43 sys.exit(1)45 # try: 46 # from svn import client, core 47 # except ImportError, e: 48 # print >>sys.stderr, '%s: cannot import Subversion Python bindings: %s' \ 49 # % (self, str(e)) 50 # sys.exit(1) 44 51 45 52 class ClientContext(object): 46 def __init__(self): 47 core.svn_config_ensure(None) 48 self.ctx = client.create_context() 49 self.providers = [ 50 client.get_simple_provider(), 51 client.get_username_provider(), 52 client.get_ssl_server_trust_file_provider(), 53 client.get_ssl_client_cert_file_provider(), 54 client.get_ssl_client_cert_pw_file_provider() 55 ] 56 self.ctx.auth_baton = core.svn_auth_open(self.providers) 57 self.ctx.config = core.svn_config_get_config(None) 58 59 def __call__(self): 60 return self.ctx 53 54 schemes = ('http', 'https', 'svn', 'svn+ssh', 'file') 55 56 def svn_path_canonicalize(self, path): 57 """Return a new path (or URL) like path, but transformed such that some types of path specification redundancies are removed. 58 59 This involves collapsing redundant "/./" elements, removing multiple adjacent separator characters, removing trailing separator characters, and possibly other semantically inoperative transformations. Convert the scheme and hostname to lowercase.""" 60 scheme, netloc, path, query, fragment = urlparse.urlsplit(path) 61 scheme = scheme.lower() 62 netloc = netloc.lower() 63 if path.startswith('/'): b = '/' 64 else: b = '' 65 path = b + '/'.join([s for s in path.split('/') if s and s != '.']) 66 return urlparse.urlunsplit((scheme, netloc, path, query, fragment)) 67 68 def urljoin(self, *args): 69 urls = [urlparse.urlsplit(arg) for arg in args] 70 if not urls: return '' 71 72 schemes = [url[0] for url in urls] 73 schemes.reverse() 74 for i, s in enumerate(schemes): 75 if s and s in self.schemes: 76 scheme = s 77 index = i 78 break 79 else: 80 scheme = '' 81 index = len(urls) - 1 82 83 netlocs = [url[1] for url in urls] 84 netlocs.reverse() 85 for i, s in enumerate(netlocs[:index + 1]): 86 if s: 87 netloc = s 88 index = i 89 break 90 else: 91 netloc = '' 92 93 path = posixpath.join(*[url[2] for url in urls][len(urls) - 1 - index:]) 94 95 query = fragment = '' 96 return urlparse.urlunsplit((scheme, netloc, path, query, fragment)) 97 98 # scheme = self.last_nonempty([url[0] for url in urls if url[0] in self.schemes]) 99 # netloc = self.last_nonempty([url[1] for url in urls]) 100 # path = posixpath.join(*[url[2] for url in urls]) 101 # query = self.last_nonempty([url[3] for url in urls]) 102 # fragment = self.last_nonempty([url[4] for url in urls]) 103 104 # return urlparse.urlunsplit((scheme, netloc, path, query, fragment)) 105 106 def last_nonempty(self, list, default = ''): 107 list.reverse() 108 for i in list: 109 if i: return i 110 else: 111 return default 112 113 # def __init__(self): 114 # core.svn_config_ensure(None) 115 # self.ctx = client.create_context() 116 # self.providers = [ 117 # client.get_simple_provider(), 118 # client.get_username_provider(), 119 # client.get_ssl_server_trust_file_provider(), 120 # client.get_ssl_client_cert_file_provider(), 121 # client.get_ssl_client_cert_pw_file_provider() 122 # ] 123 # self.ctx.auth_baton = core.svn_auth_open(self.providers) 124 # self.ctx.config = core.svn_config_get_config(None) 125 126 # def __call__(self): 127 # return self.ctx 61 128 62 129 def cwd(): … … 95 162 config = True, 96 163 with_version_directory = True, 97 cleanup = False): 164 cleanup = False, 165 head_version = None): 98 166 self.config = config 99 167 self.with_version_directory = with_version_directory 100 168 self.cleanup = cleanup 169 self.head_version = head_version 170 self.env() 171 172 def env(self): 173 if self.head_version is None: 174 self.set_head_version(os.getenv('CMTHEADVERSION', 'HEAD')) 175 # print >>sys.stderr, 'env: set head_version: %s' % self.head_version 176 177 def set_head_version(self, version): 178 self.head_version = str(version).replace('<package>', '%(package)s').replace('<PACKAGE>', '%(PACKAGE)s').replace('<revision>', '%(revision)i') 179 180 def eval_head_version(self, m): 181 return self.head_version % \ 182 {'package' : m.package, 183 'PACKAGE' : m.package.upper(), 184 'revision' : m.info.last_changed_rev} 101 185 102 186 def write(self, p, version): … … 124 208 #print >> sys.stderr, 'generate:', p 125 209 curdir = cwd() 126 cmd = 'cmt -disable_warnings' 210 # cmd = 'cmt -disable_warnings' 211 cmd = 'cmt -q' 127 212 if self.with_version_directory: 128 213 cmd += ' -with_version_directory' … … 138 223 if sc != 0: sc = 1 139 224 cd(curdir) 140 return sc 225 # return sc 226 return 0 141 227 142 228 def configure(self, path, version): … … 157 243 self.module = module 158 244 self.init = False 245 class info(object): 246 last_changed_rev = 0 247 self.info = info 248 # print >>sys.stderr, 'init module %s: last_changed_rev %i' % \ 249 # (self.module, self.info.last_changed_rev) 159 250 160 251 class Checkout(object): … … 167 258 version_dir = None, 168 259 directory = None, 169 offset = None,260 offset = '', 170 261 modules = []): 171 262 self.url = url … … 182 273 def reposLayout(self): 183 274 if self.url is None: 184 try: 185 self.url = os.environ['SVNROOT'] 186 except KeyError: 187 pass 275 self.url = os.getenv('SVNROOT', '') 276 # try: 277 # self.url = os.environ['SVNROOT'] 278 # except KeyError: 279 # pass 188 280 if self.trunk is None: 189 try: 190 self.trunk = os.environ['SVNTRUNK'] 191 except KeyError: 192 self.trunk = 'trunk' 281 self.trunk = os.getenv('SVNTRUNK', 'trunk') 193 282 if self.tags is None: 194 try: 195 self.tags = os.environ['SVNTAGS'] 196 except KeyError: 197 self.tags = 'tags' 283 self.tags = os.getenv('SVNTAGS', 'tags') 198 284 if self.branches is None: 199 try: 200 self.branches = os.environ['SVNBRANCHES'] 201 except KeyError: 202 self.branches = 'branches' 285 self.branches = os.getenv('SVNBRANCHES', 'branches') 203 286 204 287 def cmtRepos(self): … … 214 297 self.path = path 215 298 self.info = info 299 pool.info = info 216 300 217 301 def cmp(self, path1, path2, client_context): 218 outfile = tempfile.TemporaryFile('wb') 219 errfile = tempfile.TemporaryFile('wb') 220 outb = outfile.tell() 221 errb = errfile.tell() 222 client.diff([] 223 , path1 224 , self.head_revision 225 , path2 226 , self.head_revision 227 , True, True, False 228 , outfile 229 , errfile 230 , client_context()) 231 232 # position at the end of the file 233 outfile.seek(0, 2) 234 errfile.seek(0, 2) 235 oute = outfile.tell() 236 erre = errfile.tell() 237 238 if erre > errb: return 2 239 elif oute > outb: return 1 302 cmd = 'svn diff %s %s' % (path1, path2) 303 # cmd = 'svn diff --summarize %s %s' % (path1, path2) 304 sc, out = commands.getstatusoutput(cmd) 305 if sc != 0: 306 return 2 307 if out: return 1 240 308 else: return 0 309 310 # outfile = tempfile.TemporaryFile('wb') 311 # errfile = tempfile.TemporaryFile('wb') 312 # outb = outfile.tell() 313 # errb = errfile.tell() 314 # client.diff([] 315 # , path1 316 # , self.head_revision 317 # , path2 318 # , self.head_revision 319 # , True, True, False 320 # , outfile 321 # , errfile 322 # , client_context()) 323 324 # # position at the end of the file 325 # outfile.seek(0, 2) 326 # errfile.seek(0, 2) 327 # oute = outfile.tell() 328 # erre = errfile.tell() 329 330 # if erre > errb: return 2 331 # elif oute > outb: return 1 332 # else: return 0 241 333 242 334 def trunk_tag(self, module, client_context): … … 246 338 its last_changed_rev is greater than the trunk created_rev, None otherwise. 247 339 """ 248 try: 249 trunk = core.svn_path_canonicalize(posixpath.join(module.url, self.trunk)) 250 client.info(trunk, 251 self.head_revision, 252 self.head_revision, 253 self.info_receiver, 254 False, 255 client_context()) 340 trunk = posixpath.join(module.url, self.trunk) 341 # trunk = posixpath.normpath(posixpath.join(module.url, self.trunk)) 342 cmd = 'svn info %s' % trunk 343 sc, out = commands.getstatusoutput(cmd) 344 if sc != 0: 345 return None 346 p = r'last\s+changed\s+rev:\s+(?P<rev>\d+)' 347 m = re.search(p, out, re.I) 348 if m: 349 class info(object): pass 350 info.last_changed_rev = int(m.group('rev')) 351 self.info_receiver(trunk, info, module) 352 # self.info_receiver(trunk, info, None) 353 # print >>sys.stderr, '%s: last_changed_rev %i' % \ 354 # (trunk, self.info.last_changed_rev) 355 # last_changed_rev = int(m.group('rev')) 356 else: 357 return None 358 359 tags = posixpath.join(module.url, self.tags) 360 # tags = posixpath.normpath(posixpath.join(module.url, self.tags)) 361 cmd = 'svn ls -v %s' % tags 362 sc, out = commands.getstatusoutput(cmd) 363 if sc != 0: 364 return None 365 366 tags_dirent = [line.split() for line in out.splitlines()] 367 rev_tag = dict([(int(line[0]), line[-1].rstrip(posixpath.sep)) for line in tags_dirent if line[-1].endswith(posixpath.sep)]) 368 revs = rev_tag.keys() 369 revs.sort() 370 revs.reverse() 371 372 for rev in revs: 373 if rev < self.info.last_changed_rev: break 374 # if rev < last_changed_rev: break 375 tag = posixpath.join(tags, rev_tag[rev]) 376 # tag = posixpath.normpath(posixpath.join(tags, rev_tag[rev])) 377 if 0 == self.cmp(trunk, tag, client_context): 378 return rev_tag[rev] 379 380 return None 381 382 # try: 383 # trunk = core.svn_path_canonicalize(posixpath.join(module.url, self.trunk)) 384 # client.info(trunk, 385 # self.head_revision, 386 # self.head_revision, 387 # self.info_receiver, 388 # False, 389 # client_context()) 256 390 257 tags = core.svn_path_canonicalize(posixpath.join(module.url, self.tags))258 tags_dirent = client.ls(tags,259 self.head_revision,260 False,261 client_context())391 # tags = core.svn_path_canonicalize(posixpath.join(module.url, self.tags)) 392 # tags_dirent = client.ls(tags, 393 # self.head_revision, 394 # False, 395 # client_context()) 262 396 263 rev_tag = dict([(tags_dirent[p].created_rev, p) for p in tags_dirent if tags_dirent[p].kind == core.svn_node_dir])264 revs = rev_tag.keys()265 revs.sort()266 revs.reverse()267 268 for rev in revs:269 if rev < self.info.last_changed_rev: break270 tag = core.svn_path_canonicalize(posixpath.join(tags, rev_tag[rev]))271 if 0 == self.cmp(trunk, tag, client_context):272 return rev_tag[rev]273 274 return None275 except core.SubversionException, e:276 return None397 # rev_tag = dict([(tags_dirent[p].created_rev, p) for p in tags_dirent if tags_dirent[p].kind == core.svn_node_dir]) 398 # revs = rev_tag.keys() 399 # revs.sort() 400 # revs.reverse() 401 402 # for rev in revs: 403 # if rev < self.info.last_changed_rev: break 404 # tag = core.svn_path_canonicalize(posixpath.join(tags, rev_tag[rev])) 405 # if 0 == self.cmp(trunk, tag, client_context): 406 # return rev_tag[rev] 407 408 # return None 409 # except core.SubversionException, e: 410 # return None 277 411 278 412 def initialize(self, cmt_context, client_context): 279 413 sc = 0 280 self.head_revision = core.svn_opt_revision_t() 281 self.head_revision.kind = core.svn_opt_revision_head 414 # self.head_revision = core.svn_opt_revision_t() 415 # self.head_revision.kind = core.svn_opt_revision_head 416 417 # canonicalize: 418 self.url = client_context.svn_path_canonicalize(self.url) 419 self.offset = client_context.svn_path_canonicalize(self.offset) 282 420 283 421 for m in self.modules: 284 if urlparse.urlparse(m.module)[0] in ('http', 'https', 285 'svn', 'svn+ssh', 'file'): 286 m.url = m.module 287 m.path = posixpath.basename(m.module) 288 elif posixpath.isabs(m.module): 289 m.url = urlparse.urljoin('file://', m.module) 290 m.path = posixpath.basename(m.module) 291 else: 292 if self.url is None: 293 self.cmtRepos() 294 295 if self.offset is None: 296 m.url = posixpath.join(self.url, m.module) 297 else: 298 m.url = posixpath.join(self.url, self.offset, m.module) 299 m.path = os.path.join(*m.module.split(posixpath.sep)) 422 m.module = client_context.svn_path_canonicalize(m.module) 423 m.url = client_context.urljoin(self.url, self.offset, m.module) 424 425 if urlparse.urlparse(m.url)[0] not in client_context.schemes: 426 error('%s: Not a valid Subversion URL' % m.url) 427 sc += 1; continue 428 # print >>sys.stderr, '%s: URL constructed from %s %s %s' % \ 429 # (m.url, `self.url`, `self.offset`, `m.module`) 430 431 m.package = posixpath.basename(m.url) 300 432 301 433 if self.directory is not None: 302 434 m.path = os.path.normpath(self.directory) 303 435 else: 436 scheme, netloc, path, query, fragment = urlparse.urlsplit(m.module) 437 if not scheme and not netloc: 438 m.path = os.path.normpath(os.path.join(*path.split(posixpath.sep))) 439 else: 440 m.path = posixpath.basename(m.url) 441 304 442 if self.version is None: 305 443 m.head = True 306 m.version = self.trunk_tag(m, client_context) or 'HEAD' 444 m.version = self.trunk_tag(m, client_context) or \ 445 cmt_context.eval_head_version(m) 446 # print >>sys.stderr, 'set version: %s' % m.version 307 447 else: 308 448 m.head = False … … 312 452 m.URL = [posixpath.join(m.url, self.trunk)] 313 453 else: 314 # m.url = posixpath.join(m.url, self.tags, m.version)315 454 m.URL = [posixpath.join(m.url, p, m.version) 316 455 for p in (self.tags, self.branches)] 317 #m.URL = [posixpath.join(m.url, self.tags, m.version), 318 # posixpath.join(m.url, self.branches, m.version)] 319 #m.url = core.svn_path_canonicalize(m.url) 320 m.URL = [core.svn_path_canonicalize(url) for url in m.URL] 456 m.URL = [client_context.svn_path_canonicalize(url) for url in m.URL] 457 # m.URL = [core.svn_path_canonicalize(url) for url in m.URL] 321 458 322 459 if cmt_context.with_version_directory: … … 341 478 err = [] 342 479 for url in m.URL: 343 try: 344 #print 'client.checkout2:', url, m.path 345 result_rev = client.checkout2(url, 346 m.path, 347 self.head_revision, 348 self.head_revision, 349 True, 350 True, 351 client_context()) 352 except core.SubversionException, e: 480 # cmd = 'svn checkout -q %s %s 2>/dev/null' % (url, m.path) 481 cmd = 'svn checkout -q %s %s' % (url, m.path) 482 sc, e = commands.getstatusoutput(cmd) 483 #sc = os.system(cmd) 484 if 0 == sc: 485 # try: 486 # #print 'client.checkout2:', url, m.path 487 # result_rev = client.checkout2(url, 488 # m.path, 489 # self.head_revision, 490 # self.head_revision, 491 # True, 492 # True, 493 # client_context()) 494 # except core.SubversionException, e: 495 # err.append(e) 496 # continue 497 done = True 498 break 499 else: 353 500 err.append(e) 354 501 continue 355 done = True356 break357 502 358 503 if not done: 359 504 for e in err: 360 505 error(e) 361 #print >> sys.stderr, e 362 sc += 1 506 # #print >> sys.stderr, e 507 # sc += 1 508 # print >> sys.stderr, 'Failed to checkout %s into %s.' % \ 509 # (' or '.join(m.URL), m.path) 510 sc += 1 363 511 continue 364 512 -
CMT/HEAD/mgr/requirements
r502 r513 475 475 476 476 macro svn_checkout_command "" \ 477 Linux " /usr/bin/python ${CMTROOT}/mgr/cmt_svn_checkout.py " \477 Linux "python ${CMTROOT}/mgr/cmt_svn_checkout.py " \ 478 478 Darwin "python ${CMTROOT}/mgr/cmt_svn_checkout.py " \ 479 479 VisualC "python %CMTROOT%\mgr\cmt_svn_checkout.py "
Note: See TracChangeset
for help on using the changeset viewer.