Changeset 648 for CMT

Timestamp:

Jul 9, 2013, 12:08:28 PM (11 years ago)

Author:

rybkin

Message:

See C.L. 507

Location:

CMT/HEAD

Files:

• ChangeLog (modified) (1 diff)
• mgr/cmt_svn_checkout.py (modified) (6 diffs)

CMT/HEAD/ChangeLog

@@ +1 @@
+2013-07-09    <rybkin@lal.in2p3.fr> 507
+
+        * mgr/cmt_svn_checkout.py: Use a different algorithm in the class Checkout
+        function trunk_tag in order to optimise the performance, suggested by
+        Wim Lavrijsen. Add --debug command line option and SVNDEBUG environment
+        variable to facilitate debugging
+
 2013-06-25    <rybkin@lal.in2p3.fr> 506
 

CMT/HEAD/mgr/cmt_svn_checkout.py

@@ -15 +15 @@
       --with_version_directory create version directory upon PATH checkout (default)
       --url=URL                checkout PATH from repository URL
+      --debug                  print lots of debugging information
   -h, --help                   display this help and exit
       --version                output version information and exit
@@ -23 +24 @@
 """
 
-__version__ = '0.6.0'
-__date__ = 'Fri Mar 09 2012'
+__version__ = '0.7.0'
+__date__ = 'Tue Jul 09 2013'
 __author__ = 'Grigory Rybkin'
 
@@ -40 +41 @@
 import tempfile
 import re
+
+import logging
 
 self = 'cmt_svn_checkout.py'
@@ -378 +381 @@
         """Attempt to determine the tag of the module's trunk.
         
-        Return the tag if its contents are the same as those of the trunk, and
-        its last_changed_rev is greater than the trunk created_rev, None otherwise.
+        Return the tag if its files are copied from the trunk and
+        their last_changed_rev numbers are the same as those of the trunk files.
         """
         trunk = posixpath.join(module.url, self.trunk)
 #        trunk = posixpath.normpath(posixpath.join(module.url, self.trunk))
+        cmd = 'svn ls -vR %s' % trunk
+        sc, out = Utils.getstatusoutput(cmd)
+        logger.debug('%s\n%s' % (cmd, out))
+        if sc != 0:
+            return None
+        trunk_dirent = [line.split() for line in out.splitlines()]
+        trunk_revs = dict([(line[-1], int(line[0])) for line in trunk_dirent if not line[-1].endswith(posixpath.sep)])
+        logger.debug('%s' % trunk_revs)
+
+        curdir = posixpath.curdir + posixpath.sep
+        for line in trunk_dirent:
+            if line[-1] == curdir:
+                class info(object): pass
+                info.last_changed_rev = int(line[0])
+                self.info_receiver(trunk, info, module)
+                logger.debug('last_changed_rev: %d' % info.last_changed_rev)
+                break
+
+#        cmd = 'svn info %s' % trunk
+#         p = r'last\s+changed\s+rev:\s+(?P<rev>\d+)'
+#         m = re.search(p, out, re.I)
+#         if m:
+#             class info(object): pass
+#             info.last_changed_rev = int(m.group('rev'))
+#             self.info_receiver(trunk, info, module)
+# #            self.info_receiver(trunk, info, None)
+# #             print >>sys.stderr, '%s: last_changed_rev %i' % \
+# #                   (trunk, self.info.last_changed_rev)
+# #            last_changed_rev = int(m.group('rev'))
+#         else:
+#             return None
+
+        tags = posixpath.join(module.url, self.tags)
+#        tags = posixpath.normpath(posixpath.join(module.url, self.tags))
+        cmd = 'svn ls -vR %s' % tags
+#        cmd = 'svn ls -v %s' % tags
+        sc, out = Utils.getstatusoutput(cmd)
+        logger.debug('%s\n%s' % (cmd, out))
+        if sc != 0:
+            return None
+        tags_dirent = [line.split() for line in out.splitlines()]
+        tags_revs = dict()
+        for ent in tags_dirent:
+            try:
+                tag, path = ent[-1].split(posixpath.sep, 1)
+            except ValueError:
+                continue
+            if tag not in tags_revs:
+                tags_revs[tag] = dict()
+            if path and not path.endswith(posixpath.sep):
+                # assume there are no empty directories in the tag
+                tags_revs[tag][path] = int(ent[0])
+        logger.debug('%s' % tags_revs)
+
         cmd = 'svn info %s' % trunk
         sc, out = Utils.getstatusoutput(cmd)
+        logger.debug('%s\n%s' % (cmd, out))
         if sc != 0:
             return None
-        p = r'last\s+changed\s+rev:\s+(?P<rev>\d+)'
+        p = r'repository\s+root:\s+(?P<root>%s://\S+)' % \
+            ('(?:' + '|'.join(map(re.escape, client_context.schemes)) + ')')
         m = re.search(p, out, re.I)
+        logger.debug('pattern: %r' % (p))
         if m:
-            class info(object): pass
-            info.last_changed_rev = int(m.group('rev'))
-            self.info_receiver(trunk, info, module)
-#            self.info_receiver(trunk, info, None)
-#             print >>sys.stderr, '%s: last_changed_rev %i' % \
-#                   (trunk, self.info.last_changed_rev)
-#            last_changed_rev = int(m.group('rev'))
+            root = m.group('root')
         else:
             return None
-
-        tags = posixpath.join(module.url, self.tags)
-#        tags = posixpath.normpath(posixpath.join(module.url, self.tags))
-        cmd = 'svn ls -v %s' % tags
+        logger.debug('root: %s' % root)
+        trunk_path = trunk[len(root):]
+        tags_path = tags[len(root):]
+        logger.debug('trunk_path: %s' % trunk_path)
+        logger.debug('tags_path: %s' % tags_path)
+        offset = len(trunk) - len(root) + len(posixpath.sep)
+        
+        # Usually, a tag is created as a server-side copy.
+        # Sometimes, a tag is created as a copy of WC (working copy)
+        # after commit.
+        # Below, we try to take into account the latter case.
+        cmd = 'svn log -v -q %s' % tags
         sc, out = Utils.getstatusoutput(cmd)
+        logger.debug('%s\n%s' % (cmd, out))
         if sc != 0:
             return None
-
-        tags_dirent = [line.split() for line in out.splitlines()]
-        rev_tag = dict([(int(line[0]), line[-1].rstrip(posixpath.sep)) for line in tags_dirent if line[-1].endswith(posixpath.sep)])
-        revs = rev_tag.keys()
-        revs.sort()
-        revs.reverse()
-
-        for rev in revs:
-            if rev < self.info.last_changed_rev: break
-#            if rev < last_changed_rev: break
-            tag = posixpath.join(tags, rev_tag[rev])
-#            tag = posixpath.normpath(posixpath.join(tags, rev_tag[rev]))
-            if 0 == self.cmp(trunk, tag, client_context):
-                return rev_tag[rev]
+        p = re.compile(
+            r'^-{5,}$\s^r\d+.+$\s^Changed paths:$\s^\s+A\s+%s%s(?P<tag>[^%s]+)\s+\(from %s:\d+\)$(?P<replaced>(?:\s^\s+(?:R|A|D)\s+%s%s(?P=tag)%s(?P<path>.+)(?:\s+\(from %s%s(?P=path):\d+\))?$)*)' % (re.escape(tags_path), posixpath.sep, posixpath.sep, re.escape(trunk_path), re.escape(tags_path), posixpath.sep, posixpath.sep, re.escape(trunk_path), posixpath.sep)
+            , re.M
+            )
+        tags_copied = list()
+        tags_replaced_revs = dict()
+        for m in p.finditer(out):
+            logger.debug('tag: %s replaced: %r' % (m.group('tag'), m.group('replaced')))
+            tags_copied.append(m.group('tag'))
+            for line in m.group('replaced').strip().splitlines():
+                l = line.split()
+                if len(l) == 2: continue # action code D - deleted paths
+                repl = l[3].rstrip(')')
+                i = repl.rindex(':')
+                path = repl[offset:i]
+                rev = int(repl[i + 1:])
+                logger.debug('path: %s rev: %d' % (path, rev))
+
+                if m.group('tag') not in tags_replaced_revs:
+                    tags_replaced_revs[m.group('tag')] = dict()
+                if path and not path.endswith(posixpath.sep):
+                    # assume there are no empty directories in the tag
+                    tags_replaced_revs[m.group('tag')][path] = rev
+
+        logger.debug('copied: %s' % tags_copied)
+        logger.debug('replaced: %s' % tags_replaced_revs)
+
+        for t in tags_revs.keys():
+            if t not in tags_copied:
+                del tags_revs[t]
+                logger.debug('%s: Not a trunk copy' % t)
+
+        for t in tags_replaced_revs:
+            if t in tags_revs:
+                tags_revs[t].update(tags_replaced_revs[t])
+                
+        for tag in tags_revs:
+            logger.debug('Compare: %s -> %s' % (tag, tags_revs[tag]))
+            if trunk_revs == tags_revs[tag]:
+                return tag
 
         return None
+
+#         tags_dirent = [line.split() for line in out.splitlines()]
+#         rev_tag = dict([(int(line[0]), line[-1].rstrip(posixpath.sep)) for line in tags_dirent if line[-1].endswith(posixpath.sep)])
+#         revs = rev_tag.keys()
+#         revs.sort()
+#         revs.reverse()
+
+#         for rev in revs:
+#             logger.debug('rev: %s' % rev)
+#             if rev < self.info.last_changed_rev: break
+# #            if rev < last_changed_rev: break
+#             tag = posixpath.join(tags, rev_tag[rev])
+# #            tag = posixpath.normpath(posixpath.join(tags, rev_tag[rev]))
+#             logger.debug('comparing: %s %s(%d)' % (trunk, tag, rev))
+#             if 0 == self.cmp(trunk, tag, client_context):
+#                 return rev_tag[rev]
+
+#         return None
 
 #         try:
@@ -572 +679 @@
                                     "offset=", "no_config",
                                     "with_version_directory",
-                                    "without_version_directory", "url="])
+                                    "without_version_directory", "url=", "debug"])
     except getopt.error, e:
         print >>sys.stderr, '%s: %s' % (self, str(e))
         print >>sys.stderr, "Try '%s --help' for more information." % self
         return 1
+
+    global logger
+    logging.basicConfig()
+    logger = logging.getLogger(self)
+    logger.setLevel(logging.INFO)
+    if os.getenv('SVNDEBUG'):
+        logger.setLevel(logging.DEBUG)
 
     cmt_context = CmtContext()
@@ -605 +719 @@
         elif o in ("--url",):
             checkout.url = v
+        elif o in ("--debug",):
+            logger.setLevel(logging.DEBUG)
 
     if not args: