Update Mercurial repository support to be compatible with Python 3 and remove support for Mercurial < 5.1 (#33784).

Patch by Harald Klimach (user:haraldkl), Olivier Houdas (user:olivier.houdas@geoconcept.com), Jakob Haufe (user:sur5r), and Sean Baggaley (user:NotTheActualSean).


git-svn-id: https://svn.redmine.org/redmine/trunk@23513 e93f8b46-1217-0410-a6f0-8f06a7374b81

lib/redmine/scm/adapters/mercurial/redminehelper.py

@@ -5,6 +5,9 @@
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
+
+# [Nomadia-changes] Patch from Redmine.org #33784 : adapt to Python 3.0
+
 """helper commands for Redmine to reduce the number of hg calls
 
 To test this extension, please try::
@@ -45,17 +48,20 @@ Output example of rhmanifest::
       </repository>
     </rhmanifest>
 """
-import re, time, cgi, urllib
+import re, time, html, urllib
 from mercurial import cmdutil, commands, node, error, hg, registrar
 
 cmdtable = {}
 command = registrar.command(cmdtable) if hasattr(registrar, 'command') else cmdutil.command(cmdtable)
 
-_x = cgi.escape
-_u = lambda s: cgi.escape(urllib.quote(s))
+_x = lambda s: html.escape(s.decode('utf-8')).encode('utf-8')
+_u = lambda s: html.escape(urllib.parse.quote(s)).encode('utf-8')
+
+def unquoteplus(*args, **kwargs):
+    return urllib.parse.unquote_to_bytes(*args, **kwargs).replace(b'+', b' ')
 
 def _changectx(repo, rev):
-    if isinstance(rev, str):
+    if isinstance(rev, bytes):
        rev = repo.lookup(rev)
     if hasattr(repo, 'changectx'):
         return repo.changectx(rev)
@@ -70,10 +76,10 @@ def _tip(ui, repo):
         except TypeError:  # Mercurial < 1.1
             return repo.changelog.count() - 1
     tipctx = _changectx(repo, tiprev())
-    ui.write('<tip revision="%d" node="%s"/>\n'
+    ui.write(b'<tip revision="%d" node="%s"/>\n'
              % (tipctx.rev(), _x(node.hex(tipctx.node()))))
 
-_SPECIAL_TAGS = ('tip',)
+_SPECIAL_TAGS = (b'tip',)
 
 def _tags(ui, repo):
     # see mercurial/commands.py:tags
@@ -84,7 +90,7 @@ def _tags(ui, repo):
             r = repo.changelog.rev(n)
         except error.LookupError:
             continue
-        ui.write('<tag revision="%d" node="%s" name="%s"/>\n'
+        ui.write(b'<tag revision="%d" node="%s" name="%s"/>\n'
                  % (r, _x(node.hex(n)), _u(t)))
 
 def _branches(ui, repo):
@@ -104,136 +110,147 @@ def _branches(ui, repo):
             return repo.branchheads(branch)
     def lookup(rev, n):
         try:
-            return repo.lookup(rev)
+            return repo.lookup(str(rev).encode('utf-8'))
         except RuntimeError:
             return n
     for t, n, r in sorted(iterbranches(), key=lambda e: e[2], reverse=True):
         if lookup(r, n) in branchheads(t):
-            ui.write('<branch revision="%d" node="%s" name="%s"/>\n'
+            ui.write(b'<branch revision="%d" node="%s" name="%s"/>\n'
                      % (r, _x(node.hex(n)), _u(t)))
 
-def _manifest(ui, repo, path, rev):
+def _manifest(ui, repo, path, rev, path_encoding):
     ctx = _changectx(repo, rev)
-    ui.write('<manifest revision="%d" path="%s">\n'
-             % (ctx.rev(), _u(path)))
+    ui.write(b'<manifest revision="%d" path="%s">\n'
+             % (ctx.rev(), _u(path.decode(path_encoding))))
 
     known = set()
-    pathprefix = (path.rstrip('/') + '/').lstrip('/')
+    pathprefix = (path.decode(path_encoding).rstrip('/') + '/').lstrip('/')
     for f, n in sorted(ctx.manifest().iteritems(), key=lambda e: e[0]):
-        if not f.startswith(pathprefix):
+        fstr = f.decode(path_encoding)
+        if not fstr.startswith(pathprefix):
              continue
-        name = re.sub(r'/.*', '/', f[len(pathprefix):])
+        name = re.sub(r'/.*', '/', fstr[len(pathprefix):])
         if name in known:
             continue
         known.add(name)
 
         if name.endswith('/'):
-            ui.write('<dir name="%s"/>\n'
-                     % _x(urllib.quote(name[:-1])))
+            ui.write(b'<dir name="%s"/>\n'
+                     % _x(urllib.parse.quote(name[:-1]).encode('utf-8')))
         else:
             fctx = repo.filectx(f, fileid=n)
             tm, tzoffset = fctx.date()
-            ui.write('<file name="%s" revision="%d" node="%s" '
-                     'time="%d" size="%d"/>\n'
+            ui.write(b'<file name="%s" revision="%d" node="%s" '
+                     b'time="%d" size="%d"/>\n'
                      % (_u(name), fctx.rev(), _x(node.hex(fctx.node())),
                         tm, fctx.size(), ))
 
-    ui.write('</manifest>\n')
+    ui.write(b'</manifest>\n')
 
-@command('rhannotate',
-         [('r', 'rev', '', 'revision'),
-          ('u', 'user', None, 'list the author (long with -v)'),
-          ('n', 'number', None, 'list the revision number (default)'),
-          ('c', 'changeset', None, 'list the changeset'),
+@command(b'rhannotate',
+         [(b'r', b'rev', b'', b'revision'),
+          (b'u', b'user', None, b'list the author (long with -v)'),
+          (b'n', b'number', None, b'list the revision number (default)'),
+          (b'c', b'changeset', None, b'list the changeset'),
          ],
-         'hg rhannotate [-r REV] [-u] [-n] [-c] FILE...')
+         b'hg rhannotate [-r REV] [-u] [-n] [-c] FILE...')
 def rhannotate(ui, repo, *pats, **opts):
-    rev = urllib.unquote_plus(opts.pop('rev', None))
+    rev = unquoteplus(opts.pop('rev', b''))
     opts['rev'] = rev
-    return commands.annotate(ui, repo, *map(urllib.unquote_plus, pats), **opts)
+    return commands.annotate(ui, repo, *map(unquoteplus, pats), **opts)
 
-@command('rhcat',
-               [('r', 'rev', '', 'revision')],
-               'hg rhcat ([-r REV] ...) FILE...')
+@command(b'rhcat',
+               [(b'r', b'rev', b'', b'revision')],
+               b'hg rhcat ([-r REV] ...) FILE...')
 def rhcat(ui, repo, file1, *pats, **opts):
-    rev = urllib.unquote_plus(opts.pop('rev', None))
+    rev = unquoteplus(opts.pop('rev', b''))
     opts['rev'] = rev
-    return commands.cat(ui, repo, urllib.unquote_plus(file1), *map(urllib.unquote_plus, pats), **opts)
+    return commands.cat(ui, repo, unquoteplus(file1), *map(unquoteplus, pats), **opts)
 
-@command('rhdiff',
-               [('r', 'rev', [], 'revision'),
-                ('c', 'change', '', 'change made by revision')],
-               'hg rhdiff ([-c REV] | [-r REV] ...) [FILE]...')
+@command(b'rhdiff',
+               [(b'r', b'rev', [], b'revision'),
+                (b'c', b'change', b'', b'change made by revision')],
+               b'hg rhdiff ([-c REV] | [-r REV] ...) [FILE]...')
 def rhdiff(ui, repo, *pats, **opts):
     """diff repository (or selected files)"""
     change = opts.pop('change', None)
     if change:  # add -c option for Mercurial<1.1
         base = _changectx(repo, change).parents()[0].rev()
-        opts['rev'] = [str(base), change]
+        opts['rev'] = [base, change]
     opts['nodates'] = True
-    return commands.diff(ui, repo, *map(urllib.unquote_plus, pats), **opts)
+    return commands.diff(ui, repo, *map(unquoteplus, pats), **opts)
 
-@command('rhlog',
+@command(b'rhlog',
                    [
-                    ('r', 'rev', [], 'show the specified revision'),
-                    ('b', 'branch', [],
-                       'show changesets within the given named branch'),
-                    ('l', 'limit', '',
-                         'limit number of changes displayed'),
-                    ('d', 'date', '',
-                         'show revisions matching date spec'),
-                    ('u', 'user', [],
-                      'revisions committed by user'),
-                    ('', 'from', '',
-                      ''),
-                    ('', 'to', '',
-                      ''),
-                    ('', 'rhbranch', '',
-                      ''),
-                    ('', 'template', '',
-                       'display with template')],
-                   'hg rhlog [OPTION]... [FILE]')
+                    (b'r', b'rev', [], b'show the specified revision'),
+                    (b'b', b'branch', [],
+                       b'show changesets within the given named branch'),
+                    (b'l', b'limit', b'',
+                         b'limit number of changes displayed'),
+                    (b'd', b'date', b'',
+                         b'show revisions matching date spec'),
+                    (b'u', b'user', [],
+                      b'revisions committed by user'),
+                    (b'', b'from', b'',
+                      b''),
+                    (b'', b'to', b'',
+                      b''),
+                    (b'', b'rhbranch', b'',
+                      b''),
+                    (b'', b'template', b'',
+                       b'display with template')],
+                   b'hg rhlog [OPTION]... [FILE]')
 def rhlog(ui, repo, *pats, **opts):
     rev      = opts.pop('rev')
     bra0     = opts.pop('branch')
-    from_rev = urllib.unquote_plus(opts.pop('from', None))
-    to_rev   = urllib.unquote_plus(opts.pop('to'  , None))
-    bra      = urllib.unquote_plus(opts.pop('rhbranch', None))
-    from_rev = from_rev.replace('"', '\\"')
-    to_rev   = to_rev.replace('"', '\\"')
-    if hg.util.version() >= '1.6':
-      opts['rev'] = ['"%s":"%s"' % (from_rev, to_rev)]
+    from_rev = unquoteplus(opts.pop('from', b''))
+    to_rev   = unquoteplus(opts.pop('to'  , b''))
+    bra      = unquoteplus(opts.pop('rhbranch', b''))
+    from_rev = from_rev.replace(b'"', b'\\"')
+    to_rev   = to_rev.replace(b'"', b'\\"')
+    if (from_rev != b'') or (to_rev != b''):
+        if from_rev != b'':
+            quotefrom = b'"%s"' % (from_rev)
         else:
-      opts['rev'] = ['%s:%s' % (from_rev, to_rev)]
+            quotefrom = from_rev
+        if to_rev != b'':
+            quoteto = b'"%s"' % (to_rev)
+        else:
+            quoteto = to_rev
+        opts['rev'] = [b'%s:%s' % (quotefrom, quoteto)]
+        opts['rev'] = rev
+    if (bra != b''):
         opts['branch'] = [bra]
-    return commands.log(ui, repo, *map(urllib.unquote_plus, pats), **opts)
+    return commands.log(ui, repo, *map(unquoteplus, pats), **opts)
 
-@command('rhmanifest',
-                   [('r', 'rev', '', 'show the specified revision')],
-                   'hg rhmanifest [-r REV] [PATH]')
-def rhmanifest(ui, repo, path='', **opts):
+@command(b'rhmanifest',
+                   [(b'r', b'rev', b'', b'show the specified revision')],
+                   b'hg rhmanifest -r REV [PATH]')
+def rhmanifest(ui, repo, path=b'', **opts):
     """output the sub-manifest of the specified directory"""
-    ui.write('<?xml version="1.0"?>\n')
-    ui.write('<rhmanifest>\n')
-    ui.write('<repository root="%s">\n' % _u(repo.root))
+    ui.write(b'<?xml version="1.0"?>\n')
+    ui.write(b'<rhmanifest>\n')
+    ui.write(b'<repository root="%s">\n' % _u(repo.root))
     try:
-        _manifest(ui, repo, urllib.unquote_plus(path), urllib.unquote_plus(opts.get('rev')))
+        path_encoding=ui.config(b'redminehelper',b'path_encoding',b'utf-8')
+        path_encoding=bytearray(path_encoding).decode('ascii')
+        _manifest(ui, repo, unquoteplus(path), unquoteplus(opts.get('rev')), path_encoding)
     finally:
-        ui.write('</repository>\n')
-        ui.write('</rhmanifest>\n')
+        ui.write(b'</repository>\n')
+        ui.write(b'</rhmanifest>\n')
 
-@command('rhsummary',[], 'hg rhsummary')
+@command(b'rhsummary', [], b'hg rhsummary')
 def rhsummary(ui, repo, **opts):
     """output the summary of the repository"""
-    ui.write('<?xml version="1.0"?>\n')
-    ui.write('<rhsummary>\n')
-    ui.write('<repository root="%s">\n' % _u(repo.root))
+    ui.write(b'<?xml version="1.0"?>\n')
+    ui.write(b'<rhsummary>\n')
+    ui.write(b'<repository root="%s">\n' % _u(repo.root))
     try:
         _tip(ui, repo)
         _tags(ui, repo)
         _branches(ui, repo)
         # TODO: bookmarks in core (Mercurial>=1.8)
     finally:
-        ui.write('</repository>\n')
-        ui.write('</rhsummary>\n')
+        ui.write(b'</repository>\n')
+        ui.write(b'</rhsummary>\n')
 

lib/redmine/scm/adapters/mercurial_adapter.rb

@@ -50,7 +50,7 @@ module Redmine
           end
 
           def client_available
-            client_version_above?([1, 2])
+            client_version_above?([5, 1])
           end
 
           def hgversion
@@ -153,13 +153,13 @@ module Redmine
 
           entries = Entries.new
           as_ary(manifest['dir']).each do |e|
-            n = scm_iconv('UTF-8', @path_encoding, CGI.unescape(e['name']))
+            n = CGI.unescape(e['name'])
             p = "#{path_prefix}#{n}"
             entries << Entry.new(:name => n, :path => p, :kind => 'dir')
           end
 
           as_ary(manifest['file']).each do |e|
-            n = scm_iconv('UTF-8', @path_encoding, CGI.unescape(e['name']))
+            n = CGI.unescape(e['name'])
             p = "#{path_prefix}#{n}"
             lr = Revision.new(:revision => e['revision'], :scmid => e['node'],
                               :identifier => e['node'],
@@ -326,6 +326,7 @@ module Redmine
           full_args = ["-R#{repo_path}", '--encoding=utf-8']
           # don't use "--config=<value>" form for compatibility with ancient Mercurial
           full_args << '--config' << "extensions.redminehelper=#{HG_HELPER_EXT}"
+          full_args << '--config' << "redminehelper.path_encoding=#{@path_encoding}"
           full_args << '--config' << 'diff.git=false'
           full_args += args
           ret =

test/unit/repository_mercurial_test.rb

@@ -168,7 +168,7 @@ class RepositoryMercurialTest < ActiveSupport::TestCase
       @repository.fetch_changesets
       @project.reload
       assert_equal NUM_REV, @repository.changesets.count
-      assert_equal 53, @repository.filechanges.count
+      assert_equal 47, @repository.filechanges.count
       rev0 = @repository.changesets.find_by_revision('0')
       assert_equal "Initial import.\nThe repository contains 3 files.",
                    rev0.comments
@@ -261,13 +261,13 @@ class RepositoryMercurialTest < ActiveSupport::TestCase
         @repository.latest_changesets(
           '/sql_escape/percent%dir/percent%file1.txt', nil
         )
-      assert_equal %w|30 11 10 9|, changesets.collect(&:revision)
+      assert_equal %w|11 10 9|, changesets.collect(&:revision)
 
       changesets =
         @repository.latest_changesets(
           '/sql_escape/underscore_dir/understrike_file.txt', nil
         )
-      assert_equal %w|30 12 9|, changesets.collect(&:revision)
+      assert_equal %w|12 9|, changesets.collect(&:revision)
 
       changesets = @repository.latest_changesets('README', nil)
       assert_equal %w|31 30 28 17 8 6 1 0|, changesets.collect(&:revision)
@@ -284,7 +284,7 @@ class RepositoryMercurialTest < ActiveSupport::TestCase
 
       path = 'sql_escape/percent%dir'
       changesets = @repository.latest_changesets(path, nil)
-      assert_equal %w|30 13 11 10 9|, changesets.collect(&:revision)
+      assert_equal %w|13 11 10 9|, changesets.collect(&:revision)
 
       changesets = @repository.latest_changesets(path, '11')
       assert_equal %w|11 10 9|, changesets.collect(&:revision)
@@ -294,7 +294,7 @@ class RepositoryMercurialTest < ActiveSupport::TestCase
 
       path = 'sql_escape/underscore_dir'
       changesets = @repository.latest_changesets(path, nil)
-      assert_equal %w|30 13 12 9|, changesets.collect(&:revision)
+      assert_equal %w|13 12 9|, changesets.collect(&:revision)
 
       changesets = @repository.latest_changesets(path, '12')
       assert_equal %w|12 9|, changesets.collect(&:revision)