Release v1.0.5

Fixes mnauw/git-remote-hg#58

.github/workflows/ci.yml

@@ -1,7 +1,9 @@
 name: CI
 
 on:
-  push:
+  # push:
+  # save cycles; disable on push, enable manual trigger
+  workflow_dispatch:
 
 jobs:
   test:

README.asciidoc

@@ -71,13 +71,31 @@ the same commits:
 % git config --global remote-hg.hg-git-compat true
 --------------------------------------
 
+****
+mnauw's note; The above is not quite the case, it only ever has been (somewhat)
+if an undocumented debug mode (`debugextrainmessage` setting) was enabled
+in (likely somewhat patched) `hg-git`.  And as of `hg-git` v1.2.0 the latter is
+no longer considered.  In fact, `hg-git` creates git commits with additional hg
+metadata stored in so-called "extra commit headers". The latter might be seen by
+`git log --format=raw` or `git cat-file -p <commitid>`, but are otherwise mostly
+only used internally by the git suite (for signatures).  While they are supported
+by `dulwich`'s API (which is a python git implementation), there is, however,
+limited to no support for those in git "porcelain or plumbing" commands. In
+particular, `git fast-export` and `git fast-import` do not consider these, so a
+`gitremote-helpers` tool is then also out of luck.  Incidentally, it also
+follows that a `git fast-export | git fast-import` "clone" approach would also
+lose such extra metadata, and likewise so for e.g. `git filter-repo`.
+
+All in all, this mode is not quite recommended.
+If the concern here is not so much `hg-git` compatibility but rather "hg-git-hg
+round-trip fidelity", then see the discussion below on `check-hg-commits` setting.
+****
+
 == Notes ==
 
 Remember to run `git gc --aggressive` after cloning a repository, especially if
 it's a big one. Otherwise lots of space will be wasted.
 
-The newest supported version of Mercurial is 6.2, the oldest one is 2.4.
-
 === Pushing branches ===
 
 To push a branch, you need to use the 'branches/' prefix:
@@ -369,7 +387,8 @@ up elsewhere as expected (regardless of conversion mapping or ABI).
 
 Note that identifying and re-using the hg changeset relies on metadata
 (`refs/notes/hg` and marks files) that is not managed or maintained by any
-git-to-git fetch (or clone).
+git-to-git fetch (or clone) (as that is only automatically so for `refs/heads/`,
+though it could be pushed manually).
 As such (and as said), this approach aims for plain-and-simple safety, but only
 within a local scope (git repo).
 

doc/git-remote-hg.txt

@@ -104,6 +104,20 @@ the invalid '~'
 % git config --global remote-hg.ignore-name ~
 --------------------------------------
 
+Even though the "gitdir" is configurable (using `GIT_DIR`), git does not accept
+certain pathname components, e.g. `.git` or `.gitmodules` (case-insensitive).
+Problems arise if the hg repo contains such pathnames, and recent git versions
+will reject this in a very hard way.  So these pathnames are now mapped
+from "hg space" to "git space" in a one-to-one way, where (e.g.)
+`.git[0 or more suffix]` is mapped to `.git[1 or more suffix]` (obviously by
+appending or removing a suffix). The "suffix" in question defaults to `_`,
+but can be configured using
+
+--------------------------------------
+% git config --global remote-hg.dotfile-suffix _
+--------------------------------------
+
+
 NOTES
 -----
 

git-hg-helper

@@ -97,11 +97,27 @@ def debug(msg, *args):
 def log(msg, *args):
     logger.log(logging.LOG, msg, *args)
 
+# new style way to import a source file
+def _imp_load_source(module_name, file_path):
+    import importlib.util
+    loader = importlib.machinery.SourceFileLoader(module_name, file_path)
+    spec = importlib.util.spec_from_loader(module_name, loader)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)
+    return module
+
 def import_sibling(mod, filename):
-    import imp
     mydir = os.path.dirname(__file__)
     sys.dont_write_bytecode = True
-    return imp.load_source(mod, os.path.join(mydir, filename))
+    vi = sys.version_info
+    ff = os.path.join(mydir, filename)
+    if vi.major >= 3 and vi.minor >= 5:
+        return _imp_load_source(mod, ff)
+    else:
+        import imp
+        return imp.load_source(mod, ff)
+
 
 class GitHgRepo:
 
@@ -146,7 +162,7 @@ class GitHgRepo:
         process = self.start_cmd(args, **kwargs)
         output = process.communicate()[0]
         if check and process.returncode != 0:
-            die(b'command failed: %s' % b' '.join([compat.to_b(a) for a in cmd]))
+            die(b'git command failed: %s' % b' '.join([compat.to_b(a) for a in args]))
         return output
 
     def get_config(self, config, getall=False):
@@ -227,9 +243,12 @@ class GitHgRepo:
                         warn(b'failed to find local hg for remote %s' % (r))
                         continue
                 else:
+                    npath = os.path.abspath(hg_path)
+                    # use relative path if possible
+                    if check_version(4, 2):
+                        npath = os.path.join(b'..', b'..', b'..', b'.hg')
                     # make sure the shared path is always up-to-date
-                    util.writefile(os.path.join(local_hg, b'sharedpath'),
+                    util.writefile(os.path.join(local_hg, b'sharedpath'), npath)
-                        os.path.abspath(hg_path))
                 self.hg_repos[r] = os.path.join(local_path)
 
         log('%s determined hg_repos %s', self.identity(), self.hg_repos)
@@ -555,6 +574,43 @@ class GcCommand(SubCommand):
                 gm.store()
 
 
+class MapFileCommand(SubCommand):
+
+    def argumentparser(self):
+        usage = '%%(prog)s %s [options] <remote>' % (self.subcommand)
+        p = argparse.ArgumentParser(usage=usage)
+        p.add_argument('--output', required=True,
+            help='mapfile to write')
+        p.epilog = textwrap.dedent("""\
+        Writes a so-called git-mapfile, as used internally by hg-git.
+        This files consists of lines of format `<githexsha> <hghexsha>`.
+
+        As such, the result could be used to coax hg-git in some manner.
+        However, as git-remote-hg and hg-git may (likely) produce different
+        commits (either git or hg), mixed use of both tools is not recommended.
+        """)
+        return p
+
+    def do(self, options, args):
+        remotehg = import_sibling('remotehg', 'git-remote-hg')
+
+        if not args or len(args) != 1:
+            self.usage('expect 1 remote')
+
+        remote = args[0]
+        hgpath = remotehg.select_marks_dir(remote, self.githgrepo.gitdir, False)
+        puts(b"Loading hg marks ...")
+        hgm = remotehg.Marks(os.path.join(hgpath, b'marks-hg'), None)
+        puts(b"Loading git marks ...")
+        gm = GitMarks(os.path.join(hgpath, b'marks-git'))
+        puts(b"Writing mapfile ...")
+        with open(options.output, 'wb') as f:
+            for c, m in gm.marks.items():
+                hgc = hgm.rev_marks.get(m, None)
+                if hgc:
+                    f.write(b'%s %s\n' % (c, hgc))
+
+
 class SubRepoCommand(SubCommand):
 
     def writestate(repo, state):
@@ -921,6 +977,7 @@ def get_subcommands():
         b'repo': RepoCommand,
         b'gc':  GcCommand,
         b'sub': SubRepoCommand,
+        b'mapfile': MapFileCommand,
         b'help' : HelpCommand
     }
     # add remote named subcommands
@@ -943,6 +1000,7 @@ def do_usage():
     gc      \t perform maintenance and consistency cleanup on repo tracking marks
     sub     \t manage subrepos
     repo    \t show local hg repo backing a remote
+    mapfile \t dump a hg-git git-mapfile
 
     If the subcommand is the name of a remote hg repo, then any remaining arguments
     are considered a "hg command", e.g. hg heads, or thg, and it is then executed

git-remote-hg

@@ -122,6 +122,27 @@ else:
         urlparse = staticmethod(_urlparse)
         urljoin = staticmethod(_urljoin)
 
+# new style way to import a source file
+def _imp_load_source(module_name, file_path):
+    import importlib.util
+    loader = importlib.machinery.SourceFileLoader(module_name, file_path)
+    spec = importlib.util.spec_from_loader(module_name, loader)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)
+    return module
+
+def import_sibling(mod, filename):
+    mydir = os.path.dirname(__file__)
+    sys.dont_write_bytecode = True
+    vi = sys.version_info
+    ff = os.path.join(mydir, filename)
+    if vi.major >= 3 and vi.minor >= 5:
+        return _imp_load_source(mod, ff)
+    else:
+        import imp
+        return imp.load_source(mod, ff)
+
 #
 # If you want to see Mercurial revisions as Git commit notes:
 # git config core.notesRef refs/notes/hg
@@ -417,7 +438,7 @@ def export_file(ctx, fname):
         puts(b"data %d" % len(d))
         puts(f.data())
 
-    path = fix_file_path(f.path())
+    path = fixup_path_to_git(fix_file_path(f.path()))
     return (gitmode(f.flags()), mark, path)
 
 def get_filechanges(repo, ctx, parent):
@@ -497,6 +518,51 @@ def fixup_user(user):
 
     return b'%s <%s>' % (name, mail)
 
+# (recent) git fast-import does not accept .git or .gitmodule component names
+# (anywhere, case-insensitive)
+# in any case, surprising things may happen, so add some front-end replacement magic;
+# transform of (hg) .git(0 or more suffix) to (git) .git(1 or more suffix)
+# (likewise so for any invalid git keyword)
+def fixup_dotfile_path(path, suffix, add):
+    def subst(part):
+        if (not part) or part[0] != ord(b'.'):
+            return part
+        for prefix in (b'.git', b'.gitmodules'):
+            pl = len(prefix)
+            tail = len(part) - pl
+            if tail < 0:
+                continue
+            if part[0:pl].lower() == prefix and part[pl:] == suffix * tail:
+                if add:
+                    return part + suffix
+                elif tail == 0:
+                    # .git should not occur in git space
+                    # so complain
+                    if pl == 3:
+                        die('invalid path component %s' % part)
+                    else:
+                        # but .gitmodules might
+                        # leave as-is, it is handled/ignored elsewhere
+                        return part
+                else:
+                    return part[0:-1]
+        return part
+    # quick optimization check;
+    if (not path) or (path[0] != ord(b'.') and path.find(b'/.') < 0):
+        return path
+    sep = b'/'
+    return sep.join((subst(part) for part in path.split(sep)))
+
+def fixup_path_to_git(path):
+    if not dotfile_suffix:
+        return path
+    return fixup_dotfile_path(path, dotfile_suffix, True)
+
+def fixup_path_from_git(path):
+    if not dotfile_suffix:
+        return path
+    return fixup_dotfile_path(path, dotfile_suffix, False)
+
 def updatebookmarks(repo, peer):
     remotemarks = peer.listkeys(b'bookmarks')
 
@@ -578,16 +644,16 @@ def get_repo(url, alias):
             os.makedirs(dirname)
 
         local_path = os.path.join(dirname, b'clone')
+        kwargs = {}
+        hg_path = os.path.join(shared_path, b'.hg')
         if check_version(4, 2):
-            if not os.path.exists(local_path):
+            kwargs = {'relative': True}
-                hg.share(myui, shared_path, local_path, update=False, relative=True)
+            hg_path = os.path.join(b'..', b'..', b'..', b'.hg')
+        if not os.path.exists(local_path):
+            hg.share(myui, shared_path, local_path, update=False, **kwargs)
         else:
-            if not os.path.exists(local_path):
+            # make sure the shared path is always up-to-date
-                hg.share(myui, shared_path, local_path, update=False)
+            util.writefile(os.path.join(local_path, b'.hg', b'sharedpath'), hg_path)
-            else:
-                # make sure the shared path is always up-to-date
-                hg_path = os.path.join(shared_path, b'.hg')
-                util.writefile(os.path.join(local_path, b'.hg', b'sharedpath'), hg_path)
 
         repo = hg.repository(myui, local_path)
         try:
@@ -693,6 +759,7 @@ def export_ref(repo, name, kind, head):
                 if rename:
                     renames.append((rename[0], f))
 
+            # NOTE no longer used in hg-git, a HG:rename extra header is used
             for e in renames:
                 extra_msg += b"rename : %s => %s\n" % e
 
@@ -727,7 +794,7 @@ def export_ref(repo, name, kind, head):
                 puts(b"merge :%u" % (rev_to_mark(parents[1])))
 
         for f in removed:
-            puts(b"D %s" % (fix_file_path(f)))
+            puts(b"D %s" % fixup_path_to_git(fix_file_path(f)))
         for f in modified_final:
             puts(b"M %s :%u %s" % f)
         puts()
@@ -1020,6 +1087,7 @@ def parse_commit(parser):
         else:
             die(b'Unknown file command: %s' % line)
         path = c_style_unescape(path)
+        path = fixup_path_from_git(path)
         files[path] = files.get(path, {})
         files[path].update(f)
 
@@ -1127,11 +1195,17 @@ def parse_commit(parser):
         # add some extra that hg-git adds (almost) unconditionally
         # see also https://foss.heptapod.net/mercurial/hg-git/-/merge_requests/211
         # NOTE it could be changed to another value below
-        extra[b'hg-git-rename-source'] = b'git'
+        # actually, it is *almost* unconditionally, and only done if the commit
+        # is deduced to originate in git.  However, the latter is based on
+        # presence/absence of HG markers in commit "extra headers".
+        # The latter can not be handled here, and so this can not be correctly
+        # reproduced.
+        # extra[b'hg-git-rename-source'] = b'git'
         i = data.find(b'\n--HG--\n')
         if i >= 0:
             tmp = data[i + len(b'\n--HG--\n'):].strip()
             for k, v in [e.split(b' : ', 1) for e in tmp.split(b'\n')]:
+                # NOTE no longer used in hg-git, a HG:rename extra header is used
                 if k == b'rename':
                     old, new = v.split(b' => ', 1)
                     files[new]['rename'] = old
@@ -1602,10 +1676,7 @@ def do_push_refspec(parser, refspec, revs):
             tmpfastexport = open(os.path.join(marksdir, b'git-fast-export-%d' % (os.getpid())), 'w+b')
             subprocess.check_call(cmd, stdin=None, stdout=tmpfastexport)
             try:
-                import imp
+                ctx.hghelper = import_sibling('hghelper', 'git-hg-helper')
-                sys.dont_write_bytecode = True
-                ctx.hghelper = imp.load_source('hghelper', \
-                    os.path.join(os.path.dirname(__file__), 'git-hg-helper'))
                 ctx.hghelper.init_git(gitdir)
                 ctx.gitmarks = ctx.hghelper.GitMarks(tmpmarks)
                 # let processing know it should not bother pushing if not requested
@@ -1842,6 +1913,7 @@ def main(args):
     global capability_push
     global remove_username_quotes
     global marksdir
+    global dotfile_suffix
 
     marks = None
     is_tmp = False
@@ -1861,6 +1933,7 @@ def main(args):
     track_branches = get_config_bool('remote-hg.track-branches', True)
     capability_push = get_config_bool('remote-hg.capability-push', True)
     remove_username_quotes = get_config_bool('remote-hg.remove-username-quotes', True)
+    dotfile_suffix = get_config('remote-hg.dotfile-suffix').strip() or b'_'
     force_push = False
 
     if hg_git_compat:

setup.py

@@ -3,7 +3,7 @@
 import setuptools
 
 # strip leading v
-version = 'v1.0.4'[1:]
+version = 'v1.0.5'[1:]
 
 # check for released version
 assert (len(version) > 0)

test/helper.t

@@ -99,7 +99,7 @@ test_expect_success 'subcommand repo - with local proxy' '
 	test_cmp expected actual
 '
 
-test_expect_success 'subcommands hg-rev and git-rev' '
+test_expect_success 'subcommands hg-rev and git-rev and mapfile' '
 	test_when_finished "rm -rf gitrepo* hgrepo*" &&
 
 	setup_repos &&
@@ -110,7 +110,9 @@ test_expect_success 'subcommands hg-rev and git-rev' '
 	test -s rev-HEAD &&
 	git-hg-helper hg-rev `cat rev-HEAD` > hg-HEAD &&
 	git-hg-helper git-rev `cat hg-HEAD` > git-HEAD &&
-	test_cmp rev-HEAD git-HEAD
+	git-hg-helper mapfile --output mapfile origin &&
+	test_cmp rev-HEAD git-HEAD &&
+	grep "`cat rev-HEAD` `cat hg-HEAD`" mapfile
 	)
 '
 

test/hg-git.t

@@ -101,6 +101,8 @@ setup () {
 	[remote-hg]
 		hg-git-compat = true
 		track-branches = false
+		# directly use local repo to avoid push (and hence phase issues)
+		shared-marks = false
 	EOF
 
 	export HGEDITOR=true

test/main-push.t

@@ -1,7 +1,8 @@
+#!/bin/bash
+
 CAPABILITY_PUSH=t
 
-test -n "$TEST_DIRECTORY" || TEST_DIRECTORY=$(dirname $0)/
+. ./main.t
-. "$TEST_DIRECTORY"/main.t
 
 
 # .. and some push mode only specific tests

test/main.t

@@ -268,6 +268,41 @@ test_expect_success 'strip' '
 	test_cmp actual expected
 '
 
+test_expect_success 'dotfiles' '
+	test_when_finished "rm -rf hgrepo gitrepo" &&
+
+	(
+	hg init hgrepo &&
+	cd hgrepo &&
+
+	echo one >.git &&
+	echo ONE >.GIT &&
+	mkdir a && echo two > a/.gitmodules &&
+	hg add .git .GIT a/.gitmodules &&
+	hg commit -m zero
+	) &&
+
+	git clone "hg::hgrepo" gitrepo &&
+	test_cmp gitrepo/.git_ hgrepo/.git &&
+	test_cmp gitrepo/.GIT_ hgrepo/.GIT &&
+	test_cmp gitrepo/a/.gitmodules_ hgrepo/a/.gitmodules &&
+
+	(
+	cd gitrepo &&
+	echo three >.git_ &&
+	echo THREE >.GIT &&
+	echo four >a/.gitmodules_ &&
+	git add .git_ .GIT_ a/.gitmodules_ &&
+	git commit -m one &&
+	git push
+	) &&
+
+	hg -R hgrepo update &&
+	test_cmp gitrepo/.git_ hgrepo/.git &&
+	test_cmp gitrepo/.GIT_ hgrepo/.GIT &&
+	test_cmp gitrepo/a/.gitmodules_ hgrepo/a/.gitmodules
+'
+
 test_expect_success 'remote push with master bookmark' '
 	test_when_finished "rm -rf hgrepo gitrepo*" &&