Release v1.0.1

... to ensure setup.py does not trip some later time.

Fixes mnauw/git-remote-hg#25

.travis.yml

@@ -5,7 +5,7 @@ install:
     then pip install -q Mercurial${HG_VERSION+==$HG_VERSION};
     else pip install -q http://selenic.com/repo/hg/archive/tip.tar.gz;
     fi
-  - pip install -q dulwich hg-git || true
+  - pip install -q dulwich hg-git==0.6.1 || true
 
 before_script:
   - hg --version || true
@@ -20,6 +20,9 @@ matrix:
     - env: HG_VERSION=2.8.2
     - env: HG_VERSION=2.7.2
     - env: HG_VERSION=3.0
+    - env: HG_VERSION=3.5.2
+    - env: HG_VERSION=3.6.3
+    - env: HG_VERSION=3.7
     - env: HG_VERSION=dev
     - python: 2.7
     - python: 2.6

Makefile

@@ -1,6 +1,9 @@
 prefix := $(HOME)
 
-all:
+bindir := $(prefix)/bin
+mandir := $(prefix)/share/man/man1
+
+all: doc
 
 doc: doc/git-remote-hg.1
 
@@ -10,14 +13,29 @@ test:
 doc/git-remote-hg.1: doc/git-remote-hg.txt
 	a2x -d manpage -f manpage $<
 
+clean:
+	$(RM) doc/git-remote-hg.1
+
 D = $(DESTDIR)
 
 install:
-	install -D -m 755 git-remote-hg \
+	install -d -m 755 $(D)$(bindir)/
-		$(D)$(prefix)/bin/git-remote-hg
+	install -m 755 git-remote-hg $(D)$(bindir)/git-remote-hg
 
 install-doc: doc
-	install -D -m 644 doc/git-remote-hg.1 \
+	install -d -m 755 $(D)$(mandir)/
-		$(D)$(prefix)/share/man/man1/git-remote-hg.1
+	install -m 644 doc/git-remote-hg.1 $(D)$(mandir)/git-remote-hg.1
 
-.PHONY: all test
+pypi:
+	version=`git describe --tags ${REV}` && \
+		sed -i "s/version = .*/version = '$$version'/" setup.py
+	-rm -rf dist build
+	python setup.py sdist bdist_wheel
+
+pypi-upload:
+	twine upload dist/*
+
+pypi-test:
+	twine upload --repository-url https://test.pypi.org/legacy/ dist/*
+
+.PHONY: all test install install-doc clean pypy pypy-upload

README.asciidoc

@@ -9,12 +9,38 @@ git clone "hg::http://selenic.com/repo/hello"
 To enable this, simply add the 'git-remote-hg' script anywhere in your `$PATH`:
 
 --------------------------------------
-wget https://raw.github.com/felipec/git-remote-hg/master/git-remote-hg -O ~/bin/git-remote-hg
+wget https://raw.github.com/mnauw/git-remote-hg/master/git-remote-hg -O ~/bin/git-remote-hg
 chmod +x ~/bin/git-remote-hg
 --------------------------------------
 
+In Windows, you also need to put and an NTFS symbolic link named `python2.exe` somewhere
+on your `$PATH` pointing to your Python 2 executable:
+
+--------------------------------------
+mklink <path to link> <path to python.exe>
+--------------------------------------
+
 That's it :)
 
+Obviously you will need Mercurial installed.
+
+****
+At present, this "working copy"/fork <<add-features, adds the following features>>
+(and I would prefer it is indeed rather a "working copy"
+to be appropriately merged upstream):
+
+* eliminates a number of <<limitations, limitations>> as mentioned below
+* properly annotates copy/rename when pushing new commits to Mercurial
+* adds a 'git-hg-helper' script than can aid in the git-hg interaction workflow
+* provides enhanced bidirectional git-hg safety
+* avoids clutter of `refs/hg/...` by keeping these implementation details really private
+* more robust and efficient fetching, especially so when fetching or cloning from multiple
+  Mercurial clones which will only process changesets not yet fetched from elsewhere
+  (as opposed to processing everything all over again)
+
+See sections below or sidemarked notes for more details.
+****
+
 == Configuration ==
 
 If you want to see Mercurial revisions as Git commit notes:
@@ -89,6 +115,23 @@ However, some of these features require very new versions of 'git-remote-hg',
 so you might have better luck simply specifying the username and password in
 the URL.
 
+=== Submodules ===
+
+Hg repositories can be used as git submodule, but this requires to allow the hg procotol to be used by git submodule commands:
+
+--------------------------------------
+git config protocol.hg.allow always
+--------------------------------------
+
+Or adding manually the following to your git configuration file:
+
+--------------------------------------
+[protocol "hg"]
+        allow = always
+--------------------------------------
+
+This can be done per-repository, every time after a clone, or globally in the global .gitconfig (using the --global command-line option).
+
 === Caveats ===
 
 The only major incompatibility is that Git octopus merges (a merge with more
@@ -105,20 +148,272 @@ Closed branches are not supported; they are not shown and you can't close or
 reopen. Additionally in certain rare situations a synchronization issue can
 occur (https://github.com/felipec/git/issues/65[Bug #65]).
 
+[[limitations]]
 Limitations of the remote-helpers' framework apply. In particular, these
 commands don't work:
 
 * `git push origin :branch-to-delete`
-* `git push origin old:new` (it will push 'old') (Git v2.0)
+* `git push origin old:new` (it will push 'old') (patches available)
-* `git push --dry-run origin branch` (it will push) (Git v2.0)
+* `git push --dry-run origin branch` (it will push) (patches available)
+
+****
+Another limitation is that if `git log` reports a rename, this will not survive
+the push and Mercurial will not be aware of a rename (and similarly so for copy).
+Though Mercurial would know about it if you *manually* ran `git-format-patch`
+followed by a `hg apply -s`, which is not the nice way to go obviously.
+
+Actually, scratch the limitations above ascribed to the remote-helpers framework.
+They are not limitations of the framework, but are due to how the original
+implementation of 'git-remote-hg' interacts with it.
+Using the remote-helpers framework in only a slightly different way has none
+of the above limitations.  See the <<no-limitations, relevant section>>
+below for more details.
+****
 
 == Other projects ==
 
 There are other 'git-remote-hg' projects out there, do not confuse this one,
-this is the one distributed officially by the Git project:
+this is the one distributed officially by the Git project
+(_though actually no longer so nowadays_):
 
 * https://github.com/msysgit/msysgit/wiki/Guide-to-git-remote-hg[msysgit's git-remote-hg]
 * https://github.com/rfk/git-remote-hg[rfk's git-remote-hg]
 
 For a comparison between these and other projects go
 https://github.com/felipec/git/wiki/Comparison-of-git-remote-hg-alternatives[here].
+
+[[no-limitations]]
+== Limitations (or not) ==
+
+If interested in some of technical details behind this explanation, then also
+see the relevant section in 'git-remote-hg' manpage.  Otherwise, the general
+idea is presented here.
+
+More precisely and simply, the <<limitations, mentioned limitations>> are indeed
+limitations of the `export` capability of
+https://www.kernel.org/pub/software/scm/git/docs/gitremote-helpers.html[gitremote-helpers(1)]
+framework.  However, the framework also supports a `push` capability and when this
+is used appropriately in the remote helper the aforementioned limitations do not apply.
+In the case of `export` capability, git-core will internally invoke `git-fast-export`
+and the helper will process this data and hand over generated changesets to Mercurial.
+In the case of `push` capability, git informs the helper what (refs) should go where,
+and the helper is free to ponder about this and take the required action, such as
+to invoke `git-fast-export` itself (with suitable options) and process its output
+the same way as before (and over to Mercurial).
+
+And so;
+
+* `git push origin :branch-to-delete` will delete the bookmark `branch-to-delete` on remote
+* `git push --dry-run origin branch` will not touch the remote
+(or any local state, except for local helper proxy repo)
+* `git push origin old:new` will push `old` onto `new` in the remote
+* `git push origin <history-with-copy/rename>` will push copy/rename aware Mercurial revisions
+
+To tweak how 'git-remote-hg' decides on a copy/rename, use e.g:
+--------------------------------------
+% git config --global remote-hg.fast-export-options '-M -C -C'
+--------------------------------------
+
+[[add-features]]
+== Additional Features ==
+
+=== Miscellaneous Tweaks ===
+Other than <<no-limitations, removing the limitations>> as mentioned above,
+a number of issues (either so reported in
+https://github.com/felipec/git-remote-hg/issues[issue tracking] or not) have been
+addressed here, e.g. notes handling, `fetch --prune` support, correctly fetching
+after a `strip` on remote repo, tracking remote changes to import (if any) in a
+safe, robust and efficient way, etc.  Some of these have been highlighted above.
+
+For example, the `refs/hg/...` refs are really an implementation detail
+that need not clutter up the (visible) ref space.  So, in as much as they
+are still relevant, these are now kept elsewhere out of sight.
+If somehow your workflow relies on having these in the old place:
+--------------------------------------
+% git config --global remote-hg.show-private-refs true
+--------------------------------------
+
+More importantly, a significantly more efficient workflow is achieved using
+one set of shared marks files for all remotes (which also forces a local repo
+to use an internal proxy clone).
+The practical consequence is that fetching from a newly added remote hg repo
+does not require another (lengthy) complete import
+(as the original clone) but will only fetch additional changesets (if any).
+The same goes for subsequent fetching from any hg remote; what was fetched
+and imported from some remote need not be imported again from another.
+Operating in this shared mode also has the added advantage
+of correctly pushing after a `strip` on a remote.
+This shared-marks-files behaviour is the default on a fresh repo clone.  It can
+also be enabled on an existing one by the following setting.
+--------------------------------------
+% git config --global remote-hg.shared-marks true
+--------------------------------------
+Note, however, that one should then perform a fetch from each relevant remote
+to fully complete the conversion (prior to subsequent pushing).
+
+Some Mercurial names (of branches, bookmarks, tags) may not be a valid git
+refname. See e.g. `man git-check-ref-format` for a rather involved set of rules.
+Moreover, while a slash `/` is allowed, it is not supported to have both a `parent`
+and `parent/child` branch (though only the latter is allowed). So, pending some
+"nice" bidirectional encoding (not even e.g. URL encoding is safe actually), it is more
+likely that only a few instances (out of a whole Mercurial repo) are
+problematic.  This could be handled by a single-branch clone and/or configuring
+a suitable refspec.  However, it might be more convenient to simply filter out a
+few unimportant pesky cases, which can be done by configuring a regural
+expression in following setting:
+--------------------------------------
+% git config remote-hg.ignore-name nasty/nested/name
+--------------------------------------
+Recall also that a config setting can be provided at clone time
+(command line using `--config` option).
+
+--------------------------------------
+% git config --global remote-hg.remove-username-quotes false
+--------------------------------------
+
+By default, for backwards compatibility with earlier versions,
+git-remote-hg removes quotation marks from git usernames
+(e.g. 'Raffaello "Raphael" Sanzio da Urbino <raphael@example.com>'
+would become 'Raffaello Raphael Sanzio da Urbino
+<raphael@example.com>').  This breaks round-trip compatibility; a git
+commit by an author with quotes would become an hg commit without,
+and if re-imported into git, would get a different SHA1.
+
+To restore round-trip compatibility (at the cost of backwards
+compatibility with commits converted by older versions of
+git-remote-hg), turn 'remote-hg.remove-username-quotes' off.
+
+=== Helper Commands ===
+
+Beyond that, a 'git-hg-helper' script has been added that can aid in the git-hg
+interaction workflow with a number of subcommands that are not in the purview of
+a remote helper.  This is similar to e.g. 'git-svn' being a separate program
+altogether.  These subcommands
+
+* provide conversion from a hg changeset id to a git commit hash, or vice versa
+* provide consistency and cleanup maintenance on internal `git-remote-hg` metadata marks
+* provide optimization of git marks of a fetch-only remote
+
+See the helper script commands' help description for further details.
+It should simply be installed (`$PATH` accessible) next to 'git-remote-hg'.
+Following git alias is probably also convenient as it allows invoking the helper
+as `git hg`:
+--------------------------------------
+% git config --global alias.hg '!git-hg-helper'
+--------------------------------------
+
+With that in place, running `git hg gc <remote>` after initial fetch from (large)
+<remote> will save quite some space in the git marks file.  Not to mention some time
+each time it is loaded and saved again (upon fetch).  If the remote is ever pushed
+to, the marks file will similarly be squashed, but for a fetch-only <remote>
+the aforementioned command will do.  It may also be needed to run aforementioned
+command after a `git gc` has been performed.  You will notice the need
+when `git-fast-import` or `git-fast-export` complain about not finding objects ;-)
+
+In addition, the helper also provides support routines for `git-remote-hg` that
+provide for increased (or at least safer) git-hg bidirectionality.
+
+Before explaining how it helps, let's first elaborate on what is really
+meant by the above _bidirectionality_ since it can be regarded in 2 directions.
+From the git repo point of view, one can push to a hg repo and then fetch (or
+clone) back to git. Or one could have fetched a changeset from some hg repo and
+then push this back to (another) hg clone.  So what happens in either case? In the
+former case, from git to hg and then back, things work out ok whether or not in
+hg-git compatibility mode.  In the latter case, it is very likely (but
+ultimately not guaranteed) that it works out in hg-git compatibility mode, and far
+less likely otherwise.
+
+Most approaches on bidirectionality try to go for the "mapping" way.
+That is, find a way to map all Mercurial (meta)data somewhere into git;
+in the commit message, or in non-standard ways in extra headers in commit objects
+(e.g. the latest hg-git approach).  The upside of this is that such a git repo can be
+cloned to another git repo, and then one can push back into hg which will/should
+turn out ok.  The downside is setting up such a mapping in the first place,
+avoiding the slightest error in translating authors, timestamps etc,
+and maintaining all that whenever there is some Mercurial API/ABI breakage.
+
+The approach here is to consider a typical git-hg interaction workflow and to
+ensure simple/safe bidirectionality in such a setting.  That is, you are (obviously)
+in a situation having to deal with some Mercurial repo and quite probably
+with various clones as well. The objective is to fetch from these repos/clones,
+work in git and then push back.  And in the latter case, one needs to make sure
+that hg changesets from one hg clone end up *exactly* that way in another hg
+clone (or the git-hg bridge usage might not be so appreciated).  Such pushes are
+probably not recommended workflow practice, but no accidents or issues should
+arise from any push in these circumstances. There is less interest in this setting,
+however, for (git-wise) cloning around the derived git repo.
+
+Now, depending on your workflow and to ensure the above behaves well,
+following setting can be enabled as preferred:
+
+--------------------------------------
+% git config --global remote-hg.check-hg-commits fail
+% git config --global remote-hg.check-hg-commits push
+--------------------------------------
+
+If not set, the behaviour is as before; pushing a commit based on hg changeset
+will again transform the latter into a new hg changeset which may or may not
+match the original (as described above).
+If set to `fail`, it will reject and abort the push.
+If set to `push`, it will re-use the original changeset in a Mercurial native
+way (rather than creating a new one).  The latter guarantees the changeset ends
+up elsewhere as expected (regardless of conversion mapping or ABI).
+
+Note that identifying and re-using the hg changeset relies on metadata
+(`refs/notes/hg` and marks files) that is not managed or maintained by any
+git-to-git fetch (or clone).
+As such (and as said), this approach aims for plain-and-simple safety, but only
+within a local scope (git repo).
+
+=== Mercurial Subrepository Support ===
+
+Both Git and Mercurial support a submodule/subrepo system.
+In case of Git, URLs are managed in `.gitmodules`, submodule state is tracked
+in tree objects and only Git submodules are supported.
+Mercurial manages URLs in `.hgsub`, records subrepo state in `.hgsubstate` and
+supports Git, Mercurial and Subversion subrepos (at time of writing).
+Merely the latter diversity in subrepo types shows that somehow mapping Mercurial
+"natively" to git submodules is not quite evident.  Moreover, while one might
+conceivably devise such a mapping restricted to git and hg subrepos, any such would
+seem error-prone and fraught with all sorts of tricky cases and inconvenient
+workflow handling (innovative robust suggestions are welcome though ...)
+
+So, rather than overtaking the plumbing and ending up with stuffed drain further on,
+the approach here is (again) to keep it plain-and-simple.  That is, provide some
+git-ish look-and-feel helper script commands for setting up and manipulating
+subrepos.  And so (if the alias mentioned above has been defined), `git hg sub`
+provides commands similar to `git submodule` that accomplish what is otherwise
+taken care of by the Mercurial subrepo support.
+The latter is obviously extended to be git-aware in that e.g. a Mercurial subrepo
+is cloned as a git-hg subrepo and translation back-and-forth between hg changeset id
+and git commit hash is also performed where needed.  There is no support though
+for Subversion subrepos.
+
+As with the other commands, see the help description for the proper details,
+but the following example session may clarify the principle:
+
+--------------------------------------
+% git clone hg::hgparentrepo
+# bring in subrepos in proper location:
+% git hg sub update
+# do some work
+% git pull --rebase origin
+# update subrepo state:
+% git hg sub update
+# do work in subrepo and push
+% ( cd subrepo && git push origin HEAD:master )
+# fetch to update refs/notes/hg (or enable remote-hg.push-updates-notes)
+% ( cd subrepo && git fetch origin )
+# update .hgsubstate to subrepo HEAD:
+% git hg sub upstate
+% git add .hgsubstate
+# add more, commit and push as intended
+--------------------------------------
+
+Note that the refspec `HEAD:master` is needed if working with detached `HEAD`
+in subrepo, and that pushing such refspec is actually supported now in a git-hg subrepo
+as explained <<no-limitations, earlier>>.
+
+== Contributing ==
+
+Please file an issue with some patches or a pull-request.

doc/.gitignore

@@ -0,0 +1 @@
+git-remote-hg.1

doc/SubmittingPatches

@@ -0,0 +1,5 @@
+Please send your patches using `git format-patch` to the mailing list:
+git-fc@googlegroups.com.
+
+Make sure all the tests pass by running `make test`, and if possible add a new
+test to exercise the code you are submitting.

doc/git-remote-hg.txt

@@ -58,6 +58,50 @@ If you want 'git-remote-hg' to be compatible with 'hg-git', and generate exactly
 % git config --global remote-hg.hg-git-compat true
 --------------------------------------
 
+If you would like (why?) the old behaviour (export capability)
+where various limitations apply:
+
+--------------------------------------
+% git config --global remote-hg.capability-push false
+--------------------------------------
+
+In the new behaviour, performing a git push will make git search for and detect
+file rename and copy and turn this into Mercurial commit metadata.  To tweak how this
+detection happens, e.g. have it search even more:
+
+--------------------------------------
+% git config --global remote-hg.fast-export-options '-M -C -C'
+--------------------------------------
+
+The default otherwise is simply `-M -C`.  See also e.g.
+https://www.kernel.org/pub/software/scm/git/docs/git-log.html[git-log(1) manpage]
+for more details on the options used to tweak this.
+
+As the old refs/hg/... are actually an implementation detail, they are now
+maintained not so visibly.  If that, however, would be preferred:
+
+--------------------------------------
+% git config --global remote-hg.show-private-refs true
+--------------------------------------
+
+Use of shared marks files is the default in a new repo, but can also be enabled
+for an existing repo:
+
+--------------------------------------
+% git config --global remote-hg.shared-marks true
+--------------------------------------
+
+Note that one should perform a fetch from each remote to properly complete the
+conversion to shared marks files.
+
+Mercurial name(s) (of a branch or bookmark) that are not a valid git refname,
+can be ignored by configuring a suitable regular expression, e.g. avoiding
+the invalid '~'
+
+--------------------------------------
+% git config --global remote-hg.ignore-name ~
+--------------------------------------
+
 NOTES
 -----
 
@@ -119,3 +163,49 @@ would only see the latest head.
 Closed branches are not supported; they are not shown and you can't close or
 reopen. Additionally in certain rare situations a synchronization issue can
 occur (https://github.com/felipec/git/issues/65[Bug #65]).
+
+TECHNICAL DISCUSSION
+--------------------
+
+As `git-remote-hg` is a developer tool after all, it might be interesting to know a
+bit about what is going on behind the scenes, without necessarily going into all the
+details.
+
+So let's first have a look in the `.git/hg` directory, which typically
+contains a subdirectory for each remote Mercurial repo alias, as well as a `.hg`
+subdirectory.  If the Mercurial repo is a local one, it will (again typically)
+only contain a `marks-git` and a `marks-hg` file.  If the repo is a remote one,
+then the `clone` contains, well, a local clone of the remote.  However, all
+these clones share storage through the `.hg` directory mentioned previously (so
+they do not add up separately). During a fetch/push, the local (proxy) repo is
+used as an intermediate stage. If you would also prefer such an intermediate
+stage for local repos, then setting the environment variable
+`GIT_REMOTE_HG_TEST_REMOTE` will also use a proxy repo clone for a local repo.
+
+As for the marks files, `marks-git` is created and used by `git-fast-export`
+and `git-fast-import` and contains a mapping from mark to commit hash, where a
+mark is essentially a plain number.  `marks-hg` similarly contains a (JSON) based
+mapping between such mark and hg revision hash.  Together they provide for a
+(consistent) view of the synchronization state of things.
+
+When operating with shared-marks files, the `marks-git` and `marks-hg` files
+are shared among all repos.  As such, they are then found in the `.git/hg`
+directory (rather than a repo subdirectory).
+As there is really only one hg repository
+(the shared storage "union bag" in `.git/hg/.hg`), only 1 set of marks files
+should track the mapping between commit hash and revision hash.
+Each individual remote then only adds some metadata (e.g regarding heads).
+
+Upon a fetch, the helper uses the `marks-hg` file to decide what is already present
+and what not.  The required parts are then retrieved from Mercurial and turned
+into a `git-fast-import` stream as expected by `import` capability of
+https://www.kernel.org/pub/software/scm/git/docs/gitremote-helpers.html[gitremote-helpers(1)].
+
+Upon a push, the helper has specified the `push` capability in the new approach, and
+so git will provide a list of refspecs indicating what should go where.
+If the refspecs indicates a remote delete, it is performed appropriately the Mercurial way.
+If it is a regular push, then git-fast-export is invoked (using the existing `marks-git`)
+and the stream is processed and turned into Mercurial commits (along with bookmarks, etc).
+If the refspec specifies a `src:dest` rename, then the requested remote refname is tracked
+accordingly.  If a dry-run is requested, no remote is touched and no (marks) state of
+the run is retained.

git-hg-helper

@@ -0,0 +1,959 @@
+#!/usr/bin/env python2
+#
+# Copyright (c) 2016 Mark Nauwelaerts
+#
+
+from mercurial import hg, ui, commands, util
+from mercurial import context, subrepo
+
+import re
+import sys
+import os
+import subprocess
+import argparse
+import textwrap
+import logging
+import threading
+
+# thanks go to git-remote-helper for some helper functions
+
+def die(msg, *args):
+    sys.stderr.write('ERROR: %s\n' % (msg % args))
+    sys.exit(1)
+
+def warn(msg, *args):
+    sys.stderr.write('WARNING: %s\n' % (msg % args))
+
+def info(msg, *args):
+    logger.info(msg, *args)
+
+def debug(msg, *args):
+    logger.debug(msg, *args)
+
+def log(msg, *args):
+    logger.log(logging.LOG, msg, *args)
+
+def import_sibling(mod, filename):
+    import imp
+    mydir = os.path.dirname(__file__)
+    sys.dont_write_bytecode = True
+    return imp.load_source(mod, os.path.join(mydir, filename))
+
+class GitHgRepo:
+
+    def __init__(self, topdir=None, gitdir=None):
+        if gitdir != None:
+            self.gitdir = gitdir
+            self.topdir = os.path.join(gitdir, '..') # will have to do
+        else:
+            self.topdir = None
+            if not topdir:
+                topdir = self.run_cmd(['rev-parse', '--show-cdup']).strip()
+                if not topdir:
+                    if not os.path.exists('.git'):
+                        # now we lost where we are
+                        raise Exception('failed to determine topdir')
+                    topdir = '.'
+            self.topdir = topdir
+            self.gitdir = self.run_cmd(['rev-parse', '--git-dir']).strip()
+            if not self.gitdir:
+                raise Exception('failed to determine gitdir')
+            # the above was run in topdir
+            if not os.path.isabs(self.gitdir):
+                self.gitdir = os.path.join(self.topdir, self.gitdir)
+        self.hg_repos = {}
+
+    def identity(self):
+        return '[%s|%s]' % (os.getcwd(), self.topdir)
+
+    def start_cmd(self, args, **kwargs):
+        cmd = ['git'] + args
+        popen_options = { 'cwd': self.topdir,
+            'stdout': subprocess.PIPE, 'stderr': subprocess.PIPE }
+        popen_options.update(kwargs)
+        log('%s running cmd %s with options %s', self.identity(),
+            cmd, popen_options)
+        process = subprocess.Popen(cmd, **popen_options)
+        return process
+
+    # run a git cmd in repo dir, captures stdout and stderr by default
+    # override in kwargs if otherwise desired
+    def run_cmd(self, args, check=False, **kwargs):
+        process = self.start_cmd(args, **kwargs)
+        output = process.communicate()[0]
+        if check and process.returncode != 0:
+            die('command failed: %s', ' '.join(cmd))
+        return output
+
+    def get_config(self, config, getall=False):
+        get = { True : '--get-all', False: '--get' }
+        cmd = ['git', 'config', get[getall] , config]
+        return self.run_cmd(['config', get[getall] , config], stderr=None)
+
+    def get_config_bool(self, config, default=False):
+        value = self.get_config(config).rstrip('\n')
+        if value == "true":
+            return True
+        elif value == "false":
+            return False
+        else:
+            return default
+
+    def get_hg_repo_url(self, remote):
+        url = self.get_config('remote.%s.url' % (remote))
+        if url and url[0:4] == 'hg::':
+            url = url[4:].strip()
+        else:
+            url = None
+        return url
+
+    def get_hg_rev(self, commit):
+        hgrev = self.run_cmd(['notes', '--ref', 'refs/notes/hg', 'show', commit])
+        return hgrev
+
+    def rev_parse(self, ref):
+        args = [ref] if not isinstance(ref, list) else ref
+        args[0:0] = ['rev-parse', '--verify', '-q']
+        return self.run_cmd(args).strip()
+
+    def update_ref(self, ref, value):
+        self.run_cmd(['update-ref', '-m', 'update by helper', ref, value])
+        # let's check it happened
+        return git_rev_parse(ref) == git_rev_parse(value)
+
+    def cat_file(self,  ref):
+        return self.run_cmd(['cat-file', '-p', ref])
+
+    def get_git_commit(self, rev):
+        remotehg = import_sibling('remotehg', 'git-remote-hg')
+        for r in self.get_hg_repos():
+            try:
+                hgpath = remotehg.select_marks_dir(r, self.gitdir, False)
+                m = remotehg.Marks(os.path.join(hgpath, 'marks-hg'), None)
+                mark = m.from_rev(rev)
+                m = GitMarks(os.path.join(hgpath, 'marks-git'))
+                return m.to_rev(mark)
+            except:
+                pass
+
+    # returns dict: (alias: local hg repo dir)
+    def get_hg_repos(self):
+        # minor caching
+        if self.hg_repos:
+            return self.hg_repos
+
+        # check any local hg repo to see if rev is in there
+        shared_path = os.path.join(self.gitdir, 'hg')
+        hg_path = os.path.join(shared_path, '.hg')
+        if os.path.exists(shared_path):
+            repos = os.listdir(shared_path)
+            for r in repos:
+                # skip the shared repo
+                if r == '.hg':
+                    continue
+                # only dirs
+                if not os.path.isdir(os.path.join(shared_path, r)):
+                    continue
+                local_path = os.path.join(shared_path, r, 'clone')
+                local_hg = os.path.join(local_path, '.hg')
+                if not os.path.exists(local_hg):
+                    # could be a local repo without proxy, fetch url
+                    local_path = self.get_hg_repo_url(r)
+                    if not local_path:
+                        warn('failed to find local hg for remote %s', r)
+                        continue
+                else:
+                    # make sure the shared path is always up-to-date
+                    util.writefile(os.path.join(local_hg, 'sharedpath'),
+                        os.path.abspath(hg_path))
+                self.hg_repos[r] = os.path.join(local_path)
+
+        log('%s determined hg_repos %s', self.identity(), self.hg_repos)
+        return self.hg_repos
+
+    # returns hg repo object
+    def get_hg_repo(self, r):
+        repos = self.get_hg_repos()
+        if r in repos:
+            local_path = repos[r]
+            hushui = ui.ui()
+            hushui.setconfig('ui', 'interactive', 'off')
+            hushui.fout = open(os.devnull, 'w')
+            return hg.repository(hushui, local_path)
+
+    def find_hg_repo(self, rev):
+        repos = self.get_hg_repos()
+        for r in repos:
+            srepo = self.get_hg_repo(r)
+            # if this one had it, we are done
+            if srepo and rev in srepo and srepo[rev]:
+                return srepo
+
+    def rev_describe(self, rev):
+        result = self.run_cmd(['describe', rev]) or \
+            self.run_cmd(['describe', '--tags', rev]) or \
+            self.run_cmd(['describe', '--contains', rev]) or \
+            self.run_cmd(['describe', '--all', '--always', rev])
+        return result.strip()
+
+    class dictmemctx(context.memctx):
+
+        def __init__(self, repo, files):
+            p1, p2, data = repo[None], '0' * 40, ''
+            context.memctx.__init__(self, repo, (p1, p2),
+                data, files.keys(), self.getfilectx)
+            self.files = files
+            self.remotehg = import_sibling('remotehg', 'git-remote-hg')
+            self.remotehg.hg_version = hg_version
+
+        def __contains__(self, item):
+            return item in self.files
+
+        def getfilectx(self, repo, memctx, path):
+            is_exec = is_link = rename = False
+            return self.remotehg.make_memfilectx(repo, memctx, path, self.files[path],
+                is_link, is_exec, rename)
+
+    def read(self, relpath, rev=None):
+        rev = rev if rev else ':0'
+        obj = '%s:%s' % (rev, relpath)
+        # might complain bitterly to stderr if no subrepos so let's not show that
+        return self.run_cmd(['show', obj])
+
+    # see also subrepo.state
+    def state(self, remote='origin', rev=None):
+        """return a state dict, mapping subrepo paths configured in .hgsub
+        to tuple: (source from .hgsub, revision from .hgsubstate, kind
+        (key in types dict))
+        """
+
+        # obtain relevant files' content from specified revision
+        files = { }
+        for f in ('.hgsub', '.hgsubstate'):
+            files[f] = self.read(f)
+        log('state files for %s in revision %s:\n%s', remote, rev, files)
+
+        # wrap in a context and delegate to subrepo
+        # (rather than duplicating the admittedly simple parsing here)
+        repo = self.get_hg_repo(remote)
+        if not repo:
+            die('no hg repo for alias %s' % remote)
+        ctx = self.dictmemctx(repo, files)
+        # helpers moved around 4.6
+        if hasattr(subrepo, 'state'):
+          state = subrepo.state(ctx, ui.ui())
+        else:
+          from mercurial import subrepoutil
+          state = subrepoutil.state(ctx, ui.ui())
+        log('obtained state %s', state)
+
+        # now filter on type and resolve relative urls
+        import posixpath
+        resolved = {}
+        for s in state:
+            src, rev, kind = state[s]
+            if not kind in ('hg', 'git'):
+                warn('skipping unsupported subrepo type %s' % kind)
+                continue
+            if not util.url(src).isabs():
+                parent = self.get_hg_repo_url(remote)
+                if not parent:
+                    die('could not determine repo url of %s' % remote)
+                parent = util.url(parent)
+                parent.path = posixpath.join(parent.path or '', src)
+                parent.path = posixpath.normpath(parent.path)
+                src = str(parent)
+            # translate to git view url
+            if kind == 'hg':
+                src = 'hg::' + src
+            resolved[s] = (src.strip(), rev or '', kind)
+        log('resolved state %s', resolved)
+        return resolved
+
+
+class SubCommand:
+
+    def __init__(self, subcmdname, githgrepo):
+        self.subcommand = subcmdname
+        self.githgrepo = githgrepo
+        self.argparser = self.argumentparser()
+
+    def argumentparser(self):
+        return argparse.ArgumentParser()
+
+    def get_remote(self, args):
+        if len(args):
+            return (args[0], args[1:])
+        else:
+            self.usage('missing argument: <remote-alias>')
+
+    def get_remote_url_hg(self, remote):
+        url = self.githgrepo.get_hg_repo_url(remote)
+        if not url:
+            self.usage('%s is not a remote hg repository' % (remote))
+        return url
+
+    def execute(self, args):
+        (self.options, self.args) = self.argparser.parse_known_args(args)
+        self.do(self.options, self.args)
+
+    def usage(self, msg):
+        if msg:
+            self.argparser.error(msg)
+        else:
+            self.argparser.print_usage(sys.stderr)
+            sys.exit(2)
+
+    def do(self, options, args):
+        pass
+
+
+class HgRevCommand(SubCommand):
+
+    def argumentparser(self):
+        usage = '%%(prog)s %s [options] <commit-ish>' % (self.subcommand)
+        p = argparse.ArgumentParser(usage=usage)
+        p.epilog = textwrap.dedent("""\
+        Determines the hg revision corresponding to <commit-ish>.
+        """)
+        return p
+
+    def do(self, options, args):
+        if len(args):
+            hgrev = self.githgrepo.get_hg_rev(args[0])
+            if hgrev:
+                print hgrev
+
+
+class GitMarks:
+
+    def __init__(self, path):
+      self.path = path
+      self.clear()
+      self.load()
+
+    def clear(self):
+        self.marks = {}
+        self.rev_marks = {}
+
+    def load(self):
+        if not os.path.exists(self.path):
+            return
+
+        for l in file(self.path):
+            m, c = l.strip().split(' ', 2)
+            m = int(m[1:])
+            self.marks[c] = m
+            self.rev_marks[m] = c
+
+    def store(self):
+        marks = self.rev_marks.keys()
+        marks.sort()
+        with open(self.path, 'w') as f:
+            for m in marks:
+                f.write(':%d %s\n' % (m, self.rev_marks[m]))
+
+    def from_rev(self, rev):
+        return self.marks[rev]
+
+    def to_rev(self, mark):
+        return str(self.rev_marks[mark])
+
+
+class GitRevCommand(SubCommand):
+
+    def argumentparser(self):
+        usage = '%%(prog)s %s [options] <revision>' % (self.subcommand)
+        p = argparse.ArgumentParser(usage=usage)
+        p.epilog = textwrap.dedent("""\
+        Determines the git commit id corresponding to hg <revision>.
+        """)
+        return p
+
+    def do(self, options, args):
+        if len(args):
+            rev = args[0]
+            gitcommit = self.githgrepo.get_git_commit(rev)
+            if gitcommit:
+                print gitcommit
+
+
+class GcCommand(SubCommand):
+
+    def argumentparser(self):
+        usage = '%%(prog)s %s [options] <remote>...' % (self.subcommand)
+        p = argparse.ArgumentParser(usage=usage, \
+            formatter_class=argparse.RawDescriptionHelpFormatter)
+        p.add_argument('--check-hg', action='store_true',
+            help='also prune invalid hg revisions')
+        p.add_argument('-n', '--dry-run', action='store_true',
+            help='do not actually update any metadata files')
+        p.epilog = textwrap.dedent("""\
+        Performs cleanup on <remote>'s marks files and ensures these are consistent
+        (never affecting or touching any git repository objects or history).
+        The marks files are considered consistent if they "join"
+        on the :mark number (along with a valid git commit id).
+
+        This command can be useful in following scenarios:
+        * following a git gc command;
+          this could prune objects and lead to (then) invalid commit ids in marks
+          (in which case git-fast-export or git-fast-import would complain bitterly).
+          Such pruning is more likely to happen with remote hg repos with multiple heads.
+        * cleaning marks-git of a fetch-only remote;
+          git-fast-import (used during fetch) also dumps non-commit SHA-1 in the marks file,
+          so the latter can become pretty large. It will reduce in size either by a push
+          (git-fast-export only dumps commit objects) or by running this helper command.
+        """)
+        return p
+
+    def print_commits(self, gm, dest):
+        for c in gm.marks.keys():
+            dest.write(c + '\n')
+        dest.flush()
+        dest.close()
+
+    def do(self, options, args):
+        remotehg = import_sibling('remotehg', 'git-remote-hg')
+
+        hg_repos = self.githgrepo.get_hg_repos()
+        if not args:
+            self.usage('no remote specified')
+        for remote in args:
+            if not remote in hg_repos:
+                self.usage('%s is not a valid hg remote' % (remote))
+            hgpath = remotehg.select_marks_dir(remote, self.githgrepo.gitdir, False)
+            print "Loading hg marks ..."
+            hgm = remotehg.Marks(os.path.join(hgpath, 'marks-hg'), None)
+            print "Loading git marks ..."
+            gm = GitMarks(os.path.join(hgpath, 'marks-git'))
+            repo = hg.repository(ui.ui(), hg_repos[remote]) if options.check_hg else None
+            # git-gc may have dropped unreachable commits
+            # (in particular due to multiple hg head cases)
+            # need to drop those so git-fast-export or git-fast-import does not complain
+            print "Performing garbage collection on git commits ..."
+            process = self.githgrepo.start_cmd(['cat-file', '--batch-check'], \
+                stdin=subprocess.PIPE)
+            thread = threading.Thread(target=self.print_commits, args=(gm, process.stdin))
+            thread.start()
+            git_marks = set({})
+            for l in process.stdout:
+                sp = l.strip().split(' ', 2)
+                if sp[1] == 'commit':
+                    git_marks.add(gm.from_rev(sp[0]))
+            thread.join()
+            # reduce down to marks that are common to both
+            print "Computing marks intersection ..."
+            common_marks = set(hgm.rev_marks.keys()).intersection(git_marks)
+            hg_rev_marks = {}
+            git_rev_marks = {}
+            for m in common_marks:
+                if repo and not hgm.rev_marks[m] in repo:
+                        continue
+                hg_rev_marks[m] = hgm.rev_marks[m]
+                git_rev_marks[m] = gm.rev_marks[m]
+            # common marks will not not include any refs/notes/hg
+            # let's not discard those casually, though they are not vital
+            print "Including notes commits ..."
+            revlist = self.githgrepo.start_cmd(['rev-list', 'refs/notes/hg'])
+            for l in revlist.stdout.readlines():
+                c = l.strip()
+                m = gm.marks.get(c, 0)
+                if m:
+                    git_rev_marks[m] = c
+            # also save last-note mark
+            if hgm.last_note:
+                git_rev_marks[hgm.last_note] = gm.rev_marks[hgm.last_note]
+            # some status report
+            if len(hgm.rev_marks) != len(hg_rev_marks):
+                print "Trimmed hg marks from #%d down to #%d" % (len(hgm.rev_marks), len(hg_rev_marks))
+            if len(gm.rev_marks) != len(git_rev_marks):
+                print "Trimmed git marks from #%d down to #%d" % (len(gm.rev_marks), len(git_rev_marks))
+            # marks-hg tips irrelevant nowadays
+            # now update and store
+            if not options.dry_run:
+                # hg marks
+                print "Writing hg marks ..."
+                hgm.rev_marks = hg_rev_marks
+                hgm.marks = {}
+                for mark, rev in hg_rev_marks.iteritems():
+                    hgm.marks[rev] = mark
+                hgm.store()
+                # git marks
+                print "Writing git marks ..."
+                gm.rev_marks = git_rev_marks
+                gm.marks = {}
+                for mark, rev in git_rev_marks.iteritems():
+                    gm.marks[rev] = mark
+                gm.store()
+
+
+class SubRepoCommand(SubCommand):
+
+    def writestate(repo, state):
+        """rewrite .hgsubstate in (outer) repo with these subrepo states"""
+        lines = ['%s %s\n' % (state[s][1], s) for s in sorted(state)
+                                                    if state[s][1] != nullstate[1]]
+        repo.wwrite('.hgsubstate', ''.join(lines), '')
+
+    def argumentparser(self):
+        #usage = '%%(prog)s %s [options] <remote>...' % (self.subcommand)
+        # argparse.ArgumentParser(parents=[common])
+        # top-level
+        p = argparse.ArgumentParser(
+            formatter_class=argparse.RawDescriptionHelpFormatter)
+        p.add_argument('--quiet', '-q', action='store_true')
+        p.add_argument('--recursive', '-r', action='store_true',
+            help='recursive execution in each submodule')
+        p.add_argument('--remote', metavar='ALIAS', default='origin',
+            help='remote alias to use as base for relative url')
+        p.epilog = textwrap.dedent("""\
+        A group of (sub-)subcommands that aid in handling Mercurial subrepos
+        as managed by .hgsub and .hgsubstate.  The commands are (where applicable)
+        loosely modeled after the corresponding and similar git-submodule command,
+        though otherwise there is no relation whatsoever with the git-submodule system.
+        Each command supports recursive execution into subrepos of subrepos,
+        and the considered (top-level) repos can be restricted by a list of subrepo
+        paths as arguments.  For most commands, the state files are examined
+        in the parent repo's index.
+
+        Though depending on your workflow, the \'update\' command will likely
+        suffice, along with possibly an occasional \'upstate\'.
+        """)
+        # subparsers
+        sub = p.add_subparsers()
+        # common
+        common = argparse.ArgumentParser(add_help=False)
+        common.add_argument('paths', nargs=argparse.REMAINDER)
+        # update
+        p_update = sub.add_parser('update', parents=[common],
+            help='update subrepos to state as specified in parent repo')
+        p_update.set_defaults(func=self.cmd_update)
+        p_update.add_argument('--rebase', action='store_true',
+            help='rebase the subrepo current branch onto recorded superproject commit')
+        p_update.add_argument('--merge', action='store_true',
+            help='merge recorded superproject commit into the current branch of subrepo')
+        p_update.add_argument('--force', action='store_true',
+            help='throw away local changes when switching to a different commit')
+        p_update.epilog = textwrap.dedent("""\
+        Brings all subrepos in the state as specified by the parent repo.
+        If not yet present in the subpath, the subrepo is cloned from the URL
+        specified by the parent repo (resolving relative path wrt a parent repo URL).
+        If already present, and target state revision is ancestor of current HEAD,
+        no update is done.  Otherwise, a checkout to the required revision is done
+        (optionally forcibly so).  Alternatively, a rebase or merge is performed if
+        so requested by the corresponding option.
+        """)
+        # foreach
+        p_foreach = sub.add_parser('foreach',
+            help='evaluate shell command in each checked out subrepo')
+        p_foreach.set_defaults(func=self.cmd_foreach)
+        p_foreach.add_argument('command', help='shell command')
+        p_foreach.epilog = textwrap.dedent("""\
+        The command is executed in the subrepo directory and has access to
+        the variables $path, $toplevel, $sha1, $rev and $kind.
+        $path is the subrepo directory relative to the parent project.
+        $toplevel is the absolute path to the top-level of the parent project.
+        $rev is the state revision info as set in .hgsubstate whereas
+        $sha1 is the git commit corresponding to it (which may be identical
+        if subrepo is a git repo).  $kind is the type of subrepo, e.g. git or hg.
+        Unless given --quiet, the (recursively collected) submodule path is printed
+        before evaluating the command.  A non-zero return from the command in
+        any submodule causes the processing to terminate.
+        """)
+        # add state
+        p_upstate = sub.add_parser('upstate', parents=[common],
+            help='update .hgsubstate to current HEAD of subrepo')
+        p_upstate.set_defaults(func=self.cmd_upstate)
+        p_upstate.epilog = textwrap.dedent("""\
+        Rather than dealing with the index the .hgsubstate file is simply and only
+        edited to reflect the current HEAD of (all or selected) subrepos
+        (and any adding to index and committing is left to user's discretion).
+        """)
+        # status
+        p_status = sub.add_parser('status', parents=[common],
+            help='show the status of the subrepos')
+        p_status.add_argument('--cached', action='store_true',
+            help='show index .hgsubstate revision if != subrepo HEAD')
+        p_status.set_defaults(func=self.cmd_status)
+        p_status.epilog = textwrap.dedent("""\
+        This will print the SHA-1 of the currently checked out commit for each
+        subrepo, along with the subrepo path and the output of git describe for
+        the SHA-1.  Each SHA-1 will be prefixed with - if the submodule is not
+        yet set up (i.e. cloned) and + if the currently checked out submodule commit
+        does not match the SHA-1 found in the parent repo index state.
+        """)
+        # sync
+        p_sync = sub.add_parser('sync', parents=[common],
+            help='synchronize subrepo\'s remote URL configuration with parent configuration')
+        p_sync.set_defaults(func=self.cmd_sync)
+        p_sync.epilog = textwrap.dedent("""\
+        The subrepo's .git/config URL configuration is set to the value specified in
+        the parent's .hgstate (with relative path suitable resolved wrt parent project).
+        """)
+        return p
+
+    def git_hg_repo_try(self, path):
+        try:
+            return GitHgRepo(topdir=path)
+        except:
+            return None
+
+    def run_cmd(self, options, repo, *args, **kwargs):
+        # no output if quiet, otherwise show output by default
+        # unless the caller specified something explicitly
+        if hasattr(options, 'quiet') and options.quiet:
+            kwargs['stdout'] = os.devnull
+        elif 'stdout' not in kwargs:
+            kwargs['stdout'] = None
+        # show errors unless caller decide some way
+        if 'stderr' not in kwargs:
+            kwargs['stderr'] = None
+        repo.run_cmd(*args, **kwargs)
+
+    class subcontext(dict):
+        __getattr__= dict.__getitem__
+        __setattr__= dict.__setitem__
+        __delattr__= dict.__delitem__
+        def __init__(self, repo):
+            # recursion level
+            self.level = 0
+            # super repo object
+            self.repo = repo
+            # sub repo object (if any)
+            self.subrepo = None
+            # (recursively) collected path of subrepo
+            self.subpath = None
+            # relative path of subrepo wrt super repo
+            self.relpath = None
+
+    def do(self, options, args):
+        if args:
+            # all arguments are registered, so should not have leftover
+            # could be that main arguments were given to subcommands
+            warn('unparsed arguments: %s' % ' '.join(args))
+        log('running subcmd options %s, args %s', options, args)
+        # establish initial operation ctx
+        ctx = self.subcontext(self.githgrepo)
+        self.do_operation(options, args, ctx)
+
+    def do_operation(self, options, args, ctx):
+        log('running %s with options %s in context %s', \
+            options.func, options, ctx)
+        subrepos = ctx.repo.state(options.remote)
+        paths = subrepos.keys()
+        selabspaths = None
+        if ctx.level == 0 and hasattr(options, 'paths') and options.paths:
+            selabspaths = [ os.path.abspath(p) for p in options.paths ]
+        log('level %s selected paths %s', ctx.level, selabspaths)
+        for p in paths:
+            # prep context
+            ctx.relpath = p
+            ctx.subpath = os.path.join(ctx.repo.topdir, p)
+            # check if selected
+            abspath = os.path.abspath(ctx.subpath)
+            if selabspaths != None and abspath not in selabspaths:
+                log('skipping subrepo abspath %s' % abspath)
+                continue
+            ctx.subrepo = self.git_hg_repo_try(ctx.subpath)
+            ctx.state = subrepos[p]
+            # perform operation
+            log('exec for context %s', ctx)
+            options.func(options, args, ctx)
+            if not ctx.subrepo:
+                ctx.subrepo = self.git_hg_repo_try(ctx.subpath)
+            # prep recursion (only into git-hg subrepos)
+            if ctx.subrepo and options.recursive and ctx.state[2] == 'hg':
+                newctx = self.subcontext(ctx.subrepo)
+                newctx.level = ctx.level + 1
+                self.do_operation(options, args, newctx)
+
+    def get_git_commit(self, ctx):
+        src, rev, kind = ctx.state
+        if kind == 'hg':
+            gitcommit = ctx.subrepo.get_git_commit(rev)
+            if not gitcommit:
+                die('could not determine git commit for %s; a fetch may update notes' % rev)
+        else:
+            gitcommit = rev
+        return gitcommit
+
+    def cmd_upstate(self, options, args, ctx):
+        if not ctx.subrepo:
+            return
+        src, orig, kind = ctx.state
+        gitcommit = ctx.subrepo.rev_parse('HEAD')
+        if not gitcommit:
+            die('could not determine current HEAD state in %s' % ctx.subrepo.topdir)
+        rev = gitcommit
+        if kind == 'hg':
+            rev = ctx.subrepo.get_hg_rev(gitcommit)
+            if not rev:
+                die('could not determine hg changeset for commit %s' % gitcommit)
+        else:
+            rev = gitcommit
+        # obtain state from index
+        state_path = os.path.join(ctx.repo.topdir, '.hgsubstate')
+        # should have this, since we have subrepo (state) in the first place ...
+        if not os.path.exists(state_path):
+            die('no .hgsubstate found in repo %s' % ctx.repo.topdir)
+        if orig != rev:
+            short = ctx.subrepo.rev_parse(['--short', gitcommit])
+            print "Updating %s to %s [git %s]" % (ctx.subpath, rev, short)
+            # replace and update index
+            with open(state_path, 'r') as f:
+                state = f.read()
+            state = re.sub('.{40} %s' % (ctx.relpath), '%s %s' % (rev, ctx.relpath), state)
+            with open(state_path, 'wb') as f:
+                f.write(state)
+
+    def cmd_foreach(self, options, args, ctx):
+        if not ctx.subrepo:
+            return
+        if not options.quiet:
+            print 'Entering %s' % ctx.subpath
+            sys.stdout.flush()
+        newenv = os.environ.copy()
+        newenv['path'] = ctx.relpath
+        newenv['sha1'] = self.get_git_commit(ctx)
+        newenv['toplevel'] = os.path.abspath(ctx.repo.topdir)
+        newenv['rev'] = ctx.state[1]
+        newenv['kind'] = ctx.state[2]
+        proc = subprocess.Popen(options.command, shell=True, cwd=ctx.subpath, env=newenv)
+        proc.wait()
+        if proc.returncode != 0:
+            die('stopping at %s; script returned non-zero status' % ctx.subpath)
+
+    def cmd_update(self, options, args, ctx):
+        if not ctx.subrepo:
+            src, _, _ = ctx.state
+            self.run_cmd(options, ctx.repo, ['clone', src, ctx.subpath], cwd=None)
+            ctx.subrepo = self.git_hg_repo_try(ctx.subpath)
+            if not ctx.subrepo:
+                die('subrepo %s setup clone failed', ctx.subpath)
+            # force (detached) checkout of target commit following clone
+            cmd = [ 'checkout', '-q' ]
+        else:
+            self.run_cmd(options, ctx.subrepo, ['fetch', 'origin'], check=True)
+            cmd = []
+        # check if subrepo is up-to-date,
+        # i.e. if target commit is ancestor of HEAD
+        # (output never to be shown)
+        gitcommit = self.get_git_commit(ctx)
+        newrev = ctx.subrepo.run_cmd(['rev-list', gitcommit, '^HEAD']).strip()
+        if newrev and not cmd:
+            if options.force:
+                self.run_cmd(options, ctx.subrepo, ['reset', '--hard', 'HEAD'],
+                    check=True)
+            if options.rebase:
+                cmd = [ 'rebase' ]
+            elif options.merge:
+                cmd = [ 'merge' ]
+            else:
+                # we know about the detached consequences ... keep it a bit quiet
+                cmd = [ 'checkout', '-q' ]
+        if cmd:
+            cmd.append(gitcommit)
+            self.run_cmd(options, ctx.subrepo, cmd, check=True)
+
+    def cmd_status(self, options, args, ctx):
+        if not ctx.subrepo:
+            state = '-'
+            revname = ''
+            _, gitcommit, kind = ctx.state
+            if kind != 'git':
+                gitcommit += '[hg] '
+        else:
+            gitcommit = self.get_git_commit(ctx)
+            head = ctx.subrepo.rev_parse('HEAD')
+            if head == gitcommit:
+                state = ' '
+            else:
+                state = '+'
+                # option determines what to print
+                if not options.cached:
+                    gitcommit = head
+            revname = ctx.subrepo.rev_describe(gitcommit)
+            if revname:
+                revname = ' (%s)' % revname
+        print "%s%s %s%s" % (state, gitcommit, ctx.subpath, revname)
+
+    def cmd_sync(self, options, args, ctx):
+        if not ctx.subrepo:
+            return
+        src, _, _ = ctx.state
+        self.run_cmd(options, ctx.subrepo, \
+            ['config', 'remote.%s.url' % (options.remote), src])
+
+
+class RepoCommand(SubCommand):
+
+    def argumentparser(self):
+        usage = '%%(prog)s %s [options] <remote>...' % (self.subcommand)
+        p = argparse.ArgumentParser(usage=usage)
+        p.epilog = textwrap.dedent("""\
+        Determines the local hg repository of <remote>.
+        This can either be a separate and independent local hg repository
+        or a local proxy repo (within the .git directory).
+        """)
+        return p
+
+    def do(self, options, args):
+        (remote, args) = self.get_remote(args)
+        repos = self.githgrepo.get_hg_repos()
+        if remote in repos:
+            print repos[remote].rstrip('/')
+
+
+class HgCommand(SubCommand):
+
+    def argumentparser(self):
+        usage = '%%(prog)s %s <hg-command>...' % (self.subcommand)
+        p = argparse.ArgumentParser(usage=usage)
+        hgdir = self.githgrepo.get_hg_repos()[self.subcommand]
+        p.epilog = textwrap.dedent("""\
+        Executes <hg-command> on the backing repository of %s (%s)
+        (by supplying it with the standard -R option).
+        """ % (self.subcommand, hgdir))
+        return p
+
+    def do(self, options, args):
+        # subcommand name is already a known valid alias of hg repo
+        remote = self.subcommand
+        repos = self.githgrepo.get_hg_repos()
+        if len(args) and remote in repos:
+            if args[0].find('hg') < 0:
+                args.insert(0, 'hg')
+            args[1:1] = ['-R', repos[remote]]
+            p = subprocess.Popen(args, stdout=None)
+            p.wait()
+        else:
+            if len(args):
+                self.usage('invalid repo: %s' % remote)
+            else:
+                self.usage('missing command')
+
+
+class HelpCommand(SubCommand):
+
+    def do(self, options, args):
+        if len(args):
+            cmd = args[0]
+            if cmd in subcommands:
+                p = subcommands[cmd].argumentparser()
+                p.print_help(sys.stderr)
+                return
+        do_usage()
+
+
+def get_subcommands():
+    commands = {
+        'hg-rev': HgRevCommand,
+        'git-rev': GitRevCommand,
+        'repo': RepoCommand,
+        'gc':  GcCommand,
+        'sub': SubRepoCommand,
+        'help' : HelpCommand
+    }
+    # add remote named subcommands
+    repos = githgrepo.get_hg_repos()
+    for r in repos:
+        if not r in commands:
+            commands[r] = HgCommand
+    # now turn into instances
+    for c in commands:
+        commands[c] = commands[c](c, githgrepo)
+    return commands
+
+
+def do_usage():
+    usage = textwrap.dedent("""
+    git-hg-helper subcommands:
+
+    hg-rev  \t show hg revision corresponding to a git revision
+    git-rev \t find git revision corresponding to a hg revision
+    gc      \t perform maintenance and consistency cleanup on repo tracking marks
+    sub     \t manage subrepos
+    repo    \t show local hg repo backing a remote
+
+    If the subcommand is the name of a remote hg repo, then any remaining arguments
+    are considered a "hg command", e.g. hg heads, or thg, and it is then executed
+    with -R set appropriately to the local hg repo backing the specified remote.
+    Do note, however, that the local proxy repos are not maintained as exact mirrors
+    of their respective remote, and also use shared storage.  As such, depending
+    on the command, the result may not be exactly as could otherwise be expected
+    (e.g. might involve more heads, etc).
+
+    Available hg remotes:
+    """)
+    for r in githgrepo.get_hg_repos():
+        usage += '\t%s\n' % (r)
+    usage += '\n'
+    sys.stderr.write(usage)
+    sys.stderr.flush()
+    sys.exit(2)
+
+def init_git(gitdir=None):
+    global githgrepo
+
+    try:
+        githgrepo = GitHgRepo(gitdir=gitdir)
+    except Exception, e:
+        die(str(e))
+
+def init_logger():
+    global logger
+
+    # setup logging
+    logging.LOG = 5
+    logging.addLevelName(logging.LOG, 'LOG')
+    envlevel = os.environ.get('GIT_HG_HELPER_DEBUG', 'WARN')
+    loglevel = logging.getLevelName(envlevel)
+    logging.basicConfig(level=loglevel, \
+        format='%(asctime)-15s %(levelname)s %(message)s')
+    logger = logging.getLogger()
+
+def init_version():
+    global hg_version
+
+    try:
+        version, _, extra = util.version().partition('+')
+        version = list(int(e) for e in version.split('.'))
+        if extra:
+            version[-1] += 1
+        hg_version = tuple(version)
+    except:
+        hg_version = None
+
+def check_version(*check):
+    if not hg_version:
+        return True
+    return hg_version >= check
+
+def main(argv):
+    global subcommands
+
+    # as an alias, cwd is top dir, change again to original directory
+    reldir = os.environ.get('GIT_PREFIX')
+    if reldir:
+        os.chdir(reldir)
+
+    # init repo dir
+    # we will take over dir management ...
+    init_git(os.environ.pop('GIT_DIR', None))
+
+    subcommands = get_subcommands()
+
+    cmd = ''
+    if len(argv) > 1:
+        cmd = argv[1]
+    argv = argv[2:]
+    if cmd in subcommands:
+        c = subcommands[cmd]
+        c.execute(argv)
+    else:
+        do_usage()
+
+init_logger()
+init_version()
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))

setup.py

@@ -0,0 +1,46 @@
+# git-remote-hg setuptools script
+
+import setuptools
+import subprocess
+import sys
+import os
+
+# strip leading v
+version = 'v1.0.1'
+
+# check for released version
+assert (len(version) > 0)
+assert (version.find('-') < 0)
+
+long_description = \
+"""
+'git-remote-hg' is a gitremote protocol helper for Mercurial.
+It allows you to clone, fetch and push to and from Mercurial repositories as if
+they were Git ones using a hg::some-url URL.
+
+See the homepage for much more explanation.
+"""
+
+CLASSIFIERS = [
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 2",
+    "Programming Language :: Python :: 2.7",
+    "License :: OSI Approved",
+    "License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+]
+
+setuptools.setup(name="git-remote-hg",
+      version=version,
+      author="Mark Nauwelaerts",
+      author_email="mnauw@users.sourceforge.net",
+      url="http://github.com/mnauw/git-remote-hg",
+      description="access hg repositories as git remotes",
+      long_description=long_description,
+      license="GPLv2",
+      keywords="git hg mercurial",
+      scripts=["git-remote-hg", "git-hg-helper"],
+      classifiers=CLASSIFIERS
+     )
+

test/Makefile

@@ -1,6 +1,6 @@
 RM ?= rm -f
 
-T = $(wildcard *.t)
+T = main.t main-push.t bidi.t helper.t
 TEST_DIRECTORY := $(CURDIR)
 
 export TEST_DIRECTORY

test/bidi.t

@@ -8,7 +8,8 @@
 
 test_description='Test bidirectionality of remote-hg'
 
-. ./test-lib.sh
+test -n "$TEST_DIRECTORY" || TEST_DIRECTORY=$(dirname $0)/
+. "$TEST_DIRECTORY"/test-lib.sh
 
 if ! test_have_prereq PYTHON
 then
@@ -50,7 +51,7 @@ hg_push () {
 }
 
 hg_log () {
-	hg -R $1 log --graph --debug
+	hg -R $1 log --debug
 }
 
 setup () {
@@ -203,8 +204,9 @@ test_expect_success 'hg branch' '
 	: Back to the common revision &&
 	(cd hgrepo && hg checkout default) &&
 
-	hg_log hgrepo > expected &&
+	# fetch does not affect phase, but pushing now does
-	hg_log hgrepo2 > actual &&
+	hg_log hgrepo | grep -v phase > expected &&
+	hg_log hgrepo2 | grep -v phase > actual &&
 
 	test_cmp expected actual
 '
@@ -231,10 +233,12 @@ test_expect_success 'hg tags' '
 	) &&
 
 	hg_push hgrepo gitrepo &&
-	hg_clone gitrepo hgrepo2 &&
+	# pushing a fetched tag is a problem ...
+	{ hg_clone gitrepo hgrepo2 || true ; } &&
 
-	hg_log hgrepo > expected &&
+	# fetch does not affect phase, but pushing now does
-	hg_log hgrepo2 > actual &&
+	hg_log hgrepo | grep -v phase > expected &&
+	hg_log hgrepo2 | grep -v phase > actual &&
 
 	test_cmp expected actual
 '

test/helper.t

@@ -0,0 +1,547 @@
+#!/bin/sh
+#
+# Copyright (c) 2016 Mark Nauwelaerts
+#
+# Base commands from hg-git tests:
+# https://bitbucket.org/durin42/hg-git/src
+#
+
+test_description='Test git-hg-helper'
+
+test -n "$TEST_DIRECTORY" || TEST_DIRECTORY=$(dirname $0)/
+. "$TEST_DIRECTORY"/test-lib.sh
+
+if ! test_have_prereq PYTHON
+then
+	skip_all='skipping remote-hg tests; python not available'
+	test_done
+fi
+
+if ! python2 -c 'import mercurial' > /dev/null 2>&1
+then
+	skip_all='skipping remote-hg tests; mercurial not available'
+	test_done
+fi
+
+setup () {
+	cat > "$HOME"/.hgrc <<-EOF &&
+	[ui]
+	username = H G Wells <wells@example.com>
+	[extensions]
+	mq =
+	strip =
+	[subrepos]
+	git:allowed = true
+	EOF
+
+	GIT_AUTHOR_DATE="2007-01-01 00:00:00 +0230" &&
+	GIT_COMMITTER_DATE="$GIT_AUTHOR_DATE" &&
+	export GIT_COMMITTER_DATE GIT_AUTHOR_DATE
+}
+
+setup
+
+setup_repos () {
+	(
+	hg init hgrepo &&
+	cd hgrepo &&
+	echo zero > content &&
+	hg add content &&
+	hg commit -m zero
+	) &&
+
+	git clone hg::hgrepo gitrepo
+}
+
+test_expect_success 'subcommand help' '
+	test_when_finished "rm -rf gitrepo* hgrepo*" &&
+
+	setup_repos &&
+
+	(
+	cd gitrepo &&
+	test_expect_code 2 git-hg-helper help 2> ../help
+	)
+	# remotes should be in help output
+	grep origin help
+'
+
+git config --global remote-hg.shared-marks false
+test_expect_success 'subcommand repo - no local proxy' '
+	test_when_finished "rm -rf gitrepo* hgrepo*" &&
+
+	setup_repos &&
+
+	(
+	cd hgrepo &&
+	pwd >../expected
+	) &&
+
+	(
+	cd gitrepo &&
+	git-hg-helper repo origin > ../actual
+	) &&
+
+	test_cmp expected actual
+'
+
+git config --global --unset remote-hg.shared-marks
+
+GIT_REMOTE_HG_TEST_REMOTE=1 &&
+export GIT_REMOTE_HG_TEST_REMOTE
+
+test_expect_success 'subcommand repo - with local proxy' '
+	test_when_finished "rm -rf gitrepo* hgrepo*" &&
+
+	setup_repos &&
+
+	(
+	cd gitrepo &&
+	export gitdir=`git rev-parse --git-dir`
+	# trick to normalize path
+	( cd $gitdir/hg/origin/clone && pwd ) >../expected &&
+	( cd `git-hg-helper repo origin` && pwd ) > ../actual
+	) &&
+
+	test_cmp expected actual
+'
+
+test_expect_success 'subcommands hg-rev and git-rev' '
+	test_when_finished "rm -rf gitrepo* hgrepo*" &&
+
+	setup_repos &&
+
+	(
+	cd gitrepo &&
+	git rev-parse HEAD > rev-HEAD &&
+	test -s rev-HEAD &&
+	git-hg-helper hg-rev `cat rev-HEAD` > hg-HEAD &&
+	git-hg-helper git-rev `cat hg-HEAD` > git-HEAD &&
+	test_cmp rev-HEAD git-HEAD
+	)
+'
+
+test_expect_success 'subcommand gc' '
+	test_when_finished "rm -rf gitrepo* hgrepo*" &&
+
+	(
+	hg init hgrepo &&
+	cd hgrepo &&
+	echo zero > content &&
+	hg add content &&
+	hg commit -m zero
+	echo one > content &&
+	hg commit -m one &&
+	echo two > content &&
+	hg commit -m two &&
+	echo three > content &&
+	hg commit -m three
+	) &&
+
+	git clone hg::hgrepo gitrepo &&
+
+	(
+	cd hgrepo &&
+	hg strip -r 1 &&
+	echo four > content &&
+	hg commit -m four
+	) &&
+
+	(
+	cd gitrepo &&
+	git fetch origin &&
+	git reset --hard origin/master &&
+	git gc &&
+	git-hg-helper gc --check-hg origin > output &&
+	cat output &&
+	grep "hg marks" output &&
+	grep "git marks" output
+	)
+'
+
+test_expect_success 'subcommand [some-repo]' '
+	test_when_finished "rm -rf gitrepo* hgrepo*" &&
+
+	setup_repos &&
+
+	(
+	cd hgrepo &&
+	echo one > content &&
+	hg commit -m one
+	) &&
+
+	(
+	cd gitrepo &&
+	git fetch origin
+	) &&
+
+	hg log -R hgrepo > expected &&
+	# not inside gitrepo; test shared path handling
+	GIT_DIR=gitrepo/.git git-hg-helper origin log > actual
+
+	test_cmp expected actual
+'
+
+setup_repo () {
+    kind=$1 &&
+    repo=$2 &&
+    $kind init $repo &&
+    (
+    cd $repo &&
+    echo zero > content_$repo &&
+    $kind add content_$repo &&
+    $kind commit -m zero_$repo
+    )
+}
+
+check () {
+	echo $3 > expected &&
+	git --git-dir=$1/.git log --format='%s' -1 $2 > actual &&
+	test_cmp expected actual
+}
+
+check_branch () {
+	if test -n "$3"
+	then
+		echo $3 > expected &&
+		hg -R $1 log -r $2 --template '{desc}\n' > actual &&
+		test_cmp expected actual
+	else
+		hg -R $1 branches > out &&
+		! grep $2 out
+	fi
+}
+
+test_expect_success 'subcommand sub initial update (hg and git subrepos)' '
+	test_when_finished "rm -rf gitrepo* hgrepo*" &&
+
+	setup_repo hg hgrepo &&
+	(
+	cd hgrepo &&
+	setup_repo hg sub_hg_a &&
+	setup_repo hg sub_hg_b &&
+	setup_repo git sub_git &&
+	echo "sub_hg_a = sub_hg_a" > .hgsub &&
+	echo "sub_hg_b = sub_hg_b" >> .hgsub &&
+	echo "sub_git = [git]sub_git" >> .hgsub &&
+	hg add .hgsub &&
+	hg commit -m substate
+	)
+
+	git clone hg::hgrepo gitrepo &&
+
+	(
+	cd gitrepo &&
+	git-hg-helper sub update --force &&
+	test -f content_hgrepo &&
+	test -f sub_hg_a/content_sub_hg_a &&
+	test -f sub_hg_b/content_sub_hg_b &&
+	test -f sub_git/content_sub_git
+	) &&
+
+	check gitrepo HEAD substate &&
+	check gitrepo/sub_hg_a HEAD zero_sub_hg_a &&
+	check gitrepo/sub_hg_b HEAD zero_sub_hg_b &&
+	check gitrepo/sub_git HEAD zero_sub_git
+'
+
+setup_subrepos () {
+	setup_repo hg hgrepo &&
+	(
+	cd hgrepo &&
+	setup_repo hg sub_hg_a &&
+		(
+		cd sub_hg_a &&
+		setup_repo hg sub_hg_a_x &&
+		echo "sub_hg_a_x = sub_hg_a_x" > .hgsub &&
+		hg add .hgsub &&
+		hg commit -m substate_hg_a
+		) &&
+	setup_repo hg sub_hg_b &&
+		(
+		cd sub_hg_b &&
+		setup_repo git sub_git &&
+		echo "sub_git = [git]sub_git" > .hgsub &&
+		hg add .hgsub &&
+		hg commit -m substate_hg_b
+		) &&
+	echo "sub_hg_a = sub_hg_a" > .hgsub &&
+	echo "sub_hg_b = sub_hg_b" >> .hgsub &&
+	hg add .hgsub &&
+	hg commit -m substate
+	)
+}
+
+test_expect_success 'subcommand sub initial recursive update' '
+	test_when_finished "rm -rf gitrepo* hgrepo*" &&
+
+	setup_subrepos &&
+
+	git clone hg::hgrepo gitrepo &&
+
+	(
+	cd gitrepo &&
+	git-hg-helper sub --recursive update --force &&
+	test -f content_hgrepo &&
+	test -f sub_hg_a/content_sub_hg_a &&
+	test -f sub_hg_a/sub_hg_a_x/content_sub_hg_a_x &&
+	test -f sub_hg_b/content_sub_hg_b &&
+	test -f sub_hg_b/sub_git/content_sub_git
+	) &&
+
+	check gitrepo HEAD substate &&
+	check gitrepo/sub_hg_a HEAD substate_hg_a &&
+	check gitrepo/sub_hg_b HEAD substate_hg_b &&
+	check gitrepo/sub_hg_a/sub_hg_a_x HEAD zero_sub_hg_a_x &&
+	check gitrepo/sub_hg_b/sub_git HEAD zero_sub_git
+'
+
+test_sub_update () {
+	export option=$1
+
+	setup_subrepos &&
+
+	git clone hg::hgrepo gitrepo &&
+
+	(
+	cd gitrepo &&
+	git-hg-helper sub --recursive update --force
+	) &&
+
+	(
+	cd hgrepo &&
+		(
+		 cd sub_hg_a &&
+			(
+			cd sub_hg_a_x &&
+			echo one > content_sub_hg_a_x &&
+			hg commit -m one_sub_hg_a_x
+			) &&
+		hg commit -m substate_updated_hg_a
+		) &&
+	hg commit -m substate_updated
+	) &&
+
+	(
+	cd gitrepo &&
+	git fetch origin &&
+	git merge origin/master &&
+	git-hg-helper sub --recursive update --force $option &&
+	test -f content_hgrepo &&
+	test -f sub_hg_a/content_sub_hg_a &&
+	test -f sub_hg_a/sub_hg_a_x/content_sub_hg_a_x &&
+	test -f sub_hg_b/content_sub_hg_b &&
+	test -f sub_hg_b/sub_git/content_sub_git
+	) &&
+
+	check gitrepo HEAD substate_updated &&
+	check gitrepo/sub_hg_a HEAD substate_updated_hg_a &&
+	check gitrepo/sub_hg_b HEAD substate_hg_b &&
+	check gitrepo/sub_hg_a/sub_hg_a_x HEAD one_sub_hg_a_x &&
+	check gitrepo/sub_hg_b/sub_git HEAD zero_sub_git
+}
+
+test_expect_success 'subcommand sub subsequent recursive update' '
+	test_when_finished "rm -rf gitrepo* hgrepo*" &&
+
+	test_sub_update
+'
+
+test_expect_success 'subcommand sub subsequent recursive update -- rebase' '
+	test_when_finished "rm -rf gitrepo* hgrepo*" &&
+
+	test_sub_update --rebase
+'
+
+test_expect_success 'subcommand sub subsequent recursive update -- merge' '
+	test_when_finished "rm -rf gitrepo* hgrepo*" &&
+
+	test_sub_update --merge
+'
+
+check_foreach_vars () {
+	cat $1 | while read kind sha1 rev path remainder
+	do
+	    ok=0
+	    if test "$kind" = "hg" ; then
+			if test "$sha1" != "$rev" ; then
+				ok=1
+			fi
+	    else
+			if test "$sha1" = "$rev" ; then
+				ok=1
+			fi
+	    fi
+	    test $ok -eq 1 || echo "invalid $kind $sha1 $rev $path"
+	    test $ok -eq 1 || return 1
+	done &&
+
+	return 0
+}
+
+test_sub_foreach () {
+	setup_subrepos &&
+
+	git clone hg::hgrepo gitrepo &&
+
+	(
+	cd gitrepo &&
+	git-hg-helper sub --recursive update --force &&
+	git-hg-helper sub --recursive --quiet foreach 'echo $kind $sha1 $rev $path $toplevel' > output &&
+	cat output &&
+	echo 1 > expected_git &&
+	grep -c ^git output > actual_git &&
+	test_cmp expected_git actual_git &&
+	echo 3 > expected_hg &&
+	grep -c ^hg output > actual_hg &&
+	test_cmp expected_hg actual_hg &&
+	grep '\(hg\|git\) [0-9a-f]* [0-9a-f]* sub[^ ]* /.*' output > actual &&
+	test_cmp output actual &&
+	check_foreach_vars output
+	)
+}
+
+test_expect_success 'subcommand sub foreach' '
+	test_when_finished "rm -rf gitrepo* hgrepo*" &&
+
+	test_sub_foreach
+'
+
+test_expect_success 'subcommand sub sync' '
+	test_when_finished "rm -rf gitrepo* hgrepo*" &&
+
+	setup_repo hg hgrepo &&
+	(
+	cd hgrepo &&
+	setup_repo hg sub_hg &&
+	echo "sub_hg = sub_hg" > .hgsub &&
+	hg add .hgsub &&
+	hg commit -m substate
+	)
+
+	git clone hg::hgrepo gitrepo &&
+
+	(
+	cd gitrepo &&
+	git-hg-helper sub update --force &&
+
+		(
+		cd sub_hg &&
+		grep url .git/config > ../expected &&
+		git config remote.origin.url foobar &&
+		grep foobar .git/config
+		) &&
+
+	git-hg-helper sub sync &&
+	grep url sub_hg/.git/config > actual &&
+	test_cmp expected actual
+	)
+'
+
+test_expect_success 'subcommand sub addstate' '
+	test_when_finished "rm -rf gitrepo* hgrepo*" &&
+
+	setup_repo hg hgrepo &&
+	(
+	cd hgrepo &&
+	setup_repo hg sub_hg &&
+	setup_repo git sub_git &&
+	echo "sub_hg = sub_hg" > .hgsub &&
+	echo "sub_git = [git]sub_git" >> .hgsub &&
+	hg add .hgsub &&
+	hg commit -m substate
+	)
+
+	git clone hg::hgrepo gitrepo &&
+
+	(
+	cd gitrepo &&
+	git-hg-helper sub update --force &&
+
+		(
+		cd sub_hg &&
+		echo one > content_sub_hg &&
+		git add content_sub_hg &&
+		git commit -m one_sub_hg &&
+		# detached HEAD
+		git push origin HEAD:master &&
+		# also fetch to ensure notes are updated
+		git fetch origin
+		) &&
+
+		(
+		cd sub_git &&
+		echo one > content_sub_git &&
+		git add content_sub_git &&
+		git commit -m one_sub_git &&
+		# detached HEAD; push revision to other side ... anywhere
+		git push origin HEAD:refs/heads/new
+		)
+	) &&
+
+	(
+	cd gitrepo &&
+	git-hg-helper sub upstate &&
+	git diff &&
+	git status --porcelain | grep .hgsubstate &&
+	git add .hgsubstate &&
+	git commit -m update_sub &&
+	git push origin master
+	) &&
+
+	hg clone hgrepo hgclone &&
+
+	(
+	cd hgclone &&
+	hg update
+	) &&
+
+	check_branch hgclone default update_sub &&
+	check_branch hgclone/sub_hg default one_sub_hg &&
+	check hgclone/sub_git HEAD one_sub_git
+'
+
+test_expect_success 'subcommand sub status' '
+	test_when_finished "rm -rf gitrepo* hgrepo*" &&
+
+	setup_repo hg hgrepo &&
+	(
+	cd hgrepo &&
+	setup_repo hg sub_hg_a &&
+	setup_repo hg sub_hg_b &&
+	setup_repo git sub_git &&
+	echo "sub_hg_a = sub_hg_a" > .hgsub &&
+	echo "sub_hg_b = sub_hg_b" >> .hgsub &&
+	echo "sub_git = [git]sub_git" >> .hgsub &&
+	hg add .hgsub &&
+	hg commit -m substate
+	)
+
+	git clone hg::hgrepo gitrepo &&
+
+	(
+	cd gitrepo &&
+	git-hg-helper sub update sub_hg_a --force &&
+	git-hg-helper sub update sub_git --force &&
+		(
+		# advance and add a tag to the git repo
+		cd sub_git &&
+		echo one > content_sub_git &&
+		git add content_sub_git &&
+		git commit -m one_sub_git &&
+		git tag feature-a
+		) &&
+
+	git-hg-helper sub status --cached > output &&
+	cat output &&
+	grep "^ .*sub_hg_a (.*master.*)$" output &&
+	grep "^-.*sub_hg_b$" output &&
+	grep "^+.*sub_git (feature-a~1)$" output &&
+	git-hg-helper sub status sub_git > output &&
+	cat output &&
+	grep "^+.*sub_git (feature-a)$" output > actual &&
+	test_cmp output actual
+	)
+'
+
+test_done

test/hg-git.t

@@ -8,7 +8,8 @@
 
 test_description='Test remote-hg output compared to hg-git'
 
-. ./test-lib.sh
+test -n "$TEST_DIRECTORY" || TEST_DIRECTORY=$(dirname $0)/
+. "$TEST_DIRECTORY"/test-lib.sh
 
 if ! test_have_prereq PYTHON
 then
@@ -22,12 +23,26 @@ then
 	test_done
 fi
 
-if ! python2 -c 'import hggit' > /dev/null 2>&1
+if python2 -c 'import hggit' > /dev/null 2>&1
 then
+	hggit=hggit
+elif python2 -c 'import hgext.git' > /dev/null 2>&1
+then
+	hggit=hgext.git
+else
 	skip_all='skipping remote-hg tests; hg-git not available'
 	test_done
 fi
 
+hg_version=$(python2 -c 'from mercurial import util; print util.version()')
+
+case $hg_version in
+3.0*+*)
+	skip_all='skipping remote-hg tests; unsuported version of hg by hg-git'
+	test_done
+	;;
+esac
+
 # clone to a git repo with git
 git_clone_git () {
 	git clone -q "hg::$1" $2 &&
@@ -100,8 +115,7 @@ setup () {
 	debugrawcommit = -d "0 0"
 	tag = -d "0 0"
 	[extensions]
-	hgext.bookmarks =
+	$hggit =
-	hggit =
 	graphlog =
 	EOF
 	git config --global receive.denycurrentbranch warn

test/main-push.t

@@ -0,0 +1,314 @@
+CAPABILITY_PUSH=t
+
+test -n "$TEST_DIRECTORY" || TEST_DIRECTORY=$(dirname $0)/
+. "$TEST_DIRECTORY"/main.t
+
+
+# .. and some push mode only specific tests
+
+test_expect_success 'remote delete bookmark' '
+	test_when_finished "rm -rf hgrepo* gitrepo*" &&
+
+	(
+	hg init hgrepo &&
+	cd hgrepo &&
+	echo zero > content &&
+	hg add content &&
+	hg commit -m zero
+	hg bookmark feature-a
+	) &&
+
+	git clone "hg::hgrepo" gitrepo &&
+	check_bookmark hgrepo feature-a zero &&
+
+	(
+	cd gitrepo &&
+	git push --quiet origin :feature-a
+	) &&
+
+	check_bookmark hgrepo feature-a ''
+'
+
+test_expect_success 'source:dest bookmark' '
+	test_when_finished "rm -rf hgrepo gitrepo" &&
+
+	(
+	hg init hgrepo &&
+	cd hgrepo &&
+	echo zero > content &&
+	hg add content &&
+	hg commit -m zero
+	) &&
+
+	git clone "hg::hgrepo" gitrepo &&
+
+	(
+	cd gitrepo &&
+	echo one > content &&
+	git commit -a -m one &&
+	git push --quiet origin master:feature-b &&
+	git push --quiet origin master^:refs/heads/feature-a
+	) &&
+
+	check_bookmark hgrepo feature-a zero &&
+	check_bookmark hgrepo feature-b one &&
+
+	(
+	cd gitrepo &&
+	git push --quiet origin master:feature-a
+	) &&
+
+	check_bookmark hgrepo feature-a one
+'
+
+setup_check_hg_commits_repo () {
+        (
+	rm -rf hgrepo* &&
+	hg init hgrepo &&
+	cd hgrepo &&
+	echo zero > content &&
+	hg add content &&
+	hg commit -m zero
+	) &&
+
+	git clone "hg::hgrepo" gitrepo &&
+	hg clone hgrepo hgrepo.second &&
+
+	(
+	cd gitrepo &&
+	git remote add second hg::../hgrepo.second &&
+	git fetch second
+	) &&
+
+	(
+	cd hgrepo &&
+	echo one > content &&
+	hg commit -m one &&
+	echo two > content &&
+	hg commit -m two &&
+	echo three > content &&
+	hg commit -m three &&
+	hg move content content-move &&
+	hg commit -m moved &&
+	hg move content-move content &&
+	hg commit -m restored
+        )
+}
+
+# a shared bag would make all of the following pretty trivial
+git config --global remote-hg.shared-marks false
+
+git config --global remote-hg.check-hg-commits fail
+test_expect_success 'check-hg-commits with fail mode' '
+	test_when_finished "rm -rf gitrepo* hgrepo*" &&
+
+	setup_check_hg_commits_repo &&
+
+	(
+	cd gitrepo &&
+	git fetch origin &&
+	git reset --hard origin/master &&
+	! git push second master 2>../error
+	)
+
+	cat error &&
+	grep rejected error | grep hg
+'
+
+git config --global remote-hg.check-hg-commits push
+# codepath for push is slightly different depending on shared proxy involved
+# so tweak to test both
+check_hg_commits_push () {
+	test_when_finished "rm -rf gitrepo* hgrepo*" &&
+
+	setup_check_hg_commits_repo &&
+
+	(
+	cd gitrepo &&
+	git fetch origin &&
+	git reset --hard origin/master &&
+	git push second master 2> ../error
+	) &&
+
+	cat error &&
+	grep "hg changeset" error &&
+
+	hg log -R hgrepo > expected &&
+	hg log -R hgrepo.second | grep -v bookmark > actual &&
+	test_cmp expected actual
+}
+
+unset GIT_REMOTE_HG_TEST_REMOTE
+test_expect_success 'check-hg-commits with push mode - no local proxy' '
+	check_hg_commits_push
+'
+
+GIT_REMOTE_HG_TEST_REMOTE=1 &&
+export GIT_REMOTE_HG_TEST_REMOTE
+test_expect_success 'check-hg-commits with push mode - with local proxy' '
+	check_hg_commits_push
+'
+
+setup_check_shared_marks_repo () {
+        (
+	rm -rf hgrepo* &&
+	hg init hgrepo &&
+	cd hgrepo &&
+	echo zero > content &&
+	hg add content &&
+	hg commit -m zero
+	) &&
+
+	git clone "hg::hgrepo" gitrepo &&
+
+	(
+	cd gitrepo &&
+	git remote add second hg::../hgrepo &&
+	git fetch second
+	)
+}
+
+check_marks () {
+	dir=$1
+
+	ls -al $dir &&
+	if test "$2" = "y"
+	then
+		test -f $dir/marks-git && test -f $dir/marks-hg
+	else
+		test ! -f $dir/marks-git && test ! -f $dir/marks-hg
+	fi
+}
+
+# cleanup setting
+git config --global --unset remote-hg.shared-marks
+
+test_expect_success 'shared-marks unset' '
+	test_when_finished "rm -rf gitrepo* hgrepo*" &&
+
+	setup_check_shared_marks_repo &&
+
+	(
+	cd gitrepo &&
+	check_marks .git/hg y &&
+	check_marks .git/hg/origin n &&
+	check_marks .git/hg/second n
+	)
+'
+
+test_expect_success 'shared-marks set to unset' '
+	test_when_finished "rm -rf gitrepo* hgrepo*" &&
+
+	git config --global remote-hg.shared-marks true &&
+	setup_check_shared_marks_repo &&
+
+	(
+	cd gitrepo &&
+	check_marks .git/hg y &&
+	check_marks .git/hg/origin n &&
+	check_marks .git/hg/second n
+	) &&
+
+	git config --global remote-hg.shared-marks false &&
+	(
+		cd gitrepo &&
+		git fetch origin &&
+		check_marks .git/hg n &&
+		check_marks .git/hg/origin y &&
+		check_marks .git/hg/second y
+	)
+'
+
+test_expect_success 'shared-marks unset to set' '
+	test_when_finished "rm -rf gitrepo* hgrepo*" &&
+
+	git config --global remote-hg.shared-marks false &&
+	setup_check_shared_marks_repo &&
+
+	(
+	cd gitrepo &&
+	check_marks .git/hg n &&
+	check_marks .git/hg/origin y &&
+	check_marks .git/hg/second y
+	) &&
+
+	git config --global --unset remote-hg.shared-marks &&
+	(
+		cd gitrepo &&
+		git fetch origin &&
+		check_marks .git/hg n &&
+		check_marks .git/hg/origin y &&
+		check_marks .git/hg/second y
+	) &&
+
+	git config --global remote-hg.shared-marks true &&
+	(
+		cd gitrepo &&
+		git fetch origin &&
+		check_marks .git/hg y &&
+		check_marks .git/hg/origin n &&
+		check_marks .git/hg/second n
+	)
+'
+
+test_expect_success 'push with renamed executable preserves executable bit' '
+	test_when_finished "rm -rf hgrepo gitrepo*" &&
+
+	hg init hgrepo &&
+
+	(
+	git init gitrepo &&
+	cd gitrepo &&
+	git remote add origin "hg::../hgrepo" &&
+	echo one > content &&
+	chmod a+x content &&
+	git add content &&
+	git commit -a -m one &&
+	git mv content content2 &&
+	git commit -a -m two &&
+	git push origin master
+	) &&
+
+	(
+	cd hgrepo &&
+	hg update &&
+	stat content2 >expected &&
+	# umask mileage might vary
+	grep -- -r.xr.xr.x expected
+	)
+'
+
+test_expect_success 'push with submodule' '
+	test_when_finished "rm -rf sub hgrepo gitrepo*" &&
+
+	hg init hgrepo &&
+
+	(
+	git init sub &&
+	cd sub &&
+	: >empty &&
+	git add empty &&
+	git commit -m init
+	) &&
+
+	(
+	git init gitrepo &&
+	cd gitrepo &&
+	git submodule add ../sub sub &&
+	git remote add origin "hg::../hgrepo" &&
+	git commit -a -m sub &&
+	git push origin master
+	) &&
+
+	(
+	cd hgrepo &&
+	hg update &&
+	expected="[git-remote-hg: skipped import of submodule at $(git -C ../sub rev-parse HEAD)]"
+	test "$expected" = "$(cat sub)"
+	)
+'
+
+# cleanup setting
+git config --global --unset remote-hg.shared-marks
+
+test_done

test/main.t

@@ -8,9 +8,18 @@
 
 test_description='Test remote-hg'
 
-test -n "$TEST_DIRECTORY" || TEST_DIRECTORY=${0%/*}/../../t
+test -n "$TEST_DIRECTORY" || TEST_DIRECTORY=$(dirname $0)/
 . "$TEST_DIRECTORY"/test-lib.sh
 
+if test "$CAPABILITY_PUSH" = "t"
+then
+	git config --global remote-hg.capability-push true
+	git config --global remote-hg.push-updates-notes true
+	git config --global remote-hg.fast-export-options '-C -C -M'
+else
+	git config --global remote-hg.capability-push false
+fi
+
 if ! test_have_prereq PYTHON
 then
 	skip_all='skipping remote-hg tests; python not available'
@@ -81,9 +90,6 @@ check_push () {
 		'non-fast-forward')
 			grep "^ ! \[rejected\] *${branch} -> ${branch} (non-fast-forward)$" error || ref_ret=1
 			;;
-		'fetch-first')
-			grep "^ ! \[rejected\] *${branch} -> ${branch} (fetch first)$" error || ref_ret=1
-			;;
 		'forced-update')
 			grep "^ + [a-f0-9]*\.\.\.[a-f0-9]* *${branch} -> ${branch} (forced update)$" error || ref_ret=1
 			;;
@@ -176,7 +182,7 @@ test_expect_success 'update bookmark' '
 	git checkout --quiet devel &&
 	echo devel > content &&
 	git commit -a -m devel &&
-	git push --quiet
+	git push --quiet origin devel
 	) &&
 
 	check_bookmark hgrepo devel devel
@@ -442,7 +448,7 @@ test_expect_success 'remote update bookmark diverge' '
 	echo diverge > content &&
 	git commit -a -m diverge &&
 	check_push 1 <<-\EOF
-	diverge:fetch-first
+	diverge:non-fast-forward
 	EOF
 	) &&
 
@@ -467,6 +473,52 @@ test_expect_success 'remote new bookmark multiple branch head' '
 # cleanup previous stuff
 rm -rf hgrepo
 
+testcopyrenamedesc='push commits with copy and rename'
+testcopyrename='
+	test_when_finished "rm -rf gitrepo hgrepo" &&
+
+	(
+	hg init hgrepo &&
+	cd hgrepo &&
+	echo zero > content &&
+	hg add content &&
+	hg commit -m zero
+	) &&
+
+	git clone "hg::hgrepo" gitrepo &&
+
+	(
+	cd gitrepo &&
+	cp content content-copy &&
+	# recent git-fast-export is (too) picky in recognizing copies
+	# although git-log is not as picky;
+	# since https://github.com/git/git/commit/8096e1d385660c159d9d47e69b2be63cf22e4f31
+	# a copy is only marked if source filed not modified as well
+	# (though destination file can be modified)
+	echo one >> content-copy &&
+	git add content content-copy &&
+	git commit -m copy &&
+	git mv content-copy content-moved
+	git commit -m moved &&
+	git push origin master
+	) &&
+
+	(
+	hg -R hgrepo update &&
+	test_cmp gitrepo/content hgrepo/content
+	test_cmp gitrepo/content-moved hgrepo/content-moved
+	cd hgrepo &&
+	test `hg log -f content-moved | grep -c changeset` -eq 3
+	)
+'
+
+if test "$CAPABILITY_PUSH" = "t"
+then
+test_expect_success "$testcopyrenamedesc" "$testcopyrename"
+else
+test_expect_failure "$testcopyrenamedesc" "$testcopyrename"
+fi
+
 test_expect_success 'fetch special filenames' '
 	test_when_finished "rm -rf hgrepo gitrepo && LC_ALL=C" &&
 
@@ -618,6 +670,19 @@ test_expect_success 'remote big push' '
 	EOF
 	) &&
 
+	if test "$CAPABILITY_PUSH" = "t"
+	then
+		# cap push handles refs one by one
+		# so it will push all requested it can
+		check_branch hgrepo default six &&
+		check_branch hgrepo good_branch eight &&
+		check_branch hgrepo bad_branch "bad branch" &&
+		check_branch hgrepo new_branch ten &&
+		check_bookmark hgrepo good_bmark three &&
+		check_bookmark hgrepo bad_bmark1 one &&
+		check_bookmark hgrepo bad_bmark2 one &&
+		check_bookmark hgrepo new_bmark six
+	else
 		check_branch hgrepo default one &&
 		check_branch hgrepo good_branch "good branch" &&
 		check_branch hgrepo bad_branch "bad branch" &&
@@ -626,9 +691,10 @@ test_expect_success 'remote big push' '
 		check_bookmark hgrepo bad_bmark1 one &&
 		check_bookmark hgrepo bad_bmark2 one &&
 		check_bookmark hgrepo new_bmark ''
+	fi
 '
 
-test_expect_success 'remote big push fetch first' '
+test_expect_success 'remote big push non fast forward' '
 	test_when_finished "rm -rf hgrepo gitrepo*" &&
 
 	(
@@ -677,18 +743,30 @@ test_expect_success 'remote big push fetch first' '
 	check_push 1 --all <<-\EOF &&
 	master
 	good_bmark
-	bad_bmark:fetch-first
+	bad_bmark:non-fast-forward
-	branches/bad_branch:festch-first
+	branches/bad_branch:non-fast-forward
 	EOF
 
 	git fetch &&
 
+        if test "$CAPABILITY_PUSH" = "t"
+        then
+                # cap push handles refs one by one
+		# so it will already have pushed some above previously
+		# (and master is a fake one that jumps around a bit)
+		check_push 1 --all <<-\EOF
+		master:non-fast-forward
+		bad_bmark:non-fast-forward
+		branches/bad_branch:non-fast-forward
+		EOF
+	else
 		check_push 1 --all <<-\EOF
 		master
 		good_bmark
 		bad_bmark:non-fast-forward
 		branches/bad_branch:non-fast-forward
 		EOF
+	fi
 	)
 '
 
@@ -722,7 +800,7 @@ test_expect_failure 'remote big push force' '
 	check_bookmark hgrepo new_bmark six
 '
 
-test_expect_failure 'remote big push dry-run' '
+test_expect_success 'remote big push dry-run' '
 	test_when_finished "rm -rf hgrepo gitrepo*" &&
 
 	setup_big_push
@@ -783,6 +861,78 @@ test_expect_success 'remote double failed push' '
 	test_expect_code 1 git push
 	)
 '
+test_expect_success 'fetch prune' '
+	test_when_finished "rm -rf gitrepo hgrepo" &&
+
+	(
+	hg init hgrepo &&
+	cd hgrepo &&
+	echo zero > content &&
+	hg add content &&
+	hg commit -m zero &&
+	echo feature-a > content &&
+	hg commit -m feature-a
+	hg bookmark feature-a
+	) &&
+
+	git clone "hg::hgrepo" gitrepo &&
+	check gitrepo origin/feature-a feature-a &&
+
+	(
+	cd hgrepo &&
+	hg bookmark -d feature-a
+	) &&
+
+	(
+	cd gitrepo &&
+	git fetch --prune origin
+	git branch -a > out &&
+	! grep feature-a out
+	)
+'
+
+test_expect_success 'fetch multiple independent histories' '
+	test_when_finished "rm -rf gitrepo hgrepo" &&
+
+	(
+	hg init hgrepo &&
+	cd hgrepo &&
+	echo zero > content &&
+	hg add content &&
+	hg commit -m zero &&
+	hg up -r null &&
+	echo another > ocontent &&
+	hg add ocontent &&
+	hg commit -m one
+	) &&
+
+	# -r 1 acts as master
+	(
+	git init --bare gitrepo && cd gitrepo &&
+	git remote add origin hg::../hgrepo &&
+	git fetch origin refs/heads/*:refs/heads/*
+	) &&
+
+	(
+	cd hgrepo &&
+	hg up 0 &&
+	echo two > content &&
+	hg commit -m two
+	) &&
+
+	# now master already exists
+	# -r 2 becomes master head which has rev 0 as ancestor
+	# so when importing (parentless) rev 0, a reset is needed
+	# (to ensure rev 0 is not given a parent commit)
+	(
+	cd gitrepo &&
+	git fetch origin &&
+	git log --format="%s" origin/master > ../actual
+	) &&
+
+	hg -R hgrepo log -r . -f --template "{desc}\n" > expected &&
+	test_cmp actual expected
+'
 
 test_expect_success 'clone remote with null bookmark, then push' '
 	test_when_finished "rm -rf gitrepo* hgrepo*" &&
@@ -828,7 +978,8 @@ test_expect_success 'notes' '
 	test_cmp expected actual
 '
 
-test_expect_failure 'push updates notes' '
+testpushupdatesnotesdesc='push updates notes'
+testpushupdatesnotes='
 	test_when_finished "rm -rf hgrepo gitrepo" &&
 
 	(
@@ -853,6 +1004,38 @@ test_expect_failure 'push updates notes' '
 	test_cmp expected actual
 '
 
+if test "$CAPABILITY_PUSH" = "t"
+then
+test_expect_success "$testpushupdatesnotesdesc" "$testpushupdatesnotes"
+else
+test_expect_failure "$testpushupdatesnotesdesc" "$testpushupdatesnotes"
+fi
+
+test_expect_success 'push bookmark without changesets' '
+	test_when_finished "rm -rf hgrepo gitrepo" &&
+
+	(
+	hg init hgrepo &&
+	cd hgrepo &&
+	echo one > content &&
+	hg add content &&
+	hg commit -m one
+	) &&
+
+	git clone "hg::hgrepo" gitrepo &&
+
+	(
+	cd gitrepo &&
+	echo two > content &&
+	git commit -a -m two &&
+	git push origin master &&
+	git branch feature-a &&
+	git push origin feature-a
+	) &&
+
+	check_bookmark hgrepo feature-a two
+'
+
 test_expect_success 'pull tags' '
 	test_when_finished "rm -rf hgrepo gitrepo" &&
 
@@ -1024,4 +1207,29 @@ test_expect_success 'clone replace directory with a file' '
 	check_files gitrepo "dir_or_file"
 '
 
+test_expect_success 'clone can ignore invalid refnames' '
+	test_when_finished "rm -rf hgrepo gitrepo" &&
+
+	(
+	hg init hgrepo &&
+	cd hgrepo &&
+
+	touch test.txt &&
+	hg add test.txt &&
+	hg commit -m master &&
+	hg branch parent &&
+	echo test >test.txt &&
+	hg commit -m test &&
+	hg branch parent/child &&
+	echo test1 >test.txt &&
+	hg commit -m test1
+	) &&
+
+	git clone -c remote-hg.ignore-name=child "hg::hgrepo" gitrepo &&
+	check_files gitrepo "test.txt"
+'
+
+if test "$CAPABILITY_PUSH" != "t"
+then
 test_done
+fi