merge with branch 1.x

scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/web/HgCGIServlet.java

@@ -56,15 +56,20 @@ import sonia.scm.web.cgi.CGIExecutor;
 import sonia.scm.web.cgi.CGIExecutorFactory;
 import sonia.scm.web.cgi.EnvList;
 
+//~--- JDK imports ------------------------------------------------------------
+
+import java.io.File;
+import java.io.IOException;
+
+import java.util.Enumeration;
+import java.util.Map;
+
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import javax.servlet.http.HttpSession;
-import java.io.File;
-import java.io.IOException;
 import java.util.Base64;
-import java.util.Enumeration;
 
 /**
  *
@@ -74,6 +79,8 @@ import java.util.Enumeration;
 public class HgCGIServlet extends HttpServlet implements ScmProviderHttpServlet
 {
 
+  private static final String ENV_PYTHON_HTTPS_VERIFY = "PYTHONHTTPSVERIFY";
+
   /** Field description */
   public static final String ENV_REPOSITORY_NAME = "REPO_NAME";
 
@@ -83,6 +90,8 @@ public class HgCGIServlet extends HttpServlet implements ScmProviderHttpServlet
   /** Field description */
   public static final String ENV_REPOSITORY_ID = "SCM_REPOSITORY_ID";
 
+  private static final String ENV_HTTP_POST_ARGS = "SCM_HTTP_POST_ARGS";
+
   /** Field description */
   public static final String ENV_SESSION_PREFIX = "SCM_";
 
@@ -268,11 +277,22 @@ public class HgCGIServlet extends HttpServlet implements ScmProviderHttpServlet
       directory.getAbsolutePath());
 
     // add hook environment
+    Map<String, String> environment = executor.getEnvironment().asMutableMap();
+    if (handler.getConfig().isDisableHookSSLValidation()) {
+      // disable ssl validation
+      // Issue 959: https://goo.gl/zH5eY8
+      environment.put(ENV_PYTHON_HTTPS_VERIFY, "0");
+    }
+
+    // enable experimental httppostargs protocol of mercurial
+    // Issue 970: https://goo.gl/poascp
+    environment.put(ENV_HTTP_POST_ARGS, String.valueOf(handler.getConfig().isEnableHttpPostArgs()));
+
     //J-
     HgEnvironment.prepareEnvironment(
-      executor.getEnvironment().asMutableMap(),
+      environment,
       handler,
-      hookManager, 
+      hookManager,
       request
     );
     //J+

scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/web/HgPermissionFilter.java

@@ -33,13 +33,23 @@
 
 package sonia.scm.web;
 
+//~--- non-JDK imports --------------------------------------------------------
+
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableSet;
 import sonia.scm.config.ScmConfiguration;
+import sonia.scm.repository.Repository;
 import sonia.scm.repository.spi.ScmProviderHttpServlet;
 import sonia.scm.web.filter.PermissionFilter;
 
+import sonia.scm.repository.HgRepositoryHandler;
+
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import java.util.Set;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
 
 /**
  * Permission filter for mercurial repositories.
@@ -51,14 +61,48 @@ public class HgPermissionFilter extends PermissionFilter
   
   private static final Set<String> READ_METHODS = ImmutableSet.of("GET", "HEAD", "OPTIONS", "TRACE");
 
-  public HgPermissionFilter(ScmConfiguration configuration, ScmProviderHttpServlet delegate)
+  private final HgRepositoryHandler repositoryHandler;
+
+  public HgPermissionFilter(ScmConfiguration configuration, ScmProviderHttpServlet delegate, HgRepositoryHandler repositoryHandler)
   {
     super(configuration, delegate);
+    this.repositoryHandler = repositoryHandler;
+  }
+
+  @Override
+  public void service(HttpServletRequest request, HttpServletResponse response, Repository repository) throws IOException, ServletException {
+    super.service(wrapRequestIfRequired(request), response, repository);
+  }
+
+  @VisibleForTesting
+  HttpServletRequest wrapRequestIfRequired(HttpServletRequest request) {
+    if (isHttpPostArgsEnabled()) {
+      return new HgServletRequest(request);
+    }
+    return request;
   }
 
   @Override
   public boolean isWriteRequest(HttpServletRequest request)
   {
-    return !READ_METHODS.contains(request.getMethod());
+    if (isHttpPostArgsEnabled()) {
+      return isHttpPostArgsWriteRequest(request);
+    }
+    return isDefaultWriteRequest(request);
+  }
+
+  private boolean isHttpPostArgsEnabled() {
+    return repositoryHandler.getConfig().isEnableHttpPostArgs();
+  }
+
+  private boolean isHttpPostArgsWriteRequest(HttpServletRequest request) {
+    return WireProtocol.isWriteRequest(request);
+  }
+
+  private boolean isDefaultWriteRequest(HttpServletRequest request) {
+    if (READ_METHODS.contains(request.getMethod())) {
+      return WireProtocol.isWriteRequest(request);
+    }
+    return true;
   }
 }

scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/web/HgPermissionFilterFactory.java

@@ -12,10 +12,12 @@ import javax.inject.Inject;
 public class HgPermissionFilterFactory implements ScmProviderHttpServletDecoratorFactory {
 
   private final ScmConfiguration configuration;
+  private final HgRepositoryHandler repositoryHandler;
 
   @Inject
-  public HgPermissionFilterFactory(ScmConfiguration configuration) {
+  public HgPermissionFilterFactory(ScmConfiguration configuration, HgRepositoryHandler repositoryHandler) {
     this.configuration = configuration;
+    this.repositoryHandler = repositoryHandler;
   }
 
   @Override
@@ -25,6 +27,6 @@ public class HgPermissionFilterFactory implements ScmProviderHttpServletDecorato
 
   @Override
   public ScmProviderHttpServlet createDecorator(ScmProviderHttpServlet delegate) {
-    return new HgPermissionFilter(configuration, delegate);
+    return new HgPermissionFilter(configuration, delegate,repositoryHandler);
   }
 }

scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/web/HgServletInputStream.java

@@ -0,0 +1,55 @@
+package sonia.scm.web;
+
+import com.google.common.base.Preconditions;
+
+import javax.servlet.ServletInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+
+/**
+ * HgServletInputStream is a wrapper around the original {@link ServletInputStream} and provides some extra
+ * functionality to support the mercurial client.
+ */
+public class HgServletInputStream extends ServletInputStream {
+
+  private final ServletInputStream original;
+  private ByteArrayInputStream captured;
+
+  HgServletInputStream(ServletInputStream original) {
+    this.original = original;
+  }
+
+  /**
+   * Reads the given amount of bytes from the stream and captures them, if the {@link #read()} methods is called the
+   * captured bytes are returned before the rest of the stream.
+   *
+   * @param size amount of bytes to read
+   *
+   * @return byte array
+   *
+   * @throws IOException if the method is called twice
+   */
+  public byte[] readAndCapture(int size) throws IOException {
+    Preconditions.checkState(captured == null, "readAndCapture can only be called once per request");
+
+    // TODO should we enforce a limit? to prevent OOM?
+    byte[] bytes = new byte[size];
+    original.read(bytes);
+    captured = new ByteArrayInputStream(bytes);
+
+    return bytes;
+  }
+
+  @Override
+  public int read() throws IOException {
+    if (captured != null && captured.available() > 0) {
+       return captured.read();
+    }
+    return original.read();
+  }
+
+  @Override
+  public void close() throws IOException {
+    original.close();
+  }
+}

scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/web/HgServletRequest.java

@@ -0,0 +1,31 @@
+package sonia.scm.web;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletRequestWrapper;
+import java.io.IOException;
+
+/**
+ * {@link HttpServletRequestWrapper} which adds some functionality in order to support the mercurial client.
+ */
+public final class HgServletRequest extends HttpServletRequestWrapper {
+
+  private HgServletInputStream hgServletInputStream;
+
+  /**
+   * Constructs a request object wrapping the given request.
+   *
+   * @param request
+   * @throws IllegalArgumentException if the request is null
+   */
+  public HgServletRequest(HttpServletRequest request) {
+    super(request);
+  }
+
+  @Override
+  public HgServletInputStream getInputStream() throws IOException {
+    if (hgServletInputStream == null) {
+      hgServletInputStream = new HgServletInputStream(super.getInputStream());
+    }
+    return hgServletInputStream;
+  }
+}

scm-plugins/scm-hg-plugin/src/main/java/sonia/scm/web/WireProtocol.java

@@ -0,0 +1,236 @@
+/**
+ * Copyright (c) 2018, Sebastian Sdorra
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ * 3. Neither the name of SCM-Manager; nor the names of its
+ *    contributors may be used to endorse or promote products derived from this
+ *    software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * http://bitbucket.org/sdorra/scm-manager
+ *
+ */
+
+
+package sonia.scm.web;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.base.Throwables;
+import com.google.common.collect.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import sonia.scm.util.HttpUtil;
+
+import javax.servlet.http.HttpServletRequest;
+import java.io.IOException;
+import java.util.*;
+
+/**
+ * WireProtocol provides methods for handling the mercurial wire protocol.
+ *
+ * @see <a href="https://goo.gl/WaVJzw">Mercurial Wire Protocol</a>
+ */
+public final class WireProtocol {
+
+  private static final Logger LOG = LoggerFactory.getLogger(WireProtocol.class);
+
+  private static final Set<String> READ_COMMANDS = ImmutableSet.of(
+    "batch", "between", "branchmap", "branches", "capabilities", "changegroup", "changegroupsubset", "clonebundles",
+    "getbundle", "heads", "hello", "listkeys", "lookup", "known", "stream_out",
+    // could not find lheads in the wireprotocol description but mercurial 4.5.2 uses it for clone
+    "lheads"
+  );
+
+  private static final Set<String> WRITE_COMMANDS = ImmutableSet.of(
+    "pushkey", "unbundle"
+  );
+
+  private WireProtocol() {
+  }
+
+  /**
+   * Returns {@code true} if the request is a write request. The method will always return {@code true}, expect for the
+   * following cases:
+   *
+   * - no command was specified with the request (is required for the hgweb ui)
+   * - the command in the query string was found in the list of read request
+   * - if query string contains the batch command, then all commands specified in X-HgArg headers must be
+   *   in the list of read requests
+   * - in case of enabled HttpPostArgs protocol and query string container the batch command, the header X-HgArgs-Post
+   *   is read and the commands which are specified in the body from 0 to the value of X-HgArgs-Post must be in the list
+   *   of read requests
+   *
+   * @param request http request
+   *
+   * @return {@code true} for write requests.
+   */
+  public static boolean isWriteRequest(HttpServletRequest request) {
+    List<String> commands = commandsOf(request);
+    boolean write = isWriteRequest(commands);
+    LOG.trace("mercurial request {} is write: {}", commands, write);
+    return write;
+  }
+
+  @VisibleForTesting
+  static boolean isWriteRequest(List<String> commands) {
+    return !READ_COMMANDS.containsAll(commands);
+  }
+
+  @VisibleForTesting
+  static List<String> commandsOf(HttpServletRequest request) {
+    List<String> listOfCmds = Lists.newArrayList();
+
+    String cmd = getCommandFromQueryString(request);
+    if (cmd != null) {
+      listOfCmds.add(cmd);
+      if (isBatchCommand(cmd)) {
+        parseHgArgHeaders(request, listOfCmds);
+        handleHttpPostArgs(request, listOfCmds);
+      }
+    }
+    return Collections.unmodifiableList(listOfCmds);
+  }
+
+  private static void handleHttpPostArgs(HttpServletRequest request, List<String> listOfCmds) {
+    int hgArgsPostSize = request.getIntHeader("X-HgArgs-Post");
+    if (hgArgsPostSize > 0) {
+
+      if (request instanceof HgServletRequest) {
+        HgServletRequest hgRequest = (HgServletRequest) request;
+
+        parseHttpPostArgs(listOfCmds, hgArgsPostSize, hgRequest);
+      } else {
+        throw new IllegalArgumentException("could not process the httppostargs protocol without HgServletRequest");
+      }
+
+    }
+  }
+
+  private static void parseHttpPostArgs(List<String> listOfCmds, int hgArgsPostSize, HgServletRequest hgRequest) {
+    try {
+      byte[] bytes = hgRequest.getInputStream().readAndCapture(hgArgsPostSize);
+      // we use iso-8859-1 for encoding, because the post args are normally http headers which are using iso-8859-1
+      // see https://tools.ietf.org/html/rfc7230#section-3.2.4
+      String hgArgs = new String(bytes, Charsets.ISO_8859_1);
+      String decoded = decodeValue(hgArgs);
+      parseHgCommandHeader(listOfCmds, decoded);
+    } catch (IOException ex) {
+      throw Throwables.propagate(ex);
+    }
+  }
+
+  private static void parseHgArgHeaders(HttpServletRequest request, List<String> listOfCmds) {
+    Enumeration headerNames = request.getHeaderNames();
+    while (headerNames.hasMoreElements()) {
+      String header = (String) headerNames.nextElement();
+      parseHgArgHeader(request, listOfCmds, header);
+    }
+  }
+
+  private static void parseHgArgHeader(HttpServletRequest request, List<String> listOfCmds, String header) {
+    if (isHgArgHeader(header)) {
+      String value = getHeaderDecoded(request, header);
+      parseHgArgValue(listOfCmds, value);
+    }
+  }
+
+  private static void parseHgArgValue(List<String> listOfCmds, String value) {
+    if (isHgArgCommandHeader(value)) {
+      parseHgCommandHeader(listOfCmds, value);
+    }
+  }
+
+  private static void parseHgCommandHeader(List<String> listOfCmds, String value) {
+    String[] cmds = value.substring(5).split(";");
+    for (String cmd : cmds ) {
+      String normalizedCmd = normalize(cmd);
+      int index = normalizedCmd.indexOf(' ');
+      if (index > 0) {
+        listOfCmds.add(normalizedCmd.substring(0, index));
+      } else {
+        listOfCmds.add(normalizedCmd);
+      }
+    }
+  }
+
+  private static String normalize(String cmd) {
+    return cmd.trim().toLowerCase(Locale.ENGLISH);
+  }
+
+  private static boolean isHgArgCommandHeader(String value) {
+    return value.startsWith("cmds=");
+  }
+
+  private static String getHeaderDecoded(HttpServletRequest request, String header) {
+    return decodeValue(request.getHeader(header));
+  }
+
+  private static String decodeValue(String value) {
+    return HttpUtil.decode(Strings.nullToEmpty(value));
+  }
+
+  private static boolean isHgArgHeader(String header) {
+    return header.toLowerCase(Locale.ENGLISH).startsWith("x-hgarg-");
+  }
+
+  private static boolean isBatchCommand(String cmd) {
+    return "batch".equalsIgnoreCase(cmd);
+  }
+
+  private static String getCommandFromQueryString(HttpServletRequest request) {
+    // we can't use getParameter, because this would inspect the body for form parameters as well
+    Multimap<String, String> queryParameterMap = createQueryParameterMap(request);
+
+    Collection<String> cmd = queryParameterMap.get("cmd");
+    Preconditions.checkArgument(cmd.size() <= 1, "found more than one cmd query parameter");
+    Iterator<String> iterator = cmd.iterator();
+
+    String command = null;
+    if (iterator.hasNext()) {
+      command = iterator.next();
+    }
+    return command;
+  }
+
+  private static Multimap<String,String> createQueryParameterMap(HttpServletRequest request) {
+    Multimap<String,String> parameterMap = HashMultimap.create();
+
+    String queryString = request.getQueryString();
+    if (!Strings.isNullOrEmpty(queryString)) {
+
+      String[] parameters = queryString.split("&");
+      for (String parameter : parameters) {
+        int index = parameter.indexOf('=');
+        if (index > 0) {
+          parameterMap.put(parameter.substring(0, index), parameter.substring(index + 1));
+        } else {
+          parameterMap.put(parameter, "true");
+        }
+      }
+
+    }
+
+    return parameterMap;
+  }
+}