Mirror LFS files for git (#2075)

If a mirrored git repository uses LFS, SCM-Manager will now also load the binaries, so that the mirrored repository can be used without missing LFS files.

gradle/changelog/mirror_lfs.yaml

@@ -0,0 +1,2 @@
+- type: added
+  description: Mirror LFS files for git ([#2075](https://github.com/scm-manager/scm-manager/pull/2075))

scm-core/src/main/java/sonia/scm/net/HttpConnectionOptions.java

@@ -33,6 +33,9 @@ import javax.annotation.Nullable;
 import javax.net.ssl.KeyManager;
 import java.net.URL;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 
@@ -65,6 +68,7 @@ public final class HttpConnectionOptions {
   private int connectionTimeout = DEFAULT_CONNECTION_TIMEOUT;
   private int readTimeout = DEFAULT_READ_TIMEOUT;
   private boolean ignoreProxySettings = false;
+  private Map<String, String> connectionProperties = new HashMap<>();
 
   /**
    * Returns optional local proxy configuration.
@@ -86,6 +90,10 @@ public final class HttpConnectionOptions {
     return Optional.empty();
   }
 
+  public Map<String, String> getConnectionProperties() {
+    return Collections.unmodifiableMap(connectionProperties);
+  }
+
   /**
    * Disable certificate validation.
    * <b>WARNING:</b> This option should only be used for internal test.
@@ -162,4 +170,15 @@ public final class HttpConnectionOptions {
     this.ignoreProxySettings = true;
     return this;
   }
+
+  /**
+   * Add a request property that will be converted to headers in the request.
+   * @param key The property (aka header) name (eg. "User-Agent").
+   * @param value The value of the property.
+   * @return {@code this}
+   */
+  public HttpConnectionOptions addRequestProperty(String key, String value) {
+    connectionProperties.put(key, value);
+    return this;
+  }
 }

scm-core/src/main/java/sonia/scm/net/HttpURLConnectionFactory.java

@@ -181,6 +181,8 @@ public final class HttpURLConnectionFactory {
       if (connection instanceof HttpsURLConnection) {
         applySSLSettings((HttpsURLConnection) connection);
       }
+      options.getConnectionProperties()
+        .forEach(connection::setRequestProperty);
       return connection;
     }
 

scm-core/src/main/java/sonia/scm/repository/api/MirrorCommandBuilder.java

@@ -58,6 +58,7 @@ public final class MirrorCommandBuilder {
   private Collection<Credential> credentials = emptyList();
   private List<PublicKey> publicKeys = emptyList();
   private MirrorFilter filter = new MirrorFilter() {};
+  private boolean ignoreLfs;
 
   @Nullable
   private ProxyConfiguration proxyConfiguration;
@@ -99,6 +100,16 @@ public final class MirrorCommandBuilder {
     return this;
   }
 
+  /**
+   * If set to <code>true</code>, lfs files will not be mirrored. Defaults to <code>false</code>.
+   * @return This builder instance
+   * @since 2.37.0
+   */
+  public MirrorCommandBuilder setIgnoreLfs(boolean ignoreLfs) {
+    this.ignoreLfs = ignoreLfs;
+    return this;
+  }
+
   /**
    * Set the proxy configuration which should be used to access the source repository of the mirror.
    * If not proxy configuration is set the global configuration should be used instead.
@@ -130,6 +141,7 @@ public final class MirrorCommandBuilder {
     mirrorCommandRequest.setFilter(filter);
     mirrorCommandRequest.setPublicKeys(publicKeys);
     mirrorCommandRequest.setProxyConfiguration(proxyConfiguration);
+    mirrorCommandRequest.setIgnoreLfs(ignoreLfs);
     Preconditions.checkArgument(mirrorCommandRequest.isValid(), "source url has to be specified");
     return mirrorCommandRequest;
   }

scm-core/src/main/java/sonia/scm/repository/api/MirrorCommandResult.java

@@ -35,11 +35,17 @@ public final class MirrorCommandResult {
   private final ResultType result;
   private final List<String> log;
   private final Duration duration;
+  private final LfsUpdateResult lfsUpdateResult;
 
   public MirrorCommandResult(ResultType result, List<String> log, Duration duration) {
+    this(result, log, duration, null);
+  }
+
+  public MirrorCommandResult(ResultType result, List<String> log, Duration duration, LfsUpdateResult lfsUpdateResult) {
     this.result = result;
     this.log = log;
     this.duration = duration;
+    this.lfsUpdateResult = lfsUpdateResult;
   }
 
   public ResultType getResult() {
@@ -54,9 +60,38 @@ public final class MirrorCommandResult {
     return duration;
   }
 
+  public LfsUpdateResult getLfsUpdateResult() {
+    return lfsUpdateResult;
+  }
+
   public enum ResultType {
     OK,
     REJECTED_UPDATES,
     FAILED
   }
+
+  public static class LfsUpdateResult {
+    private int overallCount = 0;
+    private int failureCount = 0;
+
+    public void increaseOverallCount() {
+      overallCount++;
+    }
+
+    public void increaseFailureCount() {
+      failureCount++;
+    }
+
+    public int getOverallCount() {
+      return overallCount;
+    }
+
+    public int getFailureCount() {
+      return failureCount;
+    }
+
+    public boolean hasFailures() {
+      return failureCount > 0;
+    }
+  }
 }

scm-core/src/main/java/sonia/scm/repository/spi/MirrorCommandRequest.java

@@ -53,6 +53,7 @@ public final class MirrorCommandRequest {
 
   @Nullable
   private ProxyConfiguration proxyConfiguration;
+  private boolean ignoreLfs;
 
   public String getSourceUrl() {
     return sourceUrl;
@@ -86,6 +87,14 @@ public final class MirrorCommandRequest {
     this.filter = filter;
   }
 
+  public void setIgnoreLfs(boolean ignoreLfs) {
+    this.ignoreLfs = ignoreLfs;
+  }
+
+  public boolean isIgnoreLfs() {
+    return ignoreLfs;
+  }
+
   public boolean isValid() {
     return StringUtils.isNotBlank(sourceUrl);
   }

scm-core/src/test/java/sonia/scm/net/HttpURLConnectionFactoryTest.java

@@ -345,6 +345,17 @@ class HttpURLConnectionFactoryTest {
       assertThat(keyManagers).containsOnly(keyManager);
     }
 
+    @Test
+    void shouldSetGivenRequestProperties() throws IOException {
+      HttpConnectionOptions options = new HttpConnectionOptions();
+      options.addRequestProperty("valid_until", "end of the universe");
+
+      HttpURLConnection connection = connectionFactory.create(new URL("https://hitchhiker.org"), options);
+
+      verify(connection).setRequestProperty("valid_until", "end of the universe");
+      assertThat(usedProxy).isNull();
+    }
+
     private TrustManager[] usedTrustManagers(HttpURLConnection connection) throws KeyManagementException {
       ArgumentCaptor<TrustManager[]> captor = ArgumentCaptor.forClass(TrustManager[].class);
       assertThat(connection).isInstanceOfSatisfying(

scm-it/gradle.lockfile

@@ -209,12 +209,18 @@ org.reactivestreams:reactive-streams:1.0.3=testCompileClasspath,testCompileClass
 org.slf4j:jcl-over-slf4j:1.7.30=testCompileClasspath,testCompileClasspathCopy,testRuntimeClasspath,testRuntimeClasspathCopy
 org.slf4j:slf4j-api:1.7.30=testCompileClasspath,testCompileClasspathCopy,testRuntimeClasspath,testRuntimeClasspathCopy
 org.tmatesoft.sqljet:sqljet:1.1.14=testRuntimeClasspath,testRuntimeClasspathCopy
-sonia.jgit:org.eclipse.jgit.gpg.bc:5.11.1.202105131744-r-scm1=testRuntimeClasspath,testRuntimeClasspathCopy
-sonia.jgit:org.eclipse.jgit.http.apache:5.11.1.202105131744-r-scm1=testRuntimeClasspath,testRuntimeClasspathCopy
-sonia.jgit:org.eclipse.jgit.http.server:5.11.1.202105131744-r-scm1=testRuntimeClasspath,testRuntimeClasspathCopy
-sonia.jgit:org.eclipse.jgit.lfs.server:5.11.1.202105131744-r-scm1=testRuntimeClasspath,testRuntimeClasspathCopy
-sonia.jgit:org.eclipse.jgit.lfs:5.11.1.202105131744-r-scm1=testRuntimeClasspath,testRuntimeClasspathCopy
-sonia.jgit:org.eclipse.jgit:5.11.1.202105131744-r-scm1=testCompileClasspath,testCompileClasspathCopy,testRuntimeClasspath,testRuntimeClasspathCopy
+sonia.jgit:org.eclipse.jgit.gpg.bc:5.11.1.202105131744-r-scm2=testRuntimeClasspath
+sonia.jgit:org.eclipse.jgit.gpg.bc:5.11.1.202105131744-r-scm3=testRuntimeClasspathCopy
+sonia.jgit:org.eclipse.jgit.http.apache:5.11.1.202105131744-r-scm2=testRuntimeClasspath
+sonia.jgit:org.eclipse.jgit.http.apache:5.11.1.202105131744-r-scm3=testRuntimeClasspathCopy
+sonia.jgit:org.eclipse.jgit.http.server:5.11.1.202105131744-r-scm2=testRuntimeClasspath
+sonia.jgit:org.eclipse.jgit.http.server:5.11.1.202105131744-r-scm3=testRuntimeClasspathCopy
+sonia.jgit:org.eclipse.jgit.lfs.server:5.11.1.202105131744-r-scm2=testRuntimeClasspath
+sonia.jgit:org.eclipse.jgit.lfs.server:5.11.1.202105131744-r-scm3=testRuntimeClasspathCopy
+sonia.jgit:org.eclipse.jgit.lfs:5.11.1.202105131744-r-scm2=testRuntimeClasspath
+sonia.jgit:org.eclipse.jgit.lfs:5.11.1.202105131744-r-scm3=testRuntimeClasspathCopy
+sonia.jgit:org.eclipse.jgit:5.11.1.202105131744-r-scm2=testCompileClasspath,testRuntimeClasspath
+sonia.jgit:org.eclipse.jgit:5.11.1.202105131744-r-scm3=testCompileClasspathCopy,testRuntimeClasspathCopy
 sonia.svnkit:svnkit-dav:1.10.3-scm2=testRuntimeClasspath,testRuntimeClasspathCopy
 sonia.svnkit:svnkit:1.10.3-scm2=testRuntimeClasspath,testRuntimeClasspathCopy
 empty=annotationProcessor,annotationProcessorCopy,archives,archivesCopy,compileClasspath,compileClasspathCopy,corePlugin,corePluginCopy,default,defaultCopy,itPlugin,itPluginCopy,itWebApp,itWebAppCopy,runtimeClasspath,runtimeClasspathCopy,testAnnotationProcessor,testAnnotationProcessorCopy

scm-plugins/scm-git-plugin/build.gradle

@@ -26,7 +26,7 @@ plugins {
   id 'org.scm-manager.smp' version '0.11.0'
 }
 
-def jgitVersion = '5.11.1.202105131744-r-scm1'
+def jgitVersion = '5.11.1.202105131744-r-scm3'
 
 dependencies {
   // required by scm-it

scm-plugins/scm-git-plugin/gradle.lockfile

@@ -154,13 +154,19 @@ org.slf4j:jcl-over-slf4j:1.7.30=compileClasspath,default,runtimeClasspath,runtim
 org.slf4j:slf4j-api:1.7.25=swaggerDeps
 org.slf4j:slf4j-api:1.7.30=annotationProcessor,compileClasspath,default,runtimeClasspath,runtimePluginElements,scmCoreDependency,testCompileClasspath,testRuntimeClasspath
 org.yaml:snakeyaml:1.26=swaggerDeps
-sonia.jgit:org.eclipse.jgit.gpg.bc:5.11.1.202105131744-r-scm1=compileClasspath,default,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-sonia.jgit:org.eclipse.jgit.http.apache:5.11.1.202105131744-r-scm1=compileClasspath,default,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-sonia.jgit:org.eclipse.jgit.http.server:5.11.1.202105131744-r-scm1=compileClasspath,default,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-sonia.jgit:org.eclipse.jgit.junit.http:5.11.1.202105131744-r-scm1=testCompileClasspath,testRuntimeClasspath
-sonia.jgit:org.eclipse.jgit.junit:5.11.1.202105131744-r-scm1=testCompileClasspath,testRuntimeClasspath
-sonia.jgit:org.eclipse.jgit.lfs.server:5.11.1.202105131744-r-scm1=compileClasspath,default,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-sonia.jgit:org.eclipse.jgit.lfs:5.11.1.202105131744-r-scm1=compileClasspath,default,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-sonia.jgit:org.eclipse.jgit:5.11.1.202105131744-r-scm1=compileClasspath,default,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+sonia.jgit:org.eclipse.jgit.gpg.bc:5.11.1.202105131744-r-scm1=default
+sonia.jgit:org.eclipse.jgit.gpg.bc:5.11.1.202105131744-r-scm3=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+sonia.jgit:org.eclipse.jgit.http.apache:5.11.1.202105131744-r-scm1=default
+sonia.jgit:org.eclipse.jgit.http.apache:5.11.1.202105131744-r-scm3=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+sonia.jgit:org.eclipse.jgit.http.server:5.11.1.202105131744-r-scm1=default
+sonia.jgit:org.eclipse.jgit.http.server:5.11.1.202105131744-r-scm3=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+sonia.jgit:org.eclipse.jgit.junit.http:5.11.1.202105131744-r-scm3=testCompileClasspath,testRuntimeClasspath
+sonia.jgit:org.eclipse.jgit.junit:5.11.1.202105131744-r-scm3=testCompileClasspath,testRuntimeClasspath
+sonia.jgit:org.eclipse.jgit.lfs.server:5.11.1.202105131744-r-scm1=default
+sonia.jgit:org.eclipse.jgit.lfs.server:5.11.1.202105131744-r-scm3=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+sonia.jgit:org.eclipse.jgit.lfs:5.11.1.202105131744-r-scm1=default
+sonia.jgit:org.eclipse.jgit.lfs:5.11.1.202105131744-r-scm3=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+sonia.jgit:org.eclipse.jgit:5.11.1.202105131744-r-scm1=default
+sonia.jgit:org.eclipse.jgit:5.11.1.202105131744-r-scm3=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 sonia.scm:scm-webapp:2.32.2-SNAPSHOT=scmServer
 empty=archives,optionalPlugin,plugin

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitMirrorCommand.java

@@ -61,6 +61,7 @@ import sonia.scm.repository.api.MirrorFilter;
 import sonia.scm.repository.api.MirrorFilter.Result;
 import sonia.scm.repository.api.UsernamePasswordCredential;
 import sonia.scm.store.ConfigurationStore;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
 
 import javax.inject.Inject;
 import java.io.IOException;
@@ -86,16 +87,18 @@ import static sonia.scm.repository.api.MirrorCommandResult.ResultType.OK;
 import static sonia.scm.repository.api.MirrorCommandResult.ResultType.REJECTED_UPDATES;
 
 /**
- * Implementation of the mirror command for git. This implementation makes use of a special
- * "ref" called <code>mirror</code>. A synchronization works in principal in the following way:
+ * Implementation of the mirror command for git.
+ *
+ * The general workflow is the same for the first call and all subsequent updates and looks like this:
+ *
  * <ol>
- *   <li>The mirror reference is updated. This is done by calling the jgit equivalent of
- *     <pre>git fetch -pf <source url> "refs/heads/*:refs/mirror/heads/*" "refs/tags/*:refs/mirror/tags/*"</pre>
- *   </li>
- *   <li>These updates are then presented to the filter. Here single updates can be rejected.
- *     Such rejected updates have to be reverted in the mirror, too.
- *   </li>
- *   <li>Accepted ref updates are copied to the "normal" refs.</li>
+ *   <li>Create a local working copy for the repository.</li>
+ *   <li>Fetch updates from the source and override all local refs in the working copy
+ *     (like <code>git fetch "refs/heads/*:refs/heads/*" "refs/tags/*:refs/tags/*"</code>).</li>
+ *   <li>Iterate the updates and decide, whether to keep each single update or not. If an update is rejected,
+ *     the reference is set to its old oid (or deleted, if this is a new reference).</li>
+ *   <li>Push the changed references from the working copy to the repository.</li>
+ *   <li>Release the working copy.</li>
  * </ol>
  */
 public class GitMirrorCommand extends AbstractGitCommand implements MirrorCommand {
@@ -108,6 +111,7 @@ public class GitMirrorCommand extends AbstractGitCommand implements MirrorComman
   private final GitWorkingCopyFactory workingCopyFactory;
   private final GitHeadModifier gitHeadModifier;
   private final GitRepositoryConfigStoreProvider storeProvider;
+  private final LfsLoader lfsLoader;
 
   @Inject
   GitMirrorCommand(GitContext context,
@@ -116,7 +120,8 @@ public class GitMirrorCommand extends AbstractGitCommand implements MirrorComman
                    GitTagConverter gitTagConverter,
                    GitWorkingCopyFactory workingCopyFactory,
                    GitHeadModifier gitHeadModifier,
-                   GitRepositoryConfigStoreProvider storeProvider) {
+                   GitRepositoryConfigStoreProvider storeProvider,
+                   LfsLoader lfsLoader) {
     super(context);
     this.mirrorHttpConnectionProvider = mirrorHttpConnectionProvider;
     this.converterFactory = converterFactory;
@@ -124,6 +129,7 @@ public class GitMirrorCommand extends AbstractGitCommand implements MirrorComman
     this.workingCopyFactory = workingCopyFactory;
     this.gitHeadModifier = gitHeadModifier;
     this.storeProvider = storeProvider;
+    this.lfsLoader = lfsLoader;
   }
 
   @Override
@@ -163,6 +169,7 @@ public class GitMirrorCommand extends AbstractGitCommand implements MirrorComman
     private final Git git;
 
     private final Collection<String> deletedRefs = new ArrayList<>();
+    private final MirrorCommandResult.LfsUpdateResult lfsUpdateResult = new MirrorCommandResult.LfsUpdateResult();
 
     private FetchResult fetchResult;
     private GitFilterContext filterContext;
@@ -185,7 +192,7 @@ public class GitMirrorCommand extends AbstractGitCommand implements MirrorComman
         result = FAILED;
         LOG.info("got exception while trying to synchronize mirror for repository {}", context.getRepository(), e);
         mirrorLog.add("failed to synchronize: " + e.getMessage());
-        return new MirrorCommandResult(FAILED, mirrorLog, stopwatch.stop().elapsed());
+        return new MirrorCommandResult(FAILED, mirrorLog, stopwatch.stop().elapsed(), lfsUpdateResult);
       }
     }
 
@@ -198,7 +205,7 @@ public class GitMirrorCommand extends AbstractGitCommand implements MirrorComman
       if (fetchResult.getTrackingRefUpdates().isEmpty()) {
         LOG.trace("No updates found for mirror repository {}", repository);
         mirrorLog.add("No updates found");
-        return new MirrorCommandResult(result, mirrorLog, stopwatch.stop().elapsed());
+        return new MirrorCommandResult(result, mirrorLog, stopwatch.stop().elapsed(), lfsUpdateResult);
       } else {
         handleBranches();
         handleTags();
@@ -206,14 +213,15 @@ public class GitMirrorCommand extends AbstractGitCommand implements MirrorComman
 
       if (!defaultBranchSelector.isChanged()) {
         mirrorLog.add("No effective changes detected");
-        return new MirrorCommandResult(result, mirrorLog, stopwatch.stop().elapsed());
+        return new MirrorCommandResult(result, mirrorLog, stopwatch.stop().elapsed(), lfsUpdateResult);
       }
 
       defaultBranchSelector.newDefaultBranch().ifPresent(this::setNewDefaultBranch);
 
       String[] pushRefSpecs = generatePushRefSpecs().toArray(new String[0]);
       push(pushRefSpecs);
-      return new MirrorCommandResult(result, mirrorLog, stopwatch.stop().elapsed());
+      ResultType finalResult = lfsUpdateResult.hasFailures()? FAILED: result;
+      return new MirrorCommandResult(finalResult, mirrorLog, stopwatch.stop().elapsed(), lfsUpdateResult);
     }
 
     private void setNewDefaultBranch(String newDefaultBranch) {
@@ -357,6 +365,10 @@ public class GitMirrorCommand extends AbstractGitCommand implements MirrorComman
           LOG.trace("updating {} ref in {}: {}", refType.typeForLog, GitMirrorCommand.this.repository, targetRef);
           defaultBranchSelector.accepted(refType, referenceName);
           logger.logChange(ref, referenceName, getUpdateType(ref));
+
+          if (!mirrorCommandRequest.isIgnoreLfs()) {
+            lfsLoader.inspectTree(ref.getNewObjectId(), mirrorCommandRequest, git.getRepository(), mirrorLog, lfsUpdateResult, repository);
+          }
         }
       }
 

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/LfsLoader.java

@@ -0,0 +1,177 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package sonia.scm.repository.spi;
+
+import org.eclipse.jgit.lfs.Lfs;
+import org.eclipse.jgit.lfs.LfsPointer;
+import org.eclipse.jgit.lfs.Protocol;
+import org.eclipse.jgit.lfs.SmudgeFilter;
+import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
+import org.eclipse.jgit.lfs.lib.LfsPointerFilter;
+import org.eclipse.jgit.lib.ConfigConstants;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.revwalk.RevWalk;
+import org.eclipse.jgit.transport.http.HttpConnectionFactory;
+import org.eclipse.jgit.treewalk.TreeWalk;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import sonia.scm.repository.api.MirrorCommandResult.LfsUpdateResult;
+import sonia.scm.store.BlobStore;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.List;
+
+class LfsLoader {
+
+  private static final Logger LOG = LoggerFactory.getLogger(LfsLoader.class);
+
+  private final LfsBlobStoreFactory lfsBlobStoreFactory;
+  private final MirrorHttpConnectionProvider mirrorHttpConnectionProvider;
+
+  @Inject
+  LfsLoader(LfsBlobStoreFactory lfsBlobStoreFactory, MirrorHttpConnectionProvider mirrorHttpConnectionProvider) {
+    this.lfsBlobStoreFactory = lfsBlobStoreFactory;
+    this.mirrorHttpConnectionProvider = mirrorHttpConnectionProvider;
+  }
+
+  void inspectTree(ObjectId newObjectId,
+                   MirrorCommandRequest mirrorCommandRequest,
+                   Repository gitRepository,
+                   List<String> mirrorLog,
+                   LfsUpdateResult lfsUpdateResult,
+                   sonia.scm.repository.Repository repository) {
+    String mirrorUrl = mirrorCommandRequest.getSourceUrl();
+    EntryHandler entryHandler = new EntryHandler(repository, gitRepository, mirrorCommandRequest, mirrorLog, lfsUpdateResult);
+
+    try {
+      gitRepository
+        .getConfig()
+        .setString(ConfigConstants.CONFIG_SECTION_LFS, null, ConfigConstants.CONFIG_KEY_URL, computeLfsUrl(mirrorUrl));
+
+      TreeWalk treeWalk = new TreeWalk(gitRepository);
+      treeWalk.setFilter(new LfsPointerFilter());
+
+      RevWalk revWalk = new RevWalk(gitRepository);
+      revWalk.markStart(revWalk.parseCommit(newObjectId));
+
+      for (RevCommit commit : revWalk) {
+        treeWalk.reset();
+        treeWalk.addTree(commit.getTree());
+        while (treeWalk.next()) {
+          entryHandler.handleTreeEntry(treeWalk);
+        }
+      }
+    } catch (Exception e) {
+      LOG.warn("failed to load lfs files", e);
+      mirrorLog.add("Failed to load lfs files:");
+      mirrorLog.add(e.getMessage());
+      lfsUpdateResult.increaseFailureCount();
+    }
+  }
+
+  private String computeLfsUrl(String mirrorUrl) {
+    if (mirrorUrl.endsWith(".git")) {
+      return mirrorUrl + Protocol.INFO_LFS_ENDPOINT;
+    } else {
+      return mirrorUrl + ".git" + Protocol.INFO_LFS_ENDPOINT;
+    }
+  }
+
+  private class EntryHandler {
+
+    private final BlobStore lfsBlobStore;
+    private final Repository gitRepository;
+    private final List<String> mirrorLog;
+    private final LfsUpdateResult lfsUpdateResult;
+    private final sonia.scm.repository.Repository repository;
+    private final HttpConnectionFactory httpConnectionFactory;
+
+    private EntryHandler(sonia.scm.repository.Repository repository,
+                         Repository gitRepository,
+                         MirrorCommandRequest mirrorCommandRequest,
+                         List<String> mirrorLog,
+                         LfsUpdateResult lfsUpdateResult) {
+      this.lfsBlobStore = lfsBlobStoreFactory.getLfsBlobStore(repository);
+      this.repository = repository;
+      this.gitRepository = gitRepository;
+      this.mirrorLog = mirrorLog;
+      this.lfsUpdateResult = lfsUpdateResult;
+      this.httpConnectionFactory = mirrorHttpConnectionProvider.createHttpConnectionFactory(mirrorCommandRequest, mirrorLog);
+    }
+
+    private void handleTreeEntry(TreeWalk treeWalk) {
+      try (InputStream is = gitRepository.open(treeWalk.getObjectId(0), Constants.OBJ_BLOB).openStream()) {
+        LfsPointer lfsPointer = LfsPointer.parseLfsPointer(is);
+        AnyLongObjectId oid = lfsPointer.getOid();
+
+        if (lfsBlobStore.get(oid.name()) == null) {
+          Path tempFilePath = loadLfsFile(lfsPointer);
+          storeLfsBlob(oid, tempFilePath);
+          Files.delete(tempFilePath);
+        }
+      } catch (Exception e) {
+        LOG.warn("failed to load lfs file", e);
+        mirrorLog.add("Failed to load lfs file:");
+        mirrorLog.add(e.getMessage());
+        lfsUpdateResult.increaseFailureCount();
+      }
+    }
+
+    private Path loadLfsFile(LfsPointer lfsPointer) throws IOException {
+      lfsUpdateResult.increaseOverallCount();
+      LOG.trace("trying to load lfs file '{}' for repository {}", lfsPointer.getOid(), repository);
+      mirrorLog.add(String.format("Loading lfs file with id '%s'", lfsPointer.getOid().name()));
+      Lfs lfs = new Lfs(gitRepository);
+      lfs.getMediaFile(lfsPointer.getOid());
+
+      Collection<Path> paths = SmudgeFilter.downloadLfsResource(
+        lfs,
+        gitRepository,
+        httpConnectionFactory,
+        lfsPointer
+      );
+      return paths.iterator().next();
+    }
+
+    private void storeLfsBlob(AnyLongObjectId oid, Path tempFilePath) throws IOException {
+      LOG.trace("temporary lfs file: {}", tempFilePath);
+      Files.copy(
+        tempFilePath,
+        lfsBlobStore
+          .create(oid.name())
+          .getOutputStream()
+      );
+    }
+  }
+}

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/MirrorHttpConnectionProvider.java

@@ -24,12 +24,15 @@
 
 package sonia.scm.repository.spi;
 
+import org.eclipse.jgit.transport.UserAgent;
 import org.eclipse.jgit.transport.http.HttpConnectionFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import sonia.scm.net.HttpConnectionOptions;
 import sonia.scm.net.HttpURLConnectionFactory;
 import sonia.scm.repository.api.Pkcs12ClientCertificateCredential;
+import sonia.scm.repository.api.UsernamePasswordCredential;
+import sonia.scm.util.HttpUtil;
 import sonia.scm.web.ScmHttpConnectionFactory;
 
 import javax.inject.Inject;
@@ -37,8 +40,10 @@ import javax.net.ssl.KeyManager;
 import javax.net.ssl.KeyManagerFactory;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.security.GeneralSecurityException;
 import java.security.KeyStore;
+import java.util.Base64;
 import java.util.List;
 
 class MirrorHttpConnectionProvider {
@@ -59,6 +64,13 @@ class MirrorHttpConnectionProvider {
       .ifPresent(c -> options.withKeyManagers(createKeyManagers(c, log)));
     mirrorCommandRequest.getProxyConfiguration()
       .ifPresent(options::withProxyConfiguration);
+    mirrorCommandRequest.getCredential(UsernamePasswordCredential.class)
+      .ifPresent(credential -> {
+        String encodedAuth = Base64.getEncoder().encodeToString((credential.username() + ":" + new String(credential.password())).getBytes(StandardCharsets.UTF_8));
+        String authHeaderValue = "Basic " + encodedAuth;
+        options.addRequestProperty("Authorization", authHeaderValue);
+      });
+    options.addRequestProperty(HttpUtil.HEADER_USERAGENT, "git-lfs/2");
 
     return new ScmHttpConnectionFactory(httpURLConnectionFactory, options);
   }

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/GitLfsLockApiDetector.java

@@ -24,18 +24,40 @@
 
 package sonia.scm.web;
 
+import lombok.extern.slf4j.Slf4j;
 import sonia.scm.plugin.Extension;
 
 import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.core.MediaType;
+import java.util.Arrays;
 
+@Slf4j
 @Extension
 public class GitLfsLockApiDetector implements ScmClientDetector {
 
-  public static final String LOCK_APPLICATION_TYPE = "application/vnd.git-lfs+json";
+  private static final String APPLICATION_TYPE = "application";
+  private static final String LFS_VND_SUB_TYPE = "vnd.git-lfs+json";
 
   @Override
   public boolean isScmClient(HttpServletRequest request, UserAgent userAgent) {
-    return LOCK_APPLICATION_TYPE.equals(request.getHeader("Content-Type"))
-      || LOCK_APPLICATION_TYPE.equals(request.getHeader("Accept"));
+    return isLfsType(request, "Content-Type")
+      || isLfsType(request, "Accept");
+  }
+
+  private boolean isLfsType(HttpServletRequest request, String name) {
+    String headerValue = request.getHeader(name);
+
+    if (headerValue == null) {
+      return false;
+    }
+
+    log.trace("checking '{}' header with value '{}'", name, headerValue);
+
+    return Arrays.stream(headerValue.split(",\\s*"))
+      .anyMatch(v -> {
+        MediaType headerMediaType = MediaType.valueOf(v);
+        return APPLICATION_TYPE.equals(headerMediaType.getType())
+          && LFS_VND_SUB_TYPE.equals(headerMediaType.getSubtype());
+      });
   }
 }

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/GitLfsObjectApiDetector.java

@@ -0,0 +1,43 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package sonia.scm.web;
+
+import lombok.extern.slf4j.Slf4j;
+import sonia.scm.plugin.Extension;
+
+import javax.servlet.http.HttpServletRequest;
+import java.util.regex.Pattern;
+
+@Slf4j
+@Extension
+public class GitLfsObjectApiDetector implements ScmClientDetector {
+
+  private static final Pattern OBJECT_PATH_PATTERN = Pattern.compile("/[^/]*/repo/[^/]*/[^/]*\\.git/info/lfs/objects/.*");
+
+  @Override
+  public boolean isScmClient(HttpServletRequest request, UserAgent userAgent) {
+    return OBJECT_PATH_PATTERN.matcher(request.getRequestURI()).matches();
+  }
+}

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitMirrorCommandTest.java

@@ -64,6 +64,7 @@ import javax.net.ssl.TrustManager;
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
 import java.util.function.Consumer;
@@ -74,9 +75,12 @@ import static java.util.Collections.singletonList;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertThrows;
 import static org.mockito.AdditionalMatchers.not;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.argThat;
 import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 import static sonia.scm.repository.api.MirrorCommandResult.ResultType.FAILED;
@@ -104,6 +108,8 @@ public class GitMirrorCommandTest extends AbstractGitCommandTestBase {
 
   private final GitRepositoryConfigStoreProvider storeProvider = mock(GitRepositoryConfigStoreProvider.class);
   private final ConfigurationStore<GitRepositoryConfig> configurationStore = mock(ConfigurationStore.class);
+  private final LfsLoader lfsLoader = mock(LfsLoader.class);
+
   private final GitRepositoryConfig gitRepositoryConfig = new GitRepositoryConfig();
 
   @Before
@@ -136,11 +142,12 @@ public class GitMirrorCommandTest extends AbstractGitCommandTestBase {
       gitTagConverter,
       workingCopyFactory,
       gitHeadModifier,
-      storeProvider);
+      storeProvider,
+      lfsLoader);
   }
 
   @Before
-  public void initializeStore() {
+  public void initializeStores() {
     when(storeProvider.get(repository)).thenReturn(configurationStore);
     when(configurationStore.get()).thenReturn(gitRepositoryConfig);
   }
@@ -833,6 +840,50 @@ public class GitMirrorCommandTest extends AbstractGitCommandTestBase {
     }));
   }
 
+  @Test
+  public void shouldCallLfsLoader() {
+    callMirrorCommand();
+
+    Arrays.stream(new String[] {
+      "a8495c0335a13e6e432df90b3727fa91943189a7",
+      "d81ad6c63d7e2162308d69637b339dedd1d9201c",
+      "2f95f02d9c568594d31e78464bd11a96c62e3f91",
+      "91b99de908fcd04772798a31c308a64aea1a5523",
+      "03ca33468c2094249973d0ca11b80243a20de368",
+      "1fcebf45a215a43f0713a57b807d55e8387a6d70",
+      "383b954b27e052db6880d57f1c860dc208795247",
+      "35597e9e98fe53167266583848bfef985c2adb27",
+      "3f76a12f08a6ba0dc988c68b7f0b2cd190efc3c4",
+      "86a6645eceefe8b9a247db5eb16e3d89a7e6e6d1"
+      // one revision is missing here ("fcd0ef1831e4002ac43ea539f4094334c79ea9ec"), because this is iterated twice, what is hard to test
+    }).forEach(expectedRevision ->
+      verify(lfsLoader)
+      .inspectTree(eq(ObjectId.fromString(expectedRevision)), any(), any(), any(), any(), eq(repository)));
+  }
+
+  @Test
+  public void shouldMarkMirrorAsFailedIfLfsFileFailes() {
+    doAnswer(invocation -> {
+      invocation.getArgument(4, MirrorCommandResult.LfsUpdateResult.class).increaseFailureCount();
+      return null;
+    })
+      .when(lfsLoader)
+      .inspectTree(eq(ObjectId.fromString("a8495c0335a13e6e432df90b3727fa91943189a7")), any(), any(), any(), any(), eq(repository));
+
+
+    MirrorCommandResult mirrorCommandResult = callMirrorCommand();
+
+    assertThat(mirrorCommandResult.getResult()).isEqualTo(FAILED);
+  }
+
+  @Test
+  public void shouldNotCallLfsLoaderIfDeactivated() {
+    callMirrorCommand(repositoryDirectory.getAbsolutePath(), c -> c.setIgnoreLfs(true));
+
+    verify(lfsLoader, never())
+      .inspectTree(any(), any(), any(), any(), any(), any());
+  }
+
   public static class DefaultBranchSelectorTest {
 
     public static final List<String> BRANCHES = asList("master", "one", "two", "three");

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/MirrorHttpConnectionProviderTest.java

@@ -36,6 +36,7 @@ import org.mockito.junit.jupiter.MockitoExtension;
 import sonia.scm.net.HttpConnectionOptions;
 import sonia.scm.net.HttpURLConnectionFactory;
 import sonia.scm.net.ProxyConfiguration;
+import sonia.scm.repository.api.SimpleUsernamePasswordCredential;
 
 import java.io.IOException;
 import java.net.URL;
@@ -79,6 +80,25 @@ class MirrorHttpConnectionProviderTest {
     assertThat(value.getProxyConfiguration()).containsSame(proxy);
   }
 
+  @Test
+  void shouldConfigureAuthorizationHeader() throws IOException {
+    MirrorCommandRequest request = new MirrorCommandRequest();
+    request.setCredentials(List.of(new SimpleUsernamePasswordCredential("dent", "yellow".toCharArray())));
+
+    HttpConnectionOptions value = create(request);
+
+    assertThat(value.getConnectionProperties()).containsEntry("Authorization", "Basic ZGVudDp5ZWxsb3c=");
+  }
+
+  @Test
+  void shouldSetUserAgentHeader() throws IOException {
+    MirrorCommandRequest request = new MirrorCommandRequest();
+
+    HttpConnectionOptions value = create(request);
+
+    assertThat(value.getConnectionProperties()).containsEntry("User-Agent", "git-lfs/2");
+  }
+
   private HttpConnectionOptions create(MirrorCommandRequest request) throws IOException {
     List<String> log = new ArrayList<>();
 

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/GitLfsLockApiDetectorTest.java

@@ -0,0 +1,79 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package sonia.scm.web;
+
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import javax.servlet.http.HttpServletRequest;
+import java.util.stream.Stream;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+class GitLfsLockApiDetectorTest {
+
+  @Mock
+  private HttpServletRequest request;
+
+  private static Stream<Arguments> testParameters() {
+    return Stream.of(
+      Arguments.of("text/html, image/gif, image/jpeg", false),
+      Arguments.of("text/html, image/gif, image/jpeg, *; q=.2, */*; q=.2", false),
+      Arguments.of("*", false),
+      Arguments.of("application/vnd.git-lfs+json; charset=utf-8", true),
+      Arguments.of("application/vnd.git-lfs+json", true)
+    );
+  }
+
+  @ParameterizedTest
+  @MethodSource("testParameters")
+  void shouldHandleContentTypeHeaderCorrectly(String headerValue, boolean expected) {
+    when(request.getHeader("Content-Type"))
+      .thenReturn(headerValue);
+
+    boolean result = new GitLfsLockApiDetector().isScmClient(request, null);
+
+    assertThat(result).isEqualTo(expected);
+  }
+
+  @ParameterizedTest
+  @MethodSource("testParameters")
+  void shouldHandleAcceptHeaderCorrectly(String headerValue, boolean expected) {
+    when(request.getHeader("Content-Type"))
+      .thenReturn(null);
+    when(request.getHeader("Accept"))
+      .thenReturn(headerValue);
+
+    boolean result = new GitLfsLockApiDetector().isScmClient(request, null);
+
+    assertThat(result).isEqualTo(expected);
+  }
+}

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/GitLfsObjectApiDetectorTest.java

@@ -0,0 +1,63 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package sonia.scm.web;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import javax.servlet.http.HttpServletRequest;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+class GitLfsObjectApiDetectorTest {
+
+  @Mock
+  private HttpServletRequest request;
+
+  @Test
+  void shouldAcceptObjectRequest() {
+    when(request.getRequestURI())
+      .thenReturn("/scm/repo/scmadmin/lfs.git/info/lfs/objects/abc123");
+
+    boolean result = new GitLfsObjectApiDetector().isScmClient(request, null);
+
+    assertThat(result).isTrue();
+  }
+
+  @Test
+  void shouldRejectFakeObjectRequest() {
+    when(request.getRequestURI())
+      .thenReturn("/scm/repo/scmadmin/lfs.git/code/info/lfs/objects/abc123");
+
+    boolean result = new GitLfsObjectApiDetector().isScmClient(request, null);
+
+    assertThat(result).isFalse();
+  }
+}