Detect and load lfs files

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitBrowseCommand.java

@@ -58,6 +58,7 @@ import sonia.scm.repository.GitUtil;
 import sonia.scm.repository.Repository;
 import sonia.scm.repository.SubRepository;
 import sonia.scm.util.Util;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -86,18 +87,20 @@ public class GitBrowseCommand extends AbstractGitCommand
    */
   private static final Logger logger =
     LoggerFactory.getLogger(GitBrowseCommand.class);
+  private final LfsBlobStoreFactory lfsBlobStoreFactory;
 
   //~--- constructors ---------------------------------------------------------
 
   /**
    * Constructs ...
-   *
    *  @param context
    * @param repository
+   * @param lfsBlobStoreFactory
    */
-  public GitBrowseCommand(GitContext context, Repository repository)
+  public GitBrowseCommand(GitContext context, Repository repository, LfsBlobStoreFactory lfsBlobStoreFactory)
   {
     super(context, repository);
+    this.lfsBlobStoreFactory = lfsBlobStoreFactory;
   }
 
   //~--- get methods ----------------------------------------------------------
@@ -375,7 +378,7 @@ public class GitBrowseCommand extends AbstractGitCommand
     Map<String, SubRepository> subRepositories;
     try ( ByteArrayOutputStream baos = new ByteArrayOutputStream() )
     {
-      new GitCatCommand(context, repository).getContent(repo, revision,
+      new GitCatCommand(context, repository, lfsBlobStoreFactory).getContent(repo, revision,
         PATH_MODULES, baos);
       subRepositories = GitSubModuleParser.parse(baos.toString());
     }

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitCatCommand.java

@@ -32,7 +32,10 @@
 
 package sonia.scm.repository.spi;
 
+import org.eclipse.jgit.attributes.Attribute;
+import org.eclipse.jgit.attributes.Attributes;
 import org.eclipse.jgit.errors.MissingObjectException;
+import org.eclipse.jgit.lfs.LfsPointer;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.ObjectLoader;
@@ -42,10 +45,14 @@ import org.eclipse.jgit.revwalk.RevTree;
 import org.eclipse.jgit.revwalk.RevWalk;
 import org.eclipse.jgit.treewalk.TreeWalk;
 import org.eclipse.jgit.treewalk.filter.PathFilter;
+import org.eclipse.jgit.util.LfsFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import sonia.scm.repository.GitUtil;
+import sonia.scm.store.Blob;
+import sonia.scm.util.IOUtil;
 import sonia.scm.util.Util;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
 
 import java.io.Closeable;
 import java.io.FilterInputStream;
@@ -61,15 +68,18 @@ public class GitCatCommand extends AbstractGitCommand implements CatCommand {
 
   private static final Logger logger = LoggerFactory.getLogger(GitCatCommand.class);
 
-  public GitCatCommand(GitContext context, sonia.scm.repository.Repository repository) {
+  private final LfsBlobStoreFactory lfsBlobStoreFactory;
+
+  public GitCatCommand(GitContext context, sonia.scm.repository.Repository repository, LfsBlobStoreFactory lfsBlobStoreFactory) {
     super(context, repository);
+    this.lfsBlobStoreFactory = lfsBlobStoreFactory;
   }
 
   @Override
   public void getCatResult(CatCommandRequest request, OutputStream output) throws IOException {
     logger.debug("try to read content for {}", request);
-    try (ClosableObjectLoaderContainer closableObjectLoaderContainer = getLoader(request)) {
-      closableObjectLoaderContainer.objectLoader.copyTo(output);
+    try (Loader closableObjectLoaderContainer = getLoader(request)) {
+      closableObjectLoaderContainer.copyTo(output);
     }
   }
 
@@ -80,18 +90,18 @@ public class GitCatCommand extends AbstractGitCommand implements CatCommand {
   }
 
   void getContent(org.eclipse.jgit.lib.Repository repo, ObjectId revId, String path, OutputStream output) throws IOException {
-    try (ClosableObjectLoaderContainer closableObjectLoaderContainer = getLoader(repo, revId, path)) {
-      closableObjectLoaderContainer.objectLoader.copyTo(output);
+    try (Loader closableObjectLoaderContainer = getLoader(repo, revId, path)) {
+      closableObjectLoaderContainer.copyTo(output);
     }
   }
 
-  private ClosableObjectLoaderContainer getLoader(CatCommandRequest request) throws IOException {
+  private Loader getLoader(CatCommandRequest request) throws IOException {
     org.eclipse.jgit.lib.Repository repo = open();
     ObjectId revId = getCommitOrDefault(repo, request.getRevision());
     return getLoader(repo, revId, request.getPath());
   }
 
-  private ClosableObjectLoaderContainer getLoader(Repository repo, ObjectId revId, String path) throws IOException {
+  private Loader getLoader(Repository repo, ObjectId revId, String path) throws IOException {
     TreeWalk treeWalk = new TreeWalk(repo);
     treeWalk.setRecursive(Util.nonNull(path).contains("/"));
 
@@ -116,21 +126,69 @@ public class GitCatCommand extends AbstractGitCommand implements CatCommand {
     treeWalk.setFilter(PathFilter.create(path));
 
     if (treeWalk.next() && treeWalk.getFileMode(0).getObjectType() == Constants.OBJ_BLOB) {
+      Attributes attributes = LfsFactory.getAttributesForPath(repo, path, entry);
+
+      Attribute filter = attributes.get("filter");
+      if (filter != null && "lfs".equals(filter.getValue())) {
+        return loadFromLfsStore(repo, treeWalk, revWalk);
+      }
+
       ObjectId blobId = treeWalk.getObjectId(0);
       ObjectLoader loader = repo.open(blobId);
 
-      return new ClosableObjectLoaderContainer(loader, treeWalk, revWalk);
+      return new GitObjectLoaderWrapper(loader, treeWalk, revWalk);
     } else {
       throw notFound(entity("Path", path).in("Revision", revId.getName()).in(repository));
     }
   }
 
-  private static class ClosableObjectLoaderContainer implements Closeable {
+  private Loader loadFromLfsStore(Repository repo, TreeWalk treeWalk, RevWalk revWalk) throws IOException {
+    ObjectId blobId = treeWalk.getObjectId(0);
+    LfsPointer lfsPointer;
+    try (InputStream is = repo.open(blobId, Constants.OBJ_BLOB).openStream()) {
+      lfsPointer = LfsPointer.parseLfsPointer(is);
+    }
+    Blob blob = lfsBlobStoreFactory.getLfsBlobStore(repository).get(lfsPointer.getOid().getName());
+    GitUtil.release(revWalk);
+    GitUtil.release(treeWalk);
+    return new BlobLoader(blob);
+  }
+
+  private interface Loader extends Closeable {
+    void copyTo(OutputStream output) throws IOException;
+
+    InputStream openStream() throws IOException;
+  }
+
+  private static class BlobLoader implements Loader {
+    private final InputStream inputStream;
+
+    private BlobLoader(Blob blob) throws IOException {
+      this.inputStream = blob.getInputStream();
+    }
+
+    @Override
+    public void copyTo(OutputStream output) throws IOException {
+      IOUtil.copy(inputStream, output);
+    }
+
+    @Override
+    public InputStream openStream() {
+      return inputStream;
+    }
+
+    @Override
+    public void close() throws IOException {
+      this.inputStream.close();
+    }
+  }
+
+  private static class GitObjectLoaderWrapper implements Loader {
     private final ObjectLoader objectLoader;
     private final TreeWalk treeWalk;
     private final RevWalk revWalk;
 
-    private ClosableObjectLoaderContainer(ObjectLoader objectLoader, TreeWalk treeWalk, RevWalk revWalk) {
+    private GitObjectLoaderWrapper(ObjectLoader objectLoader, TreeWalk treeWalk, RevWalk revWalk) {
       this.objectLoader = objectLoader;
       this.treeWalk = treeWalk;
       this.revWalk = revWalk;
@@ -141,14 +199,22 @@ public class GitCatCommand extends AbstractGitCommand implements CatCommand {
       GitUtil.release(revWalk);
       GitUtil.release(treeWalk);
     }
+
+    public void copyTo(OutputStream output) throws IOException {
+      this.objectLoader.copyTo(output);
+    }
+
+    public InputStream openStream() throws IOException {
+      return objectLoader.openStream();
+    }
   }
 
   private static class InputStreamWrapper extends FilterInputStream {
 
-    private final ClosableObjectLoaderContainer container;
+    private final Loader container;
 
-    private InputStreamWrapper(ClosableObjectLoaderContainer container) throws IOException {
-      super(container.objectLoader.openStream());
+    private InputStreamWrapper(Loader container) throws IOException {
+      super(container.openStream());
       this.container = container;
     }
 

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceProvider.java

@@ -39,6 +39,7 @@ import sonia.scm.repository.Feature;
 import sonia.scm.repository.GitRepositoryHandler;
 import sonia.scm.repository.Repository;
 import sonia.scm.repository.api.Command;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
 
 import java.io.IOException;
 import java.util.EnumSet;
@@ -76,9 +77,10 @@ public class GitRepositoryServiceProvider extends RepositoryServiceProvider
 
   //~--- constructors ---------------------------------------------------------
 
-  public GitRepositoryServiceProvider(GitRepositoryHandler handler, Repository repository, GitRepositoryConfigStoreProvider storeProvider) {
+  public GitRepositoryServiceProvider(GitRepositoryHandler handler, Repository repository, GitRepositoryConfigStoreProvider storeProvider, LfsBlobStoreFactory lfsBlobStoreFactory) {
     this.handler = handler;
     this.repository = repository;
+    this.lfsBlobStoreFactory = lfsBlobStoreFactory;
     this.context = new GitContext(handler.getDirectory(repository.getId()), repository, storeProvider);
   }
 
@@ -143,7 +145,7 @@ public class GitRepositoryServiceProvider extends RepositoryServiceProvider
   @Override
   public BrowseCommand getBrowseCommand()
   {
-    return new GitBrowseCommand(context, repository);
+    return new GitBrowseCommand(context, repository, lfsBlobStoreFactory);
   }
 
   /**
@@ -155,7 +157,7 @@ public class GitRepositoryServiceProvider extends RepositoryServiceProvider
   @Override
   public CatCommand getCatCommand()
   {
-    return new GitCatCommand(context, repository);
+    return new GitCatCommand(context, repository, lfsBlobStoreFactory);
   }
 
   /**
@@ -281,11 +283,13 @@ public class GitRepositoryServiceProvider extends RepositoryServiceProvider
 //~--- fields ---------------------------------------------------------------
 
   /** Field description */
-  private GitContext context;
+  private final GitContext context;
 
   /** Field description */
-  private GitRepositoryHandler handler;
+  private final GitRepositoryHandler handler;
 
   /** Field description */
-  private Repository repository;
+  private final Repository repository;
+
+  private final LfsBlobStoreFactory lfsBlobStoreFactory;
 }

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceResolver.java

@@ -39,6 +39,7 @@ import sonia.scm.api.v2.resources.GitRepositoryConfigStoreProvider;
 import sonia.scm.plugin.Extension;
 import sonia.scm.repository.GitRepositoryHandler;
 import sonia.scm.repository.Repository;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
 
 /**
  *
@@ -49,11 +50,13 @@ public class GitRepositoryServiceResolver implements RepositoryServiceResolver {
 
   private final GitRepositoryHandler handler;
   private final GitRepositoryConfigStoreProvider storeProvider;
+  private final LfsBlobStoreFactory lfsBlobStoreFactory;
 
   @Inject
-  public GitRepositoryServiceResolver(GitRepositoryHandler handler, GitRepositoryConfigStoreProvider storeProvider) {
+  public GitRepositoryServiceResolver(GitRepositoryHandler handler, GitRepositoryConfigStoreProvider storeProvider, LfsBlobStoreFactory lfsBlobStoreFactory) {
     this.handler = handler;
     this.storeProvider = storeProvider;
+    this.lfsBlobStoreFactory = lfsBlobStoreFactory;
   }
 
   @Override
@@ -61,7 +64,7 @@ public class GitRepositoryServiceResolver implements RepositoryServiceResolver {
     GitRepositoryServiceProvider provider = null;
 
     if (GitRepositoryHandler.TYPE_NAME.equalsIgnoreCase(repository.getType())) {
-      provider = new GitRepositoryServiceProvider(handler, repository, storeProvider);
+      provider = new GitRepositoryServiceProvider(handler, repository, storeProvider, lfsBlobStoreFactory);
     }
 
     return provider;

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitBrowseCommandTest.java

@@ -171,6 +171,6 @@ public class GitBrowseCommandTest extends AbstractGitCommandTestBase {
   }
 
   private GitBrowseCommand createCommand() {
-    return new GitBrowseCommand(createContext(), repository);
+    return new GitBrowseCommand(createContext(), repository, null);
   }
 }

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitCatCommandTest.java

@@ -39,12 +39,18 @@ import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import sonia.scm.NotFoundException;
 import sonia.scm.repository.GitRepositoryConfig;
+import sonia.scm.store.Blob;
+import sonia.scm.store.BlobStore;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
 
+import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 
 import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 /**
  * Unit tests for {@link GitCatCommand}.
@@ -136,7 +142,7 @@ public class GitCatCommandTest extends AbstractGitCommandTestBase {
     CatCommandRequest request = new CatCommandRequest();
     request.setPath("b.txt");
 
-    InputStream catResultStream = new GitCatCommand(createContext(), repository).getCatResultStream(request);
+    InputStream catResultStream = new GitCatCommand(createContext(), repository, null).getCatResultStream(request);
 
     assertEquals('b', catResultStream.read());
     assertEquals('\n', catResultStream.read());
@@ -145,13 +151,38 @@ public class GitCatCommandTest extends AbstractGitCommandTestBase {
     catResultStream.close();
   }
 
+  @Test
+  public void testLfsStream() throws IOException {
+    LfsBlobStoreFactory lfsBlobStoreFactory = mock(LfsBlobStoreFactory.class);
+    BlobStore blobStore = mock(BlobStore.class);
+    Blob blob = mock(Blob.class);
+    when(lfsBlobStoreFactory.getLfsBlobStore(repository)).thenReturn(blobStore);
+    when(blobStore.get("d2252bd9fde1bb2ae7531b432c48262c3cbe4df4376008986980de40a7c9cf8b"))
+      .thenReturn(blob);
+    when(blob.getInputStream()).thenReturn(new ByteArrayInputStream(new byte[]{'i', 's'}));
+
+    CatCommandRequest request = new CatCommandRequest();
+    request.setRevision("lfs-test");
+    request.setPath("lfs-image.png");
+
+    InputStream catResultStream = new GitCatCommand(createContext(), repository, lfsBlobStoreFactory)
+      .getCatResultStream(request);
+
+    assertEquals('i', catResultStream.read());
+    assertEquals('s', catResultStream.read());
+
+    assertEquals(-1, catResultStream.read());
+
+    catResultStream.close();
+  }
+
   private String execute(CatCommandRequest request) throws IOException {
     String content = null;
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
 
     try
     {
-      new GitCatCommand(createContext(), repository).getCatResult(request,
+      new GitCatCommand(createContext(), repository, null).getCatResult(request,
                         baos);
     }
     finally