Merged in feature/cat_with_lfs (pull request #324)

cat with lfs

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/GitUtil.java

@@ -42,7 +42,10 @@ import com.google.common.collect.Multimap;
 import org.eclipse.jgit.api.FetchCommand;
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.attributes.Attribute;
+import org.eclipse.jgit.attributes.Attributes;
 import org.eclipse.jgit.diff.DiffFormatter;
+import org.eclipse.jgit.lfs.LfsPointer;
 import org.eclipse.jgit.lib.AnyObjectId;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.ObjectId;
@@ -55,6 +58,7 @@ import org.eclipse.jgit.transport.FetchResult;
 import org.eclipse.jgit.transport.RefSpec;
 import org.eclipse.jgit.treewalk.TreeWalk;
 import org.eclipse.jgit.util.FS;
+import org.eclipse.jgit.util.LfsFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import sonia.scm.ContextEntry;
@@ -65,10 +69,12 @@ import sonia.scm.web.GitUserAgentProvider;
 import javax.servlet.http.HttpServletRequest;
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 
+import static java.util.Optional.empty;
 import static java.util.Optional.of;
 
 //~--- JDK imports ------------------------------------------------------------
@@ -731,6 +737,20 @@ public final class GitUtil
     }
   }
 
+  public static Optional<LfsPointer> getLfsPointer(org.eclipse.jgit.lib.Repository repo, String path, RevCommit commit, TreeWalk treeWalk) throws IOException {
+    Attributes attributes = LfsFactory.getAttributesForPath(repo, path, commit);
+
+    Attribute filter = attributes.get("filter");
+    if (filter != null && "lfs".equals(filter.getValue())) {
+      ObjectId blobId = treeWalk.getObjectId(0);
+      try (InputStream is = repo.open(blobId, Constants.OBJ_BLOB).openStream()) {
+        return of(LfsPointer.parseLfsPointer(is));
+      }
+    } else {
+      return empty();
+    }
+  }
+
   //~--- methods --------------------------------------------------------------
 
   /**

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitBrowseCommand.java

@@ -38,6 +38,7 @@ package sonia.scm.repository.spi;
 import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import org.eclipse.jgit.lfs.LfsPointer;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.ObjectLoader;
@@ -57,13 +58,17 @@ import sonia.scm.repository.GitSubModuleParser;
 import sonia.scm.repository.GitUtil;
 import sonia.scm.repository.Repository;
 import sonia.scm.repository.SubRepository;
+import sonia.scm.store.Blob;
+import sonia.scm.store.BlobStore;
 import sonia.scm.util.Util;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import static sonia.scm.ContextEntry.ContextBuilder.entity;
 import static sonia.scm.NotFoundException.notFound;
@@ -86,18 +91,20 @@ public class GitBrowseCommand extends AbstractGitCommand
    */
   private static final Logger logger =
     LoggerFactory.getLogger(GitBrowseCommand.class);
+  private final LfsBlobStoreFactory lfsBlobStoreFactory;
 
   //~--- constructors ---------------------------------------------------------
 
   /**
    * Constructs ...
-   *
+   *  @param context
-   * @param context
    * @param repository
+   * @param lfsBlobStoreFactory
    */
-  public GitBrowseCommand(GitContext context, Repository repository)
+  public GitBrowseCommand(GitContext context, Repository repository, LfsBlobStoreFactory lfsBlobStoreFactory)
   {
     super(context, repository);
+    this.lfsBlobStoreFactory = lfsBlobStoreFactory;
   }
 
   //~--- get methods ----------------------------------------------------------
@@ -167,7 +174,7 @@ public class GitBrowseCommand extends AbstractGitCommand
    * @throws IOException
    */
   private FileObject createFileObject(org.eclipse.jgit.lib.Repository repo,
-    BrowseCommandRequest request, ObjectId revId, TreeWalk treeWalk)
+                                      BrowseCommandRequest request, ObjectId revId, TreeWalk treeWalk)
     throws IOException {
 
     FileObject file = new FileObject();
@@ -195,7 +202,6 @@ public class GitBrowseCommand extends AbstractGitCommand
       ObjectLoader loader = repo.open(treeWalk.getObjectId(0));
 
       file.setDirectory(loader.getType() == Constants.OBJ_TREE);
-      file.setLength(loader.getSize());
 
       // don't show message and date for directories to improve performance
       if (!file.isDirectory() &&!request.isDisableLastCommit())
@@ -203,6 +209,16 @@ public class GitBrowseCommand extends AbstractGitCommand
         logger.trace("fetch last commit for {} at {}", path, revId.getName());
         RevCommit commit = getLatestCommit(repo, revId, path);
 
+        Optional<LfsPointer> lfsPointer = GitUtil.getLfsPointer(repo, path, commit, treeWalk);
+
+        if (lfsPointer.isPresent()) {
+          BlobStore lfsBlobStore = lfsBlobStoreFactory.getLfsBlobStore(repository);
+          Blob blob = lfsBlobStore.get(lfsPointer.get().getOid().getName());
+          file.setLength(blob.getSize());
+        } else {
+          file.setLength(loader.getSize());
+        }
+
         if (commit != null)
         {
           file.setLastModified(GitUtil.getCommitTime(commit));
@@ -232,7 +248,7 @@ public class GitBrowseCommand extends AbstractGitCommand
    * @return
    */
   private RevCommit getLatestCommit(org.eclipse.jgit.lib.Repository repo,
-    ObjectId revId, String path)
+                                    ObjectId revId, String path)
   {
     RevCommit result = null;
     RevWalk walk = null;
@@ -339,7 +355,7 @@ public class GitBrowseCommand extends AbstractGitCommand
   }
 
   private FileObject findFirstMatch(org.eclipse.jgit.lib.Repository repo,
-                        BrowseCommandRequest request, ObjectId revId, TreeWalk treeWalk) throws IOException {
+                                    BrowseCommandRequest request, ObjectId revId, TreeWalk treeWalk) throws IOException {
     String[] pathElements = request.getPath().split("/");
     int currentDepth = 0;
     int limit = pathElements.length;
@@ -364,7 +380,7 @@ public class GitBrowseCommand extends AbstractGitCommand
   @SuppressWarnings("unchecked")
   private Map<String,
     SubRepository> getSubRepositories(org.eclipse.jgit.lib.Repository repo,
-      ObjectId revision)
+                                      ObjectId revision)
     throws IOException {
     if (logger.isDebugEnabled())
     {
@@ -375,7 +391,7 @@ public class GitBrowseCommand extends AbstractGitCommand
     Map<String, SubRepository> subRepositories;
     try ( ByteArrayOutputStream baos = new ByteArrayOutputStream() )
     {
-      new GitCatCommand(context, repository).getContent(repo, revision,
+      new GitCatCommand(context, repository, lfsBlobStoreFactory).getContent(repo, revision,
         PATH_MODULES, baos);
       subRepositories = GitSubModuleParser.parse(baos.toString());
     }
@@ -389,7 +405,7 @@ public class GitBrowseCommand extends AbstractGitCommand
   }
 
   private SubRepository getSubRepository(org.eclipse.jgit.lib.Repository repo,
-    ObjectId revId, String path)
+                                         ObjectId revId, String path)
     throws IOException {
     Map<String, SubRepository> subRepositories = subrepositoryCache.get(revId);
 

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitCatCommand.java

@@ -33,6 +33,7 @@
 package sonia.scm.repository.spi;
 
 import org.eclipse.jgit.errors.MissingObjectException;
+import org.eclipse.jgit.lfs.LfsPointer;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.ObjectLoader;
@@ -45,13 +46,18 @@ import org.eclipse.jgit.treewalk.filter.PathFilter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import sonia.scm.repository.GitUtil;
+import sonia.scm.store.Blob;
+import sonia.scm.store.BlobStore;
+import sonia.scm.util.IOUtil;
 import sonia.scm.util.Util;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
 
 import java.io.Closeable;
 import java.io.FilterInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.Optional;
 
 import static sonia.scm.ContextEntry.ContextBuilder.entity;
 import static sonia.scm.NotFoundException.notFound;
@@ -61,15 +67,18 @@ public class GitCatCommand extends AbstractGitCommand implements CatCommand {
 
   private static final Logger logger = LoggerFactory.getLogger(GitCatCommand.class);
 
-  public GitCatCommand(GitContext context, sonia.scm.repository.Repository repository) {
+  private final LfsBlobStoreFactory lfsBlobStoreFactory;
+
+  public GitCatCommand(GitContext context, sonia.scm.repository.Repository repository, LfsBlobStoreFactory lfsBlobStoreFactory) {
     super(context, repository);
+    this.lfsBlobStoreFactory = lfsBlobStoreFactory;
   }
 
   @Override
   public void getCatResult(CatCommandRequest request, OutputStream output) throws IOException {
     logger.debug("try to read content for {}", request);
-    try (ClosableObjectLoaderContainer closableObjectLoaderContainer = getLoader(request)) {
+    try (Loader closableObjectLoaderContainer = getLoader(request)) {
-      closableObjectLoaderContainer.objectLoader.copyTo(output);
+      closableObjectLoaderContainer.copyTo(output);
     }
   }
 
@@ -80,18 +89,18 @@ public class GitCatCommand extends AbstractGitCommand implements CatCommand {
   }
 
   void getContent(org.eclipse.jgit.lib.Repository repo, ObjectId revId, String path, OutputStream output) throws IOException {
-    try (ClosableObjectLoaderContainer closableObjectLoaderContainer = getLoader(repo, revId, path)) {
+    try (Loader closableObjectLoaderContainer = getLoader(repo, revId, path)) {
-      closableObjectLoaderContainer.objectLoader.copyTo(output);
+      closableObjectLoaderContainer.copyTo(output);
     }
   }
 
-  private ClosableObjectLoaderContainer getLoader(CatCommandRequest request) throws IOException {
+  private Loader getLoader(CatCommandRequest request) throws IOException {
     org.eclipse.jgit.lib.Repository repo = open();
     ObjectId revId = getCommitOrDefault(repo, request.getRevision());
     return getLoader(repo, revId, request.getPath());
   }
 
-  private ClosableObjectLoaderContainer getLoader(Repository repo, ObjectId revId, String path) throws IOException {
+  private Loader getLoader(Repository repo, ObjectId revId, String path) throws IOException {
     TreeWalk treeWalk = new TreeWalk(repo);
     treeWalk.setRecursive(Util.nonNull(path).contains("/"));
 
@@ -116,21 +125,67 @@ public class GitCatCommand extends AbstractGitCommand implements CatCommand {
     treeWalk.setFilter(PathFilter.create(path));
 
     if (treeWalk.next() && treeWalk.getFileMode(0).getObjectType() == Constants.OBJ_BLOB) {
-      ObjectId blobId = treeWalk.getObjectId(0);
+      Optional<LfsPointer> lfsPointer = GitUtil.getLfsPointer(repo, path, entry, treeWalk);
-      ObjectLoader loader = repo.open(blobId);
+      if (lfsPointer.isPresent()) {
-
+        return loadFromLfsStore(treeWalk, revWalk, lfsPointer.get());
-      return new ClosableObjectLoaderContainer(loader, treeWalk, revWalk);
+      } else {
+        return loadFromGit(repo, treeWalk, revWalk);
+      }
     } else {
       throw notFound(entity("Path", path).in("Revision", revId.getName()).in(repository));
     }
   }
 
-  private static class ClosableObjectLoaderContainer implements Closeable {
+  private Loader loadFromGit(Repository repo, TreeWalk treeWalk, RevWalk revWalk) throws IOException {
+    ObjectId blobId = treeWalk.getObjectId(0);
+    ObjectLoader loader = repo.open(blobId);
+
+    return new GitObjectLoaderWrapper(loader, treeWalk, revWalk);
+  }
+
+  private Loader loadFromLfsStore(TreeWalk treeWalk, RevWalk revWalk, LfsPointer lfsPointer) throws IOException {
+    BlobStore lfsBlobStore = lfsBlobStoreFactory.getLfsBlobStore(repository);
+    Blob blob = lfsBlobStore.get(lfsPointer.getOid().getName());
+    GitUtil.release(revWalk);
+    GitUtil.release(treeWalk);
+    return new BlobLoader(blob);
+  }
+
+  private interface Loader extends Closeable {
+    void copyTo(OutputStream output) throws IOException;
+
+    InputStream openStream() throws IOException;
+  }
+
+  private static class BlobLoader implements Loader {
+    private final InputStream inputStream;
+
+    private BlobLoader(Blob blob) throws IOException {
+      this.inputStream = blob.getInputStream();
+    }
+
+    @Override
+    public void copyTo(OutputStream output) throws IOException {
+      IOUtil.copy(inputStream, output);
+    }
+
+    @Override
+    public InputStream openStream() {
+      return inputStream;
+    }
+
+    @Override
+    public void close() throws IOException {
+      this.inputStream.close();
+    }
+  }
+
+  private static class GitObjectLoaderWrapper implements Loader {
     private final ObjectLoader objectLoader;
     private final TreeWalk treeWalk;
     private final RevWalk revWalk;
 
-    private ClosableObjectLoaderContainer(ObjectLoader objectLoader, TreeWalk treeWalk, RevWalk revWalk) {
+    private GitObjectLoaderWrapper(ObjectLoader objectLoader, TreeWalk treeWalk, RevWalk revWalk) {
       this.objectLoader = objectLoader;
       this.treeWalk = treeWalk;
       this.revWalk = revWalk;
@@ -141,14 +196,22 @@ public class GitCatCommand extends AbstractGitCommand implements CatCommand {
       GitUtil.release(revWalk);
       GitUtil.release(treeWalk);
     }
+
+    public void copyTo(OutputStream output) throws IOException {
+      this.objectLoader.copyTo(output);
+    }
+
+    public InputStream openStream() throws IOException {
+      return objectLoader.openStream();
+    }
   }
 
   private static class InputStreamWrapper extends FilterInputStream {
 
-    private final ClosableObjectLoaderContainer container;
+    private final Loader container;
 
-    private InputStreamWrapper(ClosableObjectLoaderContainer container) throws IOException {
+    private InputStreamWrapper(Loader container) throws IOException {
-      super(container.objectLoader.openStream());
+      super(container.openStream());
       this.container = container;
     }
 

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceProvider.java

@@ -39,6 +39,7 @@ import sonia.scm.repository.Feature;
 import sonia.scm.repository.GitRepositoryHandler;
 import sonia.scm.repository.Repository;
 import sonia.scm.repository.api.Command;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
 
 import java.io.IOException;
 import java.util.EnumSet;
@@ -76,9 +77,10 @@ public class GitRepositoryServiceProvider extends RepositoryServiceProvider
 
   //~--- constructors ---------------------------------------------------------
 
-  public GitRepositoryServiceProvider(GitRepositoryHandler handler, Repository repository, GitRepositoryConfigStoreProvider storeProvider) {
+  public GitRepositoryServiceProvider(GitRepositoryHandler handler, Repository repository, GitRepositoryConfigStoreProvider storeProvider, LfsBlobStoreFactory lfsBlobStoreFactory) {
     this.handler = handler;
     this.repository = repository;
+    this.lfsBlobStoreFactory = lfsBlobStoreFactory;
     this.context = new GitContext(handler.getDirectory(repository.getId()), repository, storeProvider);
   }
 
@@ -143,7 +145,7 @@ public class GitRepositoryServiceProvider extends RepositoryServiceProvider
   @Override
   public BrowseCommand getBrowseCommand()
   {
-    return new GitBrowseCommand(context, repository);
+    return new GitBrowseCommand(context, repository, lfsBlobStoreFactory);
   }
 
   /**
@@ -155,7 +157,7 @@ public class GitRepositoryServiceProvider extends RepositoryServiceProvider
   @Override
   public CatCommand getCatCommand()
   {
-    return new GitCatCommand(context, repository);
+    return new GitCatCommand(context, repository, lfsBlobStoreFactory);
   }
 
   /**
@@ -281,11 +283,13 @@ public class GitRepositoryServiceProvider extends RepositoryServiceProvider
 //~--- fields ---------------------------------------------------------------
 
   /** Field description */
-  private GitContext context;
+  private final GitContext context;
 
   /** Field description */
-  private GitRepositoryHandler handler;
+  private final GitRepositoryHandler handler;
 
   /** Field description */
-  private Repository repository;
+  private final Repository repository;
+
+  private final LfsBlobStoreFactory lfsBlobStoreFactory;
 }

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceResolver.java

@@ -39,6 +39,7 @@ import sonia.scm.api.v2.resources.GitRepositoryConfigStoreProvider;
 import sonia.scm.plugin.Extension;
 import sonia.scm.repository.GitRepositoryHandler;
 import sonia.scm.repository.Repository;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
 
 /**
  *
@@ -49,11 +50,13 @@ public class GitRepositoryServiceResolver implements RepositoryServiceResolver {
 
   private final GitRepositoryHandler handler;
   private final GitRepositoryConfigStoreProvider storeProvider;
+  private final LfsBlobStoreFactory lfsBlobStoreFactory;
 
   @Inject
-  public GitRepositoryServiceResolver(GitRepositoryHandler handler, GitRepositoryConfigStoreProvider storeProvider) {
+  public GitRepositoryServiceResolver(GitRepositoryHandler handler, GitRepositoryConfigStoreProvider storeProvider, LfsBlobStoreFactory lfsBlobStoreFactory) {
     this.handler = handler;
     this.storeProvider = storeProvider;
+    this.lfsBlobStoreFactory = lfsBlobStoreFactory;
   }
 
   @Override
@@ -61,7 +64,7 @@ public class GitRepositoryServiceResolver implements RepositoryServiceResolver {
     GitRepositoryServiceProvider provider = null;
 
     if (GitRepositoryHandler.TYPE_NAME.equalsIgnoreCase(repository.getType())) {
-      provider = new GitRepositoryServiceProvider(handler, repository, storeProvider);
+      provider = new GitRepositoryServiceProvider(handler, repository, storeProvider, lfsBlobStoreFactory);
     }
 
     return provider;

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitBrowseCommandTest.java

@@ -171,6 +171,6 @@ public class GitBrowseCommandTest extends AbstractGitCommandTestBase {
   }
 
   private GitBrowseCommand createCommand() {
-    return new GitBrowseCommand(createContext(), repository);
+    return new GitBrowseCommand(createContext(), repository, null);
   }
 }

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitCatCommandTest.java

@@ -39,12 +39,18 @@ import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import sonia.scm.NotFoundException;
 import sonia.scm.repository.GitRepositoryConfig;
+import sonia.scm.store.Blob;
+import sonia.scm.store.BlobStore;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
 
+import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 
 import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 /**
  * Unit tests for {@link GitCatCommand}.
@@ -136,7 +142,7 @@ public class GitCatCommandTest extends AbstractGitCommandTestBase {
     CatCommandRequest request = new CatCommandRequest();
     request.setPath("b.txt");
 
-    InputStream catResultStream = new GitCatCommand(createContext(), repository).getCatResultStream(request);
+    InputStream catResultStream = new GitCatCommand(createContext(), repository, null).getCatResultStream(request);
 
     assertEquals('b', catResultStream.read());
     assertEquals('\n', catResultStream.read());
@@ -145,13 +151,38 @@ public class GitCatCommandTest extends AbstractGitCommandTestBase {
     catResultStream.close();
   }
 
+  @Test
+  public void testLfsStream() throws IOException {
+    LfsBlobStoreFactory lfsBlobStoreFactory = mock(LfsBlobStoreFactory.class);
+    BlobStore blobStore = mock(BlobStore.class);
+    Blob blob = mock(Blob.class);
+    when(lfsBlobStoreFactory.getLfsBlobStore(repository)).thenReturn(blobStore);
+    when(blobStore.get("d2252bd9fde1bb2ae7531b432c48262c3cbe4df4376008986980de40a7c9cf8b"))
+      .thenReturn(blob);
+    when(blob.getInputStream()).thenReturn(new ByteArrayInputStream(new byte[]{'i', 's'}));
+
+    CatCommandRequest request = new CatCommandRequest();
+    request.setRevision("lfs-test");
+    request.setPath("lfs-image.png");
+
+    InputStream catResultStream = new GitCatCommand(createContext(), repository, lfsBlobStoreFactory)
+      .getCatResultStream(request);
+
+    assertEquals('i', catResultStream.read());
+    assertEquals('s', catResultStream.read());
+
+    assertEquals(-1, catResultStream.read());
+
+    catResultStream.close();
+  }
+
   private String execute(CatCommandRequest request) throws IOException {
     String content = null;
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
 
     try
     {
-      new GitCatCommand(createContext(), repository).getCatResult(request,
+      new GitCatCommand(createContext(), repository, null).getCatResult(request,
                         baos);
     }
     finally