LFS upload POC

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitModifyCommand.java

@@ -3,6 +3,7 @@ package sonia.scm.repository.spi;
 import org.apache.commons.lang.StringUtils;
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.attributes.FilterCommandRegistry;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.Ref;
 import org.eclipse.jgit.revwalk.RevCommit;
@@ -12,6 +13,7 @@ import sonia.scm.ContextEntry;
 import sonia.scm.repository.GitWorkdirFactory;
 import sonia.scm.repository.InternalRepositoryException;
 import sonia.scm.repository.Repository;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
 
 import java.io.File;
 import java.io.IOException;
@@ -30,10 +32,12 @@ import static sonia.scm.ScmConstraintViolationException.Builder.doThrow;
 public class GitModifyCommand extends AbstractGitCommand implements ModifyCommand {
 
   private final GitWorkdirFactory workdirFactory;
+  private final LfsBlobStoreFactory lfsBlobStoreFactory;
 
-  GitModifyCommand(GitContext context, Repository repository, GitWorkdirFactory workdirFactory) {
+  GitModifyCommand(GitContext context, Repository repository, GitWorkdirFactory workdirFactory, LfsBlobStoreFactory lfsBlobStoreFactory) {
     super(context, repository);
     this.workdirFactory = workdirFactory;
+    this.lfsBlobStoreFactory = lfsBlobStoreFactory;
   }
 
   @Override
@@ -87,10 +91,17 @@ public class GitModifyCommand extends AbstractGitCommand implements ModifyComman
           throw alreadyExists(createFileContext(toBeCreated));
         }
       }
+
+      LfsBlobStoreCleanFilterFactory cleanFilterFactory = new LfsBlobStoreCleanFilterFactory(lfsBlobStoreFactory, repository, targetFile);
+
+      String registerKey = "git-lfs clean -- '" + toBeCreated + "'";
+      FilterCommandRegistry.register(registerKey, cleanFilterFactory::createFilter);
       try {
         addFileToGit(toBeCreated);
       } catch (GitAPIException e) {
         throwInternalRepositoryException("could not add new file to index", e);
+      } finally {
+        FilterCommandRegistry.unregister(registerKey);
       }
     }
 

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceProvider.java

@@ -273,7 +273,7 @@ public class GitRepositoryServiceProvider extends RepositoryServiceProvider
 
   @Override
   public ModifyCommand getModifyCommand() {
-    return new GitModifyCommand(context, repository, handler.getWorkdirFactory());
+    return new GitModifyCommand(context, repository, handler.getWorkdirFactory(), lfsBlobStoreFactory);
   }
 
   @Override

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/LfsBlobStoreCleanFilter.java

@@ -0,0 +1,101 @@
+package sonia.scm.repository.spi;
+
+import org.eclipse.jgit.attributes.FilterCommand;
+import org.eclipse.jgit.lfs.Lfs;
+import org.eclipse.jgit.lfs.LfsPointer;
+import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
+import org.eclipse.jgit.lfs.lib.LongObjectId;
+import org.eclipse.jgit.lib.Repository;
+import sonia.scm.store.Blob;
+import sonia.scm.store.BlobStore;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+import static org.eclipse.jgit.lfs.lib.Constants.LONG_HASH_FUNCTION;
+
+/**
+ * Adapted version of JGit's {@link org.eclipse.jgit.lfs.CleanFilter} to write the
+ * lfs file directly to the lfs blob store.
+ */
+public class LfsBlobStoreCleanFilter extends FilterCommand {
+
+
+  private Lfs lfsUtil;
+  private final BlobStore lfsBlobStore;
+  private final Path targetFile;
+  private final DigestOutputStream digestOutputStream;
+
+  private long size;
+
+  public LfsBlobStoreCleanFilter(Repository db, InputStream in, OutputStream out, BlobStore lfsBlobStore, Path targetFile)
+    throws IOException {
+    super(in, out);
+    lfsUtil = new Lfs(db);
+    this.lfsBlobStore = lfsBlobStore;
+    this.targetFile = targetFile;
+    Files.createDirectories(lfsUtil.getLfsTmpDir());
+
+
+   MessageDigest md ;
+    try {
+      md = MessageDigest.getInstance(LONG_HASH_FUNCTION);
+    } catch (NoSuchAlgorithmException e) {
+      throw new RuntimeException(e);
+    }
+    digestOutputStream = new DigestOutputStream(new OutputStream() {
+      @Override
+      public void write(int b) {
+      }
+    }, md);
+
+  }
+
+  @Override
+  public int run() throws IOException {
+    try {
+      byte[] buf = new byte[8192];
+      int length = in.read(buf);
+      if (length != -1) {
+        digestOutputStream.write(buf, 0, length);
+        size += length;
+        return length;
+      } else {
+        digestOutputStream.close();
+        AnyLongObjectId loid = LongObjectId.fromRaw(digestOutputStream.getMessageDigest().digest());
+
+        Blob existingBlob = lfsBlobStore.get(loid.getName());
+        if (existingBlob != null) {
+          long blobSize = existingBlob.getSize();
+          if (blobSize != size) {
+            throw new RuntimeException("lfs entry already exists for loid " + loid.getName() + " but has wrong size");
+          }
+        } else {
+          Blob newBlob = lfsBlobStore.create(loid.getName());
+          OutputStream outputStream = newBlob.getOutputStream();
+          Files.copy(targetFile, outputStream);
+          newBlob.commit();
+        }
+
+        LfsPointer lfsPointer = new LfsPointer(loid, size);
+        lfsPointer.encode(out);
+        in.close();
+        out.close();
+        return -1;
+      }
+    } catch (IOException e) {
+      if (digestOutputStream != null) {
+        digestOutputStream.close();
+      }
+      in.close();
+      out.close();
+      throw e;
+    }
+  }
+}

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/LfsBlobStoreCleanFilterFactory.java

@@ -0,0 +1,26 @@
+package sonia.scm.repository.spi;
+
+import org.eclipse.jgit.lib.Repository;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Path;
+
+public class LfsBlobStoreCleanFilterFactory {
+
+  private final LfsBlobStoreFactory blobStoreFactory;
+  private final sonia.scm.repository.Repository repository;
+  private final Path targetFile;
+
+  public LfsBlobStoreCleanFilterFactory(LfsBlobStoreFactory blobStoreFactory, sonia.scm.repository.Repository repository, Path targetFile) {
+    this.blobStoreFactory = blobStoreFactory;
+    this.repository = repository;
+    this.targetFile = targetFile;
+  }
+
+  LfsBlobStoreCleanFilter createFilter(Repository db, InputStream in, OutputStream out) throws IOException {
+    return new LfsBlobStoreCleanFilter(db, in, out, blobStoreFactory.getLfsBlobStore(repository), targetFile);
+  }
+}