LFS upload POC

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitModifyCommand.java

@@ -3,6 +3,7 @@ package sonia.scm.repository.spi;
 import org.apache.commons.lang.StringUtils;
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.attributes.FilterCommandRegistry;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.Ref;
 import org.eclipse.jgit.revwalk.RevCommit;
@@ -12,6 +13,7 @@ import sonia.scm.ContextEntry;
 import sonia.scm.repository.GitWorkdirFactory;
 import sonia.scm.repository.InternalRepositoryException;
 import sonia.scm.repository.Repository;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
 
 import java.io.File;
 import java.io.IOException;
@@ -30,10 +32,12 @@ import static sonia.scm.ScmConstraintViolationException.Builder.doThrow;
 public class GitModifyCommand extends AbstractGitCommand implements ModifyCommand {
 
   private final GitWorkdirFactory workdirFactory;
+  private final LfsBlobStoreFactory lfsBlobStoreFactory;
 
-  GitModifyCommand(GitContext context, Repository repository, GitWorkdirFactory workdirFactory) {
+  GitModifyCommand(GitContext context, Repository repository, GitWorkdirFactory workdirFactory, LfsBlobStoreFactory lfsBlobStoreFactory) {
     super(context, repository);
     this.workdirFactory = workdirFactory;
+    this.lfsBlobStoreFactory = lfsBlobStoreFactory;
   }
 
   @Override
@@ -87,10 +91,17 @@ public class GitModifyCommand extends AbstractGitCommand implements ModifyComman
           throw alreadyExists(createFileContext(toBeCreated));
         }
       }
+
+      LfsBlobStoreCleanFilterFactory cleanFilterFactory = new LfsBlobStoreCleanFilterFactory(lfsBlobStoreFactory, repository, targetFile);
+
+      String registerKey = "git-lfs clean -- '" + toBeCreated + "'";
+      FilterCommandRegistry.register(registerKey, cleanFilterFactory::createFilter);
       try {
         addFileToGit(toBeCreated);
       } catch (GitAPIException e) {
         throwInternalRepositoryException("could not add new file to index", e);
+      } finally {
+        FilterCommandRegistry.unregister(registerKey);
       }
     }
 

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceProvider.java

@@ -273,7 +273,7 @@ public class GitRepositoryServiceProvider extends RepositoryServiceProvider
 
   @Override
   public ModifyCommand getModifyCommand() {
-    return new GitModifyCommand(context, repository, handler.getWorkdirFactory());
+    return new GitModifyCommand(context, repository, handler.getWorkdirFactory(), lfsBlobStoreFactory);
   }
 
   @Override

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/LfsBlobStoreCleanFilter.java

@@ -0,0 +1,101 @@
+package sonia.scm.repository.spi;
+
+import org.eclipse.jgit.attributes.FilterCommand;
+import org.eclipse.jgit.lfs.Lfs;
+import org.eclipse.jgit.lfs.LfsPointer;
+import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
+import org.eclipse.jgit.lfs.lib.LongObjectId;
+import org.eclipse.jgit.lib.Repository;
+import sonia.scm.store.Blob;
+import sonia.scm.store.BlobStore;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+import static org.eclipse.jgit.lfs.lib.Constants.LONG_HASH_FUNCTION;
+
+/**
+ * Adapted version of JGit's {@link org.eclipse.jgit.lfs.CleanFilter} to write the
+ * lfs file directly to the lfs blob store.
+ */
+public class LfsBlobStoreCleanFilter extends FilterCommand {
+
+
+  private Lfs lfsUtil;
+  private final BlobStore lfsBlobStore;
+  private final Path targetFile;
+  private final DigestOutputStream digestOutputStream;
+
+  private long size;
+
+  public LfsBlobStoreCleanFilter(Repository db, InputStream in, OutputStream out, BlobStore lfsBlobStore, Path targetFile)
+    throws IOException {
+    super(in, out);
+    lfsUtil = new Lfs(db);
+    this.lfsBlobStore = lfsBlobStore;
+    this.targetFile = targetFile;
+    Files.createDirectories(lfsUtil.getLfsTmpDir());
+
+
+   MessageDigest md ;
+    try {
+      md = MessageDigest.getInstance(LONG_HASH_FUNCTION);
+    } catch (NoSuchAlgorithmException e) {
+      throw new RuntimeException(e);
+    }
+    digestOutputStream = new DigestOutputStream(new OutputStream() {
+      @Override
+      public void write(int b) {
+      }
+    }, md);
+
+  }
+
+  @Override
+  public int run() throws IOException {
+    try {
+      byte[] buf = new byte[8192];
+      int length = in.read(buf);
+      if (length != -1) {
+        digestOutputStream.write(buf, 0, length);
+        size += length;
+        return length;
+      } else {
+        digestOutputStream.close();
+        AnyLongObjectId loid = LongObjectId.fromRaw(digestOutputStream.getMessageDigest().digest());
+
+        Blob existingBlob = lfsBlobStore.get(loid.getName());
+        if (existingBlob != null) {
+          long blobSize = existingBlob.getSize();
+          if (blobSize != size) {
+            throw new RuntimeException("lfs entry already exists for loid " + loid.getName() + " but has wrong size");
+          }
+        } else {
+          Blob newBlob = lfsBlobStore.create(loid.getName());
+          OutputStream outputStream = newBlob.getOutputStream();
+          Files.copy(targetFile, outputStream);
+          newBlob.commit();
+        }
+
+        LfsPointer lfsPointer = new LfsPointer(loid, size);
+        lfsPointer.encode(out);
+        in.close();
+        out.close();
+        return -1;
+      }
+    } catch (IOException e) {
+      if (digestOutputStream != null) {
+        digestOutputStream.close();
+      }
+      in.close();
+      out.close();
+      throw e;
+    }
+  }
+}

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/LfsBlobStoreCleanFilterFactory.java

@@ -0,0 +1,26 @@
+package sonia.scm.repository.spi;
+
+import org.eclipse.jgit.lib.Repository;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Path;
+
+public class LfsBlobStoreCleanFilterFactory {
+
+  private final LfsBlobStoreFactory blobStoreFactory;
+  private final sonia.scm.repository.Repository repository;
+  private final Path targetFile;
+
+  public LfsBlobStoreCleanFilterFactory(LfsBlobStoreFactory blobStoreFactory, sonia.scm.repository.Repository repository, Path targetFile) {
+    this.blobStoreFactory = blobStoreFactory;
+    this.repository = repository;
+    this.targetFile = targetFile;
+  }
+
+  LfsBlobStoreCleanFilter createFilter(Repository db, InputStream in, OutputStream out) throws IOException {
+    return new LfsBlobStoreCleanFilter(db, in, out, blobStoreFactory.getLfsBlobStore(repository), targetFile);
+  }
+}

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommandTest.java

@@ -20,12 +20,14 @@ import sonia.scm.NotFoundException;
 import sonia.scm.ScmConstraintViolationException;
 import sonia.scm.repository.Person;
 import sonia.scm.repository.util.WorkdirProvider;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
 
 @SubjectAware(configuration = "classpath:sonia/scm/configuration/shiro.ini", username = "admin", password = "secret")
 public class GitModifyCommandTest extends AbstractGitCommandTestBase {
@@ -37,6 +39,8 @@ public class GitModifyCommandTest extends AbstractGitCommandTestBase {
   @Rule
   public ShiroRule shiro = new ShiroRule();
 
+  private final LfsBlobStoreFactory lfsBlobStoreFactory = mock(LfsBlobStoreFactory.class);
+
   @Test
   public void shouldCreateCommit() throws IOException, GitAPIException {
     File newFile = Files.write(temporaryFolder.newFile().toPath(), "new content".getBytes()).toFile();
@@ -296,7 +300,7 @@ public class GitModifyCommandTest extends AbstractGitCommandTestBase {
   }
 
   private GitModifyCommand createCommand() {
-    return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider()));
+    return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider()), lfsBlobStoreFactory);
   }
 
   @FunctionalInterface

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommand_LFSTest.java

@@ -0,0 +1,83 @@
+package sonia.scm.repository.spi;
+
+import com.github.sdorra.shiro.ShiroRule;
+import com.github.sdorra.shiro.SubjectAware;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import sonia.scm.repository.Person;
+import sonia.scm.repository.util.WorkdirProvider;
+import sonia.scm.store.Blob;
+import sonia.scm.store.BlobStore;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@SubjectAware(configuration = "classpath:sonia/scm/configuration/shiro.ini", username = "admin", password = "secret")
+public class GitModifyCommand_LFSTest extends AbstractGitCommandTestBase {
+
+  @Rule
+  public TemporaryFolder temporaryFolder = new TemporaryFolder();
+  @Rule
+  public BindTransportProtocolRule transportProtocolRule = new BindTransportProtocolRule();
+  @Rule
+  public ShiroRule shiro = new ShiroRule();
+
+  private final LfsBlobStoreFactory lfsBlobStoreFactory = mock(LfsBlobStoreFactory.class);
+
+  @Test
+  public void shouldCreateCommit() throws IOException, GitAPIException {
+    BlobStore blobStore = mock(BlobStore.class);
+    Blob blob = mock(Blob.class);
+    when(lfsBlobStoreFactory.getLfsBlobStore(any())).thenReturn(blobStore);
+    when(blobStore.create("fe32608c9ef5b6cf7e3f946480253ff76f24f4ec0678f3d0f07f9844cbff9601")).thenReturn(blob);
+    when(blobStore.get("fe32608c9ef5b6cf7e3f946480253ff76f24f4ec0678f3d0f07f9844cbff9601")).thenReturn(null, blob);
+    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+    when(blob.getOutputStream()).thenReturn(outputStream);
+    when(blob.getSize()).thenReturn((long) "new content".length());
+
+    File newFile = Files.write(temporaryFolder.newFile().toPath(), "new content".getBytes()).toFile();
+
+    GitModifyCommand command = createCommand();
+
+    ModifyCommandRequest request = new ModifyCommandRequest();
+    request.setCommitMessage("test commit");
+    request.addRequest(new ModifyCommandRequest.CreateFileRequest("new_lfs.png", newFile, false));
+    request.setAuthor(new Person("Dirk Gently", "dirk@holistic.det"));
+
+    String newRef = command.execute(request);
+
+    try (Git git = new Git(createContext().open())) {
+      RevCommit lastCommit = getLastCommit(git);
+      assertThat(lastCommit.getFullMessage()).isEqualTo("test commit");
+      assertThat(lastCommit.getAuthorIdent().getName()).isEqualTo("Dirk Gently");
+      assertThat(newRef).isEqualTo(lastCommit.toObjectId().name());
+    }
+
+    assertThat(outputStream.toString()).isEqualTo("new content");
+  }
+
+  private RevCommit getLastCommit(Git git) throws GitAPIException {
+    return git.log().setMaxCount(1).call().iterator().next();
+  }
+
+  private GitModifyCommand createCommand() {
+    return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider()), lfsBlobStoreFactory);
+  }
+
+  @Override
+  protected String getZippedRepositoryResource() {
+    return "sonia/scm/repository/spi/scm-git-spi-lfs-test.zip";
+  }
+}