Git import and export (#1507)

* create git bundle command

* create git unbundle command

* Apply suggestions from code review

Co-authored-by: René Pfeuffer <rene.pfeuffer@cloudogu.com>

scm-plugins/scm-git-plugin/build.gradle

@@ -35,6 +35,7 @@ dependencies {
   implementation "sonia.jgit:org.eclipse.jgit.http.server:${jgitVersion}"
   implementation "sonia.jgit:org.eclipse.jgit.lfs.server:${jgitVersion}"
   implementation "sonia.jgit:org.eclipse.jgit.gpg.bc:${jgitVersion}"
+  implementation libraries.commonsCompress
 
   testImplementation libraries.shiroUnit
 }

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitBundleCommand.java

@@ -0,0 +1,110 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package sonia.scm.repository.spi;
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import sonia.scm.ContextEntry;
+import sonia.scm.repository.api.BundleResponse;
+import sonia.scm.repository.api.ExportFailedException;
+
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.stream.Stream;
+
+public class GitBundleCommand extends AbstractGitCommand implements BundleCommand {
+
+  private static final String TAR_ARCHIVE = "tar";
+
+  public GitBundleCommand(GitContext context) {
+    super(context);
+  }
+
+  @Override
+  public BundleResponse bundle(BundleCommandRequest request) throws IOException {
+    Path repoDir = context.getDirectory().toPath();
+    if (Files.exists(repoDir)) {
+      try (OutputStream os = request.getArchive().openStream();
+           BufferedOutputStream bos = new BufferedOutputStream(os);
+           TarArchiveOutputStream taos = new TarArchiveOutputStream(bos)) {
+
+        createTarEntryForFiles("", repoDir, taos);
+        taos.finish();
+      }
+    } else {
+      throw new ExportFailedException(
+        ContextEntry.ContextBuilder.noContext(),
+        "Could not export repository. Repository directory does not exist."
+      );
+    }
+    return new BundleResponse(0);
+  }
+
+  @Override
+  public String getFileExtension() {
+    return TAR_ARCHIVE;
+  }
+
+  private void createTarEntryForFiles(String path, Path fileOrDir, TarArchiveOutputStream taos) throws IOException {
+    try (Stream<Path> files = Files.list(fileOrDir)) {
+      if (files != null) {
+        files
+          .filter(this::shouldIncludeFile)
+          .forEach(f -> bundleFileOrDir(path, f, taos));
+      }
+    }
+  }
+
+  private void bundleFileOrDir(String path, Path fileOrDir, TarArchiveOutputStream taos) {
+    try {
+      String filePath = path + "/" + fileOrDir.getFileName().toString();
+      if (Files.isDirectory(fileOrDir)) {
+        createTarEntryForFiles(filePath, fileOrDir, taos);
+      } else {
+        createArchiveEntryForFile(filePath, fileOrDir, taos);
+      }
+    } catch (IOException e) {
+      throw new ExportFailedException(
+        ContextEntry.ContextBuilder.noContext(),
+        "Could not export repository. Error on bundling files.",
+        e
+      );
+    }
+  }
+
+  private boolean shouldIncludeFile(Path filePath) {
+    return !filePath.getFileName().toString().equals("config");
+  }
+
+  private void createArchiveEntryForFile(String filePath, Path path, TarArchiveOutputStream taos) throws IOException {
+    TarArchiveEntry entry = new TarArchiveEntry(filePath);
+    entry.setSize(path.toFile().length());
+    taos.putArchiveEntry(entry);
+    Files.copy(path, taos);
+    taos.closeArchiveEntry();
+  }
+}

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceProvider.java

@@ -34,11 +34,9 @@ import java.util.EnumSet;
 import java.util.Set;
 
 /**
- *
  * @author Sebastian Sdorra
  */
-public class GitRepositoryServiceProvider extends RepositoryServiceProvider
-{
+public class GitRepositoryServiceProvider extends RepositoryServiceProvider {
 
   public static final Set<Command> COMMANDS = ImmutableSet.of(
     Command.BLAME,
@@ -56,7 +54,9 @@ public class GitRepositoryServiceProvider extends RepositoryServiceProvider
     Command.PUSH,
     Command.PULL,
     Command.MERGE,
-    Command.MODIFY
+    Command.MODIFY,
+    Command.BUNDLE,
+    Command.UNBUNDLE
   );
 
   protected static final Set<Feature> FEATURES = EnumSet.of(Feature.INCOMING_REVISION);
@@ -161,6 +161,16 @@ public class GitRepositoryServiceProvider extends RepositoryServiceProvider
     return commandInjector.getInstance(GitModifyCommand.class);
   }
 
+  @Override
+  public BundleCommand getBundleCommand() {
+    return new GitBundleCommand(context);
+  }
+
+  @Override
+  public UnbundleCommand getUnbundleCommand() {
+    return new GitUnbundleCommand(context);
+  }
+
   @Override
   public Set<Command> getSupportedCommands() {
     return COMMANDS;

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitUnbundleCommand.java

@@ -0,0 +1,80 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package sonia.scm.repository.spi;
+
+import com.google.common.io.ByteSource;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import sonia.scm.repository.api.UnbundleResponse;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+public class GitUnbundleCommand extends AbstractGitCommand implements UnbundleCommand {
+
+  private static final Logger LOG = LoggerFactory.getLogger(GitUnbundleCommand.class);
+
+  GitUnbundleCommand(GitContext context) {
+    super(context);
+  }
+
+  @Override
+  public UnbundleResponse unbundle(UnbundleCommandRequest request) throws IOException {
+    ByteSource archive = checkNotNull(request.getArchive(),"archive is required");
+    Path repositoryDir = context.getDirectory().toPath();
+    LOG.debug("archive repository {} to {}", repositoryDir, archive);
+
+    if (!Files.exists(repositoryDir)) {
+      Files.createDirectories(repositoryDir);
+    }
+
+    unbundleRepositoryFromRequest(request, repositoryDir);
+    return new UnbundleResponse(0);
+  }
+
+  private void unbundleRepositoryFromRequest(UnbundleCommandRequest request, Path repositoryDir) throws IOException {
+    try (TarArchiveInputStream tais = new TarArchiveInputStream(request.getArchive().openBufferedStream())) {
+      TarArchiveEntry entry;
+      while ((entry = tais.getNextTarEntry()) != null) {
+        Path filePath = repositoryDir.resolve(entry.getName());
+        createDirectoriesIfNestedFile(filePath);
+        Files.copy(tais, filePath, StandardCopyOption.REPLACE_EXISTING);
+      }
+    }
+  }
+
+  private void createDirectoriesIfNestedFile(Path filePath) throws IOException {
+    Path directory = filePath.getParent();
+    if (!Files.exists(directory)) {
+        Files.createDirectories(directory);
+    }
+  }
+}

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitBundleCommandTest.java

@@ -0,0 +1,103 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package sonia.scm.repository.spi;
+
+import com.google.common.io.ByteSink;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.utils.IOUtils;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class GitBundleCommandTest {
+
+  private GitBundleCommand bundleCommand;
+
+  @Test
+  void shouldBundleRepository(@TempDir Path temp) throws IOException {
+    Path repoDir = mockGitContextWithRepoDir(temp);
+    if (!Files.exists(repoDir)) {
+      Files.createDirectories(repoDir);
+    }
+    String content = "readme testdata";
+    addFileToRepoDir(repoDir, "README.md", content);
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+    BundleCommandRequest bundleCommandRequest = createBundleCommandRequest(baos);
+    bundleCommand.bundle(bundleCommandRequest);
+
+    assertStreamContainsContent(baos, content);
+  }
+
+  @Test
+  void shouldReturnTarForGitBundleCommand() {
+    bundleCommand = new GitBundleCommand(mock(GitContext.class));
+    String fileExtension = bundleCommand.getFileExtension();
+    assertThat(fileExtension).isEqualTo("tar");
+  }
+
+  private void addFileToRepoDir(Path repoDir, String filename, String content) throws IOException {
+    Path file = repoDir.resolve(filename);
+    Files.copy(new ByteArrayInputStream(content.getBytes()), file, StandardCopyOption.REPLACE_EXISTING);
+  }
+
+  private Path mockGitContextWithRepoDir(Path temp) {
+    GitContext gitContext = mock(GitContext.class);
+    bundleCommand = new GitBundleCommand(gitContext);
+    Path repoDir = Paths.get(temp.toString(), "repository");
+    when(gitContext.getDirectory()).thenReturn(repoDir.toFile());
+    return repoDir;
+  }
+
+  private BundleCommandRequest createBundleCommandRequest(ByteArrayOutputStream baos) {
+    ByteSink byteSink = new ByteSink() {
+      @Override
+      public OutputStream openStream() {
+        return baos;
+      }
+    };
+    return new BundleCommandRequest(byteSink);
+  }
+
+  private void assertStreamContainsContent(ByteArrayOutputStream baos, String content) throws IOException {
+    TarArchiveInputStream tais = new TarArchiveInputStream(new BufferedInputStream(new ByteArrayInputStream(baos.toByteArray())));
+    tais.getNextEntry();
+
+    byte[] result = IOUtils.toByteArray(tais);
+    assertThat(new String(result)).isEqualTo(content);
+  }
+}

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitUnbundleCommandTest.java

@@ -0,0 +1,103 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package sonia.scm.repository.spi;
+
+import com.google.common.io.ByteSource;
+import com.google.common.io.Files;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import javax.annotation.Nonnull;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class GitUnbundleCommandTest extends AbstractGitCommandTestBase {
+  private GitContext gitContext;
+  private GitUnbundleCommand unbundleCommand;
+
+  @BeforeEach
+  void initCommand() {
+    gitContext = mock(GitContext.class);
+    unbundleCommand = new GitUnbundleCommand(gitContext);
+  }
+
+  @Test
+  void shouldUnbundleRepositoryFiles(@TempDir Path temp) throws IOException {
+    String filePath = "test-input";
+    String fileContent = "HeartOfGold";
+    UnbundleCommandRequest unbundleCommandRequest = createUnbundleCommandRequestForFile(temp, filePath, fileContent);
+
+    unbundleCommand.unbundle(unbundleCommandRequest);
+
+    assertFileWithContentWasCreated(temp, filePath, fileContent);
+  }
+
+  @Test
+  void shouldUnbundleNestedRepositoryFiles(@TempDir Path temp) throws IOException {
+    String filePath = "objects/pack/test-input";
+    String fileContent = "hitchhiker";
+    UnbundleCommandRequest unbundleCommandRequest = createUnbundleCommandRequestForFile(temp, filePath, fileContent);
+
+    unbundleCommand.unbundle(unbundleCommandRequest);
+
+    assertFileWithContentWasCreated(temp, filePath, fileContent);
+  }
+
+  private void assertFileWithContentWasCreated(@TempDir Path temp, String filePath, String fileContent) throws IOException {
+    File createdFile = temp.resolve(filePath).toFile();
+    assertThat(createdFile).exists();
+    assertThat(Files.readLines(createdFile, StandardCharsets.UTF_8).get(0)).isEqualTo(fileContent);
+  }
+
+  private UnbundleCommandRequest createUnbundleCommandRequestForFile(Path temp, String filePath, String fileContent) throws IOException {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    TarArchiveOutputStream taos = new TarArchiveOutputStream(baos);
+    addEntry(taos, filePath, fileContent);
+    taos.finish();
+
+    when(gitContext.getDirectory()).thenReturn(temp.toFile());
+    ByteSource byteSource = ByteSource.wrap(baos.toByteArray());
+    UnbundleCommandRequest unbundleCommandRequest = new UnbundleCommandRequest(byteSource);
+    return unbundleCommandRequest;
+  }
+
+  private void addEntry(TarArchiveOutputStream taos, String name, String input) throws IOException {
+    TarArchiveEntry entry = new TarArchiveEntry(name);
+    byte[] data = input.getBytes();
+    entry.setSize(data.length);
+    taos.putArchiveEntry(entry);
+    taos.write(data);
+    taos.closeArchiveEntry();
+  }
+}