Add the repository import and export with metadata for Subversion repositories (#1501)

* Add store exporter to collect the repository metadata
* Add EnvironmentInformationXmlGenerator
* Collect export data and put into compressed tar archive output stream
* Create full repository export endpoint.
* Add full repository export to ui
* Ignore irrelevant files from config store directory
* write metadata stores to file since a baos could teardown the server memory
* Migrate store name for git lfs files (#1504)

Changes the directory name for the git LFS blob store by
removing the repository id from the store name.

This is necessary for im- and exports of lfs blob stores,
because the original name had the repository id as a part
of it and therefore the old store would not be found when
the repository is imported with another id.

Existing blob files will be moved to the new store location
by an update step.

Co-authored-by: Eduard Heimbuch <eduard.heimbuch@cloudogu.com>

* Introduce util for migrations (#1505)

With this util it is more simple to rename
or delete stores.

* Rename files in export

Co-authored-by: René Pfeuffer <rene.pfeuffer@cloudogu.com>

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/lfs/LfsBlobStoreFactory.java

@@ -21,7 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */
-    
+
 package sonia.scm.web.lfs;
 
 import com.google.inject.Inject;
@@ -32,44 +32,37 @@ import sonia.scm.store.BlobStoreFactory;
 
 /**
  * Creates {@link BlobStore} objects to store lfs objects.
- * 
+ *
  * @author Sebastian Sdorra
  * @since 1.54
  */
 @Singleton
 public class LfsBlobStoreFactory {
-  
-  private static final String GIT_LFS_REPOSITORY_POSTFIX = "-git-lfs";
-  
+
+  private static final String GIT_LFS_STORE_NAME = "git-lfs";
+
   private final BlobStoreFactory blobStoreFactory;
 
   /**
    * Create a new instance.
-   * 
+   *
    * @param blobStoreFactory blob store factory
    */
   @Inject
   public LfsBlobStoreFactory(BlobStoreFactory blobStoreFactory) {
     this.blobStoreFactory = blobStoreFactory;
   }
-  
+
   /**
    * Provides a {@link BlobStore} corresponding to the SCM Repository.
-   * <p>
-   * git-lfs repositories should generally carry the same name as their regular SCM repository counterparts. However,
-   * we have decided to store them under their IDs instead of their names, since the names might change and provide
-   * other drawbacks, as well.
-   * <p>
-   * These repositories will have {@linkplain #GIT_LFS_REPOSITORY_POSTFIX} appended to their IDs.
    *
    * @param repository The SCM Repository to provide a LFS {@link BlobStore} for.
-   * 
+   *
    * @return blob store for the corresponding scm repository
    */
-  @SuppressWarnings("unchecked")
   public BlobStore getLfsBlobStore(Repository repository) {
     return blobStoreFactory
-        .withName(repository.getId() + GIT_LFS_REPOSITORY_POSTFIX)
+        .withName(GIT_LFS_STORE_NAME)
         .forRepository(repository)
         .build();
   }

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/lfs/RemoveRepositoryIdFromBlobStoreUpdateStep.java

@@ -0,0 +1,73 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package sonia.scm.web.lfs;
+
+import sonia.scm.migration.UpdateStep;
+import sonia.scm.plugin.Extension;
+import sonia.scm.store.StoreType;
+import sonia.scm.update.RepositoryUpdateIterator;
+import sonia.scm.update.StoreUpdateStepUtilFactory;
+import sonia.scm.version.Version;
+
+import javax.inject.Inject;
+
+import static sonia.scm.version.Version.parse;
+
+@Extension
+public class RemoveRepositoryIdFromBlobStoreUpdateStep implements UpdateStep {
+
+  private final RepositoryUpdateIterator repositoryUpdateIterator;
+  private final StoreUpdateStepUtilFactory utilFactory;
+
+  @Inject
+  public RemoveRepositoryIdFromBlobStoreUpdateStep(RepositoryUpdateIterator repositoryUpdateIterator, StoreUpdateStepUtilFactory utilFactory) {
+    this.repositoryUpdateIterator = repositoryUpdateIterator;
+    this.utilFactory = utilFactory;
+  }
+
+  @Override
+  public void doUpdate()  {
+    repositoryUpdateIterator.forEachRepository(this::doUpdate);
+  }
+
+  private void doUpdate(String repositoryId) {
+    utilFactory
+      .forType(StoreType.BLOB)
+      .forName(repositoryId + "-git-lfs")
+      .forRepository(repositoryId)
+      .build()
+      .renameStore("git-lfs");
+  }
+
+  @Override
+  public Version getTargetVersion() {
+    return parse("2.0.1");
+  }
+
+  @Override
+  public String getAffectedDataType() {
+    return "sonia.scm.git.lfs";
+  }
+}

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/lfs/LfsBlobStoreFactoryTest.java

@@ -62,7 +62,7 @@ public class LfsBlobStoreFactoryTest {
     // just make sure the right parameter is passed, as properly validating the return value is nearly impossible with 
     // the return value (and should not be part of this test)
     verify(blobStoreFactory).getStore(argThat(blobStoreParameters -> {
-      assertThat(blobStoreParameters.getName()).isEqualTo("the-id-git-lfs");
+      assertThat(blobStoreParameters.getName()).isEqualTo("git-lfs");
       assertThat(blobStoreParameters.getRepositoryId()).isEqualTo("the-id");
       return true;
     }));

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/lfs/RemoveRepositoryIdFromBlobStoreUpdateStepTest.java

@@ -0,0 +1,74 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package sonia.scm.web.lfs;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Answers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.jupiter.MockitoExtension;
+import sonia.scm.store.StoreType;
+import sonia.scm.update.RepositoryUpdateIterator;
+import sonia.scm.update.StoreUpdateStepUtilFactory;
+
+import java.io.IOException;
+import java.util.function.Consumer;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.verify;
+
+@ExtendWith(MockitoExtension.class)
+class RemoveRepositoryIdFromBlobStoreUpdateStepTest {
+
+  @Mock
+  private RepositoryUpdateIterator repositoryUpdateIterator;
+  @Mock(answer = Answers.CALLS_REAL_METHODS)
+  private StoreUpdateStepUtilFactory utilFactory;
+  @Mock
+  private StoreUpdateStepUtilFactory.StoreUpdateStepUtil util;
+
+  @InjectMocks
+  private RemoveRepositoryIdFromBlobStoreUpdateStep updateStep;
+
+  @Test
+  void migrateBlobsFromOldStoreToNewStore() throws IOException {
+    Mockito.doAnswer(invocation -> {
+      invocation.getArgument(0, Consumer.class).accept("repo-id");
+      return null;
+    }).when(repositoryUpdateIterator).forEachRepository(any());
+
+    doReturn(util)
+      .when(utilFactory).build(eq(StoreType.BLOB), argThat(argument -> argument.getName().equals("repo-id-git-lfs")));
+
+    updateStep.doUpdate();
+
+    verify(util).renameStore("git-lfs");
+  }
+}