Feature repository specific data migration (#1526)

This adds a new migration mechanism for repository data. Instead of using UpdateSteps for all data migrations, repository data shall from now on be implemented with RepositoryUpdateSteps. The general logic stays the same. Executed updates are stored with the repository. Doing this, we can now execute updates on imported repositories without touching other data. This way we can import repositories even though they were exported with older versions of SCM-Manager or a plugin.

scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryImporter.java

@@ -33,8 +33,10 @@ import sonia.scm.repository.Repository;
 import sonia.scm.repository.RepositoryManager;
 import sonia.scm.repository.RepositoryPermission;
 import sonia.scm.repository.api.ImportFailedException;
+import sonia.scm.repository.api.IncompatibleEnvironmentForImportException;
 import sonia.scm.repository.api.RepositoryService;
 import sonia.scm.repository.api.RepositoryServiceFactory;
+import sonia.scm.update.UpdateEngine;
 
 import javax.inject.Inject;
 import javax.xml.bind.JAXB;
@@ -51,22 +53,26 @@ import static sonia.scm.importexport.FullScmRepositoryExporter.STORE_DATA_FILE_N
 
 public class FullScmRepositoryImporter {
 
+  @SuppressWarnings("java:S115") // we like this name here
   private static final int _1_MB = 1000000;
 
   private final RepositoryServiceFactory serviceFactory;
   private final RepositoryManager repositoryManager;
   private final ScmEnvironmentCompatibilityChecker compatibilityChecker;
   private final TarArchiveRepositoryStoreImporter storeImporter;
+  private final UpdateEngine updateEngine;
 
   @Inject
   public FullScmRepositoryImporter(RepositoryServiceFactory serviceFactory,
                                    RepositoryManager repositoryManager,
                                    ScmEnvironmentCompatibilityChecker compatibilityChecker,
-                                   TarArchiveRepositoryStoreImporter storeImporter) {
+                                   TarArchiveRepositoryStoreImporter storeImporter,
+                                   UpdateEngine updateEngine) {
     this.serviceFactory = serviceFactory;
     this.repositoryManager = repositoryManager;
     this.compatibilityChecker = compatibilityChecker;
     this.storeImporter = storeImporter;
+    this.updateEngine = updateEngine;
   }
 
   public Repository importFromStream(Repository repository, InputStream inputStream) {
@@ -113,6 +119,7 @@ public class FullScmRepositoryImporter {
       // Inside the repository tar archive stream is another tar archive.
       // The nested tar archive is wrapped in another TarArchiveInputStream inside the storeImporter
       storeImporter.importFromTarArchive(repository, tais);
+      updateEngine.update(repository.getId());
     } else {
       throw new ImportFailedException(
         ContextEntry.ContextBuilder.entity(repository).build(),
@@ -148,10 +155,7 @@ public class FullScmRepositoryImporter {
     if (environmentEntry.getName().equals(SCM_ENVIRONMENT_FILE_NAME) && !environmentEntry.isDirectory() && environmentEntry.getSize() < _1_MB) {
       boolean validEnvironment = compatibilityChecker.check(JAXB.unmarshal(new NoneClosingInputStream(tais), ScmEnvironment.class));
       if (!validEnvironment) {
-        throw new ImportFailedException(
-          ContextEntry.ContextBuilder.noContext(),
-          "Incompatible SCM-Manager environment. Could not import file."
-        );
+        throw new IncompatibleEnvironmentForImportException();
       }
     } else {
       throw new ImportFailedException(

scm-webapp/src/main/java/sonia/scm/importexport/ScmEnvironmentCompatibilityChecker.java

@@ -29,9 +29,9 @@ import org.slf4j.LoggerFactory;
 import sonia.scm.SCMContextProvider;
 import sonia.scm.plugin.PluginInformation;
 import sonia.scm.plugin.PluginManager;
+import sonia.scm.version.Version;
 
 import javax.inject.Inject;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -54,10 +54,10 @@ public class ScmEnvironmentCompatibilityChecker {
   }
 
   private boolean isCoreVersionCompatible(String currentCoreVersion, String coreVersionFromImport) {
-    boolean compatible = currentCoreVersion.equals(coreVersionFromImport);
+    boolean compatible = Version.parse(currentCoreVersion).isNewerOrEqual(coreVersionFromImport);
     if (!compatible) {
       LOG.info(
-        "SCM-Manager version is not compatible with dump. Dump can only be imported with SCM-Manager version: {}; you are running version {}",
+        "SCM-Manager version is not compatible with dump. Dump can only be imported with SCM-Manager version {} or newer; you are running version {}.",
         coreVersionFromImport,
         currentCoreVersion
       );
@@ -73,9 +73,9 @@ public class ScmEnvironmentCompatibilityChecker {
 
     for (EnvironmentPluginDescriptor plugin : environment.getPlugins().getPlugin()) {
       Optional<PluginInformation> matchingInstalledPlugin = findMatchingInstalledPlugin(currentlyInstalledPlugins, plugin);
-      if (isPluginIncompatible(plugin, matchingInstalledPlugin)) {
+      if (matchingInstalledPlugin.isPresent() && isPluginIncompatible(plugin, matchingInstalledPlugin.get())) {
         LOG.info(
-          "The installed plugin \"{}\" with version \"{}\" doesn't match the plugin data version \"{}\" from the SCM-Manager environment the dump was created with.",
+          "The installed plugin \"{}\" with version \"{}\" is older than the plugin data version \"{}\" from the SCM-Manager environment the dump was created with. Please update the plugin.",
           matchingInstalledPlugin.get().getName(),
           matchingInstalledPlugin.get().getVersion(),
           plugin.getVersion()
@@ -86,8 +86,8 @@ public class ScmEnvironmentCompatibilityChecker {
     return true;
   }
 
-  private boolean isPluginIncompatible(EnvironmentPluginDescriptor plugin, Optional<PluginInformation> matchingInstalledPlugin) {
-    return matchingInstalledPlugin.isPresent() && isPluginVersionIncompatible(plugin.getVersion(), matchingInstalledPlugin.get().getVersion());
+  private boolean isPluginIncompatible(EnvironmentPluginDescriptor plugin, PluginInformation matchingInstalledPlugin) {
+    return isPluginVersionIncompatible(plugin.getVersion(), matchingInstalledPlugin.getVersion());
   }
 
   private Optional<PluginInformation> findMatchingInstalledPlugin(List<PluginInformation> currentlyInstalledPlugins, EnvironmentPluginDescriptor plugin) {
@@ -98,6 +98,6 @@ public class ScmEnvironmentCompatibilityChecker {
   }
 
   private boolean isPluginVersionIncompatible(String previousPluginVersion, String installedPluginVersion) {
-    return !installedPluginVersion.equals(previousPluginVersion);
+    return Version.parse(installedPluginVersion).isOlder(previousPluginVersion);
   }
 }

scm-webapp/src/main/java/sonia/scm/lifecycle/modules/UpdateStepModule.java

@@ -21,11 +21,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */
-    
+
 package sonia.scm.lifecycle.modules;
 
 import com.google.inject.AbstractModule;
 import com.google.inject.multibindings.Multibinder;
+import sonia.scm.migration.RepositoryUpdateStep;
 import sonia.scm.migration.UpdateStep;
 import sonia.scm.plugin.PluginLoader;
 
@@ -40,9 +41,14 @@ public class UpdateStepModule extends AbstractModule {
   @Override
   protected void configure() {
     Multibinder<UpdateStep> updateStepBinder = Multibinder.newSetBinder(binder(), UpdateStep.class);
+    Multibinder<RepositoryUpdateStep> repositoryUdateStepBinder = Multibinder.newSetBinder(binder(), RepositoryUpdateStep.class);
     pluginLoader
       .getExtensionProcessor()
       .byExtensionPoint(UpdateStep.class)
       .forEach(stepClass -> updateStepBinder.addBinding().to(stepClass));
+    pluginLoader
+      .getExtensionProcessor()
+      .byExtensionPoint(RepositoryUpdateStep.class)
+      .forEach(stepClass -> repositoryUdateStepBinder.addBinding().to(stepClass));
   }
 }

scm-webapp/src/main/java/sonia/scm/update/UpdateEngine.java

@@ -21,22 +21,28 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */
-    
+
 package sonia.scm.update;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import sonia.scm.migration.RepositoryUpdateContext;
+import sonia.scm.migration.RepositoryUpdateStep;
 import sonia.scm.migration.UpdateException;
 import sonia.scm.migration.UpdateStep;
+import sonia.scm.migration.UpdateStepTarget;
 import sonia.scm.store.ConfigurationEntryStore;
 import sonia.scm.store.ConfigurationEntryStoreFactory;
+import sonia.scm.version.Version;
 
 import javax.inject.Inject;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Set;
 
+import static java.lang.String.format;
 import static java.util.stream.Collectors.toList;
+import static java.util.stream.Stream.concat;
 
 public class UpdateEngine {
 
@@ -44,56 +50,84 @@ public class UpdateEngine {
 
   private static final String STORE_NAME = "executedUpdates";
 
-  private final List<UpdateStep> steps;
+  private final List<UpdateStepWrapper> steps;
   private final ConfigurationEntryStore<UpdateVersionInfo> store;
+  private final ConfigurationEntryStoreFactory storeFactory;
+  private final RepositoryUpdateIterator repositoryUpdateIterator;
 
   @Inject
-  public UpdateEngine(Set<UpdateStep> steps, ConfigurationEntryStoreFactory storeFactory) {
-    this.steps = sortSteps(steps);
+  public UpdateEngine(
+    Set<UpdateStep> globalSteps,
+    Set<RepositoryUpdateStep> repositorySteps,
+    ConfigurationEntryStoreFactory storeFactory,
+    RepositoryUpdateIterator repositoryUpdateIterator
+  ) {
+    this.storeFactory = storeFactory;
+    this.repositoryUpdateIterator = repositoryUpdateIterator;
     this.store = storeFactory.withType(UpdateVersionInfo.class).withName(STORE_NAME).build();
+    this.steps = sortSteps(globalSteps, repositorySteps);
   }
 
-  private List<UpdateStep> sortSteps(Set<UpdateStep> steps) {
+  private List<UpdateStepWrapper> sortSteps(Set<UpdateStep> globalSteps, Set<RepositoryUpdateStep> repositorySteps) {
     LOG.trace("sorting available update steps:");
-    List<UpdateStep> sortedSteps = steps.stream()
+    List<UpdateStepWrapper> sortedSteps =
+      concat(
+        globalSteps.stream().filter(this::notRunYet).map(GlobalUpdateStepWrapper::new),
+        repositorySteps.stream().map(RepositoryUpdateStepWrapper::new))
       .sorted(
         Comparator
-          .comparing(UpdateStep::getTargetVersion)
-          .thenComparing(this::isCoreUpdateStep)
-          .reversed())
+          .comparing(UpdateStepWrapper::getTargetVersion)
+          .thenComparing(UpdateStepWrapper::isGlobalUpdateStep)
+          .thenComparing(UpdateEngine::isCoreUpdateStep)
+          .reversed()
+      )
       .collect(toList());
     sortedSteps.forEach(step -> LOG.trace("{} for version {}", step.getAffectedDataType(), step.getTargetVersion()));
     return sortedSteps;
   }
 
-  private boolean isCoreUpdateStep(UpdateStep updateStep) {
-    return updateStep instanceof CoreUpdateStep;
+  private static boolean isCoreUpdateStep(UpdateStepWrapper updateStep) {
+    return updateStep.isCoreUpdate();
   }
 
   public void update() {
-    steps
-      .stream()
-      .filter(this::notRunYet)
-      .forEach(this::execute);
+    steps.forEach(this::execute);
   }
 
-  private void execute(UpdateStep updateStep) {
+  public void update(String repositoryId) {
+    steps.forEach(step -> execute(step, repositoryId));
+  }
+
+  private void execute(UpdateStepWrapper updateStep) {
     try {
-      LOG.info("running update step for type {} and version {} (class {})",
-        updateStep.getAffectedDataType(),
-        updateStep.getTargetVersion(),
-        updateStep.getClass().getName()
-      );
       updateStep.doUpdate();
     } catch (Exception e) {
       throw new UpdateException(
-        String.format(
+        format(
           "could not execute update for type %s to version %s in %s",
           updateStep.getAffectedDataType(),
           updateStep.getTargetVersion(),
           updateStep.getClass()),
         e);
     }
+  }
+
+  private void execute(UpdateStepWrapper updateStep, String repositoryId) {
+    try {
+      updateStep.doUpdate(repositoryId);
+    } catch (Exception e) {
+      throw new UpdateException(
+        format(
+          "could not execute update for type %s to version %s in %s for repository id %s",
+          updateStep.getAffectedDataType(),
+          updateStep.getTargetVersion(),
+          updateStep.getClass(),
+          repositoryId),
+        e);
+    }
+  }
+
+  private void storeNewVersion(ConfigurationEntryStore<UpdateVersionInfo> store, UpdateStepTarget updateStep) {
     UpdateVersionInfo newVersionInfo = new UpdateVersionInfo(updateStep.getTargetVersion().getParsedVersion());
     store.put(updateStep.getAffectedDataType(), newVersionInfo);
   }
@@ -103,6 +137,10 @@ public class UpdateEngine {
       updateStep.getAffectedDataType(),
       updateStep.getTargetVersion()
     );
+    return notRunYet(this.store, updateStep);
+  }
+
+  private boolean notRunYet(ConfigurationEntryStore<UpdateVersionInfo> store, UpdateStepTarget updateStep) {
     UpdateVersionInfo updateVersionInfo = store.get(updateStep.getAffectedDataType());
     if (updateVersionInfo == null) {
       LOG.trace("no updates for type {} run yet; step will be executed", updateStep.getAffectedDataType());
@@ -116,4 +154,128 @@ public class UpdateEngine {
     );
     return result;
   }
+
+  private abstract static class UpdateStepWrapper implements UpdateStepTarget {
+
+    private final UpdateStepTarget delegate;
+
+    protected UpdateStepWrapper(UpdateStepTarget delegate) {
+      this.delegate = delegate;
+    }
+
+    @Override
+    public Version getTargetVersion() {
+      return delegate.getTargetVersion();
+    }
+
+    @Override
+    public String getAffectedDataType() {
+      return delegate.getAffectedDataType();
+    }
+
+    abstract boolean isGlobalUpdateStep();
+
+    abstract boolean isCoreUpdate();
+
+    @SuppressWarnings("java:S112") // we explicitly want to allow all kinds of exceptions here
+    abstract void doUpdate() throws Exception;
+
+    @SuppressWarnings("java:S112") // we explicitly want to allow all kinds of exceptions here
+    abstract void doUpdate(String repositoryId) throws Exception;
+  }
+
+  private class GlobalUpdateStepWrapper extends UpdateStepWrapper {
+    private final UpdateStep delegate;
+    private final boolean coreUpdate;
+
+    protected GlobalUpdateStepWrapper(UpdateStep delegate) {
+      super(delegate);
+      this.delegate = delegate;
+      this.coreUpdate = delegate instanceof CoreUpdateStep;
+    }
+
+    @Override
+    public Version getTargetVersion() {
+      return delegate.getTargetVersion();
+    }
+
+    @Override
+    public String getAffectedDataType() {
+      return delegate.getAffectedDataType();
+    }
+
+    public boolean isCoreUpdate() {
+      return coreUpdate;
+    }
+
+    public boolean isGlobalUpdateStep() {
+      return true;
+    }
+
+    void doUpdate() throws Exception {
+      LOG.info("running update step for type {} and version {} (class {})",
+        delegate.getAffectedDataType(),
+        delegate.getTargetVersion(),
+        delegate.getClass().getName()
+      );
+      delegate.doUpdate();
+      storeNewVersion(store, delegate);
+    }
+
+    void doUpdate(String repositoryId) {
+      // nothing to do for repositories here
+    }
+  }
+
+  private class RepositoryUpdateStepWrapper extends UpdateStepWrapper {
+
+    private final RepositoryUpdateStep delegate;
+
+    public RepositoryUpdateStepWrapper(RepositoryUpdateStep delegate) {
+      super(delegate);
+      this.delegate = delegate;
+    }
+
+    @Override
+    public boolean isGlobalUpdateStep() {
+      return false;
+    }
+
+    @Override
+    boolean isCoreUpdate() {
+      return false;
+    }
+
+    @Override
+    void doUpdate() {
+      repositoryUpdateIterator.updateEachRepository(this::doUpdate);
+    }
+
+    @Override
+    void doUpdate(String repositoryId) throws Exception {
+      if (notRunYet(repositoryId)) {
+        LOG.info("running update step for type {} and version {} (class {}) for repository id {}",
+          delegate.getAffectedDataType(),
+          delegate.getTargetVersion(),
+          delegate.getClass().getName(),
+          repositoryId
+        );
+        delegate.doUpdate(new RepositoryUpdateContext(repositoryId));
+        storeNewVersion(storeForRepository(repositoryId), delegate);
+      }
+    }
+
+    private boolean notRunYet(String repositoryId) {
+      LOG.trace("checking whether to run update step for type {} and version {} on repository id {}",
+        delegate.getAffectedDataType(),
+        delegate.getTargetVersion(),
+        repositoryId
+      );
+      return UpdateEngine.this.notRunYet(storeForRepository(repositoryId), delegate);
+    }
+
+    private ConfigurationEntryStore<UpdateVersionInfo> storeForRepository(String repositoryId) {
+      return storeFactory.withType(UpdateVersionInfo.class).withName(STORE_NAME).forRepository(repositoryId).build();
+    }
+  }
 }