Feature/mirror (#1683)

Add mirror command and extension points.

Co-authored-by: René Pfeuffer <rene.pfeuffer@cloudogu.com>
Co-authored-by: Sebastian Sdorra <sebastian.sdorra@cloudogu.com>
Co-authored-by: Konstantin Schaper <konstantin.schaper@cloudogu.com>
This commit is contained in:
Eduard Heimbuch
2021-06-04 14:05:47 +02:00
committed by GitHub
parent e55ba52ace
commit dd0975b49a
111 changed files with 6018 additions and 796 deletions

View File

@@ -24,8 +24,6 @@
package sonia.scm.repository;
//~--- non-JDK imports --------------------------------------------------------
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.slf4j.Logger;
@@ -50,40 +48,28 @@ import java.io.IOException;
import static sonia.scm.ContextEntry.ContextBuilder.entity;
//~--- JDK imports ------------------------------------------------------------
/**
*
* @author Sebastian Sdorra
*/
@Singleton
@Extension
public class SvnRepositoryHandler
extends AbstractSimpleRepositoryHandler<SvnConfig>
{
public class SvnRepositoryHandler extends AbstractSimpleRepositoryHandler<SvnConfig> {
public static final String PROPERTY_UUID = "svn.uuid";
public static final String RESOURCE_VERSION = "sonia/scm/version/scm-svn-plugin";
public static final String TYPE_DISPLAYNAME = "Subversion";
public static final String TYPE_NAME = "svn";
public static final RepositoryType TYPE = new RepositoryType(TYPE_NAME,
TYPE_DISPLAYNAME,
SvnRepositoryServiceProvider.COMMANDS);
public static final RepositoryType TYPE = new RepositoryType(TYPE_NAME, TYPE_DISPLAYNAME, SvnRepositoryServiceProvider.COMMANDS);
private static final Logger logger =
LoggerFactory.getLogger(SvnRepositoryHandler.class);
private static final Logger LOG = LoggerFactory.getLogger(SvnRepositoryHandler.class);
private SvnRepositoryHook hook;
@Inject
public SvnRepositoryHandler(ConfigurationStoreFactory storeFactory,
HookEventFacade eventFacade,
RepositoryLocationResolver repositoryLocationResolver,
PluginLoader pluginLoader)
{
PluginLoader pluginLoader) {
super(storeFactory, repositoryLocationResolver, pluginLoader);
// register logger
@@ -93,116 +79,95 @@ public class SvnRepositoryHandler
FSRepositoryFactory.setup();
// register hook
if (eventFacade != null)
{
if (eventFacade != null) {
hook = new SvnRepositoryHook(eventFacade, this);
FSHooks.registerHook(hook);
}
else if (logger.isWarnEnabled())
{
logger.warn(
} else if (LOG.isWarnEnabled()) {
LOG.warn(
"unable to register hook, beacause of missing repositorymanager");
}
}
@Override
public ImportHandler getImportHandler()
{
public ImportHandler getImportHandler() {
return new SvnImportHandler(this);
}
@Override
public RepositoryType getType()
{
public RepositoryType getType() {
return TYPE;
}
@Override
public String getVersionInformation()
{
public String getVersionInformation() {
return getStringFromResource(RESOURCE_VERSION, DEFAULT_VERSION_INFORMATION);
}
@Override
protected void create(Repository repository, File directory) throws InternalRepositoryException {
Compatibility comp = config.getCompatibility();
if (logger.isDebugEnabled())
{
StringBuilder log = new StringBuilder("create svn repository \"");
log.append(directory.getName()).append("\": pre14Compatible=");
log.append(comp.isPre14Compatible()).append(", pre15Compatible=");
log.append(comp.isPre15Compatible()).append(", pre16Compatible=");
log.append(comp.isPre16Compatible()).append(", pre17Compatible=");
log.append(comp.isPre17Compatible()).append(", with17Compatible=");
log.append(comp.isWith17Compatible());
logger.debug(log.toString());
}
SVNRepository svnRepository = null;
try
{
SVNURL url = SVNRepositoryFactory.createLocalRepository(directory, null,
true, false, comp.isPre14Compatible(),
comp.isPre15Compatible(), comp.isPre16Compatible(),
comp.isPre17Compatible(), comp.isWith17Compatible());
try {
SVNURL url = createSvnUrl(directory);
svnRepository = SVNRepositoryFactory.create(url);
String uuid = svnRepository.getRepositoryUUID(true);
if (Util.isNotEmpty(uuid))
{
if (logger.isDebugEnabled())
{
logger.debug("store repository uuid {} for {}", uuid,
if (Util.isNotEmpty(uuid)) {
if (LOG.isDebugEnabled()) {
LOG.debug("store repository uuid {} for {}", uuid,
repository.getName());
}
repository.setProperty(PROPERTY_UUID, uuid);
}
else if (logger.isWarnEnabled())
{
logger.warn("could not read repository uuid for {}",
} else if (LOG.isWarnEnabled()) {
LOG.warn("could not read repository uuid for {}",
repository.getName());
}
}
catch (SVNException ex)
{
logger.error("could not create svn repository", ex);
throw new InternalRepositoryException(entity(repository), "could not create repository", ex);
}
finally
{
} catch (SVNException ex) {
throw new InternalRepositoryException(repository, "could not create repository", ex);
} finally {
SvnUtil.closeSession(svnRepository);
}
}
/**
* Method description
*
*
* @return
*/
public SVNURL createSvnUrl(File directory) {
Compatibility comp = config.getCompatibility();
if (LOG.isDebugEnabled()) {
LOG.debug("create svn repository \"{}\": " +
"pre14Compatible={}, " +
"pre15Compatible={}, " +
"pre16Compatible={}, " +
"pre17Compatible={}, " +
"with17Compatible={}",
directory.getName(),
comp.isPre14Compatible(),
comp.isPre15Compatible(),
comp.isPre16Compatible(),
comp.isPre17Compatible(),
comp.isWith17Compatible());
}
try {
return SVNRepositoryFactory.createLocalRepository(directory, null,
true, false, comp.isPre14Compatible(),
comp.isPre15Compatible(), comp.isPre16Compatible(),
comp.isPre17Compatible(), comp.isWith17Compatible());
} catch (SVNException ex) {
throw new InternalRepositoryException(entity(File.class, directory.toString()), "could not create svn url", ex);
}
}
@Override
protected SvnConfig createInitialConfig()
{
protected SvnConfig createInitialConfig() {
return new SvnConfig();
}
//~--- get methods ----------------------------------------------------------
/**
* Method description
*
*
* @return
*/
@Override
protected Class<SvnConfig> getConfigClass()
{
protected Class<SvnConfig> getConfigClass() {
return SvnConfig.class;
}

View File

@@ -0,0 +1,146 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.repository.spi;
import com.google.common.base.Stopwatch;
import com.google.common.collect.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.tmatesoft.svn.core.SVNException;
import org.tmatesoft.svn.core.SVNURL;
import org.tmatesoft.svn.core.auth.BasicAuthenticationManager;
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
import org.tmatesoft.svn.core.auth.SVNAuthentication;
import org.tmatesoft.svn.core.auth.SVNPasswordAuthentication;
import org.tmatesoft.svn.core.auth.SVNSSLAuthentication;
import org.tmatesoft.svn.core.wc.SVNWCUtil;
import org.tmatesoft.svn.core.wc.admin.SVNAdminClient;
import sonia.scm.repository.InternalRepositoryException;
import sonia.scm.repository.api.MirrorCommandResult;
import sonia.scm.repository.api.Pkcs12ClientCertificateCredential;
import sonia.scm.repository.api.UsernamePasswordCredential;
import javax.net.ssl.TrustManager;
import java.util.ArrayList;
import java.util.Collection;
import static java.util.Arrays.asList;
import static sonia.scm.repository.api.MirrorCommandResult.ResultType.FAILED;
import static sonia.scm.repository.api.MirrorCommandResult.ResultType.OK;
public class SvnMirrorCommand extends AbstractSvnCommand implements MirrorCommand {
private static final Logger LOG = LoggerFactory.getLogger(SvnMirrorCommand.class);
private final TrustManager trustManager;
SvnMirrorCommand(SvnContext context, TrustManager trustManager) {
super(context);
this.trustManager = trustManager;
}
@Override
public MirrorCommandResult mirror(MirrorCommandRequest mirrorCommandRequest) {
SVNURL url = createUrlForLocalRepository();
return withAdminClient(mirrorCommandRequest, admin -> {
SVNURL source = SVNURL.parseURIEncoded(mirrorCommandRequest.getSourceUrl());
admin.doCompleteSynchronize(source, url);
});
}
@Override
public MirrorCommandResult update(MirrorCommandRequest mirrorCommandRequest) {
SVNURL url = createUrlForLocalRepository();
return withAdminClient(mirrorCommandRequest, admin -> admin.doSynchronize(url));
}
private MirrorCommandResult withAdminClient(MirrorCommandRequest mirrorCommandRequest, AdminConsumer consumer) {
Stopwatch stopwatch = Stopwatch.createStarted();
long beforeUpdate;
long afterUpdate;
try {
beforeUpdate = context.open().getLatestRevision();
SVNURL url = createUrlForLocalRepository();
SVNAdminClient admin = createAdminClient(url, mirrorCommandRequest);
consumer.accept(admin);
afterUpdate = context.open().getLatestRevision();
} catch (SVNException e) {
LOG.info("Could not mirror svn repository", e);
return new MirrorCommandResult(
FAILED,
asList(
"failed to synchronize. See following error message for more details:",
e.getMessage()
),
stopwatch.stop().elapsed());
}
return new MirrorCommandResult(
OK,
ImmutableList.of("Updated from revision " + beforeUpdate + " to revision " + afterUpdate),
stopwatch.stop().elapsed()
);
}
private SVNURL createUrlForLocalRepository() {
try {
return SVNURL.fromFile(context.getDirectory());
} catch (SVNException e) {
throw new InternalRepositoryException(repository, "could not create svn url for local repository", e);
}
}
private SVNAdminClient createAdminClient(SVNURL url, MirrorCommandRequest mirrorCommandRequest) {
Collection<SVNAuthentication> authentications = new ArrayList<>();
mirrorCommandRequest.getCredential(Pkcs12ClientCertificateCredential.class)
.map(c -> createTlsAuth(url, c))
.ifPresent(authentications::add);
mirrorCommandRequest.getCredential(UsernamePasswordCredential.class)
.map(c -> SVNPasswordAuthentication.newInstance(c.username(), c.password(), false, url, false))
.ifPresent(authentications::add);
ISVNAuthenticationManager authManager = new BasicAuthenticationManager(
authentications.toArray(new SVNAuthentication[authentications.size()])) {
@Override
public TrustManager getTrustManager(SVNURL url) {
return trustManager;
}
};
return new SVNAdminClient(authManager, SVNWCUtil.createDefaultOptions(true));
}
private SVNSSLAuthentication createTlsAuth(SVNURL url, Pkcs12ClientCertificateCredential c) {
return SVNSSLAuthentication.newInstance(
c.getCertificate(),
c.getPassword(),
false,
url,
true);
}
private interface AdminConsumer {
void accept(SVNAdminClient adminClient) throws SVNException;
}
}

View File

@@ -32,6 +32,7 @@ import sonia.scm.repository.SvnWorkingCopyFactory;
import sonia.scm.repository.api.Command;
import sonia.scm.repository.api.HookContextFactory;
import javax.net.ssl.TrustManager;
import java.io.IOException;
import java.util.Set;
@@ -51,7 +52,8 @@ public class SvnRepositoryServiceProvider extends RepositoryServiceProvider {
Command.UNBUNDLE,
Command.MODIFY,
Command.LOOKUP,
Command.FULL_HEALTH_CHECK
Command.FULL_HEALTH_CHECK,
Command.MIRROR
);
//J+
@@ -59,14 +61,17 @@ public class SvnRepositoryServiceProvider extends RepositoryServiceProvider {
private final SvnContext context;
private final SvnWorkingCopyFactory workingCopyFactory;
private final HookContextFactory hookContextFactory;
private final TrustManager trustManager;
SvnRepositoryServiceProvider(SvnRepositoryHandler handler,
Repository repository,
SvnWorkingCopyFactory workingCopyFactory,
HookContextFactory hookContextFactory) {
HookContextFactory hookContextFactory,
TrustManager trustManager) {
this.context = new SvnContext(repository, handler.getDirectory(repository.getId()));
this.workingCopyFactory = workingCopyFactory;
this.hookContextFactory = hookContextFactory;
this.trustManager = trustManager;
}
@Override
@@ -133,4 +138,9 @@ public class SvnRepositoryServiceProvider extends RepositoryServiceProvider {
public FullHealthCheckCommand getFullHealthCheckCommand() {
return new SvnFullHealthCheckCommand(context);
}
@Override
public MirrorCommand getMirrorCommand() {
return new SvnMirrorCommand(context, trustManager);
}
}

View File

@@ -31,20 +31,25 @@ import sonia.scm.repository.SvnRepositoryHandler;
import sonia.scm.repository.SvnWorkingCopyFactory;
import sonia.scm.repository.api.HookContextFactory;
import javax.net.ssl.TrustManager;
@Extension
public class SvnRepositoryServiceResolver implements RepositoryServiceResolver {
private final SvnRepositoryHandler handler;
private final SvnWorkingCopyFactory workingCopyFactory;
private final HookContextFactory hookContextFactory;
private final TrustManager trustManager;
@Inject
public SvnRepositoryServiceResolver(SvnRepositoryHandler handler,
SvnWorkingCopyFactory workingCopyFactory,
HookContextFactory hookContextFactory) {
HookContextFactory hookContextFactory,
TrustManager trustManager) {
this.handler = handler;
this.workingCopyFactory = workingCopyFactory;
this.hookContextFactory = hookContextFactory;
this.trustManager = trustManager;
}
@Override
@@ -52,7 +57,7 @@ public class SvnRepositoryServiceResolver implements RepositoryServiceResolver {
SvnRepositoryServiceProvider provider = null;
if (SvnRepositoryHandler.TYPE_NAME.equalsIgnoreCase(repository.getType())) {
provider = new SvnRepositoryServiceProvider(handler, repository, workingCopyFactory, hookContextFactory);
provider = new SvnRepositoryServiceProvider(handler, repository, workingCopyFactory, hookContextFactory, trustManager);
}
return provider;

View File

@@ -24,78 +24,36 @@
package sonia.scm.repository.spi;
//~--- non-JDK imports --------------------------------------------------------
import org.junit.After;
import java.io.IOException;
/**
*
* @author Sebastian Sdorra
*/
public class AbstractSvnCommandTestBase extends ZippedRepositoryTestBase
{
public class AbstractSvnCommandTestBase extends ZippedRepositoryTestBase {
/**
* Method description
*
*
* @throws IOException
*/
@After
public void close() throws IOException
{
if (context != null)
{
public void close() throws IOException {
if (context != null) {
context.close();
}
}
/**
* Method description
*
*
* @return
*/
public SvnContext createContext()
{
if (context == null)
{
public SvnContext createContext() {
if (context == null) {
context = new SvnContext(repository, repositoryDirectory);
}
return context;
}
//~--- get methods ----------------------------------------------------------
/**
* Method description
*
*
* @return
*/
@Override
protected String getType()
{
protected String getType() {
return "svn";
}
/**
* Method description
*
*
* @return
*/
@Override
protected String getZippedRepositoryResource()
{
protected String getZippedRepositoryResource() {
return "sonia/scm/repository/spi/scm-svn-spi-test.zip";
}
//~--- fields ---------------------------------------------------------------
/** Field description */
private SvnContext context;
}

View File

@@ -0,0 +1,122 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.repository.spi;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
import org.tmatesoft.svn.core.SVNException;
import org.tmatesoft.svn.core.SVNURL;
import org.tmatesoft.svn.core.auth.BasicAuthenticationManager;
import org.tmatesoft.svn.core.auth.SVNAuthentication;
import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
import org.tmatesoft.svn.core.wc.SVNWCUtil;
import org.tmatesoft.svn.core.wc.admin.SVNAdminClient;
import sonia.scm.repository.RepositoryTestData;
import sonia.scm.repository.api.MirrorCommandResult;
import sonia.scm.repository.api.SimpleUsernamePasswordCredential;
import javax.net.ssl.X509TrustManager;
import java.io.File;
import java.io.IOException;
import java.util.function.Consumer;
import static java.util.Collections.singletonList;
import static org.assertj.core.api.Assertions.assertThat;
import static sonia.scm.repository.api.MirrorCommandResult.ResultType.OK;
@RunWith(MockitoJUnitRunner.class)
public class SvnMirrorCommandTest extends AbstractSvnCommandTestBase {
@Mock
private X509TrustManager trustManager;
private SvnContext emptyContext;
@Before
public void bendContextToNewRepository() throws IOException, SVNException {
emptyContext = createEmptyContext();
}
@Test
public void shouldDoInitialMirror() {
MirrorCommandResult result = callMirror(emptyContext, repositoryDirectory, c -> {
});
assertThat(result.getResult()).isEqualTo(OK);
}
@Test
public void shouldDoMirrorUpdate() throws SVNException {
// Initialize destination repo before update
SVNAdminClient svnAdminClient = new SVNAdminClient(new BasicAuthenticationManager(new SVNAuthentication[]{}), SVNWCUtil.createDefaultOptions(false));
svnAdminClient.doInitialize(SVNURL.fromFile(repositoryDirectory), emptyContext.createUrl());
MirrorCommandResult result = callMirrorUpdate(emptyContext, repositoryDirectory);
assertThat(result.getResult()).isEqualTo(OK);
assertThat(result.getLog()).contains("Updated from revision 0 to revision 5");
}
@Test
public void shouldUseCredentials() {
MirrorCommandResult result = callMirror(emptyContext, repositoryDirectory, createCredential("svnadmin", "secret"));
assertThat(result.getResult()).isEqualTo(OK);
}
private MirrorCommandResult callMirrorUpdate(SvnContext context, File source) {
MirrorCommandRequest request = createRequest(source);
return createMirrorCommand(context).update(request);
}
private MirrorCommandResult callMirror(SvnContext context, File source, Consumer<MirrorCommandRequest> consumer) {
MirrorCommandRequest request = createRequest(source);
consumer.accept(request);
return createMirrorCommand(context).mirror(request);
}
private MirrorCommandRequest createRequest(File source) {
MirrorCommandRequest request = new MirrorCommandRequest();
request.setSourceUrl("file://" + source.getAbsolutePath());
return request;
}
private SvnMirrorCommand createMirrorCommand(SvnContext context) {
return new SvnMirrorCommand(context, trustManager);
}
private Consumer<MirrorCommandRequest> createCredential(String username, String password) {
return request -> request.setCredentials(singletonList(new SimpleUsernamePasswordCredential(username, password.toCharArray())));
}
private SvnContext createEmptyContext() throws SVNException, IOException {
File dir = tempFolder.newFolder();
SVNRepositoryFactory.createLocalRepository(dir, true, true);
return new SvnContext(RepositoryTestData.createHappyVerticalPeopleTransporter(), dir);
}
}