Nemcio/SCM-Manager
1
0
Fork 0
mirror of https://github.com/scm-manager/scm-manager.git synced 2025-11-18 03:01:05 +01:00
Code Issues Packages Projects Releases Activity

Add import protocol (#1558)

Browse Source 
Adds a protocol for repository imports (either from an URL, a dump file or a SCM-Manager repository archive).
This protocol documents single steps of an import, the time and the user and is accessible via a dedicated REST
endpoint or a simple ui.

The id of the log is added to the repository imported event, so that plugins like the landingpage or mail can link to these logs.
This commit is contained in:
René Pfeuffer
2021-02-26 13:52:29 +01:00
committed by GitHub
parent ac404b3cdd
commit 0695ca3bac
41 changed files with 1808 additions and 480 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
scm-webapp/src/main/java/sonia/scm/api/v2/resources/IndexDtoGenerator.java
View File

@@ -116,6 +116,7 @@ public class IndexDtoGenerator extends HalAppenderMapper {
builder.single(link("repositoryTypes", resourceLinks.repositoryTypeCollection().self()));
builder.single(link("namespaceStrategies", resourceLinks.namespaceStrategies().self()));
builder.single(link("repositoryRoles", resourceLinks.repositoryRoleCollection().self()));
builder.single(link("importLog", resourceLinks.repository().importLog("IMPORT_LOG_ID").replace("IMPORT_LOG_ID", "{logId}")));
} else {
builder.single(link("login", resourceLinks.authentication().jsonLogin()));
}

1
scm-webapp/src/main/java/sonia/scm/api/v2/resources/MapperModule.java
View File

@@ -90,5 +90,6 @@ public class MapperModule extends AbstractModule {
bind(ApiKeyToApiKeyDtoMapper.class).to(Mappers.getMapperClass(ApiKeyToApiKeyDtoMapper.class));
bind(RepositoryExportInformationToDtoMapper.class).to(Mappers.getMapperClass(RepositoryExportInformationToDtoMapper.class));
bind(RepositoryImportDtoToRepositoryImportParametersMapper.class).to(Mappers.getMapperClass(RepositoryImportDtoToRepositoryImportParametersMapper.class));
}
}

4
scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryExportResource.java
View File

@@ -80,8 +80,8 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import static sonia.scm.ContextEntry.ContextBuilder.entity;
import static sonia.scm.api.v2.resources.RepositoryTypeSupportChecker.checkSupport;
import static sonia.scm.api.v2.resources.RepositoryTypeSupportChecker.type;
import static sonia.scm.importexport.RepositoryTypeSupportChecker.checkSupport;
import static sonia.scm.importexport.RepositoryTypeSupportChecker.type;
public class RepositoryExportResource {

33
scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryImportDtoToRepositoryImportParametersMapper.java Normal file
View File

@@ -0,0 +1,33 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.api.v2.resources;
import org.mapstruct.Mapper;
import sonia.scm.importexport.FromUrlImporter;
@Mapper
public interface RepositoryImportDtoToRepositoryImportParametersMapper {
FromUrlImporter.RepositoryImportParameters map(RepositoryImportResource.RepositoryImportFromUrlDto dto);
}

143
scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryImportResource.java
View File

@@ -27,10 +27,8 @@ package sonia.scm.api.v2.resources;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.common.io.ByteSource;
import com.google.common.io.Files;
import com.google.inject.Inject;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.media.Content;
@@ -39,30 +37,20 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.apache.shiro.SecurityUtils;
import org.jboss.resteasy.plugins.providers.multipart.InputPart;
import org.jboss.resteasy.plugins.providers.multipart.MultipartFormDataInput;
import org.jboss.resteasy.plugins.providers.multipart.MultipartInputImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.ContextEntry;
import sonia.scm.HandlerEventType;
import sonia.scm.Type;
import sonia.scm.event.ScmEventBus;
import sonia.scm.importexport.FromBundleImporter;
import sonia.scm.importexport.FromUrlImporter;
import sonia.scm.importexport.FullScmRepositoryImporter;
import sonia.scm.importexport.RepositoryImportExportEncryption;
import sonia.scm.repository.InternalRepositoryException;
import sonia.scm.importexport.RepositoryImportLoggerFactory;
import sonia.scm.repository.Repository;
import sonia.scm.repository.RepositoryImportEvent;
import sonia.scm.repository.RepositoryManager;
import sonia.scm.repository.RepositoryPermission;
import sonia.scm.repository.RepositoryPermissions;
import sonia.scm.repository.api.Command;
import sonia.scm.repository.api.ImportFailedException;
import sonia.scm.repository.api.PullCommandBuilder;
import sonia.scm.repository.api.RepositoryService;
import sonia.scm.repository.api.RepositoryServiceFactory;
import sonia.scm.util.IOUtil;
import sonia.scm.web.VndMediaType;
import sonia.scm.web.api.DtoValidator;
@@ -71,55 +59,55 @@ import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.Pattern;
import javax.ws.rs.Consumes;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.StreamingOutput;
import javax.ws.rs.core.UriInfo;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Collections.singletonList;
import static sonia.scm.api.v2.resources.RepositoryTypeSupportChecker.checkSupport;
import static sonia.scm.api.v2.resources.RepositoryTypeSupportChecker.type;
public class RepositoryImportResource {
private static final Logger logger = LoggerFactory.getLogger(RepositoryImportResource.class);
private final RepositoryManager manager;
private final RepositoryDtoToRepositoryMapper mapper;
private final RepositoryServiceFactory serviceFactory;
private final ResourceLinks resourceLinks;
private final ScmEventBus eventBus;
private final FullScmRepositoryImporter fullScmRepositoryImporter;
private final RepositoryImportExportEncryption repositoryImportExportEncryption;
private final RepositoryImportDtoToRepositoryImportParametersMapper importParametersMapper;
private final FromUrlImporter fromUrlImporter;
private final FromBundleImporter fromBundleImporter;
private final RepositoryImportLoggerFactory importLoggerFactory;
@Inject
public RepositoryImportResource(RepositoryManager manager,
RepositoryDtoToRepositoryMapper mapper,
RepositoryServiceFactory serviceFactory,
public RepositoryImportResource(RepositoryDtoToRepositoryMapper mapper,
ResourceLinks resourceLinks,
ScmEventBus eventBus,
FullScmRepositoryImporter fullScmRepositoryImporter,
RepositoryImportExportEncryption repositoryImportExportEncryption) {
this.manager = manager;
RepositoryImportDtoToRepositoryImportParametersMapper importParametersMapper,
RepositoryImportExportEncryption repositoryImportExportEncryption, FromUrlImporter fromUrlImporter,
FromBundleImporter fromBundleImporter,
RepositoryImportLoggerFactory importLoggerFactory) {
this.mapper = mapper;
this.serviceFactory = serviceFactory;
this.resourceLinks = resourceLinks;
this.eventBus = eventBus;
this.fullScmRepositoryImporter = fullScmRepositoryImporter;
this.repositoryImportExportEncryption = repositoryImportExportEncryption;
this.importParametersMapper = importParametersMapper;
this.fromUrlImporter = fromUrlImporter;
this.fromBundleImporter = fromBundleImporter;
this.importLoggerFactory = importLoggerFactory;
}
/**
@@ -166,49 +154,13 @@ public class RepositoryImportResource {
public Response importFromUrl(@Context UriInfo uriInfo,
@Pattern(regexp = "\\w{1,10}") @PathParam("type") String type,
@Valid RepositoryImportResource.RepositoryImportFromUrlDto request) {
RepositoryPermissions.create().check();
Type t = type(manager, type);
if (!t.getName().equals(request.getType())) {
if (!type.equals(request.getType())) {
throw new WebApplicationException("type of import url and repository does not match", Response.Status.BAD_REQUEST);
}
checkSupport(t, Command.PULL);
logger.info("start {} import for external url {}", type, request.getImportUrl());
Repository repository = fromUrlImporter.importFromUrl(importParametersMapper.map(request), mapper.map(request));
Repository repository = mapper.map(request);
repository.setPermissions(singletonList(new RepositoryPermission(SecurityUtils.getSubject().getPrincipal().toString(), "OWNER", false)));
try {
repository = manager.create(
repository,
pullChangesFromRemoteUrl(request)
);
eventBus.post(new RepositoryImportEvent(HandlerEventType.MODIFY, repository, false));
return Response.created(URI.create(resourceLinks.repository().self(repository.getNamespace(), repository.getName()))).build();
} catch (Exception e) {
eventBus.post(new RepositoryImportEvent(HandlerEventType.MODIFY, repository, true));
throw e;
}
}
@VisibleForTesting
Consumer<Repository> pullChangesFromRemoteUrl(RepositoryImportFromUrlDto request) {
return repository -> {
try (RepositoryService service = serviceFactory.create(repository)) {
PullCommandBuilder pullCommand = service.getPullCommand();
if (!Strings.isNullOrEmpty(request.getUsername()) && !Strings.isNullOrEmpty(request.getPassword())) {
pullCommand
.withUsername(request.getUsername())
.withPassword(request.getPassword());
}
pullCommand.pull(request.getImportUrl());
} catch (IOException e) {
throw new InternalRepositoryException(repository, "Failed to import from remote url", e);
}
};
return Response.created(URI.create(resourceLinks.repository().self(repository.getNamespace(), repository.getName()))).build();
}
/**
@@ -253,7 +205,6 @@ public class RepositoryImportResource {
@Pattern(regexp = "\\w{1,10}") @PathParam("type") String type,
MultipartFormDataInput input,
@QueryParam("compressed") @DefaultValue("false") boolean compressed) {
RepositoryPermissions.create().check();
Repository repository = doImportFromBundle(type, input, compressed);
return Response.created(URI.create(resourceLinks.repository().self(repository.getNamespace(), repository.getName()))).build();
@@ -303,11 +254,18 @@ public class RepositoryImportResource {
public Response importFullRepository(@Context UriInfo uriInfo,
@Pattern(regexp = "\\w{1,10}") @PathParam("type") String type,
MultipartFormDataInput input) {
RepositoryPermissions.create().check();
Repository createdRepository = importFullRepositoryFromInput(input);
return Response.created(URI.create(resourceLinks.repository().self(createdRepository.getNamespace(), createdRepository.getName()))).build();
}
@GET
@Path("log/{logId}")
@Produces(MediaType.TEXT_PLAIN)
public StreamingOutput getImportLog(@PathParam("logId") String logId) throws IOException {
importLoggerFactory.checkCanReadLog(logId);
return out -> importLoggerFactory.getLog(logId, out);
}
private Repository importFullRepositoryFromInput(MultipartFormDataInput input) {
Map<String, List<InputPart>> formParts = input.getFormDataMap();
InputStream inputStream = extractInputStream(formParts);
@@ -332,25 +290,13 @@ public class RepositoryImportResource {
inputStream = decryptInputStream(inputStream, repositoryDto.getPassword());
}
Type t = type(manager, type);
checkSupport(t, Command.UNBUNDLE);
if (!type.equals(repositoryDto.getType())) {
throw new WebApplicationException("type of import url and repository does not match", Response.Status.BAD_REQUEST);
}
Repository repository = mapper.map(repositoryDto);
repository.setPermissions(singletonList(
new RepositoryPermission(SecurityUtils.getSubject().getPrincipal().toString(), "OWNER", false)
));
try {
repository = manager.create(
repository,
unbundleImport(inputStream, compressed)
);
eventBus.post(new RepositoryImportEvent(HandlerEventType.MODIFY, repository, false));
} catch (Exception e) {
eventBus.post(new RepositoryImportEvent(HandlerEventType.MODIFY, repository, true));
throw e;
}
repository = fromBundleImporter.importFromBundle(compressed, inputStream, repository);
return repository;
}
@@ -363,27 +309,6 @@ public class RepositoryImportResource {
}
}
@VisibleForTesting
Consumer<Repository> unbundleImport(InputStream inputStream, boolean compressed) {
return repository -> {
File file = null;
try (RepositoryService service = serviceFactory.create(repository)) {
file = File.createTempFile("scm-import-", ".bundle");
long length = Files.asByteSink(file).writeFrom(inputStream);
logger.info("copied {} bytes to temp, start bundle import", length);
service.getUnbundleCommand().setCompressed(compressed).unbundle(file);
} catch (IOException e) {
throw new InternalRepositoryException(repository, "Failed to import from bundle", e);
} finally {
try {
IOUtil.delete(file);
} catch (IOException ex) {
logger.warn("could not delete temporary file", ex);
}
}
};
}
private RepositoryImportFromFileDto extractRepositoryDto(Map<String, List<InputPart>> formParts) {
RepositoryImportFromFileDto repositoryDto = extractFromInputPart(formParts.get("repository"), RepositoryImportFromFileDto.class);
checkNotNull(repositoryDto, "repository data is required");

4
scm-webapp/src/main/java/sonia/scm/api/v2/resources/ResourceLinks.java
View File

@@ -377,6 +377,10 @@ class ResourceLinks {
return repositoryImportLinkBuilder.method("getRepositoryImportResource").parameters().method("importFullRepository").parameters(type).href();
}
String importLog(String importLogId) {
return repositoryImportLinkBuilder.method("getRepositoryImportResource").parameters().method("getImportLog").parameters(importLogId).href();
}
String archive(String namespace, String name) {
return repositoryLinkBuilder.method("getRepositoryResource").parameters(namespace, name).method("archive").parameters().href();
}

1
scm-webapp/src/main/java/sonia/scm/importexport/EnvironmentCheckStep.java
View File

@@ -60,6 +60,7 @@ class EnvironmentCheckStep implements ImportStep {
if (!validEnvironment) {
throw new IncompatibleEnvironmentForImportException();
}
state.getLogger().step("checked environment");
state.environmentChecked();
return true;
}

138
scm-webapp/src/main/java/sonia/scm/importexport/FromBundleImporter.java Normal file
View File

@@ -0,0 +1,138 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.importexport;
import com.google.common.io.Files;
import org.apache.shiro.SecurityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.Type;
import sonia.scm.event.ScmEventBus;
import sonia.scm.repository.ImportRepositoryHookEvent;
import sonia.scm.repository.InternalRepositoryException;
import sonia.scm.repository.Repository;
import sonia.scm.repository.RepositoryHookEvent;
import sonia.scm.repository.RepositoryImportEvent;
import sonia.scm.repository.RepositoryManager;
import sonia.scm.repository.RepositoryPermission;
import sonia.scm.repository.RepositoryPermissions;
import sonia.scm.repository.api.Command;
import sonia.scm.repository.api.RepositoryService;
import sonia.scm.repository.api.RepositoryServiceFactory;
import sonia.scm.repository.work.WorkdirProvider;
import sonia.scm.util.IOUtil;
import javax.inject.Inject;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import static java.util.Collections.singletonList;
import static sonia.scm.importexport.RepositoryImportLogger.ImportType.DUMP;
import static sonia.scm.importexport.RepositoryTypeSupportChecker.checkSupport;
import static sonia.scm.importexport.RepositoryTypeSupportChecker.type;
public class FromBundleImporter {
private static final Logger LOG = LoggerFactory.getLogger(FromBundleImporter.class);
private final RepositoryManager manager;
private final RepositoryServiceFactory serviceFactory;
private final ScmEventBus eventBus;
private final WorkdirProvider workdirProvider;
private final RepositoryImportLoggerFactory loggerFactory;
@Inject
public FromBundleImporter(RepositoryManager manager, RepositoryServiceFactory serviceFactory, ScmEventBus eventBus, WorkdirProvider workdirProvider, RepositoryImportLoggerFactory loggerFactory) {
this.manager = manager;
this.serviceFactory = serviceFactory;
this.eventBus = eventBus;
this.workdirProvider = workdirProvider;
this.loggerFactory = loggerFactory;
}
public Repository importFromBundle(boolean compressed, InputStream inputStream, Repository repository) {
RepositoryPermissions.create().check();
Type t = type(manager, repository.getType());
checkSupport(t, Command.UNBUNDLE);
repository.setPermissions(singletonList(
new RepositoryPermission(SecurityUtils.getSubject().getPrincipal().toString(), "OWNER", false)
));
RepositoryImportLogger logger = loggerFactory.createLogger();
try {
repository = manager.create(repository, unbundleImport(inputStream, compressed, logger));
} catch (Exception e) {
logger.failed(e);
eventBus.post(new RepositoryImportEvent(repository, true));
throw e;
}
eventBus.post(new RepositoryImportEvent(repository, false));
return repository;
}
private Consumer<Repository> unbundleImport(InputStream inputStream, boolean compressed, RepositoryImportLogger logger) {
return repository -> {
logger.start(DUMP, repository);
File workdir = workdirProvider.createNewWorkdir(repository.getId());
try (RepositoryService service = serviceFactory.create(repository)) {
logger.step("writing temporary dump file");
File file = File.createTempFile("scm-import-", ".bundle", workdir);
long length = Files.asByteSink(file).writeFrom(inputStream);
LOG.info("copied {} bytes to temp, start bundle import", length);
logger.step("importing repository data from dump file");
runUnbundleCommand(compressed, service, file);
logger.finished();
} catch (IOException e) {
logger.failed(e);
throw new InternalRepositoryException(repository, "Failed to import from bundle", e);
} finally {
try {
IOUtil.delete(workdir);
} catch (IOException ex) {
LOG.warn("could not delete temporary file", ex);
}
}
};
}
private void runUnbundleCommand(boolean compressed, RepositoryService service, File file) throws IOException {
AtomicReference<RepositoryHookEvent> eventSink = new AtomicReference<>();
service.getUnbundleCommand()
.setCompressed(compressed)
.setPostEventSink(eventSink::set)
.unbundle(file);
RepositoryHookEvent repositoryHookEvent = eventSink.get();
if (repositoryHookEvent != null) {
eventBus.post(new ImportRepositoryHookEvent(repositoryHookEvent));
}
}
}

132
scm-webapp/src/main/java/sonia/scm/importexport/FromUrlImporter.java Normal file
View File

@@ -0,0 +1,132 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.importexport;
import com.google.common.base.Strings;
import lombok.Getter;
import lombok.Setter;
import org.apache.shiro.SecurityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.AlreadyExistsException;
import sonia.scm.Type;
import sonia.scm.event.ScmEventBus;
import sonia.scm.repository.InternalRepositoryException;
import sonia.scm.repository.Repository;
import sonia.scm.repository.RepositoryImportEvent;
import sonia.scm.repository.RepositoryManager;
import sonia.scm.repository.RepositoryPermission;
import sonia.scm.repository.RepositoryPermissions;
import sonia.scm.repository.api.Command;
import sonia.scm.repository.api.ImportFailedException;
import sonia.scm.repository.api.PullCommandBuilder;
import sonia.scm.repository.api.RepositoryService;
import sonia.scm.repository.api.RepositoryServiceFactory;
import javax.inject.Inject;
import java.io.IOException;
import java.util.function.Consumer;
import static java.util.Collections.singletonList;
import static sonia.scm.ContextEntry.ContextBuilder.noContext;
import static sonia.scm.importexport.RepositoryImportLogger.ImportType.URL;
import static sonia.scm.importexport.RepositoryTypeSupportChecker.checkSupport;
import static sonia.scm.importexport.RepositoryTypeSupportChecker.type;
public class FromUrlImporter {
private static final Logger LOG = LoggerFactory.getLogger(FromUrlImporter.class);
private final RepositoryManager manager;
private final RepositoryServiceFactory serviceFactory;
private final ScmEventBus eventBus;
private final RepositoryImportLoggerFactory loggerFactory;
@Inject
public FromUrlImporter(RepositoryManager manager, RepositoryServiceFactory serviceFactory, ScmEventBus eventBus, RepositoryImportLoggerFactory loggerFactory) {
this.manager = manager;
this.serviceFactory = serviceFactory;
this.eventBus = eventBus;
this.loggerFactory = loggerFactory;
}
public Repository importFromUrl(RepositoryImportParameters parameters, Repository repository) {
Type t = type(manager, repository.getType());
RepositoryPermissions.create().check();
checkSupport(t, Command.PULL);
LOG.info("start {} import for external url {}", repository.getType(), parameters.getImportUrl());
repository.setPermissions(singletonList(new RepositoryPermission(SecurityUtils.getSubject().getPrincipal().toString(), "OWNER", false)));
RepositoryImportLogger logger = loggerFactory.createLogger();
Repository createdRepository;
try {
createdRepository = manager.create(
repository,
pullChangesFromRemoteUrl(parameters, logger)
);
} catch (AlreadyExistsException e) {
throw e;
} catch (Exception e) {
if (logger.started()) {
logger.failed(e);
}
eventBus.post(new RepositoryImportEvent(repository, true));
throw new ImportFailedException(noContext(), "Could not import repository from url " + parameters.getImportUrl(), e);
}
eventBus.post(new RepositoryImportEvent(createdRepository, false));
return createdRepository;
}
private Consumer<Repository> pullChangesFromRemoteUrl(RepositoryImportParameters parameters, RepositoryImportLogger logger) {
return repository -> {
logger.start(URL, repository);
try (RepositoryService service = serviceFactory.create(repository)) {
PullCommandBuilder pullCommand = service.getPullCommand();
if (!Strings.isNullOrEmpty(parameters.getUsername()) && !Strings.isNullOrEmpty(parameters.getPassword())) {
logger.step("setting username and password for pull");
pullCommand
.withUsername(parameters.getUsername())
.withPassword(parameters.getPassword());
}
logger.step("pulling repository from " + parameters.getImportUrl());
pullCommand.pull(parameters.getImportUrl());
logger.finished();
} catch (IOException e) {
throw new InternalRepositoryException(repository, "Failed to import from remote url: " + e.getMessage(), e);
}
};
}
@Getter
@Setter
public static class RepositoryImportParameters {
private String importUrl;
private String username;
private String password;
}
}

32
scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryImporter.java
View File

@@ -33,7 +33,9 @@ import org.slf4j.LoggerFactory;
import sonia.scm.ContextEntry;
import sonia.scm.event.ScmEventBus;
import sonia.scm.repository.Repository;
import sonia.scm.repository.RepositoryImportEvent;
import sonia.scm.repository.RepositoryManager;
import sonia.scm.repository.RepositoryPermissions;
import sonia.scm.repository.api.ImportFailedException;
import javax.inject.Inject;
@@ -42,8 +44,9 @@ import java.io.IOException;
import java.io.InputStream;
import static java.util.Arrays.stream;
import static sonia.scm.util.Archives.createTarInputStream;
import static sonia.scm.ContextEntry.ContextBuilder.noContext;
import static sonia.scm.importexport.RepositoryImportLogger.ImportType.FULL;
import static sonia.scm.util.Archives.createTarInputStream;
public class FullScmRepositoryImporter {
@@ -53,6 +56,7 @@ public class FullScmRepositoryImporter {
private final RepositoryManager repositoryManager;
private final RepositoryImportExportEncryption repositoryImportExportEncryption;
private final ScmEventBus eventBus;
private final RepositoryImportLoggerFactory loggerFactory;
@Inject
public FullScmRepositoryImporter(EnvironmentCheckStep environmentCheckStep,
@@ -61,15 +65,17 @@ public class FullScmRepositoryImporter {
RepositoryImportStep repositoryImportStep,
RepositoryManager repositoryManager,
RepositoryImportExportEncryption repositoryImportExportEncryption,
ScmEventBus eventBus
) {
RepositoryImportLoggerFactory loggerFactory,
ScmEventBus eventBus) {
this.repositoryManager = repositoryManager;
this.loggerFactory = loggerFactory;
this.repositoryImportExportEncryption = repositoryImportExportEncryption;
importSteps = new ImportStep[]{environmentCheckStep, metadataImportStep, storeImportStep, repositoryImportStep};
this.eventBus = eventBus;
importSteps = new ImportStep[]{environmentCheckStep, metadataImportStep, storeImportStep, repositoryImportStep};
}
public Repository importFromStream(Repository repository, InputStream inputStream, String password) {
RepositoryPermissions.create().check();
try {
if (inputStream.available() > 0) {
try (
@@ -103,8 +109,17 @@ public class FullScmRepositoryImporter {
}
}
private RepositoryImportLogger startLogger(Repository repository) {
RepositoryImportLogger logger = loggerFactory.createLogger();
logger.start(FULL, repository);
return logger;
}
private Repository run(Repository repository, TarArchiveInputStream tais) throws IOException {
ImportState state = new ImportState(repositoryManager.create(repository));
Repository createdRepository = repositoryManager.create(repository);
RepositoryImportLogger logger = startLogger(repository);
ImportState state = new ImportState(createdRepository, logger);
logger.repositoryCreated(state.getRepository());
try {
TarArchiveEntry tarArchiveEntry;
while ((tarArchiveEntry = tais.getNextTarEntry()) != null) {
@@ -112,21 +127,28 @@ public class FullScmRepositoryImporter {
handle(tais, state, tarArchiveEntry);
}
stream(importSteps).forEach(step -> step.finish(state));
state.getLogger().finished();
return state.getRepository();
} catch (RuntimeException | IOException e) {
state.getLogger().failed(e);
throw e;
} finally {
stream(importSteps).forEach(step -> step.cleanup(state));
if (state.success()) {
// send all pending events on successful import
state.getPendingEvents().forEach(eventBus::post);
eventBus.post(new RepositoryImportEvent(repository, false));
} else {
// Delete the repository if any error occurs during the import
repositoryManager.delete(state.getRepository());
eventBus.post(new RepositoryImportEvent(repository, true));
}
}
}
private void handle(TarArchiveInputStream tais, ImportState state, TarArchiveEntry currentEntry) {
state.getLogger().step("inspecting file " + currentEntry.getName());
for (ImportStep step : importSteps) {
if (step.handle(currentEntry, state, tais)) {
return;

13
scm-webapp/src/main/java/sonia/scm/importexport/ImportState.java
View File

@@ -36,6 +36,8 @@ import java.util.Optional;
class ImportState {
private final RepositoryImportLogger logger;
private Repository repository;
private boolean environmentChecked;
@@ -48,11 +50,8 @@ class ImportState {
private final List<Object> pendingEvents = new ArrayList<>();
ImportState(Repository repository) {
this.repository = repository;
}
public void setRepository(Repository repository) {
ImportState(Repository repository, RepositoryImportLogger logger) {
this.logger = logger;
this.repository = repository;
}
@@ -104,6 +103,10 @@ class ImportState {
this.pendingEvents.add(event);
}
RepositoryImportLogger getLogger() {
return logger;
}
public Collection<Object> getPendingEvents() {
return Collections.unmodifiableCollection(pendingEvents);
}

2
scm-webapp/src/main/java/sonia/scm/importexport/MetadataImportStep.java
View File

@@ -58,6 +58,7 @@ class MetadataImportStep implements ImportStep {
RepositoryMetadataXmlGenerator.RepositoryMetadata metadata = JAXB.unmarshal(new NoneClosingInputStream(inputStream), RepositoryMetadataXmlGenerator.RepositoryMetadata.class);
if (metadata != null && metadata.getPermissions() != null) {
state.setPermissions(new HashSet<>(metadata.getPermissions()));
state.getLogger().step("reading repository metadata with permissions");
} else {
state.setPermissions(Collections.emptySet());
}
@@ -69,6 +70,7 @@ class MetadataImportStep implements ImportStep {
@Override
public void finish(ImportState state) {
LOG.trace("Saving permissions for imported repository");
state.getLogger().step("setting permissions for repository from import");
importRepositoryPermissions(state.getRepository(), state.getRepositoryPermissions());
}

131
scm-webapp/src/main/java/sonia/scm/importexport/RepositoryImportLogger.java Normal file
View File

@@ -0,0 +1,131 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.importexport;
import org.apache.shiro.SecurityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.repository.Repository;
import sonia.scm.store.Blob;
import sonia.scm.store.BlobStore;
import sonia.scm.user.User;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.time.Instant;
import static java.nio.charset.StandardCharsets.UTF_8;
class RepositoryImportLogger {
private static final Logger LOG = LoggerFactory.getLogger(RepositoryImportLogger.class);
private final BlobStore logStore;
private PrintWriter print;
private Blob blob;
RepositoryImportLogger(BlobStore logStore) {
this.logStore = logStore;
}
void start(ImportType importType, Repository repository) {
User user = SecurityUtils.getSubject().getPrincipals().oneByType(User.class);
blob = logStore.create(repository.getId());
OutputStream outputStream = getBlobOutputStream();
writeUser(user, outputStream);
print = new PrintWriter(outputStream);
print.printf("Import of repository %s/%s%n", repository.getNamespace(), repository.getName());
print.printf("Repository type: %s%n", repository.getType());
print.printf("Imported from: %s%n", importType);
print.printf("Imported by %s (%s)%n", user.getId(), user.getName());
print.println();
addLogEntry("import started");
}
private void writeUser(User user, OutputStream outputStream) {
try {
outputStream.write(user.getId().getBytes(UTF_8));
outputStream.write(0);
} catch (IOException e) {
LOG.warn("Could not write user to import log blob", e);
}
}
private OutputStream getBlobOutputStream() {
try {
return blob.getOutputStream();
} catch (IOException e) {
LOG.warn("Could not create logger for import; failed to get output stream from blob", e);
return new OutputStream() {
@Override
public void write(int b) {
// this is a dummy
}
};
}
}
public void finished() {
step("import finished successfully");
writeLog();
}
public void failed(Exception e) {
step("import failed (see next log entry)");
print.println(e.getMessage());
writeLog();
}
public void repositoryCreated(Repository createdRepository) {
step("created repository: " + createdRepository.getNamespaceAndName());
}
public void step(String message) {
addLogEntry(message);
}
private void writeLog() {
print.flush();
try {
blob.commit();
} catch (IOException e) {
LOG.warn("Could not commit blob with import log", e);
}
}
private void addLogEntry(String message) {
print.printf("%s - %s%n", Instant.now(), message);
}
public boolean started() {
return blob != null;
}
enum ImportType {
FULL, URL, DUMP
}
}

93
scm-webapp/src/main/java/sonia/scm/importexport/RepositoryImportLoggerFactory.java Normal file
View File

@@ -0,0 +1,93 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.importexport;
import org.apache.shiro.SecurityUtils;
import org.apache.shiro.authz.AuthorizationException;
import org.apache.shiro.subject.Subject;
import sonia.scm.NotFoundException;
import sonia.scm.store.BlobStore;
import sonia.scm.store.BlobStoreFactory;
import sonia.scm.util.IOUtil;
import javax.inject.Inject;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import static java.nio.charset.StandardCharsets.UTF_8;
public class RepositoryImportLoggerFactory {
private final BlobStoreFactory blobStoreFactory;
@Inject
RepositoryImportLoggerFactory(BlobStoreFactory blobStoreFactory) {
this.blobStoreFactory = blobStoreFactory;
}
RepositoryImportLogger createLogger() {
return new RepositoryImportLogger(blobStoreFactory.withName("imports").build());
}
public void checkCanReadLog(String logId) throws IOException {
try (InputStream blob = getBlob(logId)) {
// nothing to read
}
}
public void getLog(String logId, OutputStream out) throws IOException {
try (InputStream log = getBlob(logId)) {
IOUtil.copy(log, out);
}
}
private InputStream getBlob(String logId) throws IOException {
BlobStore importStore = blobStoreFactory.withName("imports").build();
InputStream log = importStore
.getOptional(logId).orElseThrow(() -> new NotFoundException("Log", logId))
.getInputStream();
checkPermission(log);
return log;
}
private void checkPermission(InputStream log) throws IOException {
Subject subject = SecurityUtils.getSubject();
String logUser = readUserFrom(log);
if (!subject.isPermitted("only:admin:allowed") && !subject.getPrincipal().toString().equals(logUser)) {
throw new AuthorizationException("not permitted");
}
}
private String readUserFrom(InputStream log) throws IOException {
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
int b;
while ((b = log.read()) > 0) {
buffer.write(b);
}
return new String(buffer.toByteArray(), UTF_8);
}
}

7
scm-webapp/src/main/java/sonia/scm/importexport/RepositoryImportStep.java
View File

@@ -62,12 +62,14 @@ class RepositoryImportStep implements ImportStep {
@Override
public boolean handle(TarArchiveEntry currentEntry, ImportState state, InputStream inputStream) {
if (!currentEntry.isDirectory()) {
if (!currentEntry.isDirectory() && !currentEntry.getName().contains("/")) {
if (state.isStoreImported()) {
LOG.trace("Importing directly from tar stream (entry '{}')", currentEntry.getName());
state.getLogger().step("directly importing repository data");
unbundleRepository(state, inputStream);
} else {
LOG.debug("Temporally storing tar entry '{}' in work dir", currentEntry.getName());
LOG.debug("Temporarily storing tar entry '{}' in work dir", currentEntry.getName());
state.getLogger().step("temporarily storing repository data for later import");
Path path = saveRepositoryDataFromTarArchiveEntry(state.getRepository(), inputStream);
state.setTemporaryRepositoryBundle(path);
}
@@ -90,6 +92,7 @@ class RepositoryImportStep implements ImportStep {
private void importFromTemporaryPath(ImportState state, Path path) {
LOG.debug("Importing repository from temporary location in work dir");
state.getLogger().step("importing repository from temporary location");
try {
unbundleRepository(state, Files.newInputStream(path));
} catch (IOException e) {

26
scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryTypeSupportChecker.java → scm-webapp/src/main/java/sonia/scm/importexport/RepositoryTypeSupportChecker.java
View File

@@ -22,10 +22,11 @@
* SOFTWARE.
*/
package sonia.scm.api.v2.resources;
package sonia.scm.importexport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.BadRequestException;
import sonia.scm.Type;
import sonia.scm.repository.RepositoryHandler;
import sonia.scm.repository.RepositoryManager;
@@ -36,7 +37,9 @@ import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;
import java.util.Set;
class RepositoryTypeSupportChecker {
import static sonia.scm.ContextEntry.ContextBuilder.noContext;
public class RepositoryTypeSupportChecker {
private RepositoryTypeSupportChecker() {
}
@@ -49,7 +52,7 @@ class RepositoryTypeSupportChecker {
* @param type repository type
* @param cmd command
*/
static void checkSupport(Type type, Command cmd) {
public static void checkSupport(Type type, Command cmd) {
if (!(type instanceof RepositoryType)) {
logger.warn("type {} is not a repository type", type.getName());
throw new WebApplicationException(Response.Status.BAD_REQUEST);
@@ -60,17 +63,28 @@ class RepositoryTypeSupportChecker {
logger.warn("type {} does not support this command {}",
type.getName(),
cmd.name());
throw new WebApplicationException(Response.Status.BAD_REQUEST);
throw new IllegalTypeForImportException("type does not support command");
}
}
@SuppressWarnings("javasecurity:S5145") // the type parameter is validated in the resource to only contain valid characters (\w)
static Type type(RepositoryManager manager, String type) {
public static Type type(RepositoryManager manager, String type) {
RepositoryHandler handler = manager.getHandler(type);
if (handler == null) {
logger.warn("no handler for type {} found", type);
throw new WebApplicationException(Response.Status.NOT_FOUND);
throw new IllegalTypeForImportException("unsupported repository type: " + type);
}
return handler.getType();
}
private static class IllegalTypeForImportException extends BadRequestException {
public IllegalTypeForImportException(String message) {
super(noContext(), message);
}
@Override
public String getCode() {
return "CISPvega31";
}
}
}

7
scm-webapp/src/main/java/sonia/scm/importexport/StoreImportStep.java
View File

@@ -52,17 +52,18 @@ class StoreImportStep implements ImportStep {
public boolean handle(TarArchiveEntry entry, ImportState state, InputStream inputStream) {
if (entry.getName().equals(STORE_DATA_FILE_NAME) && !entry.isDirectory()) {
LOG.trace("Importing store from tar");
state.getLogger().step("importing stores");
// Inside the repository tar archive stream is another tar archive.
// The nested tar archive is wrapped in another TarArchiveInputStream inside the storeImporter
importStores(state.getRepository(), inputStream);
importStores(state.getRepository(), inputStream, state.getLogger());
state.storeImported();
return true;
}
return false;
}
private void importStores(Repository repository, InputStream inputStream) {
storeImporter.importFromTarArchive(repository, inputStream);
private void importStores(Repository repository, InputStream inputStream, RepositoryImportLogger logger) {
storeImporter.importFromTarArchive(repository, inputStream, logger);
updateEngine.update(repository.getId());
}
}

16
scm-webapp/src/main/java/sonia/scm/importexport/TarArchiveRepositoryStoreImporter.java
View File

@@ -47,36 +47,40 @@ public class TarArchiveRepositoryStoreImporter {
this.repositoryStoreImporter = repositoryStoreImporter;
}
public void importFromTarArchive(Repository repository, InputStream inputStream) {
public void importFromTarArchive(Repository repository, InputStream inputStream, RepositoryImportLogger logger) {
try (TarArchiveInputStream tais = new NoneClosingTarArchiveInputStream(inputStream)) {
ArchiveEntry entry = tais.getNextEntry();
while (entry != null) {
String[] entryPathParts = entry.getName().split(File.separator);
validateStorePath(repository, entryPathParts);
importStoreByType(repository, tais, entryPathParts);
importStoreByType(repository, tais, entryPathParts, logger);
entry = tais.getNextEntry();
}
} catch (IOException e) {
throw new ImportFailedException(ContextEntry.ContextBuilder.entity(repository).build(), "Could not import stores from metadata file.", e);
throw new ImportFailedException(ContextEntry.ContextBuilder.entity(repository).build(), "Could not import stores from metadata file.", e);
}
}
private void importStoreByType(Repository repository, TarArchiveInputStream tais, String[] entryPathParts) {
private void importStoreByType(Repository repository, TarArchiveInputStream tais, String[] entryPathParts, RepositoryImportLogger logger) {
String storeType = entryPathParts[1];
String storeName = entryPathParts[2];
if (isDataStore(storeType)) {
logger.step("importing data store entry for store " + storeName);
repositoryStoreImporter
.doImport(repository)
.importStore(new StoreEntryMetaData(StoreType.DATA, entryPathParts[2]))
.importEntry(entryPathParts[3], tais);
} else if (isConfigStore(storeType)){
logger.step("importing data store entry for store " + storeName);
repositoryStoreImporter
.doImport(repository)
.importStore(new StoreEntryMetaData(StoreType.CONFIG, ""))
.importEntry(entryPathParts[2], tais);
.importEntry(storeName, tais);
} else if(isBlobStore(storeType)) {
logger.step("importing blob store entry for store " + storeName);
repositoryStoreImporter
.doImport(repository)
.importStore(new StoreEntryMetaData(StoreType.BLOB, entryPathParts[2]))
.importStore(new StoreEntryMetaData(StoreType.BLOB, storeName))
.importEntry(entryPathParts[3], tais);
}
}