Nemcio/SCM-Manager
1
0
Fork 0
mirror of https://github.com/scm-manager/scm-manager.git synced 2025-11-17 18:51:10 +01:00
Code Issues Packages Projects Releases Activity

Feature/import export encryption (#1533)

Browse Source 
Add option to encrypt repository exports with a password and add possibility to decrypt them on repository import. Also make the repository export asynchronous. This implies that the repository export will be created on the server and can be downloaded multiple times. The repository export will be deleted automatically 10 days after creation.
This commit is contained in:
Eduard Heimbuch
2021-02-25 13:01:03 +01:00
committed by GitHub
parent 367d7294b8
commit db2ce98721
53 changed files with 2698 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scm-webapp/src/main/java/sonia/scm/api/v2/resources/MapperModule.java
View File

@@ -88,5 +88,7 @@ public class MapperModule extends AbstractModule {
bind(PluginDtoMapper.class).to(Mappers.getMapperClass(PluginDtoMapper.class));
bind(ApiKeyToApiKeyDtoMapper.class).to(Mappers.getMapperClass(ApiKeyToApiKeyDtoMapper.class));
bind(RepositoryExportInformationToDtoMapper.class).to(Mappers.getMapperClass(RepositoryExportInformationToDtoMapper.class));
}
}

52
scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryExportInformationDto.java Normal file
View File

@@ -0,0 +1,52 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.api.v2.resources;
import de.otto.edison.hal.HalRepresentation;
import de.otto.edison.hal.Links;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import sonia.scm.importexport.ExportStatus;
import java.time.Instant;
@Getter
@Setter
@NoArgsConstructor
@SuppressWarnings("squid:S2160") // we do not need equals for dto
public class RepositoryExportInformationDto extends HalRepresentation {
private String exporterName;
private Instant created;
private boolean withMetadata;
private boolean compressed;
private boolean encrypted;
private ExportStatus status;
RepositoryExportInformationDto(Links links) {
super(links);
}
}

70
scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryExportInformationToDtoMapper.java Normal file
View File

@@ -0,0 +1,70 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.api.v2.resources;
import com.google.common.annotations.VisibleForTesting;
import de.otto.edison.hal.Links;
import org.mapstruct.Context;
import org.mapstruct.Mapper;
import org.mapstruct.ObjectFactory;
import sonia.scm.importexport.ExportService;
import sonia.scm.importexport.ExportStatus;
import sonia.scm.importexport.RepositoryExportInformation;
import sonia.scm.repository.Repository;
import javax.inject.Inject;
import static de.otto.edison.hal.Link.link;
@Mapper
public abstract class RepositoryExportInformationToDtoMapper {
@Inject
private ResourceLinks resourceLinks;
@Inject
private ExportService exportService;
@VisibleForTesting
void setResourceLinks(ResourceLinks resourceLinks) {
this.resourceLinks = resourceLinks;
}
@VisibleForTesting
void setExportService(ExportService exportService) {
this.exportService = exportService;
}
abstract RepositoryExportInformationDto map(RepositoryExportInformation info, @Context Repository repository);
@ObjectFactory
RepositoryExportInformationDto createDto(@Context Repository repository) {
Links.Builder links = Links.linkingTo();
links.self(resourceLinks.repository().exportInfo(repository.getNamespace(), repository.getName()));
if (!exportService.isExporting(repository) && exportService.getExportInformation(repository).getStatus() == ExportStatus.FINISHED) {
links.single(link("download", resourceLinks.repository().downloadExport(repository.getNamespace(), repository.getName())));
}
return new RepositoryExportInformationDto(links.build());
}
}

369
scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryExportResource.java
View File

@@ -24,32 +24,47 @@
package sonia.scm.api.v2.resources;
import com.google.common.base.Strings;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.inject.Inject;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
import sonia.scm.BadRequestException;
import sonia.scm.ConcurrentModificationException;
import sonia.scm.NotFoundException;
import sonia.scm.Type;
import sonia.scm.importexport.ExportFileExtensionResolver;
import sonia.scm.importexport.ExportService;
import sonia.scm.importexport.FullScmRepositoryExporter;
import sonia.scm.repository.InternalRepositoryException;
import sonia.scm.importexport.RepositoryImportExportEncryption;
import sonia.scm.repository.NamespaceAndName;
import sonia.scm.repository.Repository;
import sonia.scm.repository.RepositoryManager;
import sonia.scm.repository.RepositoryPermissions;
import sonia.scm.repository.api.BundleCommandBuilder;
import sonia.scm.repository.api.Command;
import sonia.scm.repository.api.ExportFailedException;
import sonia.scm.repository.api.RepositoryService;
import sonia.scm.repository.api.RepositoryServiceFactory;
import sonia.scm.util.IOUtil;
import sonia.scm.web.VndMediaType;
import javax.validation.Valid;
import javax.validation.constraints.Pattern;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
@@ -57,7 +72,12 @@ import javax.ws.rs.core.Response;
import javax.ws.rs.core.StreamingOutput;
import javax.ws.rs.core.UriInfo;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.time.Instant;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import static sonia.scm.ContextEntry.ContextBuilder.entity;
import static sonia.scm.api.v2.resources.RepositoryTypeSupportChecker.checkSupport;
@@ -65,16 +85,35 @@ import static sonia.scm.api.v2.resources.RepositoryTypeSupportChecker.type;
public class RepositoryExportResource {
private static final String NO_PASSWORD = "";
private final RepositoryManager manager;
private final RepositoryServiceFactory serviceFactory;
private final FullScmRepositoryExporter fullScmRepositoryExporter;
private final RepositoryImportExportEncryption repositoryImportExportEncryption;
private final ExecutorService repositoryExportHandler;
private final ExportService exportService;
private final RepositoryExportInformationToDtoMapper informationToDtoMapper;
private final ExportFileExtensionResolver fileExtensionResolver;
private final ResourceLinks resourceLinks;
@Inject
public RepositoryExportResource(RepositoryManager manager,
RepositoryServiceFactory serviceFactory, FullScmRepositoryExporter fullScmRepositoryExporter) {
RepositoryServiceFactory serviceFactory,
FullScmRepositoryExporter fullScmRepositoryExporter,
RepositoryImportExportEncryption repositoryImportExportEncryption,
ExportService exportService,
RepositoryExportInformationToDtoMapper informationToDtoMapper,
ExportFileExtensionResolver fileExtensionResolver, ResourceLinks resourceLinks) {
this.manager = manager;
this.serviceFactory = serviceFactory;
this.fullScmRepositoryExporter = fullScmRepositoryExporter;
this.repositoryImportExportEncryption = repositoryImportExportEncryption;
this.exportService = exportService;
this.informationToDtoMapper = informationToDtoMapper;
this.fileExtensionResolver = fileExtensionResolver;
this.resourceLinks = resourceLinks;
this.repositoryExportHandler = this.createExportHandlerPool();
}
/**
@@ -90,7 +129,6 @@ public class RepositoryExportResource {
*/
@GET
@Path("{type: ^(?!full$)[^/]+$}")
@Consumes(VndMediaType.REPOSITORY)
@Operation(summary = "Exports the repository", description = "Exports the repository.", tags = "Repository")
@ApiResponse(
responseCode = "200",
@@ -119,7 +157,8 @@ public class RepositoryExportResource {
@DefaultValue("false") @QueryParam("compressed") boolean compressed
) {
Repository repository = getVerifiedRepository(namespace, name, type);
return exportRepository(repository, compressed);
RepositoryPermissions.export(repository).check();
return exportRepository(repository, NO_PASSWORD, compressed, false);
}
/**
@@ -134,7 +173,6 @@ public class RepositoryExportResource {
*/
@GET
@Path("full")
@Consumes(VndMediaType.REPOSITORY)
@Operation(summary = "Exports the repository", description = "Exports the repository with metadata and environment information.", tags = "Repository")
@ApiResponse(
responseCode = "200",
@@ -161,11 +199,235 @@ public class RepositoryExportResource {
@PathParam("name") String name
) {
Repository repository = getVerifiedRepository(namespace, name);
return exportFullRepository(repository);
RepositoryPermissions.export(repository).check();
return exportFullRepository(repository, NO_PASSWORD, false);
}
/**
* Exports an existing repository without additional metadata. The method can
* only be used, if the repository type supports the {@link Command#BUNDLE}.
*
* @param uriInfo uri info
* @param namespace namespace of the repository
* @param name name of the repository
* @param type type of the repository
* @param request request of repository export which contains the password
* @return response with readable stream of repository dump
* @since 2.14.0
*/
@POST
@Path("{type: ^(?!full$)[^/]+$}")
@Consumes(VndMediaType.REPOSITORY_EXPORT)
@Operation(summary = "Exports the repository", description = "Exports the repository.", tags = "Repository")
@ApiResponse(
responseCode = "200",
description = "Repository export was successful"
)
@ApiResponse(
responseCode = "401",
description = "not authenticated / invalid credentials"
)
@ApiResponse(
responseCode = "403",
description = "not authorized, the current user has no privileges to read the repository"
)
@ApiResponse(
responseCode = "409",
description = "Repository export already started."
)
@ApiResponse(
responseCode = "500",
description = "internal server error",
content = @Content(
mediaType = VndMediaType.ERROR_TYPE,
schema = @Schema(implementation = ErrorDto.class)
)
)
public Response exportRepositoryWithPassword(@Context UriInfo uriInfo,
@PathParam("namespace") String namespace,
@PathParam("name") String name,
@Pattern(regexp = "\\w{1,10}") @PathParam("type") String type,
@DefaultValue("false") @QueryParam("compressed") boolean compressed,
@Valid ExportDto request
) throws Exception {
Repository repository = getVerifiedRepository(namespace, name, type);
RepositoryPermissions.export(repository).check();
checkRepositoryIsAlreadyExporting(repository);
return exportAsync(repository, request.isAsync(), () -> {
Response response = exportRepository(repository, request.getPassword(), compressed, request.isAsync());
exportService.setExportFinished(repository);
return response;
});
}
/**
* Exports an existing repository with all additional metadata and environment information. The method can
* only be used, if the repository type supports the {@link Command#BUNDLE}.
*
* @param uriInfo uri info
* @param namespace namespace of the repository
* @param name name of the repository
* @param request request of repository export which contains the password
* @return response with readable stream of repository dump
* @since 2.14.0
*/
@POST
@Path("full")
@Consumes(VndMediaType.REPOSITORY_EXPORT)
@Operation(summary = "Exports the repository", description = "Exports the repository with metadata and environment information.", tags = "Repository")
@ApiResponse(
responseCode = "200",
description = "Repository export was successful"
)
@ApiResponse(
responseCode = "204",
description = "Repository export was started successfully"
)
@ApiResponse(
responseCode = "401",
description = "not authenticated / invalid credentials"
)
@ApiResponse(
responseCode = "403",
description = "not authorized, the current user has no privileges to read the repository"
)
@ApiResponse(
responseCode = "500",
description = "internal server error",
content = @Content(
mediaType = VndMediaType.ERROR_TYPE,
schema = @Schema(implementation = ErrorDto.class)
)
)
public Response exportFullRepositoryWithPassword(@Context UriInfo uriInfo,
@PathParam("namespace") String namespace,
@PathParam("name") String name,
@Valid ExportDto request
) throws Exception {
Repository repository = getVerifiedRepository(namespace, name);
RepositoryPermissions.export(repository).check();
checkRepositoryIsAlreadyExporting(repository);
return exportAsync(repository, request.isAsync(), () -> {
Response response = exportFullRepository(repository, request.getPassword(), request.isAsync());
exportService.setExportFinished(repository);
return response;
});
}
@DELETE
@Path("")
@Operation(summary = "Deletes repository export", description = "Deletes repository export if stored.", tags = "Repository")
@ApiResponse(
responseCode = "204",
description = "Repository export was deleted"
)
@ApiResponse(
responseCode = "500",
description = "internal server error",
content = @Content(
mediaType = VndMediaType.ERROR_TYPE,
schema = @Schema(implementation = ErrorDto.class)
)
)
public Response deleteExport(@Context UriInfo uriInfo,
@PathParam("namespace") String namespace,
@PathParam("name") String name) {
Repository repository = getVerifiedRepository(namespace, name);
RepositoryPermissions.export(repository).check();
exportService.clear(repository.getId());
return Response.noContent().build();
}
@GET
@Path("download")
@Operation(summary = "Download stored repository export", description = "Download the stored repository export.", tags = "Repository")
@ApiResponse(
responseCode = "200",
description = "Repository export was downloaded"
)
@ApiResponse(
responseCode = "404",
description = "Repository export not found"
)
@ApiResponse(
responseCode = "409",
description = "Repository export is not ready yet"
)
@ApiResponse(
responseCode = "500",
description = "internal server error",
content = @Content(
mediaType = VndMediaType.ERROR_TYPE,
schema = @Schema(implementation = ErrorDto.class)
)
)
public Response downloadExport(@Context UriInfo uriInfo,
@PathParam("namespace") String namespace,
@PathParam("name") String name) {
Repository repository = getVerifiedRepository(namespace, name);
RepositoryPermissions.export(repository).check();
checkRepositoryIsAlreadyExporting(repository);
exportService.checkExportIsAvailable(repository);
StreamingOutput output = os -> {
try (InputStream is = exportService.getData(repository)) {
IOUtil.copy(is, os);
}
};
String fileExtension = exportService.getFileExtension(repository);
return createResponse(repository, fileExtension, fileExtension.contains(".gz"), output);
}
@GET
@Produces(VndMediaType.REPOSITORY_EXPORT_INFO)
@Path("info")
@Operation(summary = "Returns stored repository export information", description = "Returns the stored repository export information.", tags = "Repository")
@ApiResponse(
responseCode = "200",
description = "Repository export information"
)
@ApiResponse(
responseCode = "404",
description = "Repository export information not found"
)
@ApiResponse(
responseCode = "500",
description = "internal server error",
content = @Content(
mediaType = VndMediaType.ERROR_TYPE,
schema = @Schema(implementation = ErrorDto.class)
)
)
public RepositoryExportInformationDto getExportInformation(@Context UriInfo uriInfo,
@PathParam("namespace") String namespace,
@PathParam("name") String name) {
Repository repository = getVerifiedRepository(namespace, name);
RepositoryPermissions.export(repository).check();
return informationToDtoMapper.map(exportService.getExportInformation(repository), repository);
}
private Response exportAsync(Repository repository, boolean async, Callable<Response> call) throws Exception {
if (async) {
repositoryExportHandler.submit(call);
return Response.status(202).header(
"SCM-Export-Download",
resourceLinks.repository().downloadExport(repository.getNamespace(), repository.getName())
).build();
} else {
return call.call();
}
}
private void checkRepositoryIsAlreadyExporting(Repository repository) {
if (exportService.isExporting(repository)) {
throw new ConcurrentModificationException(Repository.class, repository.getId());
}
}
private Repository getVerifiedRepository(String namespace, String name) {
Repository repository = manager.get(new NamespaceAndName(namespace, name));
if (repository == null) {
throw new NotFoundException(Repository.class, namespace + "/" + name);
}
RepositoryPermissions.read().check(repository);
return repository;
}
@@ -181,53 +443,65 @@ public class RepositoryExportResource {
return repository;
}
private Response exportFullRepository(Repository repository) {
StreamingOutput output = os -> fullScmRepositoryExporter.export(repository, os);
private Response exportFullRepository(Repository repository, String password, boolean async) {
boolean encrypted = !Strings.isNullOrEmpty(password);
String fileExtension = fileExtensionResolver.resolve(repository, true, true, encrypted);
if (async) {
OutputStream blobOutputStream = exportService.store(repository, true, true, encrypted);
fullScmRepositoryExporter.export(repository, blobOutputStream, password);
exportService.setExportFinished(repository);
return Response.status(204).build();
} else {
StreamingOutput output = os -> fullScmRepositoryExporter.export(repository, os, password);
return Response
.ok(output, "application/x-gzip")
.header("content-disposition", createContentDispositionHeaderValue(repository, "tar.gz"))
.build();
return Response
.ok(output, "application/x-gzip")
.header("content-disposition", createContentDispositionHeaderValue(repository, fileExtension))
.build();
}
}
private Response exportRepository(Repository repository, boolean compressed) {
StreamingOutput output;
String fileExtension;
private Response exportRepository(Repository repository, String password, boolean compressed, boolean async) {
boolean encrypted = !Strings.isNullOrEmpty(password);
try (final RepositoryService service = serviceFactory.create(repository)) {
BundleCommandBuilder bundleCommand = service.getBundleCommand();
fileExtension = resolveFileExtension(bundleCommand, compressed);
output = os -> {
try {
if (compressed) {
GzipCompressorOutputStream gzipCompressorOutputStream = new GzipCompressorOutputStream(os);
bundleCommand.bundle(gzipCompressorOutputStream);
gzipCompressorOutputStream.finish();
} else {
bundleCommand.bundle(os);
}
} catch (IOException e) {
throw new InternalRepositoryException(repository, "repository export failed", e);
}
};
String fileExtension = fileExtensionResolver.resolve(repository, false, compressed, encrypted);
if (async) {
OutputStream blobOutputStream = exportService.store(repository, false, compressed, !Strings.isNullOrEmpty(password));
OutputStream os = repositoryImportExportEncryption.optionallyEncrypt(blobOutputStream, password);
bundleRepository(os, compressed, bundleCommand);
return Response.status(204).build();
} else {
StreamingOutput output = os -> {
os = repositoryImportExportEncryption.optionallyEncrypt(os, password);
bundleRepository(os, compressed, bundleCommand);
};
return createResponse(repository, fileExtension, compressed, output);
}
} catch (IOException e) {
throw new ExportFailedException(entity(repository).build(), "repository export failed", e);
}
return createResponse(repository, fileExtension, compressed, output);
}
private Response createResponse(Repository repository, String fileExtension, boolean compressed, StreamingOutput output) {
private void bundleRepository(OutputStream os, boolean compressed, BundleCommandBuilder bundleCommand) throws IOException {
if (compressed) {
GzipCompressorOutputStream gzipCompressorOutputStream = new GzipCompressorOutputStream(os);
bundleCommand.bundle(gzipCompressorOutputStream);
gzipCompressorOutputStream.finish();
} else {
bundleCommand.bundle(os);
}
}
private Response createResponse(Repository repository, String fileExtension, boolean compressed, StreamingOutput
output) {
return Response
.ok(output, compressed ? "application/x-gzip" : MediaType.APPLICATION_OCTET_STREAM)
.header("content-disposition", createContentDispositionHeaderValue(repository, fileExtension))
.build();
}
private String resolveFileExtension(BundleCommandBuilder bundleCommand, boolean compressed) {
if (compressed) {
return bundleCommand.getFileExtension() + ".gz";
} else {
return bundleCommand.getFileExtension();
}
}
private String createContentDispositionHeaderValue(Repository repository, String fileExtension) {
String timestamp = createFormattedTimestamp();
@@ -244,6 +518,15 @@ public class RepositoryExportResource {
return Instant.now().toString().replace(":", "-").split("\\.")[0];
}
private ExecutorService createExportHandlerPool() {
return Executors.newCachedThreadPool(
new ThreadFactoryBuilder()
.setNameFormat("RepositoryExportHandler-%d")
.build()
);
}
private static class WrongTypeException extends BadRequestException {
private static final String CODE = "4hSNNTBiu1";
@@ -257,4 +540,12 @@ public class RepositoryExportResource {
return CODE;
}
}
@Getter
@Setter
@NoArgsConstructor
private static class ExportDto {
private String password;
private boolean async;
}
}

70
scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryImportResource.java
View File

@@ -32,8 +32,6 @@ import com.google.common.base.Strings;
import com.google.common.io.ByteSource;
import com.google.common.io.Files;
import com.google.inject.Inject;
import de.otto.edison.hal.Embedded;
import de.otto.edison.hal.Links;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
@@ -47,10 +45,12 @@ import org.jboss.resteasy.plugins.providers.multipart.MultipartFormDataInput;
import org.jboss.resteasy.plugins.providers.multipart.MultipartInputImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.ContextEntry;
import sonia.scm.HandlerEventType;
import sonia.scm.Type;
import sonia.scm.event.ScmEventBus;
import sonia.scm.importexport.FullScmRepositoryImporter;
import sonia.scm.importexport.RepositoryImportExportEncryption;
import sonia.scm.repository.InternalRepositoryException;
import sonia.scm.repository.Repository;
import sonia.scm.repository.RepositoryImportEvent;
@@ -58,16 +58,15 @@ import sonia.scm.repository.RepositoryManager;
import sonia.scm.repository.RepositoryPermission;
import sonia.scm.repository.RepositoryPermissions;
import sonia.scm.repository.api.Command;
import sonia.scm.repository.api.ImportFailedException;
import sonia.scm.repository.api.PullCommandBuilder;
import sonia.scm.repository.api.RepositoryService;
import sonia.scm.repository.api.RepositoryServiceFactory;
import sonia.scm.util.IOUtil;
import sonia.scm.util.ValidationUtil;
import sonia.scm.web.VndMediaType;
import sonia.scm.web.api.DtoValidator;
import javax.validation.Valid;
import javax.validation.constraints.Email;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.Pattern;
import javax.ws.rs.Consumes;
@@ -104,6 +103,7 @@ public class RepositoryImportResource {
private final ResourceLinks resourceLinks;
private final ScmEventBus eventBus;
private final FullScmRepositoryImporter fullScmRepositoryImporter;
private final RepositoryImportExportEncryption repositoryImportExportEncryption;
@Inject
public RepositoryImportResource(RepositoryManager manager,
@@ -111,13 +111,15 @@ public class RepositoryImportResource {
RepositoryServiceFactory serviceFactory,
ResourceLinks resourceLinks,
ScmEventBus eventBus,
FullScmRepositoryImporter fullScmRepositoryImporter) {
FullScmRepositoryImporter fullScmRepositoryImporter,
RepositoryImportExportEncryption repositoryImportExportEncryption) {
this.manager = manager;
this.mapper = mapper;
this.serviceFactory = serviceFactory;
this.resourceLinks = resourceLinks;
this.eventBus = eventBus;
this.fullScmRepositoryImporter = fullScmRepositoryImporter;
this.repositoryImportExportEncryption = repositoryImportExportEncryption;
}
/**
@@ -142,7 +144,7 @@ public class RepositoryImportResource {
description = "Repository import was successful",
content = @Content(
mediaType = VndMediaType.REPOSITORY,
schema = @Schema(implementation = ImportRepositoryDto.class)
schema = @Schema(implementation = ImportRepositoryFromUrlDto.class)
)
)
@ApiResponse(
@@ -163,7 +165,7 @@ public class RepositoryImportResource {
)
public Response importFromUrl(@Context UriInfo uriInfo,
@Pattern(regexp = "\\w{1,10}") @PathParam("type") String type,
@Valid RepositoryImportDto request) {
@Valid RepositoryImportResource.RepositoryImportFromUrlDto request) {
RepositoryPermissions.create().check();
Type t = type(manager, type);
@@ -192,7 +194,7 @@ public class RepositoryImportResource {
}
@VisibleForTesting
Consumer<Repository> pullChangesFromRemoteUrl(RepositoryImportDto request) {
Consumer<Repository> pullChangesFromRemoteUrl(RepositoryImportFromUrlDto request) {
return repository -> {
try (RepositoryService service = serviceFactory.create(repository)) {
PullCommandBuilder pullCommand = service.getPullCommand();
@@ -309,8 +311,9 @@ public class RepositoryImportResource {
private Repository importFullRepositoryFromInput(MultipartFormDataInput input) {
Map<String, List<InputPart>> formParts = input.getFormDataMap();
InputStream inputStream = extractInputStream(formParts);
RepositoryDto repositoryDto = extractRepositoryDto(formParts);
return fullScmRepositoryImporter.importFromStream(mapper.map(repositoryDto), inputStream);
RepositoryImportFromFileDto repositoryDto = extractRepositoryDto(formParts);
return fullScmRepositoryImporter.importFromStream(mapper.map(repositoryDto), inputStream, repositoryDto.getPassword());
}
/**
@@ -324,7 +327,8 @@ public class RepositoryImportResource {
private Repository doImportFromBundle(String type, MultipartFormDataInput input, boolean compressed) {
Map<String, List<InputPart>> formParts = input.getFormDataMap();
InputStream inputStream = extractInputStream(formParts);
RepositoryDto repositoryDto = extractRepositoryDto(formParts);
RepositoryImportFromFileDto repositoryDto = extractRepositoryDto(formParts);
inputStream = decryptInputStream(inputStream, repositoryDto.getPassword());
Type t = type(manager, type);
checkSupport(t, Command.UNBUNDLE);
@@ -349,6 +353,14 @@ public class RepositoryImportResource {
return repository;
}
private InputStream decryptInputStream(InputStream inputStream, String password) {
try {
return repositoryImportExportEncryption.decrypt(inputStream, password);
} catch (IOException e) {
throw new ImportFailedException(ContextEntry.ContextBuilder.noContext(), "import failed", e);
}
}
@VisibleForTesting
Consumer<Repository> unbundleImport(InputStream inputStream, boolean compressed) {
return repository -> {
@@ -370,8 +382,8 @@ public class RepositoryImportResource {
};
}
private RepositoryDto extractRepositoryDto(Map<String, List<InputPart>> formParts) {
RepositoryDto repositoryDto = extractFromInputPart(formParts.get("repository"), RepositoryDto.class);
private RepositoryImportFromFileDto extractRepositoryDto(Map<String, List<InputPart>> formParts) {
RepositoryImportFromFileDto repositoryDto = extractFromInputPart(formParts.get("repository"), RepositoryImportFromFileDto.class);
checkNotNull(repositoryDto, "repository data is required");
DtoValidator.validate(repositoryDto);
return repositoryDto;
@@ -416,32 +428,22 @@ public class RepositoryImportResource {
@Setter
@NoArgsConstructor
@SuppressWarnings("java:S2160")
public static class RepositoryImportDto extends RepositoryDto implements ImportRepositoryDto {
public static class RepositoryImportFromUrlDto extends RepositoryDto implements ImportRepositoryFromUrlDto {
@NotEmpty
private String importUrl;
private String username;
private String password;
RepositoryImportDto(Links links, Embedded embedded) {
super(links, embedded);
}
}
interface ImportRepositoryDto {
String getNamespace();
@Pattern(regexp = ValidationUtil.REGEX_REPOSITORYNAME)
String getName();
@NotEmpty
String getType();
@Email
String getContact();
String getDescription();
@Getter
@Setter
@NoArgsConstructor
@SuppressWarnings("java:S2160")
public static class RepositoryImportFromFileDto extends RepositoryDto implements ImportRepositoryFromFileDto {
private String password;
}
interface ImportRepositoryFromUrlDto extends CreateRepositoryDto {
@NotEmpty
String getImportUrl();
@@ -449,4 +451,8 @@ public class RepositoryImportResource {
String getPassword();
}
interface ImportRepositoryFromFileDto extends CreateRepositoryDto {
String getPassword();
}
}

1
scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryToRepositoryDtoMapper.java
View File

@@ -115,6 +115,7 @@ public abstract class RepositoryToRepositoryDtoMapper extends BaseMapper<Reposit
if (repositoryService.isSupported(Command.BUNDLE) && RepositoryPermissions.export(repository).isPermitted()) {
linksBuilder.single(link("export", resourceLinks.repository().export(repository.getNamespace(), repository.getName(), repository.getType())));
linksBuilder.single(link("fullExport", resourceLinks.repository().fullExport(repository.getNamespace(), repository.getName(), repository.getType())));
linksBuilder.single(link("exportInfo", resourceLinks.repository().exportInfo(repository.getNamespace(), repository.getName())));
}
if (repositoryService.isSupported(Command.TAGS)) {

8
scm-webapp/src/main/java/sonia/scm/api/v2/resources/ResourceLinks.java
View File

@@ -392,6 +392,14 @@ class ResourceLinks {
String fullExport(String namespace, String name, String type) {
return repositoryExportLinkBuilder.method("getRepositoryResource").parameters(namespace, name).method("export").parameters().method("exportFullRepository").parameters(type).href();
}
String downloadExport(String namespace, String name) {
return repositoryExportLinkBuilder.method("getRepositoryResource").parameters(namespace, name).method("export").parameters().method("downloadExport").parameters().href();
}
String exportInfo(String namespace, String name) {
return repositoryExportLinkBuilder.method("getRepositoryResource").parameters(namespace, name).method("export").parameters().method("getExportInformation").parameters().href();
}
}
RepositoryCollectionLinks repositoryCollection() {

59
scm-webapp/src/main/java/sonia/scm/importexport/ExportFileExtensionResolver.java Normal file
View File

@@ -0,0 +1,59 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.importexport;
import sonia.scm.repository.Repository;
import sonia.scm.repository.api.RepositoryService;
import sonia.scm.repository.api.RepositoryServiceFactory;
import javax.inject.Inject;
public class ExportFileExtensionResolver {
private final RepositoryServiceFactory serviceFactory;
@Inject
public ExportFileExtensionResolver(RepositoryServiceFactory serviceFactory) {
this.serviceFactory = serviceFactory;
}
public String resolve(Repository repository, boolean withMetadata, boolean compressed, boolean encrypted) {
StringBuilder builder = new StringBuilder();
if (withMetadata) {
builder.append("tar.gz");
} else {
try (RepositoryService service = serviceFactory.create(repository)) {
builder.append(service.getBundleCommand().getFileExtension());
}
if (compressed) {
builder.append(".gz");
}
}
if (encrypted) {
builder.append(".enc");
}
return builder.toString();
}
}

48
scm-webapp/src/main/java/sonia/scm/importexport/ExportGarbageCollector.java Normal file
View File

@@ -0,0 +1,48 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.importexport;
import sonia.scm.Initable;
import sonia.scm.SCMContextProvider;
import sonia.scm.schedule.Scheduler;
import javax.inject.Inject;
public class ExportGarbageCollector implements Initable {
private final ExportService exportService;
private final Scheduler scheduler;
@Inject
public ExportGarbageCollector(ExportService exportService, Scheduler scheduler) {
this.exportService = exportService;
this.scheduler = scheduler;
}
@Override
public void init(SCMContextProvider context) {
scheduler.schedule("0 0 6 * * ?", exportService::cleanupOutdatedExports);
}
}

199
scm-webapp/src/main/java/sonia/scm/importexport/ExportService.java Normal file
View File

@@ -0,0 +1,199 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.importexport;
import org.apache.shiro.SecurityUtils;
import sonia.scm.NotFoundException;
import sonia.scm.repository.Repository;
import sonia.scm.repository.RepositoryPermissions;
import sonia.scm.repository.api.ExportFailedException;
import sonia.scm.store.Blob;
import sonia.scm.store.BlobStore;
import sonia.scm.store.BlobStoreFactory;
import sonia.scm.store.DataStore;
import sonia.scm.store.DataStoreFactory;
import sonia.scm.user.User;
import javax.inject.Inject;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static sonia.scm.ContextEntry.ContextBuilder.entity;
public class ExportService {
static final String STORE_NAME = "repository-export";
private final BlobStoreFactory blobStoreFactory;
private final DataStoreFactory dataStoreFactory;
private final ExportFileExtensionResolver fileExtensionResolver;
@Inject
public ExportService(BlobStoreFactory blobStoreFactory, DataStoreFactory dataStoreFactory, ExportFileExtensionResolver fileExtensionResolver) {
this.blobStoreFactory = blobStoreFactory;
this.dataStoreFactory = dataStoreFactory;
this.fileExtensionResolver = fileExtensionResolver;
}
public OutputStream store(Repository repository, boolean withMetadata, boolean compressed, boolean encrypted) {
RepositoryPermissions.export(repository).check();
storeExportInformation(repository.getId(), withMetadata, compressed, encrypted);
try {
return storeNewBlob(repository.getId()).getOutputStream();
} catch (IOException e) {
throw new ExportFailedException(
entity(repository).build(),
"Could not store repository export to blob file",
e
);
}
}
public RepositoryExportInformation getExportInformation(Repository repository) {
RepositoryPermissions.export(repository).check();
RepositoryExportInformation info = createDataStore().get(repository.getId());
if (info == null) {
throw new NotFoundException(RepositoryExportInformation.class, repository.getId());
}
return info;
}
public InputStream getData(Repository repository) throws IOException {
RepositoryPermissions.export(repository).check();
Blob blob = getBlob(repository.getId());
if (blob == null) {
throw new NotFoundException(Blob.class, repository.getId());
}
return blob.getInputStream();
}
public void checkExportIsAvailable(Repository repository) {
RepositoryPermissions.export(repository).check();
if (createDataStore().get(repository.getId()) == null) {
throw new NotFoundException(RepositoryExportInformation.class, repository.getId());
}
}
public String getFileExtension(Repository repository) {
RepositoryPermissions.export(repository).check();
RepositoryExportInformation exportInfo = getExportInformation(repository);
return fileExtensionResolver.resolve(repository, exportInfo.isWithMetadata(), exportInfo.isCompressed(), exportInfo.isEncrypted());
}
public void clear(String repositoryId) {
RepositoryPermissions.export(repositoryId).check();
createDataStore().remove(repositoryId);
createBlobStore(repositoryId).clear();
}
public void setExportFinished(Repository repository) {
RepositoryPermissions.export(repository).check();
DataStore<RepositoryExportInformation> dataStore = createDataStore();
RepositoryExportInformation info = dataStore.get(repository.getId());
info.setStatus(ExportStatus.FINISHED);
dataStore.put(repository.getId(), info);
}
public boolean isExporting(Repository repository) {
RepositoryPermissions.export(repository).check();
RepositoryExportInformation info = createDataStore().get(repository.getId());
return info != null && info.getStatus() == ExportStatus.EXPORTING;
}
public void cleanupUnfinishedExports() {
DataStore<RepositoryExportInformation> dataStore = createDataStore();
List<Map.Entry<String, RepositoryExportInformation>> unfinishedExports = dataStore.getAll().entrySet().stream()
.filter(e -> e.getValue().getStatus() == ExportStatus.EXPORTING)
.collect(Collectors.toList());
for (Map.Entry<String, RepositoryExportInformation> export : unfinishedExports) {
createBlobStore(export.getKey()).clear();
RepositoryExportInformation info = dataStore.get(export.getKey());
info.setStatus(ExportStatus.INTERRUPTED);
dataStore.put(export.getKey(), info);
}
}
void cleanupOutdatedExports() {
DataStore<RepositoryExportInformation> dataStore = createDataStore();
List<String> outdatedExportIds = collectOutdatedExportIds(dataStore);
for (String id : outdatedExportIds) {
createBlobStore(id).clear();
}
outdatedExportIds.forEach(dataStore::remove);
}
private List<String> collectOutdatedExportIds(DataStore<RepositoryExportInformation> dataStore) {
List<String> outdatedExportIds = new ArrayList<>();
Instant expireDate = Instant.now().minus(10, ChronoUnit.DAYS);
dataStore
.getAll()
.entrySet()
.stream()
.filter(e -> e.getValue().getCreated().isBefore(expireDate))
.forEach(e -> outdatedExportIds.add(e.getKey()));
return outdatedExportIds;
}
private Blob storeNewBlob(String repositoryId) {
BlobStore store = createBlobStore(repositoryId);
if (!store.getAll().isEmpty()) {
store.clear();
}
return store.create(repositoryId);
}
private void storeExportInformation(String repositoryId, boolean withMetadata, boolean compressed, boolean encrypted) {
DataStore<RepositoryExportInformation> dataStore = createDataStore();
if (dataStore.get(repositoryId) != null) {
dataStore.remove(repositoryId);
}
String exporter = SecurityUtils.getSubject().getPrincipals().oneByType(User.class).getName();
RepositoryExportInformation info = new RepositoryExportInformation(exporter, Instant.now(), withMetadata, compressed, encrypted, ExportStatus.EXPORTING);
dataStore.put(repositoryId, info);
}
private Blob getBlob(String repositoryId) {
return createBlobStore(repositoryId).get(repositoryId);
}
private DataStore<RepositoryExportInformation> createDataStore() {
return dataStoreFactory.withType(RepositoryExportInformation.class).withName(STORE_NAME).build();
}
private BlobStore createBlobStore(String repositoryId) {
return blobStoreFactory.withName(STORE_NAME).forRepository(repositoryId).build();
}
}

31
scm-webapp/src/main/java/sonia/scm/importexport/ExportStatus.java Normal file
View File

@@ -0,0 +1,31 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.importexport;
public enum ExportStatus {
EXPORTING,
INTERRUPTED,
FINISHED
}

17
scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryExporter.java
View File

@@ -46,6 +46,7 @@ import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
public class FullScmRepositoryExporter {
static final String SCM_ENVIRONMENT_FILE_NAME = "scm-environment.xml";
@@ -57,6 +58,7 @@ public class FullScmRepositoryExporter {
private final TarArchiveRepositoryStoreExporter storeExporter;
private final WorkdirProvider workdirProvider;
private final RepositoryExportingCheck repositoryExportingCheck;
private final RepositoryImportExportEncryption repositoryImportExportEncryption;
@Inject
public FullScmRepositoryExporter(EnvironmentInformationXmlGenerator environmentGenerator,
@@ -64,28 +66,31 @@ public class FullScmRepositoryExporter {
RepositoryServiceFactory serviceFactory,
TarArchiveRepositoryStoreExporter storeExporter,
WorkdirProvider workdirProvider,
RepositoryExportingCheck repositoryExportingCheck) {
RepositoryExportingCheck repositoryExportingCheck,
RepositoryImportExportEncryption repositoryImportExportEncryption) {
this.environmentGenerator = environmentGenerator;
this.metadataGenerator = metadataGenerator;
this.serviceFactory = serviceFactory;
this.storeExporter = storeExporter;
this.workdirProvider = workdirProvider;
this.repositoryExportingCheck = repositoryExportingCheck;
this.repositoryImportExportEncryption = repositoryImportExportEncryption;
}
public void export(Repository repository, OutputStream outputStream) {
public void export(Repository repository, OutputStream outputStream, String password) {
repositoryExportingCheck.withExportingLock(repository, () -> {
exportInLock(repository, outputStream);
exportInLock(repository, outputStream, password);
return null;
});
}
private void exportInLock(Repository repository, OutputStream outputStream) {
private void exportInLock(Repository repository, OutputStream outputStream, String password) {
try (
RepositoryService service = serviceFactory.create(repository);
BufferedOutputStream bos = new BufferedOutputStream(outputStream);
GzipCompressorOutputStream gzos = new GzipCompressorOutputStream(bos);
TarArchiveOutputStream taos = Archives.createTarOutputStream(gzos)
OutputStream cos = repositoryImportExportEncryption.optionallyEncrypt(bos, password);
GzipCompressorOutputStream gzos = new GzipCompressorOutputStream(cos);
TarArchiveOutputStream taos = Archives.createTarOutputStream(gzos);
) {
writeEnvironmentData(taos);
writeMetadata(repository, taos);

14
scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryImporter.java
View File

@@ -41,6 +41,7 @@ import java.io.InputStream;
import static java.util.Arrays.stream;
import static sonia.scm.util.Archives.createTarInputStream;
import static sonia.scm.ContextEntry.ContextBuilder.noContext;
public class FullScmRepositoryImporter {
@@ -48,24 +49,27 @@ public class FullScmRepositoryImporter {
private final ImportStep[] importSteps;
private final RepositoryManager repositoryManager;
private final RepositoryImportExportEncryption repositoryImportExportEncryption;
@Inject
public FullScmRepositoryImporter(EnvironmentCheckStep environmentCheckStep,
MetadataImportStep metadataImportStep,
StoreImportStep storeImportStep,
RepositoryImportStep repositoryImportStep,
RepositoryManager repositoryManager
) {
RepositoryManager repositoryManager,
RepositoryImportExportEncryption repositoryImportExportEncryption) {
this.repositoryManager = repositoryManager;
this.repositoryImportExportEncryption = repositoryImportExportEncryption;
importSteps = new ImportStep[]{environmentCheckStep, metadataImportStep, storeImportStep, repositoryImportStep};
}
public Repository importFromStream(Repository repository, InputStream inputStream) {
public Repository importFromStream(Repository repository, InputStream inputStream, String password) {
try {
if (inputStream.available() > 0) {
try (
BufferedInputStream bif = new BufferedInputStream(inputStream);
GzipCompressorInputStream gcis = new GzipCompressorInputStream(bif);
InputStream cif = repositoryImportExportEncryption.decrypt(bif, password);
GzipCompressorInputStream gcis = new GzipCompressorInputStream(cif);
TarArchiveInputStream tais = createTarInputStream(gcis)
) {
return run(repository, tais);
@@ -78,7 +82,7 @@ public class FullScmRepositoryImporter {
}
} catch (IOException e) {
throw new ImportFailedException(
ContextEntry.ContextBuilder.entity(repository).build(),
noContext(),
"Could not import repository data from stream; got io exception while reading",
e
);

55
scm-webapp/src/main/java/sonia/scm/importexport/RepositoryExportInformation.java Normal file
View File

@@ -0,0 +1,55 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.importexport;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import sonia.scm.xml.XmlInstantAdapter;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import java.time.Instant;
@NoArgsConstructor
@AllArgsConstructor
@Getter
@Setter
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
public class RepositoryExportInformation {
private String exporterName;
@XmlJavaTypeAdapter(XmlInstantAdapter.class)
private Instant created;
private boolean withMetadata;
private boolean compressed;
private boolean encrypted;
private ExportStatus status;
}

195
scm-webapp/src/main/java/sonia/scm/importexport/RepositoryImportExportEncryption.java Normal file
View File

@@ -0,0 +1,195 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.importexport;
import com.google.common.base.Strings;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.crypto.Cipher;
import javax.crypto.CipherInputStream;
import javax.crypto.CipherOutputStream;
import javax.crypto.NoSuchPaddingException;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.PBEKeySpec;
import javax.crypto.spec.SecretKeySpec;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.spec.InvalidKeySpecException;
import java.security.spec.KeySpec;
import java.util.Arrays;
public class RepositoryImportExportEncryption {
private static final SecureRandom SECURE_RANDOM = new SecureRandom();
private static final String ENCRYPTION_HEADER = "SCMM_v1_";
private static final Charset ENCRYPTION_HEADER_CHARSET = StandardCharsets.ISO_8859_1;
static {
SECURE_RANDOM.setSeed(System.currentTimeMillis());
}
/**
* Returns an encrypting stream for the given origin stream, if a not-empty secret is given. Otherwise
* the original stream is returned. That is, this delegates to {@link #encrypt(OutputStream, String)} if
* a secret is given.
* @param origin The stream that should be encrypted, when a not-empty secret is given.
* @param secret The secret to use or <code>null</code> or an empty string, if no encryption should be used.
* @return An encrypted stream or <code>origin</code>, when no secret is given.
*/
public OutputStream optionallyEncrypt(OutputStream origin, @Nullable String secret) throws IOException {
if (!Strings.isNullOrEmpty(secret)) {
return encrypt(origin, secret);
} else {
return origin;
}
}
/**
* Encrypts the given stream with the given secret.
*/
public OutputStream encrypt(OutputStream origin, @Nonnull String secret) throws IOException {
byte[] salt = createSalt();
writeSaltHeader(origin, salt);
Cipher cipher = createCipher(Cipher.ENCRYPT_MODE, secret, salt);
return new CipherOutputStream(origin, cipher);
}
/**
* Returns a decrypting stream for the given input stream, if a not-empty secret is given. Otherwise
* the original stream is returned. That is, this delegated to {@link #decrypt(InputStream, String)} if
* a secret is given.
* @param stream The stream that should be decrypted, when a not-empty secret is given.
* @param secret The secret to use or <code>null</code> or an empty string, if no decryption should take place.
* @return A decrypted stream or <code>stream</code>, when no secret is given.
*/
public InputStream optionallyDecrypt(InputStream stream, @Nullable String secret) throws IOException {
if (!Strings.isNullOrEmpty(secret)) {
return decrypt(stream, secret);
} else {
return stream;
}
}
/**
* Decrypts the given stream with the given secret.
*/
public InputStream decrypt(InputStream encryptedStream, @Nonnull String secret) throws IOException {
byte[] salt = readSaltHeader(encryptedStream);
Cipher cipher = createCipher(Cipher.DECRYPT_MODE, secret, salt);
return new CipherInputStream(encryptedStream, cipher);
}
private void writeSaltHeader(OutputStream origin, byte[] salt) throws IOException {
origin.write(ENCRYPTION_HEADER.getBytes(ENCRYPTION_HEADER_CHARSET));
origin.write(salt);
}
private byte[] readSaltHeader(InputStream encryptedStream) throws IOException {
byte[] header = new byte[8];
int headerBytesRead = encryptedStream.read(header);
if (headerBytesRead != 8 || !ENCRYPTION_HEADER.equals(new String(header, ENCRYPTION_HEADER_CHARSET))) {
throw new IOException("Expected header with salt not found (\"Salted__\")");
}
byte[] salt = new byte[8];
int lengthRead = encryptedStream.read(salt);
if (lengthRead != 8) {
throw new IOException("Failed to read salt from input");
}
return salt;
}
private Cipher createCipher(int encryptMode, String key, byte[] salt) {
Cipher cipher = getCipher();
try {
cipher.init(encryptMode, getSecretKeySpec(key.toCharArray(), salt), getIvSpec(key.toCharArray(), salt));
} catch (InvalidKeyException | InvalidAlgorithmParameterException e) {
throw new IllegalStateException("Could not initialize cipher", e);
}
return cipher;
}
private Cipher getCipher() {
try {
return Cipher.getInstance("AES/CBC/PKCS5Padding");
} catch (NoSuchAlgorithmException | NoSuchPaddingException e) {
throw new IllegalStateException("Could not initialize cipher", e);
}
}
private byte[] createSalt() {
byte[] salt = new byte[8];
SECURE_RANDOM.nextBytes(salt);
return salt;
}
private SecretKeySpec getSecretKeySpec(char[] key, byte[] salt) {
SecretKey secretKey = computeSecretKey(key, salt);
return new SecretKeySpec(secretKey.getEncoded(), "AES");
}
private SecretKey computeSecretKey(char[] password, byte[] salt) {
KeySpec spec = getKeySpec(password, salt);
try {
return new SecretKeySpec(getSecretKeyFactory().generateSecret(spec).getEncoded(), "AES");
} catch (InvalidKeySpecException e) {
throw new IllegalStateException("could not create key spec", e);
}
}
private PBEKeySpec getKeySpec(char[] password, byte[] salt) {
return new PBEKeySpec(password, salt, 10000, 256);
}
@SuppressWarnings("java:S3329") // we generate the IV by deriving it from the password; this should be pseudo random enough
private IvParameterSpec getIvSpec(char[] password, byte[] salt) {
PBEKeySpec spec = new PBEKeySpec(password, salt, 1000, 256);
try {
byte[] bytes = getSecretKeyFactory().generateSecret(spec).getEncoded();
byte[] iv = Arrays.copyOfRange(bytes, 16, 16 + 16);
return new IvParameterSpec(iv);
} catch (InvalidKeySpecException e) {
throw new IllegalStateException("Could not derive from key", e);
}
}
private SecretKeyFactory getSecretKeyFactory() {
try {
return SecretKeyFactory.getInstance("PBKDF2WithHmacSHA256");
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("Could not instantiate secret key factory", e);
}
}
}

7
scm-webapp/src/main/java/sonia/scm/lifecycle/SetupContextListener.java
View File

@@ -32,6 +32,7 @@ import sonia.scm.SCMContext;
import sonia.scm.config.ScmConfiguration;
import sonia.scm.group.Group;
import sonia.scm.group.GroupManager;
import sonia.scm.importexport.ExportService;
import sonia.scm.plugin.Extension;
import sonia.scm.security.AnonymousMode;
import sonia.scm.security.PermissionAssigner;
@@ -80,17 +81,19 @@ public class SetupContextListener implements ServletContextListener {
private final PermissionAssigner permissionAssigner;
private final ScmConfiguration scmConfiguration;
private final GroupManager groupManager;
private final ExportService exportService;
@VisibleForTesting
static final String AUTHENTICATED_GROUP_DESCRIPTION = "Includes all authenticated users";
@Inject
public SetupAction(UserManager userManager, PasswordService passwordService, PermissionAssigner permissionAssigner, ScmConfiguration scmConfiguration, GroupManager groupManager) {
public SetupAction(UserManager userManager, PasswordService passwordService, PermissionAssigner permissionAssigner, ScmConfiguration scmConfiguration, GroupManager groupManager, ExportService exportService) {
this.userManager = userManager;
this.passwordService = passwordService;
this.permissionAssigner = permissionAssigner;
this.scmConfiguration = scmConfiguration;
this.groupManager = groupManager;
this.exportService = exportService;
}
@Override
@@ -105,6 +108,8 @@ public class SetupContextListener implements ServletContextListener {
if (authenticatedGroupDoesNotExists()) {
createAuthenticatedGroup();
}
exportService.cleanupUnfinishedExports();
}
private boolean anonymousUserRequiredButNotExists() {