Nemcio/SCM-Manager
1
0
Fork 0
mirror of https://github.com/scm-manager/scm-manager.git synced 2025-11-09 06:55:47 +01:00
Code Issues Packages Projects Releases Activity

Git import with lfs support (#2133)

Browse Source 
This adds the possibility to load files managed by lfs to the repository import of git repositories.

Co-authored-by: Konstantin Schaper <konstantin.schaper@cloudogu.com>
This commit is contained in:
René Pfeuffer
2022-10-25 09:14:40 +02:00
committed by GitHub
parent 96ce4cb8e6
commit 54081ccdc6
33 changed files with 673 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitMirrorCommand.java
View File

@@ -60,8 +60,8 @@ import sonia.scm.repository.api.MirrorCommandResult.ResultType;
import sonia.scm.repository.api.MirrorFilter;
import sonia.scm.repository.api.MirrorFilter.Result;
import sonia.scm.repository.api.UsernamePasswordCredential;
import sonia.scm.repository.spi.LfsLoader.LfsLoaderLogger;
import sonia.scm.store.ConfigurationStore;
import sonia.scm.web.lfs.LfsBlobStoreFactory;
import javax.inject.Inject;
import java.io.IOException;
@@ -367,7 +367,16 @@ public class GitMirrorCommand extends AbstractGitCommand implements MirrorComman
logger.logChange(ref, referenceName, getUpdateType(ref));
if (!mirrorCommandRequest.isIgnoreLfs()) {
lfsLoader.inspectTree(ref.getNewObjectId(), mirrorCommandRequest, git.getRepository(), mirrorLog, lfsUpdateResult, repository);
LfsLoaderLogger lfsLoaderLogger = new MirrorLfsLoaderLogger();
lfsLoader.inspectTree(
ref.getNewObjectId(),
git.getRepository(),
lfsLoaderLogger,
lfsUpdateResult,
repository,
mirrorHttpConnectionProvider.createHttpConnectionFactory(mirrorCommandRequest, mirrorLog),
mirrorCommandRequest.getSourceUrl()
);
}
}
}
@@ -406,6 +415,19 @@ public class GitMirrorCommand extends AbstractGitCommand implements MirrorComman
private String getUpdateType(TrackingRefUpdate trackingRefUpdate) {
return trackingRefUpdate.getResult().name().toLowerCase(Locale.ENGLISH);
}
private class MirrorLfsLoaderLogger implements LfsLoaderLogger {
@Override
public void failed(Exception e) {
mirrorLog.add("Failed to load lfs file:");
mirrorLog.add(e.getMessage());
}
@Override
public void loading(String name) {
mirrorLog.add(String.format("Loading lfs file with id '%s'", name));
}
}
}
private class LoggerWithHeader {
@@ -653,7 +675,7 @@ public class GitMirrorCommand extends AbstractGitCommand implements MirrorComman
}
private boolean isOfTypeOrEmpty(Optional<MirrorFilter.UpdateType> updateType, MirrorFilter.UpdateType type) {
return !updateType.isPresent() || updateType.get() == type;
return updateType.isEmpty() || updateType.get() == type;
}
private Optional<MirrorFilter.UpdateType> getUpdateTypeFor(ReceiveCommand receiveCommand) {

58
scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitPullCommand.java
View File

@@ -41,6 +41,7 @@ import sonia.scm.repository.GitRepositoryHandler;
import sonia.scm.repository.GitUtil;
import sonia.scm.repository.Repository;
import sonia.scm.repository.api.ImportFailedException;
import sonia.scm.repository.api.MirrorCommandResult;
import sonia.scm.repository.api.PullResponse;
import javax.inject.Inject;
@@ -54,14 +55,21 @@ public class GitPullCommand extends AbstractGitPushOrPullCommand
implements PullCommand {
private static final Logger LOG = LoggerFactory.getLogger(GitPullCommand.class);
private final PostReceiveRepositoryHookEventFactory postReceiveRepositoryHookEventFactory;
private final LfsLoader lfsLoader;
private final PullHttpConnectionProvider pullHttpConnectionProvider;
@Inject
public GitPullCommand(GitRepositoryHandler handler,
GitContext context,
PostReceiveRepositoryHookEventFactory postReceiveRepositoryHookEventFactory) {
PostReceiveRepositoryHookEventFactory postReceiveRepositoryHookEventFactory,
LfsLoader lfsLoader,
PullHttpConnectionProvider pullHttpConnectionProvider) {
super(handler, context);
this.postReceiveRepositoryHookEventFactory = postReceiveRepositoryHookEventFactory;
this.lfsLoader = lfsLoader;
this.pullHttpConnectionProvider = pullHttpConnectionProvider;
}
@Override
@@ -81,7 +89,7 @@ public class GitPullCommand extends AbstractGitPushOrPullCommand
return response;
}
private PullResponse convert(Git git, FetchResult fetch) {
private PullResponse convert(Git git, FetchResult fetch, CountingLfsLoaderLogger lfsLoaderLogger) {
long counter = 0;
for (TrackingRefUpdate tru : fetch.getTrackingRefUpdates()) {
@@ -90,7 +98,7 @@ public class GitPullCommand extends AbstractGitPushOrPullCommand
LOG.debug("received {} changesets by pull", counter);
return new PullResponse(counter);
return new PullResponse(counter, new PullResponse.LfsCount(lfsLoaderLogger.getSuccessCount(), lfsLoaderLogger.getFailureCount()));
}
private long count(Git git, TrackingRefUpdate tru) {
@@ -178,7 +186,11 @@ public class GitPullCommand extends AbstractGitPushOrPullCommand
.call();
//J+
response = convert(git, result);
CountingLfsLoaderLogger lfsLoaderLogger = new CountingLfsLoaderLogger();
if (request.isFetchLfs()) {
fetchLfs(request, git, lfsLoaderLogger);
}
response = convert(git, result, lfsLoaderLogger);
} catch
(GitAPIException ex) {
throw new ImportFailedException(
@@ -193,7 +205,45 @@ public class GitPullCommand extends AbstractGitPushOrPullCommand
return response;
}
private void fetchLfs(PullCommandRequest request, Git git, LfsLoader.LfsLoaderLogger lfsLoaderLogger) throws IOException {
open().getRefDatabase().getRefs().forEach(
ref -> lfsLoader.inspectTree(
ref.getObjectId(),
git.getRepository(),
lfsLoaderLogger,
new MirrorCommandResult.LfsUpdateResult(),
repository,
pullHttpConnectionProvider.createHttpConnectionFactory(request),
request.getRemoteUrl().toString()
)
);
}
private void firePostReceiveRepositoryHookEvent(Git git, FetchResult result) {
postReceiveRepositoryHookEventFactory.fireForFetch(git, result);
}
private static class CountingLfsLoaderLogger implements LfsLoader.LfsLoaderLogger {
private int successCount = 0;
private int failureCount = 0;
@Override
public void failed(Exception e) {
++failureCount;
}
@Override
public void loading(String name) {
++successCount;
}
public int getSuccessCount() {
return successCount;
}
public int getFailureCount() {
return failureCount;
}
}
}

58
scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/LfsLoader.java
View File

@@ -50,34 +50,39 @@ import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collection;
import java.util.List;
class LfsLoader {
private static final Logger LOG = LoggerFactory.getLogger(LfsLoader.class);
private final LfsBlobStoreFactory lfsBlobStoreFactory;
private final MirrorHttpConnectionProvider mirrorHttpConnectionProvider;
@Inject
LfsLoader(LfsBlobStoreFactory lfsBlobStoreFactory, MirrorHttpConnectionProvider mirrorHttpConnectionProvider) {
LfsLoader(LfsBlobStoreFactory lfsBlobStoreFactory) {
this.lfsBlobStoreFactory = lfsBlobStoreFactory;
this.mirrorHttpConnectionProvider = mirrorHttpConnectionProvider;
}
void inspectTree(ObjectId newObjectId,
MirrorCommandRequest mirrorCommandRequest,
Repository gitRepository,
List<String> mirrorLog,
LfsLoaderLogger mirrorLog,
LfsUpdateResult lfsUpdateResult,
sonia.scm.repository.Repository repository) {
String mirrorUrl = mirrorCommandRequest.getSourceUrl();
EntryHandler entryHandler = new EntryHandler(repository, gitRepository, mirrorCommandRequest, mirrorLog, lfsUpdateResult);
sonia.scm.repository.Repository repository,
HttpConnectionFactory httpConnectionFactory,
String url) {
EntryHandler entryHandler = new EntryHandler(repository, gitRepository, mirrorLog, lfsUpdateResult, httpConnectionFactory);
inspectTree(newObjectId, entryHandler, gitRepository, mirrorLog, lfsUpdateResult, url);
}
private void inspectTree(ObjectId newObjectId,
EntryHandler entryHandler,
Repository gitRepository,
LfsLoaderLogger mirrorLog,
LfsUpdateResult lfsUpdateResult,
String sourceUrl) {
try {
gitRepository
.getConfig()
.setString(ConfigConstants.CONFIG_SECTION_LFS, null, ConfigConstants.CONFIG_KEY_URL, computeLfsUrl(mirrorUrl));
.setString(ConfigConstants.CONFIG_SECTION_LFS, null, ConfigConstants.CONFIG_KEY_URL, computeLfsUrl(sourceUrl));
TreeWalk treeWalk = new TreeWalk(gitRepository);
treeWalk.setFilter(new LfsPointerFilter());
@@ -94,17 +99,16 @@ class LfsLoader {
}
} catch (Exception e) {
LOG.warn("failed to load lfs files", e);
mirrorLog.add("Failed to load lfs files:");
mirrorLog.add(e.getMessage());
mirrorLog.failed(e);
lfsUpdateResult.increaseFailureCount();
}
}
private String computeLfsUrl(String mirrorUrl) {
if (mirrorUrl.endsWith(".git")) {
return mirrorUrl + Protocol.INFO_LFS_ENDPOINT;
private String computeLfsUrl(String sourceUrl) {
if (sourceUrl.endsWith(".git")) {
return sourceUrl + Protocol.INFO_LFS_ENDPOINT;
} else {
return mirrorUrl + ".git" + Protocol.INFO_LFS_ENDPOINT;
return sourceUrl + ".git" + Protocol.INFO_LFS_ENDPOINT;
}
}
@@ -112,22 +116,22 @@ class LfsLoader {
private final BlobStore lfsBlobStore;
private final Repository gitRepository;
private final List<String> mirrorLog;
private final LfsLoaderLogger mirrorLog;
private final LfsUpdateResult lfsUpdateResult;
private final sonia.scm.repository.Repository repository;
private final HttpConnectionFactory httpConnectionFactory;
private EntryHandler(sonia.scm.repository.Repository repository,
Repository gitRepository,
MirrorCommandRequest mirrorCommandRequest,
List<String> mirrorLog,
LfsUpdateResult lfsUpdateResult) {
LfsLoaderLogger mirrorLog,
LfsUpdateResult lfsUpdateResult,
HttpConnectionFactory httpConnectionFactory) {
this.lfsBlobStore = lfsBlobStoreFactory.getLfsBlobStore(repository);
this.repository = repository;
this.gitRepository = gitRepository;
this.mirrorLog = mirrorLog;
this.lfsUpdateResult = lfsUpdateResult;
this.httpConnectionFactory = mirrorHttpConnectionProvider.createHttpConnectionFactory(mirrorCommandRequest, mirrorLog);
this.httpConnectionFactory = httpConnectionFactory;
}
private void handleTreeEntry(TreeWalk treeWalk) {
@@ -142,8 +146,7 @@ class LfsLoader {
}
} catch (Exception e) {
LOG.warn("failed to load lfs file", e);
mirrorLog.add("Failed to load lfs file:");
mirrorLog.add(e.getMessage());
mirrorLog.failed(e);
lfsUpdateResult.increaseFailureCount();
}
}
@@ -151,7 +154,7 @@ class LfsLoader {
private Path loadLfsFile(LfsPointer lfsPointer) throws IOException {
lfsUpdateResult.increaseOverallCount();
LOG.trace("trying to load lfs file '{}' for repository {}", lfsPointer.getOid(), repository);
mirrorLog.add(String.format("Loading lfs file with id '%s'", lfsPointer.getOid().name()));
mirrorLog.loading(lfsPointer.getOid().name());
Lfs lfs = new Lfs(gitRepository);
lfs.getMediaFile(lfsPointer.getOid());
@@ -174,4 +177,11 @@ class LfsLoader {
);
}
}
interface LfsLoaderLogger {
void failed(Exception e);
void loading(String name);
}
}

1
scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/MirrorHttpConnectionProvider.java
View File

@@ -24,7 +24,6 @@
package sonia.scm.repository.spi;
import org.eclipse.jgit.transport.UserAgent;
import org.eclipse.jgit.transport.http.HttpConnectionFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

55
scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/PullHttpConnectionProvider.java Normal file
View File

@@ -0,0 +1,55 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.repository.spi;
import com.google.common.base.Strings;
import org.eclipse.jgit.transport.http.HttpConnectionFactory;
import sonia.scm.net.HttpConnectionOptions;
import sonia.scm.net.HttpURLConnectionFactory;
import sonia.scm.web.ScmHttpConnectionFactory;
import javax.inject.Inject;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
class PullHttpConnectionProvider {
private final HttpURLConnectionFactory httpURLConnectionFactory;
@Inject
PullHttpConnectionProvider(HttpURLConnectionFactory httpURLConnectionFactory) {
this.httpURLConnectionFactory = httpURLConnectionFactory;
}
HttpConnectionFactory createHttpConnectionFactory(PullCommandRequest request) {
HttpConnectionOptions options = new HttpConnectionOptions();
if (!Strings.isNullOrEmpty(request.getUsername()) && !Strings.isNullOrEmpty(request.getPassword())) {
String encodedAuth = Base64.getEncoder().encodeToString((request.getUsername() + ":" + request.getPassword()).getBytes(StandardCharsets.UTF_8));
String authHeaderValue = "Basic " + encodedAuth;
options.addRequestProperty("Authorization", authHeaderValue);
}
return new ScmHttpConnectionFactory(httpURLConnectionFactory, options);
}
}

8
scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitIncomingCommandTest.java
View File

@@ -40,6 +40,7 @@ import java.io.IOException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.mockito.Mockito.mock;
//~--- JDK imports ------------------------------------------------------------
@@ -49,6 +50,9 @@ import static org.junit.Assert.assertNotNull;
public class GitIncomingCommandTest
extends AbstractRemoteCommandTestBase {
private final LfsLoader lfsLoader = mock(LfsLoader.class);
private final PullHttpConnectionProvider pullHttpConnectionProvider = mock(PullHttpConnectionProvider.class);
/**
* Method description
*
@@ -99,7 +103,9 @@ public class GitIncomingCommandTest
GitPullCommand pull = new GitPullCommand(
handler,
context,
postReceiveRepositoryHookEventFactory);
postReceiveRepositoryHookEventFactory,
lfsLoader,
pullHttpConnectionProvider);
PullCommandRequest req = new PullCommandRequest();
req.setRemoteRepository(outgoingRepository);
pull.pull(req);

11
scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitMirrorCommandTest.java
View File

@@ -858,18 +858,17 @@ public class GitMirrorCommandTest extends AbstractGitCommandTestBase {
// one revision is missing here ("fcd0ef1831e4002ac43ea539f4094334c79ea9ec"), because this is iterated twice, what is hard to test
}).forEach(expectedRevision ->
verify(lfsLoader)
.inspectTree(eq(ObjectId.fromString(expectedRevision)), any(), any(), any(), any(), eq(repository)));
.inspectTree(eq(ObjectId.fromString(expectedRevision)), any(), any(), any(), eq(repository), any(), any()));
}
@Test
public void shouldMarkMirrorAsFailedIfLfsFileFailes() {
public void shouldMarkMirrorAsFailedIfLfsFileFails() {
doAnswer(invocation -> {
invocation.getArgument(4, MirrorCommandResult.LfsUpdateResult.class).increaseFailureCount();
invocation.getArgument(3, MirrorCommandResult.LfsUpdateResult.class).increaseFailureCount();
return null;
})
.when(lfsLoader)
.inspectTree(eq(ObjectId.fromString("a8495c0335a13e6e432df90b3727fa91943189a7")), any(), any(), any(), any(), eq(repository));
.inspectTree(eq(ObjectId.fromString("a8495c0335a13e6e432df90b3727fa91943189a7")), any(), any(), any(), eq(repository), any(), any());
MirrorCommandResult mirrorCommandResult = callMirrorCommand();
@@ -881,7 +880,7 @@ public class GitMirrorCommandTest extends AbstractGitCommandTestBase {
callMirrorCommand(repositoryDirectory.getAbsolutePath(), c -> c.setIgnoreLfs(true));
verify(lfsLoader, never())
.inspectTree(any(), any(), any(), any(), any(), any());
.inspectTree(any(), any(), any(), any(), any(), any(), any());
}
public static class DefaultBranchSelectorTest {

10
scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModificationsCommandTest.java
View File

@@ -30,6 +30,7 @@ import org.eclipse.jgit.revwalk.RevCommit;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import sonia.scm.repository.GitConfig;
@@ -49,6 +50,11 @@ public class GitModificationsCommandTest extends AbstractRemoteCommandTestBase {
private GitModificationsCommand incomingModificationsCommand;
private GitModificationsCommand outgoingModificationsCommand;
@Mock
private LfsLoader lfsLoader;
@Mock
private PullHttpConnectionProvider pullHttpConnectionProvider;
@Before
public void init() {
incomingModificationsCommand = new GitModificationsCommand(Mockito.spy(new GitContext(incomingDirectory, incomingRepository, null, new GitConfig())));
@@ -171,7 +177,9 @@ public class GitModificationsCommandTest extends AbstractRemoteCommandTestBase {
GitPullCommand pullCommand = new GitPullCommand(
handler,
context,
postReceiveRepositoryHookEventFactory);
postReceiveRepositoryHookEventFactory,
lfsLoader,
pullHttpConnectionProvider);
PullCommandRequest pullRequest = new PullCommandRequest();
pullRequest.setRemoteRepository(incomingRepository);
pullCommand.pull(pullRequest);