Mirror LFS files for git (#2075)

If a mirrored git repository uses LFS, SCM-Manager will now also load the binaries, so that the mirrored repository can be used without missing LFS files.

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitMirrorCommandTest.java

@@ -64,6 +64,7 @@ import javax.net.ssl.TrustManager;
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
 import java.util.function.Consumer;
@@ -74,9 +75,12 @@ import static java.util.Collections.singletonList;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertThrows;
 import static org.mockito.AdditionalMatchers.not;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.argThat;
 import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 import static sonia.scm.repository.api.MirrorCommandResult.ResultType.FAILED;
@@ -104,6 +108,8 @@ public class GitMirrorCommandTest extends AbstractGitCommandTestBase {
 
   private final GitRepositoryConfigStoreProvider storeProvider = mock(GitRepositoryConfigStoreProvider.class);
   private final ConfigurationStore<GitRepositoryConfig> configurationStore = mock(ConfigurationStore.class);
+  private final LfsLoader lfsLoader = mock(LfsLoader.class);
+
   private final GitRepositoryConfig gitRepositoryConfig = new GitRepositoryConfig();
 
   @Before
@@ -136,11 +142,12 @@ public class GitMirrorCommandTest extends AbstractGitCommandTestBase {
       gitTagConverter,
       workingCopyFactory,
       gitHeadModifier,
-      storeProvider);
+      storeProvider,
+      lfsLoader);
   }
 
   @Before
-  public void initializeStore() {
+  public void initializeStores() {
     when(storeProvider.get(repository)).thenReturn(configurationStore);
     when(configurationStore.get()).thenReturn(gitRepositoryConfig);
   }
@@ -833,6 +840,50 @@ public class GitMirrorCommandTest extends AbstractGitCommandTestBase {
     }));
   }
 
+  @Test
+  public void shouldCallLfsLoader() {
+    callMirrorCommand();
+
+    Arrays.stream(new String[] {
+      "a8495c0335a13e6e432df90b3727fa91943189a7",
+      "d81ad6c63d7e2162308d69637b339dedd1d9201c",
+      "2f95f02d9c568594d31e78464bd11a96c62e3f91",
+      "91b99de908fcd04772798a31c308a64aea1a5523",
+      "03ca33468c2094249973d0ca11b80243a20de368",
+      "1fcebf45a215a43f0713a57b807d55e8387a6d70",
+      "383b954b27e052db6880d57f1c860dc208795247",
+      "35597e9e98fe53167266583848bfef985c2adb27",
+      "3f76a12f08a6ba0dc988c68b7f0b2cd190efc3c4",
+      "86a6645eceefe8b9a247db5eb16e3d89a7e6e6d1"
+      // one revision is missing here ("fcd0ef1831e4002ac43ea539f4094334c79ea9ec"), because this is iterated twice, what is hard to test
+    }).forEach(expectedRevision ->
+      verify(lfsLoader)
+      .inspectTree(eq(ObjectId.fromString(expectedRevision)), any(), any(), any(), any(), eq(repository)));
+  }
+
+  @Test
+  public void shouldMarkMirrorAsFailedIfLfsFileFailes() {
+    doAnswer(invocation -> {
+      invocation.getArgument(4, MirrorCommandResult.LfsUpdateResult.class).increaseFailureCount();
+      return null;
+    })
+      .when(lfsLoader)
+      .inspectTree(eq(ObjectId.fromString("a8495c0335a13e6e432df90b3727fa91943189a7")), any(), any(), any(), any(), eq(repository));
+
+
+    MirrorCommandResult mirrorCommandResult = callMirrorCommand();
+
+    assertThat(mirrorCommandResult.getResult()).isEqualTo(FAILED);
+  }
+
+  @Test
+  public void shouldNotCallLfsLoaderIfDeactivated() {
+    callMirrorCommand(repositoryDirectory.getAbsolutePath(), c -> c.setIgnoreLfs(true));
+
+    verify(lfsLoader, never())
+      .inspectTree(any(), any(), any(), any(), any(), any());
+  }
+
   public static class DefaultBranchSelectorTest {
 
     public static final List<String> BRANCHES = asList("master", "one", "two", "three");

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/MirrorHttpConnectionProviderTest.java

@@ -36,6 +36,7 @@ import org.mockito.junit.jupiter.MockitoExtension;
 import sonia.scm.net.HttpConnectionOptions;
 import sonia.scm.net.HttpURLConnectionFactory;
 import sonia.scm.net.ProxyConfiguration;
+import sonia.scm.repository.api.SimpleUsernamePasswordCredential;
 
 import java.io.IOException;
 import java.net.URL;
@@ -79,6 +80,25 @@ class MirrorHttpConnectionProviderTest {
     assertThat(value.getProxyConfiguration()).containsSame(proxy);
   }
 
+  @Test
+  void shouldConfigureAuthorizationHeader() throws IOException {
+    MirrorCommandRequest request = new MirrorCommandRequest();
+    request.setCredentials(List.of(new SimpleUsernamePasswordCredential("dent", "yellow".toCharArray())));
+
+    HttpConnectionOptions value = create(request);
+
+    assertThat(value.getConnectionProperties()).containsEntry("Authorization", "Basic ZGVudDp5ZWxsb3c=");
+  }
+
+  @Test
+  void shouldSetUserAgentHeader() throws IOException {
+    MirrorCommandRequest request = new MirrorCommandRequest();
+
+    HttpConnectionOptions value = create(request);
+
+    assertThat(value.getConnectionProperties()).containsEntry("User-Agent", "git-lfs/2");
+  }
+
   private HttpConnectionOptions create(MirrorCommandRequest request) throws IOException {
     List<String> log = new ArrayList<>();
 

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/GitLfsLockApiDetectorTest.java

@@ -0,0 +1,79 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package sonia.scm.web;
+
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import javax.servlet.http.HttpServletRequest;
+import java.util.stream.Stream;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+class GitLfsLockApiDetectorTest {
+
+  @Mock
+  private HttpServletRequest request;
+
+  private static Stream<Arguments> testParameters() {
+    return Stream.of(
+      Arguments.of("text/html, image/gif, image/jpeg", false),
+      Arguments.of("text/html, image/gif, image/jpeg, *; q=.2, */*; q=.2", false),
+      Arguments.of("*", false),
+      Arguments.of("application/vnd.git-lfs+json; charset=utf-8", true),
+      Arguments.of("application/vnd.git-lfs+json", true)
+    );
+  }
+
+  @ParameterizedTest
+  @MethodSource("testParameters")
+  void shouldHandleContentTypeHeaderCorrectly(String headerValue, boolean expected) {
+    when(request.getHeader("Content-Type"))
+      .thenReturn(headerValue);
+
+    boolean result = new GitLfsLockApiDetector().isScmClient(request, null);
+
+    assertThat(result).isEqualTo(expected);
+  }
+
+  @ParameterizedTest
+  @MethodSource("testParameters")
+  void shouldHandleAcceptHeaderCorrectly(String headerValue, boolean expected) {
+    when(request.getHeader("Content-Type"))
+      .thenReturn(null);
+    when(request.getHeader("Accept"))
+      .thenReturn(headerValue);
+
+    boolean result = new GitLfsLockApiDetector().isScmClient(request, null);
+
+    assertThat(result).isEqualTo(expected);
+  }
+}

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/GitLfsObjectApiDetectorTest.java

@@ -0,0 +1,63 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package sonia.scm.web;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import javax.servlet.http.HttpServletRequest;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+class GitLfsObjectApiDetectorTest {
+
+  @Mock
+  private HttpServletRequest request;
+
+  @Test
+  void shouldAcceptObjectRequest() {
+    when(request.getRequestURI())
+      .thenReturn("/scm/repo/scmadmin/lfs.git/info/lfs/objects/abc123");
+
+    boolean result = new GitLfsObjectApiDetector().isScmClient(request, null);
+
+    assertThat(result).isTrue();
+  }
+
+  @Test
+  void shouldRejectFakeObjectRequest() {
+    when(request.getRequestURI())
+      .thenReturn("/scm/repo/scmadmin/lfs.git/code/info/lfs/objects/abc123");
+
+    boolean result = new GitLfsObjectApiDetector().isScmClient(request, null);
+
+    assertThat(result).isFalse();
+  }
+}