This commit is contained in:
Eduard Heimbuch
2019-10-11 11:22:24 +02:00
24 changed files with 806 additions and 141 deletions

View File

@@ -0,0 +1,77 @@
package sonia.scm.repository.spi;
import com.google.common.io.ByteStreams;
import org.eclipse.jgit.attributes.FilterCommand;
import org.eclipse.jgit.attributes.FilterCommandRegistry;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.util.FS;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.SCMContextProvider;
import sonia.scm.plugin.Extension;
import javax.inject.Inject;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.regex.Pattern;
import static java.nio.file.StandardOpenOption.CREATE;
import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
@Extension
public class GitLfsFilterContextListener implements ServletContextListener {
public static final String GITCONFIG = "[filter \"lfs\"]\n" +
"clean = git-lfs clean -- %f\n" +
"smudge = git-lfs smudge -- %f\n" +
"process = git-lfs filter-process\n" +
"required = true\n";
public static final Pattern COMMAND_NAME_PATTERN = Pattern.compile("git-lfs (smudge|clean) -- .*");
private static final Logger LOG = LoggerFactory.getLogger(GitLfsFilterContextListener.class);
private final SCMContextProvider contextProvider;
@Inject
public GitLfsFilterContextListener(SCMContextProvider contextProvider) {
this.contextProvider = contextProvider;
}
@Override
public void contextInitialized(ServletContextEvent sce) {
Path gitconfig = contextProvider.getBaseDirectory().toPath().resolve("gitconfig");
try {
Files.write(gitconfig, GITCONFIG.getBytes(Charset.defaultCharset()), TRUNCATE_EXISTING, CREATE);
FS.DETECTED.setGitSystemConfig(gitconfig.toFile());
LOG.info("wrote git config file: {}", gitconfig);
} catch (IOException e) {
LOG.error("could not write git config in path {}; git lfs support may not work correctly", gitconfig, e);
}
FilterCommandRegistry.register(COMMAND_NAME_PATTERN, NoOpFilterCommand::new);
}
@Override
public void contextDestroyed(ServletContextEvent sce) {
FilterCommandRegistry.unregister(COMMAND_NAME_PATTERN);
}
private static class NoOpFilterCommand extends FilterCommand {
NoOpFilterCommand(Repository db, InputStream in, OutputStream out) {
super(in, out);
}
@Override
public int run() throws IOException {
ByteStreams.copy(in, out);
in.close();
out.close();
return -1;
}
}
}

View File

@@ -1,36 +1,38 @@
package sonia.scm.repository.spi;
import com.google.common.util.concurrent.Striped;
import org.apache.commons.lang.StringUtils;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.attributes.FilterCommandRegistry;
import org.eclipse.jgit.revwalk.RevCommit;
import sonia.scm.BadRequestException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.ConcurrentModificationException;
import sonia.scm.ContextEntry;
import sonia.scm.NoChangesMadeException;
import sonia.scm.repository.GitWorkdirFactory;
import sonia.scm.repository.InternalRepositoryException;
import sonia.scm.repository.Repository;
import sonia.scm.web.lfs.LfsBlobStoreFactory;
import java.io.File;
import java.io.IOException;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.util.Optional;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
import static sonia.scm.AlreadyExistsException.alreadyExists;
import static sonia.scm.ContextEntry.ContextBuilder.entity;
import static sonia.scm.NotFoundException.notFound;
import java.util.concurrent.locks.Lock;
public class GitModifyCommand extends AbstractGitCommand implements ModifyCommand {
private final GitWorkdirFactory workdirFactory;
private static final Logger LOG = LoggerFactory.getLogger(GitModifyCommand.class);
private static final Striped<Lock> REGISTER_LOCKS = Striped.lock(5);
GitModifyCommand(GitContext context, Repository repository, GitWorkdirFactory workdirFactory) {
private final GitWorkdirFactory workdirFactory;
private final LfsBlobStoreFactory lfsBlobStoreFactory;
GitModifyCommand(GitContext context, Repository repository, GitWorkdirFactory workdirFactory, LfsBlobStoreFactory lfsBlobStoreFactory) {
super(context, repository);
this.workdirFactory = workdirFactory;
this.lfsBlobStoreFactory = lfsBlobStoreFactory;
}
@Override
@@ -38,7 +40,7 @@ public class GitModifyCommand extends AbstractGitCommand implements ModifyComman
return inClone(clone -> new ModifyWorker(clone, request), workdirFactory, request.getBranch());
}
private class ModifyWorker extends GitCloneWorker<String> implements Worker {
private class ModifyWorker extends GitCloneWorker<String> implements ModifyWorkerHelper {
private final File workDir;
private final ModifyCommandRequest request;
@@ -52,52 +54,42 @@ public class GitModifyCommand extends AbstractGitCommand implements ModifyComman
@Override
String run() throws IOException {
getClone().getRepository().getFullBranch();
if (!StringUtils.isEmpty(request.getExpectedRevision())) {
if (!request.getExpectedRevision().equals(getCurrentRevision().getName())) {
throw new ConcurrentModificationException("branch", request.getBranch() == null? "default": request.getBranch());
}
if (!StringUtils.isEmpty(request.getExpectedRevision())
&& !request.getExpectedRevision().equals(getCurrentRevision().getName())) {
throw new ConcurrentModificationException("branch", request.getBranch() == null ? "default" : request.getBranch());
}
for (ModifyCommandRequest.PartialRequest r : request.getRequests()) {
r.execute(this);
}
failIfNotChanged(NoChangesMadeException::new);
failIfNotChanged(() -> new NoChangesMadeException(repository, ModifyWorker.this.request.getBranch()));
Optional<RevCommit> revCommit = doCommit(request.getCommitMessage(), request.getAuthor());
push();
return revCommit.orElseThrow(NoChangesMadeException::new).name();
return revCommit.orElseThrow(() -> new NoChangesMadeException(repository, ModifyWorker.this.request.getBranch())).name();
}
@Override
public void create(String toBeCreated, File file, boolean overwrite) throws IOException {
Path targetFile = new File(workDir, toBeCreated).toPath();
createDirectories(targetFile);
if (overwrite) {
Files.move(file.toPath(), targetFile, REPLACE_EXISTING);
} else {
public void addFileToScm(String name, Path file) {
addToGitWithLfsSupport(name, file);
}
private void addToGitWithLfsSupport(String path, Path targetFile) {
REGISTER_LOCKS.get(targetFile).lock();
try {
LfsBlobStoreCleanFilterFactory cleanFilterFactory = new LfsBlobStoreCleanFilterFactory(lfsBlobStoreFactory, repository, targetFile);
String registerKey = "git-lfs clean -- '" + path + "'";
LOG.debug("register lfs filter command factory for command '{}'", registerKey);
FilterCommandRegistry.register(registerKey, cleanFilterFactory::createFilter);
try {
Files.move(file.toPath(), targetFile);
} catch (FileAlreadyExistsException e) {
throw alreadyExists(createFileContext(toBeCreated));
addFileToGit(path);
} catch (GitAPIException e) {
throwInternalRepositoryException("could not add file to index", e);
} finally {
LOG.debug("unregister lfs filter command factory for command \"{}\"", registerKey);
FilterCommandRegistry.unregister(registerKey);
}
}
try {
addFileToGit(toBeCreated);
} catch (GitAPIException e) {
throwInternalRepositoryException("could not add new file to index", e);
}
}
@Override
public void modify(String path, File file) throws IOException {
Path targetFile = new File(workDir, path).toPath();
createDirectories(targetFile);
if (!targetFile.toFile().exists()) {
throw notFound(createFileContext(path));
}
Files.move(file.toPath(), targetFile, REPLACE_EXISTING);
try {
addFileToGit(path);
} catch (GitAPIException e) {
throwInternalRepositoryException("could not add new file to index", e);
} finally {
REGISTER_LOCKS.get(targetFile).unlock();
}
}
@@ -106,13 +98,7 @@ public class GitModifyCommand extends AbstractGitCommand implements ModifyComman
}
@Override
public void delete(String toBeDeleted) throws IOException {
Path fileToBeDeleted = new File(workDir, toBeDeleted).toPath();
try {
Files.delete(fileToBeDeleted);
} catch (NoSuchFileException e) {
throw notFound(createFileContext(toBeDeleted));
}
public void doScmDelete(String toBeDeleted) {
try {
getClone().rm().addFilepattern(removeStartingPathSeparators(toBeDeleted)).call();
} catch (GitAPIException e) {
@@ -120,6 +106,21 @@ public class GitModifyCommand extends AbstractGitCommand implements ModifyComman
}
}
@Override
public File getWorkDir() {
return workDir;
}
@Override
public Repository getRepository() {
return repository;
}
@Override
public String getBranch() {
return request.getBranch();
}
private String removeStartingPathSeparators(String path) {
while (path.startsWith(File.separator)) {
path = path.substring(1);
@@ -127,41 +128,8 @@ public class GitModifyCommand extends AbstractGitCommand implements ModifyComman
return path;
}
private void createDirectories(Path targetFile) throws IOException {
try {
Files.createDirectories(targetFile.getParent());
} catch (FileAlreadyExistsException e) {
throw alreadyExists(createFileContext(targetFile.toString()));
}
private String throwInternalRepositoryException(String message, Exception e) {
throw new InternalRepositoryException(context.getRepository(), message, e);
}
private ContextEntry.ContextBuilder createFileContext(String path) {
ContextEntry.ContextBuilder contextBuilder = entity("file", path);
if (!StringUtils.isEmpty(request.getBranch())) {
contextBuilder.in("branch", request.getBranch());
}
contextBuilder.in(context.getRepository());
return contextBuilder;
}
@Override
public void move(String sourcePath, String targetPath) {
}
private class NoChangesMadeException extends BadRequestException {
public NoChangesMadeException() {
super(ContextEntry.ContextBuilder.entity(context.getRepository()).build(), "no changes detected to branch " + ModifyWorker.this.request.getBranch());
}
@Override
public String getCode() {
return "40RaYIeeR1";
}
}
}
private String throwInternalRepositoryException(String message, Exception e) {
throw new InternalRepositoryException(context.getRepository(), message, e);
}
}

View File

@@ -273,7 +273,7 @@ public class GitRepositoryServiceProvider extends RepositoryServiceProvider
@Override
public ModifyCommand getModifyCommand() {
return new GitModifyCommand(context, repository, handler.getWorkdirFactory());
return new GitModifyCommand(context, repository, handler.getWorkdirFactory(), lfsBlobStoreFactory);
}
@Override

View File

@@ -0,0 +1,83 @@
package sonia.scm.repository.spi;
import com.google.common.io.ByteStreams;
import org.eclipse.jgit.attributes.FilterCommand;
import org.eclipse.jgit.lfs.LfsPointer;
import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
import org.eclipse.jgit.lfs.lib.Constants;
import org.eclipse.jgit.lfs.lib.LongObjectId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.store.Blob;
import sonia.scm.store.BlobStore;
import sonia.scm.util.IOUtil;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.DigestOutputStream;
/**
* Adapted version of JGit's {@link org.eclipse.jgit.lfs.CleanFilter} to write the
* lfs file directly to the lfs blob store.
*/
class LfsBlobStoreCleanFilter extends FilterCommand {
private static final Logger LOG = LoggerFactory.getLogger(LfsBlobStoreCleanFilter.class);
private final BlobStore lfsBlobStore;
private final Path targetFile;
LfsBlobStoreCleanFilter(InputStream in, OutputStream out, BlobStore lfsBlobStore, Path targetFile) {
super(in, out);
this.lfsBlobStore = lfsBlobStore;
this.targetFile = targetFile;
}
@Override
// Suppress warning for RuntimeException after check for wrong size, because mathematicians say this will never happen
@SuppressWarnings("squid:S00112")
public int run() throws IOException {
LOG.debug("running scm lfs filter for file {}", targetFile);
DigestOutputStream digestOutputStream = createDigestStream();
try {
long size = ByteStreams.copy(in, digestOutputStream);
AnyLongObjectId loid = LongObjectId.fromRaw(digestOutputStream.getMessageDigest().digest());
String hash = loid.getName();
Blob existingBlob = lfsBlobStore.get(hash);
if (existingBlob != null) {
LOG.debug("found existing lfs blob for oid {}", hash);
long blobSize = existingBlob.getSize();
if (blobSize != size) {
throw new RuntimeException("lfs entry already exists for loid " + hash + " but has wrong size");
}
} else {
LOG.debug("uploading new lfs blob for oid {}", hash);
Blob newBlob = lfsBlobStore.create(hash);
OutputStream outputStream = newBlob.getOutputStream();
Files.copy(targetFile, outputStream);
newBlob.commit();
}
LfsPointer lfsPointer = new LfsPointer(loid, size);
lfsPointer.encode(out);
return -1;
} finally {
IOUtil.close(digestOutputStream);
IOUtil.close(in);
IOUtil.close(out);
}
}
private DigestOutputStream createDigestStream() {
return new DigestOutputStream(new OutputStream() {
@Override
public void write(int b) {
// no further target here, we are just interested in the digest
}
}, Constants.newMessageDigest());
}
}

View File

@@ -0,0 +1,27 @@
package sonia.scm.repository.spi;
import org.eclipse.jgit.lib.Repository;
import sonia.scm.web.lfs.LfsBlobStoreFactory;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Path;
class LfsBlobStoreCleanFilterFactory {
private final LfsBlobStoreFactory blobStoreFactory;
private final sonia.scm.repository.Repository repository;
private final Path targetFile;
LfsBlobStoreCleanFilterFactory(LfsBlobStoreFactory blobStoreFactory, sonia.scm.repository.Repository repository, Path targetFile) {
this.blobStoreFactory = blobStoreFactory;
this.repository = repository;
this.targetFile = targetFile;
}
@SuppressWarnings("squid:S1172")
// suppress unused parameter to keep the api compatible to jgit's FilterCommandFactory
LfsBlobStoreCleanFilter createFilter(Repository db, InputStream in, OutputStream out) {
return new LfsBlobStoreCleanFilter(in, out, blobStoreFactory.getLfsBlobStore(repository), targetFile);
}
}

View File

@@ -17,15 +17,16 @@ import sonia.scm.AlreadyExistsException;
import sonia.scm.BadRequestException;
import sonia.scm.ConcurrentModificationException;
import sonia.scm.NotFoundException;
import sonia.scm.ScmConstraintViolationException;
import sonia.scm.repository.Person;
import sonia.scm.repository.util.WorkdirProvider;
import sonia.scm.web.lfs.LfsBlobStoreFactory;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
@SubjectAware(configuration = "classpath:sonia/scm/configuration/shiro.ini", username = "admin", password = "secret")
public class GitModifyCommandTest extends AbstractGitCommandTestBase {
@@ -37,6 +38,8 @@ public class GitModifyCommandTest extends AbstractGitCommandTestBase {
@Rule
public ShiroRule shiro = new ShiroRule();
private final LfsBlobStoreFactory lfsBlobStoreFactory = mock(LfsBlobStoreFactory.class);
@Test
public void shouldCreateCommit() throws IOException, GitAPIException {
File newFile = Files.write(temporaryFolder.newFile().toPath(), "new content".getBytes()).toFile();
@@ -296,7 +299,7 @@ public class GitModifyCommandTest extends AbstractGitCommandTestBase {
}
private GitModifyCommand createCommand() {
return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider()));
return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider()), lfsBlobStoreFactory);
}
@FunctionalInterface

View File

@@ -0,0 +1,116 @@
package sonia.scm.repository.spi;
import com.github.sdorra.shiro.ShiroRule;
import com.github.sdorra.shiro.SubjectAware;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.revwalk.RevCommit;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import sonia.scm.repository.Person;
import sonia.scm.repository.util.WorkdirProvider;
import sonia.scm.store.Blob;
import sonia.scm.store.BlobStore;
import sonia.scm.web.lfs.LfsBlobStoreFactory;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@SubjectAware(configuration = "classpath:sonia/scm/configuration/shiro.ini", username = "admin", password = "secret")
public class GitModifyCommand_LFSTest extends AbstractGitCommandTestBase {
@Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
@Rule
public BindTransportProtocolRule transportProtocolRule = new BindTransportProtocolRule();
@Rule
public ShiroRule shiro = new ShiroRule();
private final LfsBlobStoreFactory lfsBlobStoreFactory = mock(LfsBlobStoreFactory.class);
@Before
public void registerFilter() {
new GitLfsFilterContextListener(contextProvider).contextInitialized(null);
}
@After
public void unregisterFilter() {
new GitLfsFilterContextListener(contextProvider).contextDestroyed(null);
}
@Test
public void shouldCreateCommit() throws IOException, GitAPIException {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
String newRef = createCommit("new_lfs.png", "new content", "fe32608c9ef5b6cf7e3f946480253ff76f24f4ec0678f3d0f07f9844cbff9601", outputStream);
try (Git git = new Git(createContext().open())) {
RevCommit lastCommit = getLastCommit(git);
assertThat(lastCommit.getFullMessage()).isEqualTo("test commit");
assertThat(lastCommit.getAuthorIdent().getName()).isEqualTo("Dirk Gently");
assertThat(newRef).isEqualTo(lastCommit.toObjectId().name());
}
assertThat(outputStream.toString()).isEqualTo("new content");
}
@Test
public void shouldCreateSecondCommits() throws IOException, GitAPIException {
createCommit("new_lfs.png", "new content", "fe32608c9ef5b6cf7e3f946480253ff76f24f4ec0678f3d0f07f9844cbff9601", new ByteArrayOutputStream());
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
String newRef = createCommit("more_lfs.png", "more content", "2c2316737c9313956dfc0083da3a2a62ce259f66484f3e26440f0d1b02dd4128", outputStream);
try (Git git = new Git(createContext().open())) {
RevCommit lastCommit = getLastCommit(git);
assertThat(lastCommit.getFullMessage()).isEqualTo("test commit");
assertThat(lastCommit.getAuthorIdent().getName()).isEqualTo("Dirk Gently");
assertThat(newRef).isEqualTo(lastCommit.toObjectId().name());
}
assertThat(outputStream.toString()).isEqualTo("more content");
}
private String createCommit(String fileName, String content, String hashOfContent, ByteArrayOutputStream outputStream) throws IOException {
BlobStore blobStore = mock(BlobStore.class);
Blob blob = mock(Blob.class);
when(lfsBlobStoreFactory.getLfsBlobStore(any())).thenReturn(blobStore);
when(blobStore.create(hashOfContent)).thenReturn(blob);
when(blobStore.get(hashOfContent)).thenReturn(null, blob);
when(blob.getOutputStream()).thenReturn(outputStream);
when(blob.getSize()).thenReturn((long) content.length());
File newFile = Files.write(temporaryFolder.newFile().toPath(), content.getBytes()).toFile();
GitModifyCommand command = createCommand();
ModifyCommandRequest request = new ModifyCommandRequest();
request.setCommitMessage("test commit");
request.addRequest(new ModifyCommandRequest.CreateFileRequest(fileName, newFile, false));
request.setAuthor(new Person("Dirk Gently", "dirk@holistic.det"));
return command.execute(request);
}
private RevCommit getLastCommit(Git git) throws GitAPIException {
return git.log().setMaxCount(1).call().iterator().next();
}
private GitModifyCommand createCommand() {
return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider()), lfsBlobStoreFactory);
}
@Override
protected String getZippedRepositoryResource() {
return "sonia/scm/repository/spi/scm-git-spi-lfs-test.zip";
}
}

View File

@@ -13,15 +13,16 @@ import org.eclipse.jgit.treewalk.CanonicalTreeParser;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import sonia.scm.ScmConstraintViolationException;
import sonia.scm.repository.Person;
import sonia.scm.repository.util.WorkdirProvider;
import sonia.scm.web.lfs.LfsBlobStoreFactory;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
@SubjectAware(configuration = "classpath:sonia/scm/configuration/shiro.ini", username = "admin", password = "secret")
public class GitModifyCommand_withEmptyRepositoryTest extends AbstractGitCommandTestBase {
@@ -33,6 +34,8 @@ public class GitModifyCommand_withEmptyRepositoryTest extends AbstractGitCommand
@Rule
public ShiroRule shiro = new ShiroRule();
private final LfsBlobStoreFactory lfsBlobStoreFactory = mock(LfsBlobStoreFactory.class);
@Test
public void shouldCreateNewFileInEmptyRepository() throws IOException, GitAPIException {
File newFile = Files.write(temporaryFolder.newFile().toPath(), "new content".getBytes()).toFile();
@@ -74,7 +77,7 @@ public class GitModifyCommand_withEmptyRepositoryTest extends AbstractGitCommand
}
private GitModifyCommand createCommand() {
return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider()));
return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider()), lfsBlobStoreFactory);
}
@FunctionalInterface