Add git-lfs support

This commit is contained in:
Oliver Milke
2017-05-19 17:27:18 +02:00
parent 9b932a325e
commit 33ea2273b0
16 changed files with 914 additions and 51 deletions

View File

@@ -85,4 +85,13 @@ public interface Blob
* @throws IOException
*/
public OutputStream getOutputStream() throws IOException;
/**
*
* Returns the size (in bytes) of the blob.
* @since 1.54
*/
public long getSize();
}

View File

@@ -119,6 +119,19 @@ public final class FileBlob implements Blob
return new FileOutputStream(file);
}
@Override
public long getSize() {
if (this.file.isFile()) {
return this.file.length();
} else {
//to sum up all other cases, in which we cannot determine a size
return -1;
}
}
//~--- fields ---------------------------------------------------------------
/** Field description */

View File

@@ -37,6 +37,12 @@
<version>${jgit.version}</version>
</dependency>
<dependency>
<groupId>sonia.jgit</groupId>
<artifactId>org.eclipse.jgit.lfs.server</artifactId>
<version>${jgit.version}</version>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
@@ -51,6 +57,16 @@
<version>1.54-SNAPSHOT</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>sonia.scm</groupId>
<artifactId>scm-dao-xml</artifactId>
<version>1.52-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>sonia.scm</groupId>
<artifactId>scm-dao-xml</artifactId>
<version>1.52-SNAPSHOT</version>
</dependency>
</dependencies>

View File

@@ -35,6 +35,7 @@ package sonia.scm.web;
//~--- non-JDK imports --------------------------------------------------------
import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -127,13 +128,22 @@ public class GitPermissionFilter extends ProviderPermissionFilter
* @return
*/
@Override
protected boolean isWriteRequest(HttpServletRequest request)
{
protected boolean isWriteRequest(HttpServletRequest request) {
String uri = request.getRequestURI();
return uri.endsWith(URI_RECEIVE_PACK)
|| (uri.endsWith(URI_REF_INFO)
&& PARAMETER_VALUE_RECEIVE.equals(
request.getParameter(PARAMETER_SERVICE)));
return uri.endsWith(URI_RECEIVE_PACK) ||
(uri.endsWith(URI_REF_INFO) && PARAMETER_VALUE_RECEIVE.equals(request.getParameter(PARAMETER_SERVICE))) ||
isLfsFileUpload(request);
}
@VisibleForTesting
static boolean isLfsFileUpload(HttpServletRequest request) {
String regex = String.format("^%s%s/.+(\\.git)?/info/lfs/objects/[a-z0-9]{64}$", request.getContextPath(), GitServletModule.GIT_PATH);
return request.getRequestURI().matches(regex) && "PUT".equals(request.getMethod());
}
}

View File

@@ -49,8 +49,11 @@ import sonia.scm.plugin.ext.Extension;
public class GitServletModule extends ServletModule
{
public static final String GIT_PATH = "/git";
/** Field description */
public static final String PATTERN_GIT = "/git/*";
public static final String PATTERN_GIT = GIT_PATH + "/*";
//~--- methods --------------------------------------------------------------

View File

@@ -53,6 +53,12 @@ public class GitUserAgentProvider implements UserAgentProvider
false).basicAuthenticationCharset(
Charsets.UTF_8).build();
@VisibleForTesting
static final UserAgent GIT_LFS = UserAgent.builder("Git Lfs")
.browser(false)
.basicAuthenticationCharset(Charsets.UTF_8)
.build();
/** Field description */
@VisibleForTesting
static final UserAgent MSYSGIT = UserAgent.builder("msysGit").browser(
@@ -60,10 +66,11 @@ public class GitUserAgentProvider implements UserAgentProvider
Charsets.UTF_8).build();
/** Field description */
private static final String PREFIX = "git/";
private static final String PREFIX_REGULAR = "git/";
private static final String PREFIX_LFS = "git-lfs/";
/** Field description */
private static final String SUFFIX = "msysgit";
private static final String SUFFIX_MSYSGIT = "msysgit";
//~--- methods --------------------------------------------------------------
@@ -80,16 +87,14 @@ public class GitUserAgentProvider implements UserAgentProvider
{
UserAgent ua = null;
if (userAgentString.startsWith(PREFIX))
{
if (userAgentString.contains(SUFFIX))
{
if (userAgentString.startsWith(PREFIX_REGULAR)) {
if (userAgentString.contains(SUFFIX_MSYSGIT)) {
ua = MSYSGIT;
}
else
{
} else {
ua = GIT;
}
} else if (userAgentString.startsWith(PREFIX_LFS)) {
ua = GIT_LFS;
}
return ua;

View File

@@ -35,23 +35,31 @@ package sonia.scm.web;
//~--- non-JDK imports --------------------------------------------------------
import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.eclipse.jgit.http.server.GitServlet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.slf4j.LoggerFactory.getLogger;
import org.eclipse.jgit.lfs.lib.Constants;
import static org.eclipse.jgit.lfs.lib.Constants.CONTENT_TYPE_GIT_LFS_JSON;
import sonia.scm.repository.Repository;
import sonia.scm.repository.RepositoryProvider;
import sonia.scm.repository.RepositoryRequestListenerUtil;
import sonia.scm.util.HttpUtil;
import sonia.scm.web.lfs.servlet.LfsServletFactory;
//~--- JDK imports ------------------------------------------------------------
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import sonia.scm.repository.RepositoryException;
@@ -73,7 +81,7 @@ public class ScmGitServlet extends GitServlet
/** the logger for ScmGitServlet */
private static final Logger logger =
LoggerFactory.getLogger(ScmGitServlet.class);
getLogger(ScmGitServlet.class);
//~--- constructors ---------------------------------------------------------
@@ -90,14 +98,19 @@ public class ScmGitServlet extends GitServlet
*/
@Inject
public ScmGitServlet(GitRepositoryResolver repositoryResolver,
GitReceivePackFactory receivePackFactory,
GitRepositoryViewer repositoryViewer,
RepositoryProvider repositoryProvider,
RepositoryRequestListenerUtil repositoryRequestListenerUtil)
GitReceivePackFactory receivePackFactory,
GitRepositoryViewer repositoryViewer,
RepositoryProvider repositoryProvider,
RepositoryRequestListenerUtil repositoryRequestListenerUtil,
LfsServletFactory lfsServletFactory,
UserAgentParser userAgentParser)
{
this.repositoryProvider = repositoryProvider;
this.repositoryViewer = repositoryViewer;
this.repositoryRequestListenerUtil = repositoryRequestListenerUtil;
this.lfsServletFactory = lfsServletFactory;
this.userAgentParser = userAgentParser;
setRepositoryResolver(repositoryResolver);
setReceivePackFactory(receivePackFactory);
}
@@ -120,49 +133,44 @@ public class ScmGitServlet extends GitServlet
throws ServletException, IOException
{
String uri = HttpUtil.getStrippedURI(request);
logger.trace("--request URI: {}", uri);
if (uri.matches(REGEX_GITHTTPBACKEND))
{
sonia.scm.repository.Repository repository = repositoryProvider.get();
//decide the type of response to be presented to the client
UserAgent userAgent = userAgentParser.parse(request);
if (userAgent.isBrowser()) {
if (repository != null)
{
if (repositoryRequestListenerUtil.callListeners(request, response,
repository))
{
super.service(request, response);
}
else if (logger.isDebugEnabled())
{
renderHtmlRepositryOverview(request, response);
} else {
//service the request for a git client
final Repository repository = repositoryProvider.get();
if (repository == null) {
//repository could not be matched found the current request
super.service(request, response);
} else {
if (repositoryRequestListenerUtil.callListeners(request, response, repository)) {
handleRequest(request, response, repository);
} else if (logger.isDebugEnabled()) {
logger.debug("request aborted by repository request listener");
}
}
else
{
super.service(request, response);
}
}
else
{
printGitInformation(request, response);
}
}
/**
* Method description
*
*
*
* This method renders basic information about the repository into the response. The result is meant to be viewed by
* browser.
* @param request
* @param response
*
* @throws IOException
* @throws ServletException
*/
private void printGitInformation(HttpServletRequest request,
HttpServletResponse response)
throws ServletException, IOException
{
private void renderHtmlRepositryOverview(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
sonia.scm.repository.Repository scmRepository = repositoryProvider.get();
if (scmRepository != null)
@@ -186,6 +194,113 @@ public class ScmGitServlet extends GitServlet
}
}
/**
* Decides the type request being currently made and delegates it accordingly.
* <ul>
* <li>Batch API:</li>
* <ul>
* <li> used to provide the client with information on how handle the large files of a repository.</li>
* <li> response contains the information where to perform the actual upload and download of the large objects.</li>
* </ul>
* <li>Transfer API:</li>
* <ul>
* <li>receives and provides the actual large objects (resolves the pointer placed in the file of the working copy).</li>
* <li>invoked only after the Batch API has been questioned about what to do with the large files</li>
* </ul>
* <li>Regular HTTP Backend</li>
* <ul>
* <li>services everything that is not git-lfs.</li>
* </ul>
* </li>
* </ul>
*/
private void handleRequest(HttpServletRequest request, HttpServletResponse response, Repository repository) throws ServletException, IOException {
logger.trace("--- Repository is: {}", repository.getName());
if (isLfsBatchApiRequest(request, repository.getName())) {
logger.trace("--- detected LFS Batch API Request");
HttpServlet servlet = lfsServletFactory.createProtocolServletFor(repository, request);
servlet.service(request, response);
} else if (isLfsFileTransferRequest(request, repository.getName())) {
logger.trace("--- detected LFS File Transfer Request");
HttpServlet servlet = lfsServletFactory.createFileLfsServletFor(repository, request);
servlet.service(request, response);
} else {
logger.trace("--- seems to be regular Git HTTP backend request: {}", request.getRequestURI());
//continue to the regular HTTP Backend
super.service(request, response);
}
}
/**
* Decides whether or not a request is for the LFS Batch API,
* <p>
* - PUT or GET
* - exactly for this repository
* - Content Type is {@link Constants#HDR_APPLICATION_OCTET_STREAM}.
*
* @return Returns {@code false} if either of the conditions does not match. Returns true if all match.
*/
private static boolean isLfsFileTransferRequest(HttpServletRequest request, String repository) {
String regex = String.format("^%s%s/%s(\\.git)?/info/lfs/objects/[a-z0-9]{64}$", request.getContextPath(), GitServletModule.GIT_PATH, repository);
boolean pathMatches = request.getRequestURI().matches(regex);
boolean methodMatches = request.getMethod().equals("PUT") || request.getMethod().equals("GET");
return pathMatches && methodMatches;
}
/**
* Decides whether or not a request is for the LFS Batch API,
* <p>
* - POST
* - exactly for this repository
* - Content Type is {@link Constants#CONTENT_TYPE_GIT_LFS_JSON}.
*
* @return Returns {@code false} if either of the conditions does not match. Returns true if all match.
*/
private static boolean isLfsBatchApiRequest(HttpServletRequest request, String repository) {
String regex = String.format("^%s%s/%s(\\.git)?/info/lfs/objects/batch$", request.getContextPath(), GitServletModule.GIT_PATH, repository);
boolean pathMatches = request.getRequestURI().matches(regex);
boolean methodMatches = "POST".equals(request.getMethod());
boolean headerContentTypeMatches = isLfsContentHeaderField(request.getContentType(), CONTENT_TYPE_GIT_LFS_JSON);
boolean headerAcceptMatches = isLfsContentHeaderField(request.getHeader("Accept"), CONTENT_TYPE_GIT_LFS_JSON);
return pathMatches && methodMatches && headerContentTypeMatches && headerAcceptMatches;
}
/**
* Checks whether request is of the specific content type.
*
* @param request The HTTP request header value to be examined.
* @param expectedContentType The expected content type.
* @return Returns {@code true} if the request has the expected content type. Return {@code false} otherwise.
*/
@VisibleForTesting
static boolean isLfsContentHeaderField(String request, String expectedContentType) {
if (request == null || request.isEmpty()) {
return false;
}
String[] parts = request.split(" ");
for (String part : parts) {
if (part.startsWith(expectedContentType)) {
return true;
}
}
return false;
}
//~--- fields ---------------------------------------------------------------
/** Field description */
@@ -194,6 +309,14 @@ public class ScmGitServlet extends GitServlet
/** Field description */
private final RepositoryRequestListenerUtil repositoryRequestListenerUtil;
/** Field description */
/**
* Field description
*/
private final GitRepositoryViewer repositoryViewer;
private final LfsServletFactory lfsServletFactory;
private final UserAgentParser userAgentParser;
}

View File

@@ -0,0 +1,90 @@
package sonia.scm.web.lfs;
import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
import org.eclipse.jgit.lfs.server.LargeFileRepository;
import org.eclipse.jgit.lfs.server.Response;
import sonia.scm.store.Blob;
import sonia.scm.store.BlobStore;
import java.io.IOException;
/**
* This LargeFileRepository is used for jGit-Servlet implementation. Under the jgit LFS Servlet hood, the
* SCM-Repository API is used to implement the Repository.
*
* @since 1.54
* Created by omilke on 03.05.2017.
*/
public class ScmBlobLfsRepository implements LargeFileRepository {
private final BlobStore blobStore;
/**
* This URI is used to determine the actual URI for Upload / Download. Must be full URI (or rewritable by reverse
* proxy).
*/
private final String baseUri;
/**
* Creates a {@link ScmBlobLfsRepository} for the provided repository.
*
* @param blobStore The SCM Blobstore used for this @{@link LargeFileRepository}.
* @param baseUri This URI is used to determine the actual URI for Upload / Download. Must be full URI (or
* rewritable by reverse proxy).
*/
public ScmBlobLfsRepository(BlobStore blobStore, String baseUri) {
this.blobStore = blobStore;
this.baseUri = baseUri;
}
@Override
public Response.Action getDownloadAction(AnyLongObjectId id) {
return getAction(id);
}
@Override
public Response.Action getUploadAction(AnyLongObjectId id, long size) {
return getAction(id);
}
@Override
public Response.Action getVerifyAction(AnyLongObjectId id) {
//validation is optional. We do not support it.
return null;
}
@Override
public long getSize(AnyLongObjectId id) throws IOException {
//this needs to be size of what is will be written into the response of the download. Clients are likely to
// verify it.
Blob blob = this.blobStore.get(id.getName());
if (blob == null) {
return -1;
} else {
return blob.getSize();
}
}
/**
* Constructs the Download / Upload actions to be supplied to the client.
*/
private Response.Action getAction(AnyLongObjectId id) {
//LFS protocol has to provide the information on where to put or get the actual content, i. e.
//the actual URI for up- and download.
Response.Action a = new Response.Action();
a.href = baseUri + id.getName();
return a;
}
}

View File

@@ -0,0 +1,98 @@
package sonia.scm.web.lfs.servlet;
import com.google.common.annotations.VisibleForTesting;
import org.eclipse.jgit.lfs.server.LargeFileRepository;
import org.eclipse.jgit.lfs.server.LfsProtocolServlet;
import org.eclipse.jgit.lfs.server.fs.FileLfsServlet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.repository.Repository;
import sonia.scm.store.BlobStore;
import sonia.scm.store.BlobStoreFactory;
import sonia.scm.util.HttpUtil;
import sonia.scm.web.lfs.ScmBlobLfsRepository;
import javax.inject.Inject;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
/**
* This factory class is a helper class to provide the {@link LfsProtocolServlet} and the {@link FileLfsServlet}
* belonging to a SCM Repository.
*
* @since 1.54
* Created by omilke on 11.05.2017.
*/
public class LfsServletFactory {
private static final String GIT_LFS_REPOSITORY_POSTFIX = "-git-lfs";
private static final Logger logger = LoggerFactory.getLogger(LfsServletFactory.class);
private final BlobStoreFactory blobStoreFactory;
@Inject
public LfsServletFactory(BlobStoreFactory blobStoreFactory) {
this.blobStoreFactory = blobStoreFactory;
}
/**
* Builds the {@link LfsProtocolServlet} (jgit API) for a SCM Repository.
*
* @param repository The SCM Repository to build the servlet for.
* @param request The {@link HttpServletRequest} the used to access the SCM Repository.
* @return The {@link LfsProtocolServlet} to provide the LFS Batch API for a SCM Repository.
*/
public LfsProtocolServlet createProtocolServletFor(Repository repository, HttpServletRequest request) {
BlobStore blobStore = getBlobStore(repository);
String baseUri = buildBaseUri(repository, request);
LargeFileRepository largeFileRepository = new ScmBlobLfsRepository(blobStore, baseUri);
return new ScmLfsProtocolServlet(largeFileRepository);
}
/**
* Builds the {@link FileLfsServlet} (jgit API) for a SCM Repository.
*
* @param repository The SCM Repository to build the servlet for.
* @param request The {@link HttpServletRequest} the used to access the SCM Repository.
* @return The {@link FileLfsServlet} to provide the LFS Upload / Download API for a SCM Repository.
*/
public HttpServlet createFileLfsServletFor(Repository repository, HttpServletRequest request) {
return new ScmFileTransferServlet(getBlobStore(repository));
}
/**
* Build the complete URI, under which the File Transfer API for this repository will be will be reachable.
*
* @param repository The repository to build the File Transfer URI for.
* @param request The request to construct the complete URI from.
*/
@VisibleForTesting
static String buildBaseUri(Repository repository, HttpServletRequest request) {
return String.format("%s/git/%s.git/info/lfs/objects/", HttpUtil.getCompleteUrl(request), repository.getName());
}
/**
* Provides a {@link BlobStore} corresponding to the SCM Repository.
* <p>
* git-lfs repositories should generally carry the same name as their regular SCM repository counterparts. However,
* we have decided to store them under their IDs instead of their names, since the names might change and provide
* other drawbacks, as well.
* <p>
* These repositories will have {@linkplain #GIT_LFS_REPOSITORY_POSTFIX} appended to their IDs.
*
* @param repository The SCM Repository to provide a LFS {@link BlobStore} for.
*/
@VisibleForTesting
BlobStore getBlobStore(Repository repository) {
return blobStoreFactory.getBlobStore(repository.getId() + GIT_LFS_REPOSITORY_POSTFIX);
}
}

View File

@@ -0,0 +1,278 @@
package sonia.scm.web.lfs.servlet;
import com.google.common.annotations.VisibleForTesting;
import com.google.gson.FieldNamingPolicy;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.apache.http.HttpStatus;
import org.eclipse.jgit.lfs.errors.CorruptLongObjectException;
import org.eclipse.jgit.lfs.errors.InvalidLongObjectIdException;
import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
import org.eclipse.jgit.lfs.lib.Constants;
import org.eclipse.jgit.lfs.lib.LongObjectId;
import org.eclipse.jgit.lfs.server.LfsProtocolServlet;
import org.eclipse.jgit.lfs.server.fs.FileLfsServlet;
import org.eclipse.jgit.lfs.server.internal.LfsServerText;
import org.eclipse.jgit.util.HttpSupport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.store.Blob;
import sonia.scm.store.BlobStore;
import sonia.scm.util.IOUtil;
import javax.servlet.ServletException;
import javax.servlet.ServletInputStream;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.text.MessageFormat;
/**
* This Servlet provides the upload and download of files via git-lfs.
* <p>
* This implementation is based on {@link FileLfsServlet} but adjusted to work with
* servlet-2.5 instead of servlet-3.1.
* <p>
*
* @see FileLfsServlet
* @since 1.54
* Created by omilke on 15.05.2017.
*/
public class ScmFileTransferServlet extends HttpServlet {
private static final Logger logger = LoggerFactory.getLogger(ScmFileTransferServlet.class);
private static final long serialVersionUID = 1L;
/**
* Gson is used because the implementation was based on the jgit implementation. However the {@link LfsProtocolServlet} (which we do use in
* {@link ScmLfsProtocolServlet}) also uses Gson, which currently ties us to Gson anyway.
*/
private static Gson gson = createGson();
private final BlobStore blobStore;
public ScmFileTransferServlet(BlobStore store) {
this.blobStore = store;
}
/**
* Extracts the part after the last slash from path.
*
* @return Returns {@code null} if the part after the last slash is itself {@code null} or if its length is not 64.
*/
@VisibleForTesting
static String objectIdFromPath(String info) {
int lastSlash = info.lastIndexOf('/');
String potentialObjectId = info.substring(lastSlash + 1);
if (potentialObjectId.length() != 64) {
return null;
} else {
return potentialObjectId;
}
}
/**
* Logs the message and provides it to the client.
*
* @param response The response
* @param status The HTTP Status Code to be provided to the client.
* @param message the message to used for server-side logging. It is also provided to the client.
*/
private static void sendErrorAndLog(HttpServletResponse response, int status, String message) throws IOException {
logger.warn("Error occurred during git-lfs file transfer: {}", message);
sendError(response, status, message);
}
/**
* Logs the exception and provides only the message of the exception to the client.
*
* @param response The response
* @param status The HTTP Status Code to be provided to the client.
* @param exception An exception to used for server-side logging.
*/
private static void sendErrorAndLog(HttpServletResponse response, int status, Exception exception) throws IOException {
logger.warn("Error occurred during git-lfs file transfer.", exception);
String message = exception.getMessage();
sendError(response, status, message);
}
private static void sendError(HttpServletResponse response, int status, String message) throws IOException {
try (PrintWriter writer = response.getWriter()) {
gson.toJson(new Error(message), writer);
response.setStatus(status);
writer.flush();
}
response.flushBuffer();
}
private static Gson createGson() {
GsonBuilder gb = new GsonBuilder().setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES).setPrettyPrinting().disableHtmlEscaping();
return gb.create();
}
/**
* Provides a blob to download.
* <p>
* Actual implementation is based on <code>org.eclipse.jgit.lfs.server.fs.ObjectDownloadListener</code> and adjusted
* to non-async as we're currently on servlet-2.5.
*
* @param request servlet request
* @param response servlet response
* @throws ServletException if a servlet-specific error occurs
* @throws IOException if an I/O error occurs
*/
@Override
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
AnyLongObjectId objectId = getObjectToTransfer(request, response);
if (objectId == null) {
logInvalidObjectId(request.getRequestURI());
} else {
final String objectIdName = objectId.getName();
logger.trace("---- providing download for LFS-Oid: {}", objectIdName);
Blob savedBlob = blobStore.get(objectIdName);
if (isBlobPresent(savedBlob)) {
logger.trace("----- Object {}: providing {} bytes", objectIdName, savedBlob.getSize());
writeBlobIntoResponse(savedBlob, response);
} else {
sendErrorAndLog(response, HttpStatus.SC_NOT_FOUND, MessageFormat.format(LfsServerText.get().objectNotFound, objectIdName));
}
}
}
/**
* Receives a blob from an upload.
* <p>
* Actual implementation is based on <code>org.eclipse.jgit.lfs.server.fs.ObjectUploadListener</code> and adjusted
* to non-async as we're currently on servlet-2.5.
*
* @param request servlet request
* @param response servlet response
* @throws ServletException if a servlet-specific error occurs
* @throws IOException if an I/O error occurs
*/
@Override
protected void doPut(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
AnyLongObjectId objectId = getObjectToTransfer(request, response);
if (objectId == null) {
logInvalidObjectId(request.getRequestURI());
} else {
logger.trace("---- receiving upload for LFS-Oid: {}", objectId.getName());
readBlobFromResponse(request, response, objectId);
}
}
/**
* Extracts the {@link LongObjectId} from the request. Finishes the request, in case the {@link LongObjectId} cannot
* be extracted with an appropriate error.
*
* @throws IOException Thrown if the response could not be completed in an error case.
*/
private AnyLongObjectId getObjectToTransfer(HttpServletRequest request, HttpServletResponse response) throws IOException {
String path = request.getPathInfo();
String objectIdFromPath = objectIdFromPath(path);
if (objectIdFromPath == null) {
//ObjectId is not retrievable from URL
sendErrorAndLog(response, HttpStatus.SC_UNPROCESSABLE_ENTITY, MessageFormat.format(LfsServerText.get().invalidPathInfo, path));
return null;
} else {
try {
return LongObjectId.fromString(objectIdFromPath);
} catch (InvalidLongObjectIdException e) {
sendErrorAndLog(response, HttpStatus.SC_UNPROCESSABLE_ENTITY, e);
return null;
}
}
}
private void logInvalidObjectId(String requestURI) {
logger.warn("---- could not extract Oid from Request. Path seems to be invalid: {}", requestURI);
}
private boolean isBlobPresent(Blob savedBlob) {
return savedBlob != null && savedBlob.getSize() >= 0;
}
private void writeBlobIntoResponse(Blob savedBlob, HttpServletResponse response) throws IOException {
try (ServletOutputStream responseOutputStream = response.getOutputStream();
InputStream savedBlobInputStream = savedBlob.getInputStream()) {
response.addHeader(HttpSupport.HDR_CONTENT_LENGTH, String.valueOf(savedBlob.getSize()));
response.setContentType(Constants.HDR_APPLICATION_OCTET_STREAM);
IOUtil.copy(savedBlobInputStream, responseOutputStream);
} catch (IOException ex) {
sendErrorAndLog(response, HttpStatus.SC_INTERNAL_SERVER_ERROR, ex);
}
}
private void readBlobFromResponse(HttpServletRequest request, HttpServletResponse response, AnyLongObjectId objectId) throws IOException {
try (OutputStream blobOutputStream = blobStore.create(objectId.getName()).getOutputStream();
ServletInputStream requestInputStream = request.getInputStream()) {
IOUtil.copy(requestInputStream, blobOutputStream);
response.setContentType(Constants.CONTENT_TYPE_GIT_LFS_JSON);
response.setStatus(HttpServletResponse.SC_OK);
} catch (CorruptLongObjectException ex) {
sendErrorAndLog(response, HttpStatus.SC_BAD_REQUEST, ex);
}
}
/**
* Used for providing an error message.
*/
private static class Error {
String message;
Error(String m) {
this.message = m;
}
}
}

View File

@@ -0,0 +1,26 @@
package sonia.scm.web.lfs.servlet;
import org.eclipse.jgit.lfs.errors.LfsException;
import org.eclipse.jgit.lfs.server.LargeFileRepository;
import org.eclipse.jgit.lfs.server.LfsProtocolServlet;
/**
* Provides an implementation for the git-lfs Batch API.
*
* @since 1.54
* Created by omilke on 11.05.2017.
*/
public class ScmLfsProtocolServlet extends LfsProtocolServlet {
private final LargeFileRepository repository;
public ScmLfsProtocolServlet(LargeFileRepository largeFileRepository) {
this.repository = largeFileRepository;
}
@Override
protected LargeFileRepository getLargeFileRepository(LfsRequest request, String path) throws LfsException {
return repository;
}
}

View File

@@ -0,0 +1,48 @@
package sonia.scm.web;
import org.junit.Test;
import javax.servlet.http.HttpServletRequest;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
/**
* Created by omilke on 19.05.2017.
*/
public class GitPermissionFilterTest {
@Test
public void isLfsFileUpload() throws Exception {
HttpServletRequest mockedRequest = getRequestWithMethodAndPathInfo("PUT",
"/scm/git/git-lfs-demo.git/info/lfs/objects/8fcebeb5698230685f92028e560f8f1683ebc15ec82a620ffad5c12a3c19bdec");
assertThat((GitPermissionFilter.isLfsFileUpload(mockedRequest)), is(true));
mockedRequest = getRequestWithMethodAndPathInfo("GET",
"/scm/git/git-lfs-demo.git/info/lfs/objects/8fcebeb5698230685f92028e560f8f1683ebc15ec82a620ffad5c12a3c19bdec");
assertThat((GitPermissionFilter.isLfsFileUpload(mockedRequest)), is(false));
mockedRequest = getRequestWithMethodAndPathInfo("POST",
"/scm/git/git-lfs-demo.git/info/lfs/objects/8fcebeb5698230685f92028e560f8f1683ebc15ec82a620ffad5c12a3c19bdec");
assertThat((GitPermissionFilter.isLfsFileUpload(mockedRequest)), is(false));
mockedRequest = getRequestWithMethodAndPathInfo("POST",
"/scm/git/git-lfs-demo.git/info/lfs/objects/batch");
assertThat((GitPermissionFilter.isLfsFileUpload(mockedRequest)), is(false));
}
private HttpServletRequest getRequestWithMethodAndPathInfo(String method, String pathInfo) {
HttpServletRequest mock = mock(HttpServletRequest.class);
when(mock.getMethod()).thenReturn(method);
when(mock.getRequestURI()).thenReturn(pathInfo);
when(mock.getContextPath()).thenReturn("/scm");
return mock;
}
}

View File

@@ -58,6 +58,7 @@ public class GitUserAgentProviderTest
public void testParseUserAgent()
{
assertEquals(GitUserAgentProvider.GIT, parse("git/1.7.9.5"));
assertEquals(GitUserAgentProvider.GIT_LFS, parse("git-lfs/2.0.1 (GitHub; windows amd64; go 1.8; git 678cdbd4)"));
assertEquals(GitUserAgentProvider.MSYSGIT, parse("git/1.8.3.msysgit.0"));
assertNull(parse("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"));
}

View File

@@ -0,0 +1,26 @@
package sonia.scm.web;
import org.junit.Test;
import static org.eclipse.jgit.lfs.lib.Constants.CONTENT_TYPE_GIT_LFS_JSON;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertThat;
/**
* Created by omilke on 11.05.2017.
*/
public class ScmGitServletTest {
@Test
public void isContentTypeMatches() throws Exception {
assertThat(ScmGitServlet.isLfsContentHeaderField("application/vnd.git-lfs+json", CONTENT_TYPE_GIT_LFS_JSON), is(true));
assertThat(ScmGitServlet.isLfsContentHeaderField("application/vnd.git-lfs+json;", CONTENT_TYPE_GIT_LFS_JSON), is(true));
assertThat(ScmGitServlet.isLfsContentHeaderField("application/vnd.git-lfs+json; charset=utf-8", CONTENT_TYPE_GIT_LFS_JSON), is(true));
assertThat(ScmGitServlet.isLfsContentHeaderField("application/vnd.git-lfs-json;", CONTENT_TYPE_GIT_LFS_JSON), is(false));
assertThat(ScmGitServlet.isLfsContentHeaderField("", CONTENT_TYPE_GIT_LFS_JSON), is(false));
assertThat(ScmGitServlet.isLfsContentHeaderField(null, CONTENT_TYPE_GIT_LFS_JSON), is(false));
}
}

View File

@@ -0,0 +1,75 @@
package sonia.scm.web.lfs.servlet;
import org.junit.Test;
import sonia.scm.repository.Repository;
import sonia.scm.repository.RepositoryTestData;
import sonia.scm.store.BlobStoreFactory;
import javax.servlet.http.HttpServletRequest;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.*;
import static org.mockito.Matchers.matches;
import static org.mockito.Mockito.*;
/**
* Created by omilke on 18.05.2017.
*/
public class LfsServletFactoryTest {
@Test
public void buildBaseUri() throws Exception {
String repositoryName = "git-lfs-demo";
String result = LfsServletFactory.buildBaseUri(new Repository("", "GIT", repositoryName), RequestWithUri(repositoryName, true));
assertThat(result, is(equalTo("http://localhost:8081/scm/git/git-lfs-demo.git/info/lfs/objects/")));
//result will be with dot-gix suffix, ide
result = LfsServletFactory.buildBaseUri(new Repository("", "GIT", repositoryName), RequestWithUri(repositoryName, false));
assertThat(result, is(equalTo("http://localhost:8081/scm/git/git-lfs-demo.git/info/lfs/objects/")));
}
@Test
public void getBlobStore() throws Exception {
BlobStoreFactory blobStoreFactoryMock = mock(BlobStoreFactory.class);
//TODO #239:
RepositoryTestData repositoryTestData;
new LfsServletFactory(blobStoreFactoryMock).getBlobStore(new Repository("the-id", "GIT", "the-name"));
//just make sure the right parameter is passed, as properly validating the return value is nearly impossible with the return value (and should not be
// part of this test)
verify(blobStoreFactoryMock).getBlobStore(matches("the-id-git-lfs"));
//make sure there have been no further usages of the factory
verifyNoMoreInteractions(blobStoreFactoryMock);
}
private HttpServletRequest RequestWithUri(String repositoryName, boolean withDotGitSuffix) {
HttpServletRequest mockedRequest = mock(HttpServletRequest.class);
final String suffix;
if (withDotGitSuffix) {
suffix = ".git";
} else {
suffix = "";
}
//build from valid live request data
when(mockedRequest.getRequestURL()).thenReturn(
new StringBuffer(String.format("http://localhost:8081/scm/git/%s%s/info/lfs/objects/batch", repositoryName, suffix)));
when(mockedRequest.getRequestURI()).thenReturn(String.format("/scm/git/%s%s/info/lfs/objects/batch", repositoryName, suffix));
when(mockedRequest.getContextPath()).thenReturn("/scm");
return mockedRequest;
}
}

View File

@@ -0,0 +1,42 @@
package sonia.scm.web.lfs.servlet;
import org.junit.Test;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.junit.Assert.*;
/**
* Created by omilke on 16.05.2017.
*/
public class ScmFileTransferServletTest {
@Test
public void hasObjectId() throws Exception {
String SAMPLE_OBJECT_ID = "8fcebeb5698230685f92028e560f8f1683ebc15ec82a620ffad5c12a3c19bdec";
String path = "/git-lfs-demo.git/info/lfs/objects/" + SAMPLE_OBJECT_ID;
assertThat(ScmFileTransferServlet.objectIdFromPath(path), is(equalTo(SAMPLE_OBJECT_ID)));
path = "/" + SAMPLE_OBJECT_ID;
assertThat(ScmFileTransferServlet.objectIdFromPath(path), is(equalTo(SAMPLE_OBJECT_ID)));
path = SAMPLE_OBJECT_ID;
assertThat(ScmFileTransferServlet.objectIdFromPath(path), is(equalTo(SAMPLE_OBJECT_ID)));
String nonObjectId = "this-ist-last-to-found";
path = "/git-lfs-demo.git/info/lfs/objects/" + nonObjectId;
assertThat(ScmFileTransferServlet.objectIdFromPath(path), is(nullValue()));
nonObjectId = SAMPLE_OBJECT_ID.substring(1);
path = "/git-lfs-demo.git/info/lfs/objects/" + nonObjectId;
assertThat(ScmFileTransferServlet.objectIdFromPath(path), is(nullValue()));
nonObjectId = SAMPLE_OBJECT_ID + "X";
path = "/git-lfs-demo.git/info/lfs/objects/" + nonObjectId;
assertThat(ScmFileTransferServlet.objectIdFromPath(path), is(nullValue()));
}
}