mirror of
https://github.com/scm-manager/scm-manager.git
synced 2025-11-03 12:05:52 +01:00
Add git-lfs support
This commit is contained in:
@@ -35,6 +35,7 @@ package sonia.scm.web;
|
||||
|
||||
//~--- non-JDK imports --------------------------------------------------------
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.inject.Inject;
|
||||
import com.google.inject.Singleton;
|
||||
|
||||
@@ -127,13 +128,22 @@ public class GitPermissionFilter extends ProviderPermissionFilter
|
||||
* @return
|
||||
*/
|
||||
@Override
|
||||
protected boolean isWriteRequest(HttpServletRequest request)
|
||||
{
|
||||
protected boolean isWriteRequest(HttpServletRequest request) {
|
||||
|
||||
String uri = request.getRequestURI();
|
||||
|
||||
return uri.endsWith(URI_RECEIVE_PACK)
|
||||
|| (uri.endsWith(URI_REF_INFO)
|
||||
&& PARAMETER_VALUE_RECEIVE.equals(
|
||||
request.getParameter(PARAMETER_SERVICE)));
|
||||
return uri.endsWith(URI_RECEIVE_PACK) ||
|
||||
(uri.endsWith(URI_REF_INFO) && PARAMETER_VALUE_RECEIVE.equals(request.getParameter(PARAMETER_SERVICE))) ||
|
||||
isLfsFileUpload(request);
|
||||
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static boolean isLfsFileUpload(HttpServletRequest request) {
|
||||
String regex = String.format("^%s%s/.+(\\.git)?/info/lfs/objects/[a-z0-9]{64}$", request.getContextPath(), GitServletModule.GIT_PATH);
|
||||
return request.getRequestURI().matches(regex) && "PUT".equals(request.getMethod());
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -49,8 +49,11 @@ import sonia.scm.plugin.ext.Extension;
|
||||
public class GitServletModule extends ServletModule
|
||||
{
|
||||
|
||||
public static final String GIT_PATH = "/git";
|
||||
|
||||
/** Field description */
|
||||
public static final String PATTERN_GIT = "/git/*";
|
||||
public static final String PATTERN_GIT = GIT_PATH + "/*";
|
||||
|
||||
|
||||
//~--- methods --------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -53,6 +53,12 @@ public class GitUserAgentProvider implements UserAgentProvider
|
||||
false).basicAuthenticationCharset(
|
||||
Charsets.UTF_8).build();
|
||||
|
||||
@VisibleForTesting
|
||||
static final UserAgent GIT_LFS = UserAgent.builder("Git Lfs")
|
||||
.browser(false)
|
||||
.basicAuthenticationCharset(Charsets.UTF_8)
|
||||
.build();
|
||||
|
||||
/** Field description */
|
||||
@VisibleForTesting
|
||||
static final UserAgent MSYSGIT = UserAgent.builder("msysGit").browser(
|
||||
@@ -60,10 +66,11 @@ public class GitUserAgentProvider implements UserAgentProvider
|
||||
Charsets.UTF_8).build();
|
||||
|
||||
/** Field description */
|
||||
private static final String PREFIX = "git/";
|
||||
private static final String PREFIX_REGULAR = "git/";
|
||||
private static final String PREFIX_LFS = "git-lfs/";
|
||||
|
||||
/** Field description */
|
||||
private static final String SUFFIX = "msysgit";
|
||||
private static final String SUFFIX_MSYSGIT = "msysgit";
|
||||
|
||||
//~--- methods --------------------------------------------------------------
|
||||
|
||||
@@ -80,16 +87,14 @@ public class GitUserAgentProvider implements UserAgentProvider
|
||||
{
|
||||
UserAgent ua = null;
|
||||
|
||||
if (userAgentString.startsWith(PREFIX))
|
||||
{
|
||||
if (userAgentString.contains(SUFFIX))
|
||||
{
|
||||
if (userAgentString.startsWith(PREFIX_REGULAR)) {
|
||||
if (userAgentString.contains(SUFFIX_MSYSGIT)) {
|
||||
ua = MSYSGIT;
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
ua = GIT;
|
||||
}
|
||||
} else if (userAgentString.startsWith(PREFIX_LFS)) {
|
||||
ua = GIT_LFS;
|
||||
}
|
||||
|
||||
return ua;
|
||||
|
||||
@@ -35,23 +35,31 @@ package sonia.scm.web;
|
||||
|
||||
//~--- non-JDK imports --------------------------------------------------------
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.inject.Inject;
|
||||
import com.google.inject.Singleton;
|
||||
|
||||
import org.eclipse.jgit.http.server.GitServlet;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import static org.slf4j.LoggerFactory.getLogger;
|
||||
|
||||
|
||||
import org.eclipse.jgit.lfs.lib.Constants;
|
||||
import static org.eclipse.jgit.lfs.lib.Constants.CONTENT_TYPE_GIT_LFS_JSON;
|
||||
|
||||
import sonia.scm.repository.Repository;
|
||||
import sonia.scm.repository.RepositoryProvider;
|
||||
import sonia.scm.repository.RepositoryRequestListenerUtil;
|
||||
import sonia.scm.util.HttpUtil;
|
||||
import sonia.scm.web.lfs.servlet.LfsServletFactory;
|
||||
|
||||
//~--- JDK imports ------------------------------------------------------------
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import sonia.scm.repository.RepositoryException;
|
||||
@@ -73,7 +81,7 @@ public class ScmGitServlet extends GitServlet
|
||||
|
||||
/** the logger for ScmGitServlet */
|
||||
private static final Logger logger =
|
||||
LoggerFactory.getLogger(ScmGitServlet.class);
|
||||
getLogger(ScmGitServlet.class);
|
||||
|
||||
//~--- constructors ---------------------------------------------------------
|
||||
|
||||
@@ -90,14 +98,19 @@ public class ScmGitServlet extends GitServlet
|
||||
*/
|
||||
@Inject
|
||||
public ScmGitServlet(GitRepositoryResolver repositoryResolver,
|
||||
GitReceivePackFactory receivePackFactory,
|
||||
GitRepositoryViewer repositoryViewer,
|
||||
RepositoryProvider repositoryProvider,
|
||||
RepositoryRequestListenerUtil repositoryRequestListenerUtil)
|
||||
GitReceivePackFactory receivePackFactory,
|
||||
GitRepositoryViewer repositoryViewer,
|
||||
RepositoryProvider repositoryProvider,
|
||||
RepositoryRequestListenerUtil repositoryRequestListenerUtil,
|
||||
LfsServletFactory lfsServletFactory,
|
||||
UserAgentParser userAgentParser)
|
||||
{
|
||||
this.repositoryProvider = repositoryProvider;
|
||||
this.repositoryViewer = repositoryViewer;
|
||||
this.repositoryRequestListenerUtil = repositoryRequestListenerUtil;
|
||||
this.lfsServletFactory = lfsServletFactory;
|
||||
this.userAgentParser = userAgentParser;
|
||||
|
||||
setRepositoryResolver(repositoryResolver);
|
||||
setReceivePackFactory(receivePackFactory);
|
||||
}
|
||||
@@ -120,49 +133,44 @@ public class ScmGitServlet extends GitServlet
|
||||
throws ServletException, IOException
|
||||
{
|
||||
String uri = HttpUtil.getStrippedURI(request);
|
||||
logger.trace("--request URI: {}", uri);
|
||||
|
||||
if (uri.matches(REGEX_GITHTTPBACKEND))
|
||||
{
|
||||
sonia.scm.repository.Repository repository = repositoryProvider.get();
|
||||
//decide the type of response to be presented to the client
|
||||
UserAgent userAgent = userAgentParser.parse(request);
|
||||
if (userAgent.isBrowser()) {
|
||||
|
||||
if (repository != null)
|
||||
{
|
||||
if (repositoryRequestListenerUtil.callListeners(request, response,
|
||||
repository))
|
||||
{
|
||||
super.service(request, response);
|
||||
}
|
||||
else if (logger.isDebugEnabled())
|
||||
{
|
||||
renderHtmlRepositryOverview(request, response);
|
||||
} else {
|
||||
|
||||
//service the request for a git client
|
||||
final Repository repository = repositoryProvider.get();
|
||||
|
||||
if (repository == null) {
|
||||
|
||||
//repository could not be matched found the current request
|
||||
super.service(request, response);
|
||||
} else {
|
||||
|
||||
if (repositoryRequestListenerUtil.callListeners(request, response, repository)) {
|
||||
handleRequest(request, response, repository);
|
||||
} else if (logger.isDebugEnabled()) {
|
||||
logger.debug("request aborted by repository request listener");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
super.service(request, response);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
printGitInformation(request, response);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Method description
|
||||
*
|
||||
*
|
||||
*
|
||||
* This method renders basic information about the repository into the response. The result is meant to be viewed by
|
||||
* browser.
|
||||
* @param request
|
||||
* @param response
|
||||
*
|
||||
* @throws IOException
|
||||
* @throws ServletException
|
||||
*/
|
||||
private void printGitInformation(HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws ServletException, IOException
|
||||
{
|
||||
private void renderHtmlRepositryOverview(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
|
||||
sonia.scm.repository.Repository scmRepository = repositoryProvider.get();
|
||||
|
||||
if (scmRepository != null)
|
||||
@@ -186,6 +194,113 @@ public class ScmGitServlet extends GitServlet
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decides the type request being currently made and delegates it accordingly.
|
||||
* <ul>
|
||||
* <li>Batch API:</li>
|
||||
* <ul>
|
||||
* <li> used to provide the client with information on how handle the large files of a repository.</li>
|
||||
* <li> response contains the information where to perform the actual upload and download of the large objects.</li>
|
||||
* </ul>
|
||||
* <li>Transfer API:</li>
|
||||
* <ul>
|
||||
* <li>receives and provides the actual large objects (resolves the pointer placed in the file of the working copy).</li>
|
||||
* <li>invoked only after the Batch API has been questioned about what to do with the large files</li>
|
||||
* </ul>
|
||||
* <li>Regular HTTP Backend</li>
|
||||
* <ul>
|
||||
* <li>services everything that is not git-lfs.</li>
|
||||
* </ul>
|
||||
* </li>
|
||||
* </ul>
|
||||
*/
|
||||
private void handleRequest(HttpServletRequest request, HttpServletResponse response, Repository repository) throws ServletException, IOException {
|
||||
|
||||
logger.trace("--- Repository is: {}", repository.getName());
|
||||
if (isLfsBatchApiRequest(request, repository.getName())) {
|
||||
|
||||
logger.trace("--- detected LFS Batch API Request");
|
||||
HttpServlet servlet = lfsServletFactory.createProtocolServletFor(repository, request);
|
||||
servlet.service(request, response);
|
||||
} else if (isLfsFileTransferRequest(request, repository.getName())) {
|
||||
|
||||
logger.trace("--- detected LFS File Transfer Request");
|
||||
HttpServlet servlet = lfsServletFactory.createFileLfsServletFor(repository, request);
|
||||
servlet.service(request, response);
|
||||
} else {
|
||||
logger.trace("--- seems to be regular Git HTTP backend request: {}", request.getRequestURI());
|
||||
//continue to the regular HTTP Backend
|
||||
super.service(request, response);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decides whether or not a request is for the LFS Batch API,
|
||||
* <p>
|
||||
* - PUT or GET
|
||||
* - exactly for this repository
|
||||
* - Content Type is {@link Constants#HDR_APPLICATION_OCTET_STREAM}.
|
||||
*
|
||||
* @return Returns {@code false} if either of the conditions does not match. Returns true if all match.
|
||||
*/
|
||||
private static boolean isLfsFileTransferRequest(HttpServletRequest request, String repository) {
|
||||
|
||||
String regex = String.format("^%s%s/%s(\\.git)?/info/lfs/objects/[a-z0-9]{64}$", request.getContextPath(), GitServletModule.GIT_PATH, repository);
|
||||
boolean pathMatches = request.getRequestURI().matches(regex);
|
||||
|
||||
boolean methodMatches = request.getMethod().equals("PUT") || request.getMethod().equals("GET");
|
||||
|
||||
return pathMatches && methodMatches;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decides whether or not a request is for the LFS Batch API,
|
||||
* <p>
|
||||
* - POST
|
||||
* - exactly for this repository
|
||||
* - Content Type is {@link Constants#CONTENT_TYPE_GIT_LFS_JSON}.
|
||||
*
|
||||
* @return Returns {@code false} if either of the conditions does not match. Returns true if all match.
|
||||
*/
|
||||
private static boolean isLfsBatchApiRequest(HttpServletRequest request, String repository) {
|
||||
|
||||
String regex = String.format("^%s%s/%s(\\.git)?/info/lfs/objects/batch$", request.getContextPath(), GitServletModule.GIT_PATH, repository);
|
||||
boolean pathMatches = request.getRequestURI().matches(regex);
|
||||
|
||||
boolean methodMatches = "POST".equals(request.getMethod());
|
||||
|
||||
boolean headerContentTypeMatches = isLfsContentHeaderField(request.getContentType(), CONTENT_TYPE_GIT_LFS_JSON);
|
||||
boolean headerAcceptMatches = isLfsContentHeaderField(request.getHeader("Accept"), CONTENT_TYPE_GIT_LFS_JSON);
|
||||
|
||||
return pathMatches && methodMatches && headerContentTypeMatches && headerAcceptMatches;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether request is of the specific content type.
|
||||
*
|
||||
* @param request The HTTP request header value to be examined.
|
||||
* @param expectedContentType The expected content type.
|
||||
* @return Returns {@code true} if the request has the expected content type. Return {@code false} otherwise.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
static boolean isLfsContentHeaderField(String request, String expectedContentType) {
|
||||
|
||||
if (request == null || request.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
String[] parts = request.split(" ");
|
||||
for (String part : parts) {
|
||||
if (part.startsWith(expectedContentType)) {
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
//~--- fields ---------------------------------------------------------------
|
||||
|
||||
/** Field description */
|
||||
@@ -194,6 +309,14 @@ public class ScmGitServlet extends GitServlet
|
||||
/** Field description */
|
||||
private final RepositoryRequestListenerUtil repositoryRequestListenerUtil;
|
||||
|
||||
/** Field description */
|
||||
/**
|
||||
* Field description
|
||||
*/
|
||||
private final GitRepositoryViewer repositoryViewer;
|
||||
|
||||
private final LfsServletFactory lfsServletFactory;
|
||||
|
||||
private final UserAgentParser userAgentParser;
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package sonia.scm.web.lfs;
|
||||
|
||||
import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
|
||||
import org.eclipse.jgit.lfs.server.LargeFileRepository;
|
||||
import org.eclipse.jgit.lfs.server.Response;
|
||||
import sonia.scm.store.Blob;
|
||||
import sonia.scm.store.BlobStore;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* This LargeFileRepository is used for jGit-Servlet implementation. Under the jgit LFS Servlet hood, the
|
||||
* SCM-Repository API is used to implement the Repository.
|
||||
*
|
||||
* @since 1.54
|
||||
* Created by omilke on 03.05.2017.
|
||||
*/
|
||||
public class ScmBlobLfsRepository implements LargeFileRepository {
|
||||
|
||||
private final BlobStore blobStore;
|
||||
|
||||
/**
|
||||
* This URI is used to determine the actual URI for Upload / Download. Must be full URI (or rewritable by reverse
|
||||
* proxy).
|
||||
*/
|
||||
private final String baseUri;
|
||||
|
||||
/**
|
||||
* Creates a {@link ScmBlobLfsRepository} for the provided repository.
|
||||
*
|
||||
* @param blobStore The SCM Blobstore used for this @{@link LargeFileRepository}.
|
||||
* @param baseUri This URI is used to determine the actual URI for Upload / Download. Must be full URI (or
|
||||
* rewritable by reverse proxy).
|
||||
*/
|
||||
|
||||
public ScmBlobLfsRepository(BlobStore blobStore, String baseUri) {
|
||||
|
||||
this.blobStore = blobStore;
|
||||
this.baseUri = baseUri;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Response.Action getDownloadAction(AnyLongObjectId id) {
|
||||
|
||||
return getAction(id);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Response.Action getUploadAction(AnyLongObjectId id, long size) {
|
||||
|
||||
return getAction(id);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Response.Action getVerifyAction(AnyLongObjectId id) {
|
||||
|
||||
//validation is optional. We do not support it.
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getSize(AnyLongObjectId id) throws IOException {
|
||||
|
||||
//this needs to be size of what is will be written into the response of the download. Clients are likely to
|
||||
// verify it.
|
||||
Blob blob = this.blobStore.get(id.getName());
|
||||
if (blob == null) {
|
||||
|
||||
return -1;
|
||||
} else {
|
||||
|
||||
return blob.getSize();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs the Download / Upload actions to be supplied to the client.
|
||||
*/
|
||||
private Response.Action getAction(AnyLongObjectId id) {
|
||||
|
||||
//LFS protocol has to provide the information on where to put or get the actual content, i. e.
|
||||
//the actual URI for up- and download.
|
||||
|
||||
Response.Action a = new Response.Action();
|
||||
a.href = baseUri + id.getName();
|
||||
|
||||
return a;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package sonia.scm.web.lfs.servlet;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import org.eclipse.jgit.lfs.server.LargeFileRepository;
|
||||
import org.eclipse.jgit.lfs.server.LfsProtocolServlet;
|
||||
import org.eclipse.jgit.lfs.server.fs.FileLfsServlet;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import sonia.scm.repository.Repository;
|
||||
import sonia.scm.store.BlobStore;
|
||||
import sonia.scm.store.BlobStoreFactory;
|
||||
import sonia.scm.util.HttpUtil;
|
||||
import sonia.scm.web.lfs.ScmBlobLfsRepository;
|
||||
|
||||
import javax.inject.Inject;
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
/**
|
||||
* This factory class is a helper class to provide the {@link LfsProtocolServlet} and the {@link FileLfsServlet}
|
||||
* belonging to a SCM Repository.
|
||||
*
|
||||
* @since 1.54
|
||||
* Created by omilke on 11.05.2017.
|
||||
*/
|
||||
public class LfsServletFactory {
|
||||
|
||||
private static final String GIT_LFS_REPOSITORY_POSTFIX = "-git-lfs";
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(LfsServletFactory.class);
|
||||
|
||||
private final BlobStoreFactory blobStoreFactory;
|
||||
|
||||
@Inject
|
||||
public LfsServletFactory(BlobStoreFactory blobStoreFactory) {
|
||||
|
||||
this.blobStoreFactory = blobStoreFactory;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the {@link LfsProtocolServlet} (jgit API) for a SCM Repository.
|
||||
*
|
||||
* @param repository The SCM Repository to build the servlet for.
|
||||
* @param request The {@link HttpServletRequest} the used to access the SCM Repository.
|
||||
* @return The {@link LfsProtocolServlet} to provide the LFS Batch API for a SCM Repository.
|
||||
*/
|
||||
public LfsProtocolServlet createProtocolServletFor(Repository repository, HttpServletRequest request) {
|
||||
|
||||
BlobStore blobStore = getBlobStore(repository);
|
||||
String baseUri = buildBaseUri(repository, request);
|
||||
|
||||
LargeFileRepository largeFileRepository = new ScmBlobLfsRepository(blobStore, baseUri);
|
||||
return new ScmLfsProtocolServlet(largeFileRepository);
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the {@link FileLfsServlet} (jgit API) for a SCM Repository.
|
||||
*
|
||||
* @param repository The SCM Repository to build the servlet for.
|
||||
* @param request The {@link HttpServletRequest} the used to access the SCM Repository.
|
||||
* @return The {@link FileLfsServlet} to provide the LFS Upload / Download API for a SCM Repository.
|
||||
*/
|
||||
public HttpServlet createFileLfsServletFor(Repository repository, HttpServletRequest request) {
|
||||
|
||||
return new ScmFileTransferServlet(getBlobStore(repository));
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the complete URI, under which the File Transfer API for this repository will be will be reachable.
|
||||
*
|
||||
* @param repository The repository to build the File Transfer URI for.
|
||||
* @param request The request to construct the complete URI from.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
static String buildBaseUri(Repository repository, HttpServletRequest request) {
|
||||
|
||||
return String.format("%s/git/%s.git/info/lfs/objects/", HttpUtil.getCompleteUrl(request), repository.getName());
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides a {@link BlobStore} corresponding to the SCM Repository.
|
||||
* <p>
|
||||
* git-lfs repositories should generally carry the same name as their regular SCM repository counterparts. However,
|
||||
* we have decided to store them under their IDs instead of their names, since the names might change and provide
|
||||
* other drawbacks, as well.
|
||||
* <p>
|
||||
* These repositories will have {@linkplain #GIT_LFS_REPOSITORY_POSTFIX} appended to their IDs.
|
||||
*
|
||||
* @param repository The SCM Repository to provide a LFS {@link BlobStore} for.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
BlobStore getBlobStore(Repository repository) {
|
||||
|
||||
return blobStoreFactory.getBlobStore(repository.getId() + GIT_LFS_REPOSITORY_POSTFIX);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
package sonia.scm.web.lfs.servlet;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.gson.FieldNamingPolicy;
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import org.apache.http.HttpStatus;
|
||||
import org.eclipse.jgit.lfs.errors.CorruptLongObjectException;
|
||||
import org.eclipse.jgit.lfs.errors.InvalidLongObjectIdException;
|
||||
import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
|
||||
import org.eclipse.jgit.lfs.lib.Constants;
|
||||
import org.eclipse.jgit.lfs.lib.LongObjectId;
|
||||
import org.eclipse.jgit.lfs.server.LfsProtocolServlet;
|
||||
import org.eclipse.jgit.lfs.server.fs.FileLfsServlet;
|
||||
import org.eclipse.jgit.lfs.server.internal.LfsServerText;
|
||||
import org.eclipse.jgit.util.HttpSupport;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import sonia.scm.store.Blob;
|
||||
import sonia.scm.store.BlobStore;
|
||||
import sonia.scm.util.IOUtil;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.ServletInputStream;
|
||||
import javax.servlet.ServletOutputStream;
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.io.PrintWriter;
|
||||
import java.text.MessageFormat;
|
||||
|
||||
/**
|
||||
* This Servlet provides the upload and download of files via git-lfs.
|
||||
* <p>
|
||||
* This implementation is based on {@link FileLfsServlet} but adjusted to work with
|
||||
* servlet-2.5 instead of servlet-3.1.
|
||||
* <p>
|
||||
*
|
||||
* @see FileLfsServlet
|
||||
* @since 1.54
|
||||
* Created by omilke on 15.05.2017.
|
||||
*/
|
||||
public class ScmFileTransferServlet extends HttpServlet {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(ScmFileTransferServlet.class);
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/**
|
||||
* Gson is used because the implementation was based on the jgit implementation. However the {@link LfsProtocolServlet} (which we do use in
|
||||
* {@link ScmLfsProtocolServlet}) also uses Gson, which currently ties us to Gson anyway.
|
||||
*/
|
||||
private static Gson gson = createGson();
|
||||
|
||||
private final BlobStore blobStore;
|
||||
|
||||
public ScmFileTransferServlet(BlobStore store) {
|
||||
|
||||
this.blobStore = store;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Extracts the part after the last slash from path.
|
||||
*
|
||||
* @return Returns {@code null} if the part after the last slash is itself {@code null} or if its length is not 64.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
static String objectIdFromPath(String info) {
|
||||
|
||||
int lastSlash = info.lastIndexOf('/');
|
||||
String potentialObjectId = info.substring(lastSlash + 1);
|
||||
|
||||
if (potentialObjectId.length() != 64) {
|
||||
return null;
|
||||
|
||||
} else {
|
||||
return potentialObjectId;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Logs the message and provides it to the client.
|
||||
*
|
||||
* @param response The response
|
||||
* @param status The HTTP Status Code to be provided to the client.
|
||||
* @param message the message to used for server-side logging. It is also provided to the client.
|
||||
*/
|
||||
private static void sendErrorAndLog(HttpServletResponse response, int status, String message) throws IOException {
|
||||
|
||||
logger.warn("Error occurred during git-lfs file transfer: {}", message);
|
||||
|
||||
sendError(response, status, message);
|
||||
}
|
||||
|
||||
/**
|
||||
* Logs the exception and provides only the message of the exception to the client.
|
||||
*
|
||||
* @param response The response
|
||||
* @param status The HTTP Status Code to be provided to the client.
|
||||
* @param exception An exception to used for server-side logging.
|
||||
*/
|
||||
private static void sendErrorAndLog(HttpServletResponse response, int status, Exception exception) throws IOException {
|
||||
|
||||
logger.warn("Error occurred during git-lfs file transfer.", exception);
|
||||
String message = exception.getMessage();
|
||||
|
||||
|
||||
sendError(response, status, message);
|
||||
}
|
||||
|
||||
private static void sendError(HttpServletResponse response, int status, String message) throws IOException {
|
||||
|
||||
try (PrintWriter writer = response.getWriter()) {
|
||||
|
||||
gson.toJson(new Error(message), writer);
|
||||
|
||||
response.setStatus(status);
|
||||
writer.flush();
|
||||
}
|
||||
|
||||
response.flushBuffer();
|
||||
}
|
||||
|
||||
private static Gson createGson() {
|
||||
|
||||
GsonBuilder gb = new GsonBuilder().setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES).setPrettyPrinting().disableHtmlEscaping();
|
||||
return gb.create();
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides a blob to download.
|
||||
* <p>
|
||||
* Actual implementation is based on <code>org.eclipse.jgit.lfs.server.fs.ObjectDownloadListener</code> and adjusted
|
||||
* to non-async as we're currently on servlet-2.5.
|
||||
*
|
||||
* @param request servlet request
|
||||
* @param response servlet response
|
||||
* @throws ServletException if a servlet-specific error occurs
|
||||
* @throws IOException if an I/O error occurs
|
||||
*/
|
||||
@Override
|
||||
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
|
||||
AnyLongObjectId objectId = getObjectToTransfer(request, response);
|
||||
if (objectId == null) {
|
||||
|
||||
logInvalidObjectId(request.getRequestURI());
|
||||
} else {
|
||||
|
||||
final String objectIdName = objectId.getName();
|
||||
logger.trace("---- providing download for LFS-Oid: {}", objectIdName);
|
||||
|
||||
Blob savedBlob = blobStore.get(objectIdName);
|
||||
if (isBlobPresent(savedBlob)) {
|
||||
|
||||
logger.trace("----- Object {}: providing {} bytes", objectIdName, savedBlob.getSize());
|
||||
writeBlobIntoResponse(savedBlob, response);
|
||||
} else {
|
||||
|
||||
sendErrorAndLog(response, HttpStatus.SC_NOT_FOUND, MessageFormat.format(LfsServerText.get().objectNotFound, objectIdName));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Receives a blob from an upload.
|
||||
* <p>
|
||||
* Actual implementation is based on <code>org.eclipse.jgit.lfs.server.fs.ObjectUploadListener</code> and adjusted
|
||||
* to non-async as we're currently on servlet-2.5.
|
||||
*
|
||||
* @param request servlet request
|
||||
* @param response servlet response
|
||||
* @throws ServletException if a servlet-specific error occurs
|
||||
* @throws IOException if an I/O error occurs
|
||||
*/
|
||||
@Override
|
||||
protected void doPut(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
|
||||
|
||||
AnyLongObjectId objectId = getObjectToTransfer(request, response);
|
||||
if (objectId == null) {
|
||||
|
||||
logInvalidObjectId(request.getRequestURI());
|
||||
} else {
|
||||
|
||||
logger.trace("---- receiving upload for LFS-Oid: {}", objectId.getName());
|
||||
readBlobFromResponse(request, response, objectId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the {@link LongObjectId} from the request. Finishes the request, in case the {@link LongObjectId} cannot
|
||||
* be extracted with an appropriate error.
|
||||
*
|
||||
* @throws IOException Thrown if the response could not be completed in an error case.
|
||||
*/
|
||||
private AnyLongObjectId getObjectToTransfer(HttpServletRequest request, HttpServletResponse response) throws IOException {
|
||||
|
||||
String path = request.getPathInfo();
|
||||
|
||||
String objectIdFromPath = objectIdFromPath(path);
|
||||
if (objectIdFromPath == null) {
|
||||
|
||||
//ObjectId is not retrievable from URL
|
||||
sendErrorAndLog(response, HttpStatus.SC_UNPROCESSABLE_ENTITY, MessageFormat.format(LfsServerText.get().invalidPathInfo, path));
|
||||
return null;
|
||||
} else {
|
||||
try {
|
||||
return LongObjectId.fromString(objectIdFromPath);
|
||||
} catch (InvalidLongObjectIdException e) {
|
||||
|
||||
sendErrorAndLog(response, HttpStatus.SC_UNPROCESSABLE_ENTITY, e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void logInvalidObjectId(String requestURI) {
|
||||
|
||||
logger.warn("---- could not extract Oid from Request. Path seems to be invalid: {}", requestURI);
|
||||
}
|
||||
|
||||
private boolean isBlobPresent(Blob savedBlob) {
|
||||
|
||||
return savedBlob != null && savedBlob.getSize() >= 0;
|
||||
}
|
||||
|
||||
private void writeBlobIntoResponse(Blob savedBlob, HttpServletResponse response) throws IOException {
|
||||
|
||||
try (ServletOutputStream responseOutputStream = response.getOutputStream();
|
||||
InputStream savedBlobInputStream = savedBlob.getInputStream()) {
|
||||
|
||||
response.addHeader(HttpSupport.HDR_CONTENT_LENGTH, String.valueOf(savedBlob.getSize()));
|
||||
response.setContentType(Constants.HDR_APPLICATION_OCTET_STREAM);
|
||||
|
||||
IOUtil.copy(savedBlobInputStream, responseOutputStream);
|
||||
} catch (IOException ex) {
|
||||
|
||||
sendErrorAndLog(response, HttpStatus.SC_INTERNAL_SERVER_ERROR, ex);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private void readBlobFromResponse(HttpServletRequest request, HttpServletResponse response, AnyLongObjectId objectId) throws IOException {
|
||||
|
||||
try (OutputStream blobOutputStream = blobStore.create(objectId.getName()).getOutputStream();
|
||||
ServletInputStream requestInputStream = request.getInputStream()) {
|
||||
|
||||
IOUtil.copy(requestInputStream, blobOutputStream);
|
||||
|
||||
response.setContentType(Constants.CONTENT_TYPE_GIT_LFS_JSON);
|
||||
response.setStatus(HttpServletResponse.SC_OK);
|
||||
} catch (CorruptLongObjectException ex) {
|
||||
|
||||
sendErrorAndLog(response, HttpStatus.SC_BAD_REQUEST, ex);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Used for providing an error message.
|
||||
*/
|
||||
private static class Error {
|
||||
String message;
|
||||
|
||||
Error(String m) {
|
||||
|
||||
this.message = m;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package sonia.scm.web.lfs.servlet;
|
||||
|
||||
import org.eclipse.jgit.lfs.errors.LfsException;
|
||||
import org.eclipse.jgit.lfs.server.LargeFileRepository;
|
||||
import org.eclipse.jgit.lfs.server.LfsProtocolServlet;
|
||||
|
||||
/**
|
||||
* Provides an implementation for the git-lfs Batch API.
|
||||
*
|
||||
* @since 1.54
|
||||
* Created by omilke on 11.05.2017.
|
||||
*/
|
||||
public class ScmLfsProtocolServlet extends LfsProtocolServlet {
|
||||
|
||||
private final LargeFileRepository repository;
|
||||
|
||||
public ScmLfsProtocolServlet(LargeFileRepository largeFileRepository) {
|
||||
this.repository = largeFileRepository;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected LargeFileRepository getLargeFileRepository(LfsRequest request, String path) throws LfsException {
|
||||
return repository;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user