Nemcio/SCM-Manager
1
0
Fork 0
mirror of https://github.com/scm-manager/scm-manager.git synced 2025-11-03 12:05:52 +01:00
Code Issues Packages Projects Releases Activity

merge with branch 1.x

Browse Source
This commit is contained in:
Sebastian Sdorra
2017-06-25 19:01:33 +02:00
parent b7065bedd1 73cfbe8993
commit aec3d5d65d
81 changed files with 4617 additions and 1319 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/GitPermissionFilter.java
View File

@@ -35,6 +35,7 @@ package sonia.scm.web;
//~--- non-JDK imports --------------------------------------------------------
import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -57,7 +58,8 @@ import sonia.scm.filter.Filters;
import sonia.scm.filter.WebElement;
/**
*
* GitPermissionFilter decides if a git request requires write or read privileges.
*
* @author Sebastian Sdorra
*/
@Priority(Filters.PRIORITY_AUTHORIZATION)
@@ -65,79 +67,60 @@ import sonia.scm.filter.WebElement;
public class GitPermissionFilter extends ProviderPermissionFilter
{
/** Field description */
public static final String PARAMETER_SERVICE = "service";
private static final String PARAMETER_SERVICE = "service";
/** Field description */
public static final String PARAMETER_VALUE_RECEIVE = "git-receive-pack";
private static final String PARAMETER_VALUE_RECEIVE = "git-receive-pack";
/** Field description */
public static final String URI_RECEIVE_PACK = "git-receive-pack";
private static final String URI_RECEIVE_PACK = "git-receive-pack";
/** Field description */
public static final String URI_REF_INFO = "/info/refs";
private static final String URI_REF_INFO = "/info/refs";
private static final String METHOD_LFS_UPLOAD = "PUT";
//~--- constructors ---------------------------------------------------------
/**
* Constructs ...
* Constructs a new instance of the GitPermissionFilter.
*
* @param configuration
* @param repositoryProvider
* @param configuration scm main configuration
* @param repositoryProvider repository provider
*/
@Inject
public GitPermissionFilter(ScmConfiguration configuration,
RepositoryProvider repositoryProvider)
{
public GitPermissionFilter(ScmConfiguration configuration, RepositoryProvider repositoryProvider) {
super(configuration, repositoryProvider);
}
//~--- methods --------------------------------------------------------------
/**
* Method description
*
*
* @param request
* @param response
*
* @throws IOException
*/
@Override
protected void sendNotEnoughPrivilegesError(HttpServletRequest request,
HttpServletResponse response)
throws IOException
{
if (GitUtil.isGitClient(request))
{
protected void sendNotEnoughPrivilegesError(HttpServletRequest request, HttpServletResponse response)
throws IOException {
if (GitUtil.isGitClient(request)) {
GitSmartHttpTools.sendError(request, response,
HttpServletResponse.SC_FORBIDDEN,
ClientMessages.get(request).notEnoughPrivileges());
}
else
{
} else {
super.sendNotEnoughPrivilegesError(request, response);
}
}
//~--- get methods ----------------------------------------------------------
/**
* Method description
*
*
* @param request
*
* @return
*/
@Override
protected boolean isWriteRequest(HttpServletRequest request)
{
String uri = request.getRequestURI();
return uri.endsWith(URI_RECEIVE_PACK)
|| (uri.endsWith(URI_REF_INFO)
&& PARAMETER_VALUE_RECEIVE.equals(
request.getParameter(PARAMETER_SERVICE)));
protected boolean isWriteRequest(HttpServletRequest request) {
return isReceivePackRequest(request) ||
isReceiveServiceRequest(request) ||
isLfsFileUpload(request);
}
private boolean isReceivePackRequest(HttpServletRequest request) {
return request.getRequestURI().endsWith(URI_RECEIVE_PACK);
}
private boolean isReceiveServiceRequest(HttpServletRequest request) {
return request.getRequestURI().endsWith(URI_REF_INFO)
&& PARAMETER_VALUE_RECEIVE.equals(request.getParameter(PARAMETER_SERVICE));
}
@VisibleForTesting
private static boolean isLfsFileUpload(HttpServletRequest request) {
return METHOD_LFS_UPLOAD.equalsIgnoreCase(request.getMethod());
}
}

55
scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/GitRepositoryResolver.java
View File

@@ -35,6 +35,7 @@ package sonia.scm.web;
//~--- non-JDK imports --------------------------------------------------------
import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Inject;
import org.eclipse.jgit.errors.RepositoryNotFoundException;
@@ -63,13 +64,11 @@ import javax.servlet.http.HttpServletRequest;
*
* @author Sebastian Sdorra
*/
public class GitRepositoryResolver
implements RepositoryResolver<HttpServletRequest>
public class GitRepositoryResolver implements RepositoryResolver<HttpServletRequest>
{
/** the logger for GitRepositoryResolver */
private static final Logger logger =
LoggerFactory.getLogger(GitRepositoryResolver.class);
private static final Logger logger = LoggerFactory.getLogger(GitRepositoryResolver.class);
//~--- constructors ---------------------------------------------------------
@@ -114,20 +113,14 @@ public class GitRepositoryResolver
if (config.isValid())
{
File gitdir = new File(config.getRepositoryDirectory(), repositoryName);
if (logger.isDebugEnabled())
{
logger.debug("try to open git repository at {}", gitdir);
}
if (!gitdir.exists())
{
File gitdir = findRepository(config.getRepositoryDirectory(), repositoryName);
if (gitdir == null) {
throw new RepositoryNotFoundException(repositoryName);
}
logger.debug("try to open git repository at {}", gitdir);
repository = RepositoryCache.open(FileKey.lenient(gitdir, FS.DETECTED),
true);
repository = RepositoryCache.open(FileKey.lenient(gitdir, FS.DETECTED), true);
}
else
{
@@ -139,17 +132,39 @@ public class GitRepositoryResolver
throw new ServiceNotEnabledException();
}
}
catch (RuntimeException e)
{
throw new RepositoryNotFoundException(repositoryName, e);
}
catch (IOException e)
catch (RuntimeException | IOException e)
{
throw new RepositoryNotFoundException(repositoryName, e);
}
return repository;
}
@VisibleForTesting
File findRepository(File parentDirectory, String repositoryName) {
File repositoryDirectory = new File(parentDirectory, repositoryName);
if (repositoryDirectory.exists()) {
return repositoryDirectory;
}
if (endsWithDotGit(repositoryName)) {
String repositoryNameWithoutDotGit = repositoryNameWithoutDotGit(repositoryName);
repositoryDirectory = new File(parentDirectory, repositoryNameWithoutDotGit);
if (repositoryDirectory.exists()) {
return repositoryDirectory;
}
}
return null;
}
private boolean endsWithDotGit(String repositoryName) {
return repositoryName.endsWith(GitRepositoryHandler.DOT_GIT);
}
private String repositoryNameWithoutDotGit(String repositoryName) {
return repositoryName.substring(0, repositoryName.length() - GitRepositoryHandler.DOT_GIT.length());
}
//~--- fields ---------------------------------------------------------------

9
scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/GitServletModule.java
View File

@@ -41,6 +41,8 @@ import org.eclipse.jgit.transport.ScmTransportProtocol;
import sonia.scm.plugin.Extension;
import sonia.scm.web.lfs.LfsBlobStoreFactory;
/**
*
* @author Sebastian Sdorra
@@ -49,8 +51,11 @@ import sonia.scm.plugin.Extension;
public class GitServletModule extends ServletModule
{
public static final String GIT_PATH = "/git";
/** Field description */
public static final String PATTERN_GIT = "/git/*";
public static final String PATTERN_GIT = GIT_PATH + "/*";
//~--- methods --------------------------------------------------------------
@@ -65,6 +70,8 @@ public class GitServletModule extends ServletModule
bind(GitRepositoryResolver.class);
bind(GitReceivePackFactory.class);
bind(ScmTransportProtocol.class);
bind(LfsBlobStoreFactory.class);
// serlvelts and filters
serve(PATTERN_GIT).with(ScmGitServlet.class);

106
scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/GitUserAgentProvider.java
View File

@@ -35,63 +35,89 @@ package sonia.scm.web;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Charsets;
import com.google.common.base.Strings;
import java.util.Locale;
import sonia.scm.plugin.Extension;
/**
*
* UserAgent provider for git related clients.
* @author Sebastian Sdorra <sebastian.sdorra@gmail.com>
* @since 1.45
*/
@Extension
public class GitUserAgentProvider implements UserAgentProvider
{
public class GitUserAgentProvider implements UserAgentProvider {
private static final String PREFIX_JGIT = "jgit/";
/** Field description */
@VisibleForTesting
static final UserAgent GIT = UserAgent.builder("Git").browser(
false).basicAuthenticationCharset(
Charsets.UTF_8).build();
/** Field description */
static final UserAgent JGIT = UserAgent.builder("JGit")
.browser(false)
.basicAuthenticationCharset(Charsets.UTF_8)
.build();
private static final String PREFIX_REGULAR = "git/";
@VisibleForTesting
static final UserAgent MSYSGIT = UserAgent.builder("msysGit").browser(
false).basicAuthenticationCharset(
Charsets.UTF_8).build();
static final UserAgent GIT = UserAgent.builder("Git")
.browser(false)
.basicAuthenticationCharset(Charsets.UTF_8)
.build();
private static final String PREFIX_LFS = "git-lfs/";
@VisibleForTesting
static final UserAgent GIT_LFS = UserAgent.builder("Git Lfs")
.browser(false)
.basicAuthenticationCharset(Charsets.UTF_8)
.build();
private static final String SUFFIX_MSYSGIT = "msysgit";
@VisibleForTesting
static final UserAgent MSYSGIT = UserAgent.builder("msysGit")
.browser(false)
.basicAuthenticationCharset(Charsets.UTF_8)
.build();
/** Field description */
private static final String PREFIX = "git/";
/** Field description */
private static final String SUFFIX = "msysgit";
//~--- methods --------------------------------------------------------------
/**
* Method description
*
*
* @param userAgentString
*
* @return
*/
@Override
public UserAgent parseUserAgent(String userAgentString)
{
UserAgent ua = null;
if (userAgentString.startsWith(PREFIX))
{
if (userAgentString.contains(SUFFIX))
{
ua = MSYSGIT;
}
else
{
ua = GIT;
}
public UserAgent parseUserAgent(String userAgentString) {
String lowerUserAgent = toLower(userAgentString);
if (isJGit(lowerUserAgent)) {
return JGIT;
} else if (isMsysGit(lowerUserAgent)) {
return MSYSGIT;
} else if (isGitLFS(lowerUserAgent)) {
return GIT_LFS;
} else if (isGit(lowerUserAgent)) {
return GIT;
} else {
return null;
}
return ua;
}
private String toLower(String value) {
return Strings.nullToEmpty(value).toLowerCase(Locale.ENGLISH);
}
private boolean isJGit(String userAgent) {
return userAgent.startsWith(PREFIX_JGIT);
}
private boolean isMsysGit(String userAgent) {
return userAgent.startsWith(PREFIX_REGULAR) && userAgent.contains(SUFFIX_MSYSGIT);
}
private boolean isGitLFS(String userAgent) {
return userAgent.startsWith(PREFIX_LFS);
}
private boolean isGit(String userAgent) {
return userAgent.startsWith(PREFIX_REGULAR);
}
}

239
scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/ScmGitServlet.java
View File

@@ -35,23 +35,32 @@ package sonia.scm.web;
//~--- non-JDK imports --------------------------------------------------------
import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.eclipse.jgit.http.server.GitServlet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.slf4j.LoggerFactory.getLogger;
import org.eclipse.jgit.lfs.lib.Constants;
import static org.eclipse.jgit.lfs.lib.Constants.CONTENT_TYPE_GIT_LFS_JSON;
import sonia.scm.repository.Repository;
import sonia.scm.repository.RepositoryProvider;
import sonia.scm.repository.RepositoryRequestListenerUtil;
import sonia.scm.util.HttpUtil;
import sonia.scm.web.lfs.servlet.LfsServletFactory;
//~--- JDK imports ------------------------------------------------------------
import java.io.IOException;
import java.util.regex.Pattern;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import sonia.scm.repository.RepositoryException;
@@ -65,15 +74,15 @@ public class ScmGitServlet extends GitServlet
{
/** Field description */
public static final String REGEX_GITHTTPBACKEND =
"(?x)^/git/(.*/(HEAD|info/refs|objects/(info/[^/]+|[0-9a-f]{2}/[0-9a-f]{38}|pack/pack-[0-9a-f]{40}\\.(pack|idx))|git-(upload|receive)-pack))$";
public static final Pattern REGEX_GITHTTPBACKEND = Pattern.compile(
"(?x)^/git/(.*/(HEAD|info/refs|objects/(info/[^/]+|[0-9a-f]{2}/[0-9a-f]{38}|pack/pack-[0-9a-f]{40}\\.(pack|idx))|git-(upload|receive)-pack))$"
);
/** Field description */
private static final long serialVersionUID = -7712897339207470674L;
/** the logger for ScmGitServlet */
private static final Logger logger =
LoggerFactory.getLogger(ScmGitServlet.class);
private static final Logger logger = getLogger(ScmGitServlet.class);
//~--- constructors ---------------------------------------------------------
@@ -87,17 +96,21 @@ public class ScmGitServlet extends GitServlet
* @param repositoryViewer
* @param repositoryProvider
* @param repositoryRequestListenerUtil
* @param lfsServletFactory
*/
@Inject
public ScmGitServlet(GitRepositoryResolver repositoryResolver,
GitReceivePackFactory receivePackFactory,
GitRepositoryViewer repositoryViewer,
RepositoryProvider repositoryProvider,
RepositoryRequestListenerUtil repositoryRequestListenerUtil)
GitReceivePackFactory receivePackFactory,
GitRepositoryViewer repositoryViewer,
RepositoryProvider repositoryProvider,
RepositoryRequestListenerUtil repositoryRequestListenerUtil,
LfsServletFactory lfsServletFactory)
{
this.repositoryProvider = repositoryProvider;
this.repositoryViewer = repositoryViewer;
this.repositoryRequestListenerUtil = repositoryRequestListenerUtil;
this.lfsServletFactory = lfsServletFactory;
setRepositoryResolver(repositoryResolver);
setReceivePackFactory(receivePackFactory);
}
@@ -118,74 +131,165 @@ public class ScmGitServlet extends GitServlet
protected void service(HttpServletRequest request,
HttpServletResponse response)
throws ServletException, IOException
{
String uri = HttpUtil.getStrippedURI(request);
if (uri.matches(REGEX_GITHTTPBACKEND))
{
sonia.scm.repository.Repository repository = repositoryProvider.get();
if (repository != null)
{
if (repositoryRequestListenerUtil.callListeners(request, response,
repository))
{
super.service(request, response);
}
else if (logger.isDebugEnabled())
{
logger.debug("request aborted by repository request listener");
}
}
else
{
super.service(request, response);
}
}
else
{
printGitInformation(request, response);
{
Repository repository = repositoryProvider.get();
if (repository != null) {
handleRequest(request, response, repository);
} else {
// logger
response.sendError(HttpServletResponse.SC_NOT_FOUND);
}
}
/**
* Decides the type request being currently made and delegates it accordingly.
* <ul>
* <li>Batch API:</li>
* <ul>
* <li>used to provide the client with information on how handle the large files of a repository.</li>
* <li>response contains the information where to perform the actual upload and download of the large objects.</li>
* </ul>
* <li>Transfer API:</li>
* <ul>
* <li>receives and provides the actual large objects (resolves the pointer placed in the file of the working copy).</li>
* <li>invoked only after the Batch API has been questioned about what to do with the large files</li>
* </ul>
* <li>Regular Git Http API:</li>
* <ul>
* <li>regular git http wire protocol, use by normal git clients.</li>
* </ul>
* <li>Browser Overview:<li>
* <ul>
* <li>short repository overview for browser clients.</li>
* </ul>
* </li>
* </ul>
*/
private void handleRequest(HttpServletRequest request, HttpServletResponse response, Repository repository) throws ServletException, IOException {
logger.trace("handle git repository at {}", repository.getName());
if (isLfsBatchApiRequest(request, repository.getName())) {
HttpServlet servlet = lfsServletFactory.createProtocolServletFor(repository, request);
logger.trace("handle lfs batch api request");
handleGitLfsRequest(servlet, request, response, repository);
} else if (isLfsFileTransferRequest(request, repository.getName())) {
HttpServlet servlet = lfsServletFactory.createFileLfsServletFor(repository, request);
logger.trace("handle lfs file transfer request");
handleGitLfsRequest(servlet, request, response, repository);
} else if (isRegularGitAPIRequest(request)) {
logger.trace("handle regular git request");
// continue with the regular git Backend
handleRegularGitRequest(request, response, repository);
} else {
logger.trace("handle browser request");
handleBrowserRequest(request, response, repository);
}
}
private boolean isRegularGitAPIRequest(HttpServletRequest request) {
return REGEX_GITHTTPBACKEND.matcher(HttpUtil.getStrippedURI(request)).matches();
}
private void handleGitLfsRequest(HttpServlet servlet, HttpServletRequest request, HttpServletResponse response, Repository repository) throws ServletException, IOException {
if (repositoryRequestListenerUtil.callListeners(request, response, repository)) {
servlet.service(request, response);
} else if (logger.isDebugEnabled()) {
logger.debug("request aborted by repository request listener");
}
}
private void handleRegularGitRequest(HttpServletRequest request, HttpServletResponse response, Repository repository) throws ServletException, IOException {
if (repositoryRequestListenerUtil.callListeners(request, response, repository)) {
super.service(request, response);
} else if (logger.isDebugEnabled()) {
logger.debug("request aborted by repository request listener");
}
}
/**
* Method description
*
*
*
* This method renders basic information about the repository into the response. The result is meant to be viewed by
* browser.
* @param request
* @param response
*
* @throws IOException
* @throws ServletException
*/
private void printGitInformation(HttpServletRequest request,
HttpServletResponse response)
throws ServletException, IOException
{
sonia.scm.repository.Repository scmRepository = repositoryProvider.get();
if (scmRepository != null)
{
try
{
repositoryViewer.handleRequest(request, response, scmRepository);
}
catch (RepositoryException ex)
{
throw new ServletException("could not create repository view", ex);
}
catch (IOException ex)
{
throw new ServletException("could not create repository view", ex);
}
}
else
{
response.sendError(HttpServletResponse.SC_NOT_FOUND);
private void handleBrowserRequest(HttpServletRequest request, HttpServletResponse response, Repository repository) throws ServletException, IOException {
try {
repositoryViewer.handleRequest(request, response, repository);
} catch (RepositoryException | IOException ex) {
throw new ServletException("could not create repository view", ex);
}
}
/**
* Decides whether or not a request is for the LFS Batch API,
* <p>
* - PUT or GET
* - exactly for this repository
* - Content Type is {@link Constants#HDR_APPLICATION_OCTET_STREAM}.
*
* @return Returns {@code false} if either of the conditions does not match. Returns true if all match.
*/
private static boolean isLfsFileTransferRequest(HttpServletRequest request, String repository) {
String regex = String.format("^%s%s/%s(\\.git)?/info/lfs/objects/[a-z0-9]{64}$", request.getContextPath(), GitServletModule.GIT_PATH, repository);
boolean pathMatches = request.getRequestURI().matches(regex);
boolean methodMatches = request.getMethod().equals("PUT") || request.getMethod().equals("GET");
return pathMatches && methodMatches;
}
/**
* Decides whether or not a request is for the LFS Batch API,
* <p>
* - POST
* - exactly for this repository
* - Content Type is {@link Constants#CONTENT_TYPE_GIT_LFS_JSON}.
*
* @return Returns {@code false} if either of the conditions does not match. Returns true if all match.
*/
private static boolean isLfsBatchApiRequest(HttpServletRequest request, String repository) {
String regex = String.format("^%s%s/%s(\\.git)?/info/lfs/objects/batch$", request.getContextPath(), GitServletModule.GIT_PATH, repository);
boolean pathMatches = request.getRequestURI().matches(regex);
boolean methodMatches = "POST".equals(request.getMethod());
boolean headerContentTypeMatches = isLfsContentHeaderField(request.getContentType(), CONTENT_TYPE_GIT_LFS_JSON);
boolean headerAcceptMatches = isLfsContentHeaderField(request.getHeader("Accept"), CONTENT_TYPE_GIT_LFS_JSON);
return pathMatches && methodMatches && headerContentTypeMatches && headerAcceptMatches;
}
/**
* Checks whether request is of the specific content type.
*
* @param request The HTTP request header value to be examined.
* @param expectedContentType The expected content type.
* @return Returns {@code true} if the request has the expected content type. Return {@code false} otherwise.
*/
@VisibleForTesting
static boolean isLfsContentHeaderField(String request, String expectedContentType) {
if (request == null || request.isEmpty()) {
return false;
}
String[] parts = request.split(" ");
for (String part : parts) {
if (part.startsWith(expectedContentType)) {
return true;
}
}
return false;
}
//~--- fields ---------------------------------------------------------------
/** Field description */
@@ -194,6 +298,11 @@ public class ScmGitServlet extends GitServlet
/** Field description */
private final RepositoryRequestListenerUtil repositoryRequestListenerUtil;
/** Field description */
/**
* Field description
*/
private final GitRepositoryViewer repositoryViewer;
private final LfsServletFactory lfsServletFactory;
}

80
scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/lfs/LfsBlobStoreFactory.java Normal file
View File

@@ -0,0 +1,80 @@
/**
* Copyright (c) 2010, Sebastian Sdorra
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. Neither the name of SCM-Manager; nor the names of its
* contributors may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* http://bitbucket.org/sdorra/scm-manager
*
*/
package sonia.scm.web.lfs;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import sonia.scm.repository.Repository;
import sonia.scm.store.BlobStore;
import sonia.scm.store.BlobStoreFactory;
/**
* Creates {@link BlobStore} objects to store lfs objects.
*
* @author Sebastian Sdorra
* @since 1.54
*/
@Singleton
public class LfsBlobStoreFactory {
private static final String GIT_LFS_REPOSITORY_POSTFIX = "-git-lfs";
private final BlobStoreFactory blobStoreFactory;
/**
* Create a new instance.
*
* @param blobStoreFactory blob store factory
*/
@Inject
public LfsBlobStoreFactory(BlobStoreFactory blobStoreFactory) {
this.blobStoreFactory = blobStoreFactory;
}
/**
* Provides a {@link BlobStore} corresponding to the SCM Repository.
* <p>
* git-lfs repositories should generally carry the same name as their regular SCM repository counterparts. However,
* we have decided to store them under their IDs instead of their names, since the names might change and provide
* other drawbacks, as well.
* <p>
* These repositories will have {@linkplain #GIT_LFS_REPOSITORY_POSTFIX} appended to their IDs.
*
* @param repository The SCM Repository to provide a LFS {@link BlobStore} for.
*
* @return blob store for the corresponding scm repository
*/
public BlobStore getLfsBlobStore(Repository repository) {
return blobStoreFactory.getBlobStore(repository.getId() + GIT_LFS_REPOSITORY_POSTFIX);
}
}

97
scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/lfs/LfsStoreRemoveListener.java Normal file
View File

@@ -0,0 +1,97 @@
/**
* Copyright (c) 2010, Sebastian Sdorra
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. Neither the name of SCM-Manager; nor the names of its
* contributors may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* http://bitbucket.org/sdorra/scm-manager
*
*/
package sonia.scm.web.lfs;
import com.google.common.eventbus.Subscribe;
import com.google.inject.Inject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.EagerSingleton;
import sonia.scm.HandlerEventType;
import sonia.scm.plugin.Extension;
import sonia.scm.repository.GitRepositoryHandler;
import sonia.scm.repository.Repository;
import sonia.scm.repository.RepositoryEvent;
import sonia.scm.store.Blob;
import sonia.scm.store.BlobStore;
/**
* Listener which removes all lfs objects from a blob store, whenever its corresponding git repository gets deleted.
*
* @author Sebastian Sdorra
* @since 1.54
*/
@Extension
@EagerSingleton
public class LfsStoreRemoveListener {
private static final Logger LOG = LoggerFactory.getLogger(LfsBlobStoreFactory.class);
private final LfsBlobStoreFactory lfsBlobStoreFactory;
@Inject
public LfsStoreRemoveListener(LfsBlobStoreFactory lfsBlobStoreFactory) {
this.lfsBlobStoreFactory = lfsBlobStoreFactory;
}
/**
* Remove all object from the blob store, if the event is an delete event and the repository is a git repository.
*
* @param event repository event
*/
@Subscribe
public void handleRepositoryEvent(RepositoryEvent event) {
if ( isDeleteEvent(event) && isGitRepositoryEvent(event) ) {
removeLfsStore(event.getItem());
}
}
private boolean isDeleteEvent(RepositoryEvent event) {
return HandlerEventType.DELETE == event.getEventType();
}
private boolean isGitRepositoryEvent(RepositoryEvent event) {
return event.getItem() != null
&& event.getItem().getType().equals(GitRepositoryHandler.TYPE_NAME);
}
private void removeLfsStore(Repository repository) {
LOG.debug("remove all blobs from store, because corresponding git repository {} was removed", repository.getName());
BlobStore blobStore = lfsBlobStoreFactory.getLfsBlobStore(repository);
for ( Blob blob : blobStore.getAll() ) {
LOG.trace("remove blob {}, because repository {} was removed", blob.getId(), repository.getName());
blobStore.remove(blob);
}
}
}

90
scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/lfs/ScmBlobLfsRepository.java Normal file
View File

@@ -0,0 +1,90 @@
package sonia.scm.web.lfs;
import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
import org.eclipse.jgit.lfs.server.LargeFileRepository;
import org.eclipse.jgit.lfs.server.Response;
import sonia.scm.store.Blob;
import sonia.scm.store.BlobStore;
import java.io.IOException;
/**
* This LargeFileRepository is used for jGit-Servlet implementation. Under the jgit LFS Servlet hood, the
* SCM-Repository API is used to implement the Repository.
*
* @since 1.54
* Created by omilke on 03.05.2017.
*/
public class ScmBlobLfsRepository implements LargeFileRepository {
private final BlobStore blobStore;
/**
* This URI is used to determine the actual URI for Upload / Download. Must be full URI (or rewritable by reverse
* proxy).
*/
private final String baseUri;
/**
* Creates a {@link ScmBlobLfsRepository} for the provided repository.
*
* @param blobStore The SCM Blobstore used for this @{@link LargeFileRepository}.
* @param baseUri This URI is used to determine the actual URI for Upload / Download. Must be full URI (or
* rewritable by reverse proxy).
*/
public ScmBlobLfsRepository(BlobStore blobStore, String baseUri) {
this.blobStore = blobStore;
this.baseUri = baseUri;
}
@Override
public Response.Action getDownloadAction(AnyLongObjectId id) {
return getAction(id);
}
@Override
public Response.Action getUploadAction(AnyLongObjectId id, long size) {
return getAction(id);
}
@Override
public Response.Action getVerifyAction(AnyLongObjectId id) {
//validation is optional. We do not support it.
return null;
}
@Override
public long getSize(AnyLongObjectId id) throws IOException {
//this needs to be size of what is will be written into the response of the download. Clients are likely to
// verify it.
Blob blob = this.blobStore.get(id.getName());
if (blob == null) {
return -1;
} else {
return blob.getSize();
}
}
/**
* Constructs the Download / Upload actions to be supplied to the client.
*/
private Response.Action getAction(AnyLongObjectId id) {
//LFS protocol has to provide the information on where to put or get the actual content, i. e.
//the actual URI for up- and download.
Response.Action a = new Response.Action();
a.href = baseUri + id.getName();
return a;
}
}

76
scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/lfs/servlet/LfsServletFactory.java Normal file
View File

@@ -0,0 +1,76 @@
package sonia.scm.web.lfs.servlet;
import com.google.common.annotations.VisibleForTesting;
import org.eclipse.jgit.lfs.server.LargeFileRepository;
import org.eclipse.jgit.lfs.server.LfsProtocolServlet;
import org.eclipse.jgit.lfs.server.fs.FileLfsServlet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.repository.Repository;
import sonia.scm.store.BlobStore;
import sonia.scm.util.HttpUtil;
import sonia.scm.web.lfs.ScmBlobLfsRepository;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import sonia.scm.web.lfs.LfsBlobStoreFactory;
/**
* This factory class is a helper class to provide the {@link LfsProtocolServlet} and the {@link FileLfsServlet}
* belonging to a SCM Repository.
*
* @since 1.54
* Created by omilke on 11.05.2017.
*/
@Singleton
public class LfsServletFactory {
private static final Logger logger = LoggerFactory.getLogger(LfsServletFactory.class);
private final LfsBlobStoreFactory lfsBlobStoreFactory;
@Inject
public LfsServletFactory(LfsBlobStoreFactory lfsBlobStoreFactory) {
this.lfsBlobStoreFactory = lfsBlobStoreFactory;
}
/**
* Builds the {@link LfsProtocolServlet} (jgit API) for a SCM Repository.
*
* @param repository The SCM Repository to build the servlet for.
* @param request The {@link HttpServletRequest} the used to access the SCM Repository.
* @return The {@link LfsProtocolServlet} to provide the LFS Batch API for a SCM Repository.
*/
public LfsProtocolServlet createProtocolServletFor(Repository repository, HttpServletRequest request) {
BlobStore blobStore = lfsBlobStoreFactory.getLfsBlobStore(repository);
String baseUri = buildBaseUri(repository, request);
LargeFileRepository largeFileRepository = new ScmBlobLfsRepository(blobStore, baseUri);
return new ScmLfsProtocolServlet(largeFileRepository);
}
/**
* Builds the {@link FileLfsServlet} (jgit API) for a SCM Repository.
*
* @param repository The SCM Repository to build the servlet for.
* @param request The {@link HttpServletRequest} the used to access the SCM Repository.
* @return The {@link FileLfsServlet} to provide the LFS Upload / Download API for a SCM Repository.
*/
public HttpServlet createFileLfsServletFor(Repository repository, HttpServletRequest request) {
return new ScmFileTransferServlet(lfsBlobStoreFactory.getLfsBlobStore(repository));
}
/**
* Build the complete URI, under which the File Transfer API for this repository will be will be reachable.
*
* @param repository The repository to build the File Transfer URI for.
* @param request The request to construct the complete URI from.
*/
@VisibleForTesting
static String buildBaseUri(Repository repository, HttpServletRequest request) {
return String.format("%s/git/%s.git/info/lfs/objects/", HttpUtil.getCompleteUrl(request), repository.getName());
}
}

280
scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/lfs/servlet/ScmFileTransferServlet.java Normal file
View File

@@ -0,0 +1,280 @@
package sonia.scm.web.lfs.servlet;
import com.google.common.annotations.VisibleForTesting;
import com.google.gson.FieldNamingPolicy;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.apache.http.HttpStatus;
import org.eclipse.jgit.lfs.errors.CorruptLongObjectException;
import org.eclipse.jgit.lfs.errors.InvalidLongObjectIdException;
import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
import org.eclipse.jgit.lfs.lib.Constants;
import org.eclipse.jgit.lfs.lib.LongObjectId;
import org.eclipse.jgit.lfs.server.LfsProtocolServlet;
import org.eclipse.jgit.lfs.server.fs.FileLfsServlet;
import org.eclipse.jgit.lfs.server.internal.LfsServerText;
import org.eclipse.jgit.util.HttpSupport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.store.Blob;
import sonia.scm.store.BlobStore;
import sonia.scm.util.IOUtil;
import javax.servlet.ServletException;
import javax.servlet.ServletInputStream;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.text.MessageFormat;
/**
* This Servlet provides the upload and download of files via git-lfs.
* <p>
* This implementation is based on {@link FileLfsServlet} but adjusted to work with
* servlet-2.5 instead of servlet-3.1.
* <p>
*
* @see FileLfsServlet
* @since 1.54
* Created by omilke on 15.05.2017.
*/
public class ScmFileTransferServlet extends HttpServlet {
private static final Logger logger = LoggerFactory.getLogger(ScmFileTransferServlet.class);
private static final long serialVersionUID = 1L;
/**
* Gson is used because the implementation was based on the jgit implementation. However the {@link LfsProtocolServlet} (which we do use in
* {@link ScmLfsProtocolServlet}) also uses Gson, which currently ties us to Gson anyway.
*/
private static Gson gson = createGson();
private final BlobStore blobStore;
public ScmFileTransferServlet(BlobStore store) {
this.blobStore = store;
}
/**
* Extracts the part after the last slash from path.
*
* @return Returns {@code null} if the part after the last slash is itself {@code null} or if its length is not 64.
*/
@VisibleForTesting
static String objectIdFromPath(String info) {
int lastSlash = info.lastIndexOf('/');
String potentialObjectId = info.substring(lastSlash + 1);
if (potentialObjectId.length() != 64) {
return null;
} else {
return potentialObjectId;
}
}
/**
* Logs the message and provides it to the client.
*
* @param response The response
* @param status The HTTP Status Code to be provided to the client.
* @param message the message to used for server-side logging. It is also provided to the client.
*/
private static void sendErrorAndLog(HttpServletResponse response, int status, String message) throws IOException {
logger.warn("Error occurred during git-lfs file transfer: {}", message);
sendError(response, status, message);
}
/**
* Logs the exception and provides only the message of the exception to the client.
*
* @param response The response
* @param status The HTTP Status Code to be provided to the client.
* @param exception An exception to used for server-side logging.
*/
private static void sendErrorAndLog(HttpServletResponse response, int status, Exception exception) throws IOException {
logger.warn("Error occurred during git-lfs file transfer.", exception);
String message = exception.getMessage();
sendError(response, status, message);
}
private static void sendError(HttpServletResponse response, int status, String message) throws IOException {
try (PrintWriter writer = response.getWriter()) {
gson.toJson(new Error(message), writer);
response.setStatus(status);
writer.flush();
}
response.flushBuffer();
}
private static Gson createGson() {
GsonBuilder gb = new GsonBuilder().setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES).setPrettyPrinting().disableHtmlEscaping();
return gb.create();
}
/**
* Provides a blob to download.
* <p>
* Actual implementation is based on <code>org.eclipse.jgit.lfs.server.fs.ObjectDownloadListener</code> and adjusted
* to non-async as we're currently on servlet-2.5.
*
* @param request servlet request
* @param response servlet response
* @throws ServletException if a servlet-specific error occurs
* @throws IOException if an I/O error occurs
*/
@Override
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
AnyLongObjectId objectId = getObjectToTransfer(request, response);
if (objectId == null) {
logInvalidObjectId(request.getRequestURI());
} else {
final String objectIdName = objectId.getName();
logger.trace("---- providing download for LFS-Oid: {}", objectIdName);
Blob savedBlob = blobStore.get(objectIdName);
if (isBlobPresent(savedBlob)) {
logger.trace("----- Object {}: providing {} bytes", objectIdName, savedBlob.getSize());
writeBlobIntoResponse(savedBlob, response);
} else {
sendErrorAndLog(response, HttpStatus.SC_NOT_FOUND, MessageFormat.format(LfsServerText.get().objectNotFound, objectIdName));
}
}
}
/**
* Receives a blob from an upload.
* <p>
* Actual implementation is based on <code>org.eclipse.jgit.lfs.server.fs.ObjectUploadListener</code> and adjusted
* to non-async as we're currently on servlet-2.5.
*
* @param request servlet request
* @param response servlet response
* @throws ServletException if a servlet-specific error occurs
* @throws IOException if an I/O error occurs
*/
@Override
protected void doPut(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
AnyLongObjectId objectId = getObjectToTransfer(request, response);
if (objectId == null) {
logInvalidObjectId(request.getRequestURI());
} else {
logger.trace("---- receiving upload for LFS-Oid: {}", objectId.getName());
readBlobFromResponse(request, response, objectId);
}
}
/**
* Extracts the {@link LongObjectId} from the request. Finishes the request, in case the {@link LongObjectId} cannot
* be extracted with an appropriate error.
*
* @throws IOException Thrown if the response could not be completed in an error case.
*/
private AnyLongObjectId getObjectToTransfer(HttpServletRequest request, HttpServletResponse response) throws IOException {
String path = request.getPathInfo();
String objectIdFromPath = objectIdFromPath(path);
if (objectIdFromPath == null) {
//ObjectId is not retrievable from URL
sendErrorAndLog(response, HttpStatus.SC_UNPROCESSABLE_ENTITY, MessageFormat.format(LfsServerText.get().invalidPathInfo, path));
return null;
} else {
try {
return LongObjectId.fromString(objectIdFromPath);
} catch (InvalidLongObjectIdException e) {
sendErrorAndLog(response, HttpStatus.SC_UNPROCESSABLE_ENTITY, e);
return null;
}
}
}
private void logInvalidObjectId(String requestURI) {
logger.warn("---- could not extract Oid from Request. Path seems to be invalid: {}", requestURI);
}
private boolean isBlobPresent(Blob savedBlob) {
return savedBlob != null && savedBlob.getSize() >= 0;
}
private void writeBlobIntoResponse(Blob savedBlob, HttpServletResponse response) throws IOException {
try (ServletOutputStream responseOutputStream = response.getOutputStream();
InputStream savedBlobInputStream = savedBlob.getInputStream()) {
response.addHeader(HttpSupport.HDR_CONTENT_LENGTH, String.valueOf(savedBlob.getSize()));
response.setContentType(Constants.HDR_APPLICATION_OCTET_STREAM);
IOUtil.copy(savedBlobInputStream, responseOutputStream);
} catch (IOException ex) {
sendErrorAndLog(response, HttpStatus.SC_INTERNAL_SERVER_ERROR, ex);
}
}
private void readBlobFromResponse(HttpServletRequest request, HttpServletResponse response, AnyLongObjectId objectId) throws IOException {
Blob blob = blobStore.create(objectId.getName());
try (OutputStream blobOutputStream = blob.getOutputStream();
ServletInputStream requestInputStream = request.getInputStream()) {
IOUtil.copy(requestInputStream, blobOutputStream);
blob.commit();
response.setContentType(Constants.CONTENT_TYPE_GIT_LFS_JSON);
response.setStatus(HttpServletResponse.SC_OK);
} catch (CorruptLongObjectException ex) {
sendErrorAndLog(response, HttpStatus.SC_BAD_REQUEST, ex);
}
}
/**
* Used for providing an error message.
*/
private static class Error {
String message;
Error(String m) {
this.message = m;
}
}
}

26
scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/lfs/servlet/ScmLfsProtocolServlet.java Normal file
View File

@@ -0,0 +1,26 @@
package sonia.scm.web.lfs.servlet;
import org.eclipse.jgit.lfs.errors.LfsException;
import org.eclipse.jgit.lfs.server.LargeFileRepository;
import org.eclipse.jgit.lfs.server.LfsProtocolServlet;
/**
* Provides an implementation for the git-lfs Batch API.
*
* @since 1.54
* Created by omilke on 11.05.2017.
*/
public class ScmLfsProtocolServlet extends LfsProtocolServlet {
private final LargeFileRepository repository;
public ScmLfsProtocolServlet(LargeFileRepository largeFileRepository) {
this.repository = largeFileRepository;
}
@Override
protected LargeFileRepository getLargeFileRepository(LfsRequest request, String path) throws LfsException {
return repository;
}
}