created adapter between scm and shiro caches, see issue #781

This commit is contained in:
Sebastian Sdorra
2017-02-16 22:15:36 +01:00
parent 1b16613840
commit 731337f2ab
8 changed files with 492 additions and 250 deletions

View File

@@ -0,0 +1,113 @@
/**
* Copyright (c) 2014, Sebastian Sdorra
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. Neither the name of SCM-Manager; nor the names of its
* contributors may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* http://bitbucket.org/sdorra/scm-manager
*
*/
package sonia.scm.cache;
import com.google.common.cache.Cache;
import com.google.common.collect.Sets;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.Filter;
/**
* Implementation of basic cache methods for guava based caches.
*
* @author Sebastian Sdorra
*
* @since 1.52
*
* @param <K> key
* @param <V> value
*/
public abstract class GuavaBaseCache<K, V> {
/**
* the logger for GuavaBaseCache
*/
private static final Logger logger = LoggerFactory.getLogger(GuavaCache.class);
protected com.google.common.cache.Cache<K, V> cache;
protected CopyStrategy copyStrategy = CopyStrategy.NONE;
private final String name;
GuavaBaseCache(com.google.common.cache.Cache<K, V> cache, CopyStrategy copyStrategy, String name) {
this.cache = cache;
this.name = name;
if (copyStrategy != null) {
this.copyStrategy = copyStrategy;
}
}
//~--- methods --------------------------------------------------------------
public void clear() {
logger.debug("clear cache {}", name);
cache.invalidateAll();
}
public boolean contains(K key) {
return cache.getIfPresent(key) != null;
}
public boolean removeAll(Filter<K> filter) {
Set<K> keysToRemove = Sets.newHashSet();
for (K key : cache.asMap().keySet()) {
if (filter.accept(key)) {
keysToRemove.add(key);
}
}
boolean result = false;
if (!keysToRemove.isEmpty()) {
cache.invalidateAll(keysToRemove);
result = true;
}
return result;
}
public V get(K key) {
V value = cache.getIfPresent(key);
if (value != null) {
value = copyStrategy.copyOnRead(value);
}
return value;
}
public Cache<K, V> getWrappedCache() {
return cache;
}
}

View File

@@ -34,18 +34,6 @@ package sonia.scm.cache;
//~--- non-JDK imports --------------------------------------------------------
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Sets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.Filter;
//~--- JDK imports ------------------------------------------------------------
import java.util.Set;
/**
*
* @author Sebastian Sdorra
@@ -53,185 +41,22 @@ import java.util.Set;
* @param <K>
* @param <V>
*/
public class GuavaCache<K, V> implements Cache<K, V>
{
public class GuavaCache<K, V> extends GuavaBaseCache<K, V> implements Cache<K, V> {
/**
* the logger for GuavaCache
*/
private static final Logger logger =
LoggerFactory.getLogger(GuavaCache.class);
//~--- constructors ---------------------------------------------------------
/**
* Constructs ...
*
*
* @param configuration
*/
public GuavaCache(GuavaNamedCacheConfiguration configuration)
{
this(configuration, configuration.getName());
GuavaCache(com.google.common.cache.Cache<K, V> cache, CopyStrategy copyStrategy, String name) {
super(cache, copyStrategy, name);
}
/**
* Constructs ...
*
*
* @param configuration
* @param name
*/
public GuavaCache(GuavaCacheConfiguration configuration, String name)
{
this(GuavaCaches.create(configuration, name),
configuration.getCopyStrategy(), name);
}
/**
* Constructs ...
*
*
* @param cache
* @param copyStrategy
* @param name
*/
@VisibleForTesting
protected GuavaCache(com.google.common.cache.Cache<K, V> cache,
CopyStrategy copyStrategy, String name)
{
this.cache = cache;
this.name = name;
if (copyStrategy != null)
{
this.copyStrategy = copyStrategy;
}
}
//~--- methods --------------------------------------------------------------
/**
* Method description
*
*/
@Override
public void clear()
{
if (logger.isDebugEnabled())
{
logger.debug("clear cache {}", name);
}
cache.invalidateAll();
}
/**
* Method description
*
*
* @param key
*
* @return
*/
@Override
public boolean contains(K key)
{
return cache.getIfPresent(key) != null;
}
/**
* Method description
*
*
* @param key
* @param value
*/
@Override
public void put(K key, V value)
{
public void put(K key, V value) {
cache.put(key, copyStrategy.copyOnWrite(value));
}
/**
* Method description
*
*
* @param key
*
* @return
*/
@Override
public boolean remove(K key)
{
public boolean remove(K key) {
cache.invalidate(key);
return true;
}
/**
* Method description
*
*
* @param filter
*
* @return
*/
@Override
public boolean removeAll(Filter<K> filter)
{
Set<K> keysToRemove = Sets.newHashSet();
for (K key : cache.asMap().keySet())
{
if (filter.accept(key))
{
keysToRemove.add(key);
}
}
boolean result = false;
if (!keysToRemove.isEmpty())
{
cache.invalidateAll(keysToRemove);
result = true;
}
return result;
}
//~--- get methods ----------------------------------------------------------
/**
* Method description
*
*
* @param key
*
* @return
*/
@Override
public V get(K key)
{
V value = cache.getIfPresent(key);
if (value != null)
{
value = copyStrategy.copyOnRead(value);
}
return value;
}
//~--- fields ---------------------------------------------------------------
/** Field description */
private com.google.common.cache.Cache<K, V> cache;
/** Field description */
private CopyStrategy copyStrategy = CopyStrategy.NONE;
/** Field description */
private String name;
}
}

View File

@@ -47,66 +47,52 @@ import java.io.IOException;
import java.util.Map;
import org.apache.shiro.cache.CacheException;
/**
*
* Guava based implementation of {@link CacheManager} and {@link org.apache.shiro.cache.CacheManager}.
*
* @author Sebastian Sdorra
*/
@Singleton
public class GuavaCacheManager implements CacheManager
public class GuavaCacheManager implements CacheManager, org.apache.shiro.cache.CacheManager
{
/**
* the logger for GuavaCacheManager
*/
private static final Logger logger =
LoggerFactory.getLogger(GuavaCacheManager.class);
private static final Logger logger = LoggerFactory.getLogger(GuavaCacheManager.class);
private volatile Map<String, CacheWithConfiguration> cacheMap = Maps.newHashMap();
private GuavaCacheConfiguration defaultConfiguration;
//~--- constructors ---------------------------------------------------------
/**
* Constructs ...
*
*/
public GuavaCacheManager()
{
public GuavaCacheManager() {
this(GuavaCacheConfigurationReader.read());
}
/**
* Constructs ...
*
*
* @param config
*/
@VisibleForTesting
protected GuavaCacheManager(GuavaCacheManagerConfiguration config)
{
protected GuavaCacheManager(GuavaCacheManagerConfiguration config) {
defaultConfiguration = config.getDefaultCache();
for (GuavaNamedCacheConfiguration ncc : config.getCaches())
{
logger.debug("create cache {} from configured configuration {}",
ncc.getName(), ncc);
cacheMap.put(ncc.getName(), new GuavaCache(ncc));
for (GuavaNamedCacheConfiguration ncc : config.getCaches()) {
logger.debug("create cache {} from configured configuration {}", ncc.getName(), ncc);
cacheMap.put(ncc.getName(), new CacheWithConfiguration(
GuavaCaches.create(defaultConfiguration, ncc.getName()),
defaultConfiguration)
);
}
}
//~--- methods --------------------------------------------------------------
/**
* Method description
*
*
* @throws IOException
*/
@Override
public void close() throws IOException
{
public void close() throws IOException {
logger.info("close guava cache manager");
for (Cache c : cacheMap.values())
{
c.clear();
for (CacheWithConfiguration c : cacheMap.values()) {
c.cache.invalidateAll();
}
cacheMap.clear();
@@ -114,43 +100,44 @@ public class GuavaCacheManager implements CacheManager
//~--- get methods ----------------------------------------------------------
/**
* Method description
*
*
* @param key
* @param value
* @param name
* @param <K>
* @param <V>
*
* @return
*/
@Override
public synchronized <K, V> GuavaCache<K, V> getCache(Class<K> key,
Class<V> value, String name)
{
private synchronized <K, V> CacheWithConfiguration<K, V> getCacheWithConfiguration(String name) {
logger.trace("try to retrieve cache {}", name);
GuavaCache<K, V> cache = cacheMap.get(name);
CacheWithConfiguration<K, V> cache = cacheMap.get(name);
if (cache == null)
{
if (cache == null) {
logger.debug(
"cache {} does not exists, creating a new instance from default configuration: {}",
name, defaultConfiguration);
cache = new GuavaCache<K, V>(defaultConfiguration, name);
name, defaultConfiguration
);
cache = new CacheWithConfiguration(GuavaCaches.create(defaultConfiguration, name), defaultConfiguration);
cacheMap.put(name, cache);
}
return cache;
}
@Override
public <K, V> GuavaCache<K, V> getCache(Class<K> key, Class<V> value, String name) {
CacheWithConfiguration<K, V> cw = getCacheWithConfiguration(name);
return new GuavaCache<>(cw.cache, cw.configuration.getCopyStrategy(), name);
}
@Override
public <K, V> GuavaSecurityCache<K, V> getCache(String name) throws CacheException {
CacheWithConfiguration<K, V> cw = getCacheWithConfiguration(name);
return new GuavaSecurityCache<>(cw.cache, cw.configuration.getCopyStrategy(), name);
}
//~--- fields ---------------------------------------------------------------
private static class CacheWithConfiguration<K,V> {
private final com.google.common.cache.Cache<K,V> cache;
private final GuavaCacheConfiguration configuration;
/** Field description */
private volatile Map<String, GuavaCache> cacheMap = Maps.newHashMap();
/** Field description */
private GuavaCacheConfiguration defaultConfiguration;
private CacheWithConfiguration(com.google.common.cache.Cache<K, V> cache, GuavaCacheConfiguration configuration) {
this.cache = cache;
this.configuration = configuration;
}
}
}

View File

@@ -0,0 +1,83 @@
/**
* Copyright (c) 2014, Sebastian Sdorra
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. Neither the name of SCM-Manager; nor the names of its
* contributors may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* http://bitbucket.org/sdorra/scm-manager
*
*/
package sonia.scm.cache;
import com.google.common.cache.Cache;
import java.util.Collection;
import java.util.Set;
import org.apache.shiro.cache.CacheException;
/**
* Guava based implementation of {@link org.apache.shiro.cache.Cache}.
*
* @author Sebastian Sdorra
*
* @since 1.52
*
* @param <K>
* @param <V>
*/
public class GuavaSecurityCache<K, V> extends GuavaBaseCache<K, V> implements org.apache.shiro.cache.Cache<K, V> {
GuavaSecurityCache(Cache<K, V> cache, CopyStrategy copyStrategy, String name) {
super(cache, copyStrategy, name);
}
@Override
public V put(K key, V value) throws CacheException {
V previousValue = cache.getIfPresent(key);
cache.put(key, value);
return previousValue;
}
@Override
public V remove(K key) throws CacheException {
V previousValue = cache.getIfPresent(key);
cache.invalidate(key);
return previousValue;
}
@Override
public int size() {
return (int) cache.size();
}
@Override
public Set<K> keys() {
return cache.asMap().keySet();
}
@Override
public Collection<V> values() {
return cache.asMap().values();
}
}

View File

@@ -333,7 +333,7 @@ public class AuthorizationCollector
*
* @return
*/
AuthorizationInfo collect(PrincipalCollection principals)
public AuthorizationInfo collect(PrincipalCollection principals)
{
Preconditions.checkNotNull(principals, "principals parameter is required");

View File

@@ -159,8 +159,7 @@ public abstract class CacheManagerTestBase<C extends Cache>
* @param c1
* @param c2
*/
protected void assertIsSame(Cache<String, String> c1,
Cache<String, String> c2)
protected void assertIsSame(Cache<String, String> c1, Cache<String, String> c2)
{
assertSame(c1, c2);
}

View File

@@ -32,6 +32,8 @@
package sonia.scm.cache;
import org.junit.Assert;
/**
*
* @author Sebastian Sdorra
@@ -50,4 +52,14 @@ public class GuavaCacheManagerTest extends CacheManagerTestBase
{
return CacheTestUtil.createDefaultGuavaCacheManager();
}
@Override
protected void assertIsSame(Cache c1, Cache c2) {
Assert.assertSame(
((GuavaCache)c1).getWrappedCache(),
((GuavaCache)c2).getWrappedCache()
);
}
}

View File

@@ -0,0 +1,223 @@
/**
* Copyright (c) 2014, Sebastian Sdorra
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. Neither the name of SCM-Manager; nor the names of its
* contributors may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* http://bitbucket.org/sdorra/scm-manager
*
*/
package sonia.scm.repository;
import com.google.common.base.Stopwatch;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Provider;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.shiro.SecurityUtils;
import org.apache.shiro.authc.AuthenticationException;
import org.apache.shiro.authc.AuthenticationInfo;
import org.apache.shiro.authc.AuthenticationToken;
import org.apache.shiro.authc.SimpleAuthenticationInfo;
import org.apache.shiro.authc.UsernamePasswordToken;
import org.apache.shiro.authc.credential.AllowAllCredentialsMatcher;
import org.apache.shiro.authz.AuthorizationInfo;
import org.apache.shiro.mgt.DefaultSecurityManager;
import org.apache.shiro.realm.AuthorizingRealm;
import org.apache.shiro.subject.PrincipalCollection;
import org.apache.shiro.subject.SimplePrincipalCollection;
import org.apache.shiro.util.ThreadContext;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import static org.mockito.Mockito.*;
import org.mockito.runners.MockitoJUnitRunner;
import sonia.scm.SCMContextProvider;
import sonia.scm.Type;
import sonia.scm.cache.GuavaCacheManager;
import sonia.scm.config.ScmConfiguration;
import sonia.scm.security.AuthorizationCollector;
import sonia.scm.security.DefaultKeyGenerator;
import sonia.scm.security.KeyGenerator;
import sonia.scm.security.RepositoryPermissionResolver;
import sonia.scm.security.SecuritySystem;
import sonia.scm.user.UserTestData;
/**
*
* @author Sebastian Sdorra
*/
@RunWith(MockitoJUnitRunner.class)
public class DefaultRepositoryManagerPerfTest {
private static final int REPOSITORY_COUNT = 2000;
private static final String REPOSITORY_TYPE = "perf";
@Mock
private SCMContextProvider contextProvider;
@Mock
private RepositoryDAO repositoryDAO;
@Mock
private PreProcessorUtil preProcessorUtil;
private final ScmConfiguration configuration = new ScmConfiguration();
private final KeyGenerator keyGenerator = new DefaultKeyGenerator();
@Mock
private RepositoryHandler repositoryHandler;
private DefaultRepositoryManager repositoryManager;
@Mock
private SecuritySystem securitySystem;
private AuthorizationCollector authzCollector;
@Before
public void setUpObjectUnderTest(){
when(repositoryHandler.getType()).thenReturn(new Type(REPOSITORY_TYPE, REPOSITORY_TYPE));
Set<RepositoryHandler> handlerSet = ImmutableSet.of(repositoryHandler);
Provider<Set<RepositoryListener>> repositoryListenersProvider = new SetProvider();
Provider<Set<RepositoryHook>> repositoryHooksProvider = new SetProvider();
repositoryManager = new DefaultRepositoryManager(
configuration,
contextProvider,
keyGenerator,
repositoryDAO,
handlerSet,
repositoryListenersProvider,
repositoryHooksProvider,
preProcessorUtil
);
setUpTestRepositories();
GuavaCacheManager cacheManager = new GuavaCacheManager();
authzCollector = new AuthorizationCollector(cacheManager, repositoryDAO, securitySystem, new RepositoryPermissionResolver());
DefaultSecurityManager securityManager = new DefaultSecurityManager(new DummyRealm(authzCollector, cacheManager));
ThreadContext.bind(securityManager);
}
@After
public void tearDown(){
ThreadContext.unbindSecurityManager();
}
@Test(timeout = 6000l)
public void perfTestGetAll(){
SecurityUtils.getSubject().login(new UsernamePasswordToken("trillian", "secret"));
List<Long> times = new ArrayList<>();
for ( int i=0; i<3; i++ ) {
times.add(benchGetAll());
}
long average = calculateAverage(times);
double value = (double) average / TimeUnit.MILLISECONDS.convert(1, TimeUnit.SECONDS);
// Too bad this functionality is not exposed as a regular method call
System.out.println( String.format("%.4g s", value) );
}
private long calculateAverage(List<Long> times) {
Long sum = 0l;
if(!times.isEmpty()) {
for (Long time : times) {
sum += time;
}
return Math.round(sum.doubleValue() / times.size());
}
return sum;
}
private long benchGetAll(){
Stopwatch sw = Stopwatch.createStarted();
System.out.append("found ").append(String.valueOf(repositoryManager.getAll().size()));
sw.stop();
System.out.append(" in ").println(sw);
return sw.elapsed(TimeUnit.MILLISECONDS);
}
private void setUpTestRepositories() {
Map<String,Repository> repositories = new LinkedHashMap<>();
for ( int i=0; i<REPOSITORY_COUNT; i++ ) {
Repository repository = createTestRepository(i);
repositories.put(repository.getId(), repository);
}
when(repositoryDAO.getAll()).thenReturn(repositories.values());
}
private Repository createTestRepository(int number){
Repository repository = new Repository(keyGenerator.createKey(), REPOSITORY_TYPE, "repo-" + number);
repository.getPermissions().add(new Permission("trillian", PermissionType.READ));
return repository;
}
static class DummyRealm extends AuthorizingRealm {
private final AuthorizationCollector authzCollector;
public DummyRealm(AuthorizationCollector authzCollector, org.apache.shiro.cache.CacheManager cacheManager) {
this.authzCollector = authzCollector;
setCredentialsMatcher(new AllowAllCredentialsMatcher());
setCacheManager(cacheManager);
}
@Override
protected AuthenticationInfo doGetAuthenticationInfo(AuthenticationToken token) throws AuthenticationException {
SimplePrincipalCollection spc = new SimplePrincipalCollection(token.getPrincipal(), REPOSITORY_TYPE);
spc.add(UserTestData.createTrillian(), REPOSITORY_TYPE);
return new SimpleAuthenticationInfo(spc, REPOSITORY_TYPE);
}
@Override
protected AuthorizationInfo doGetAuthorizationInfo(PrincipalCollection principals) {
return authzCollector.collect(principals);
}
}
private static class SetProvider implements Provider {
@Override
public Object get() {
return Collections.emptySet();
}
}
}