com.google.gerrit.server.account.GroupIncludeCacheImpl Maven / Gradle / Ivy
The newest version!
// Copyright (C) 2011 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.account;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.entities.Account;
import com.google.gerrit.entities.AccountGroup;
import com.google.gerrit.entities.InternalGroup;
import com.google.gerrit.index.IndexConfig;
import com.google.gerrit.proto.Protos;
import com.google.gerrit.server.cache.CacheModule;
import com.google.gerrit.server.cache.proto.Cache.AllExternalGroupsProto;
import com.google.gerrit.server.cache.proto.Cache.AllExternalGroupsProto.ExternalGroupProto;
import com.google.gerrit.server.cache.serialize.CacheSerializer;
import com.google.gerrit.server.cache.serialize.StringCacheSerializer;
import com.google.gerrit.server.group.db.Groups;
import com.google.gerrit.server.logging.Metadata;
import com.google.gerrit.server.logging.TraceContext;
import com.google.gerrit.server.logging.TraceContext.TraceTimer;
import com.google.gerrit.server.update.RetryHelper;
import com.google.inject.Inject;
import com.google.inject.Module;
import com.google.inject.Singleton;
import com.google.inject.TypeLiteral;
import com.google.inject.name.Named;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
/** Tracks group inclusions in memory for efficient access. */
@Singleton
public class GroupIncludeCacheImpl implements GroupIncludeCache {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private static final String PARENT_GROUPS_NAME = "groups_bysubgroup";
private static final String GROUPS_WITH_MEMBER_NAME = "groups_bymember";
private static final String EXTERNAL_NAME = "groups_external";
private static final String PERSISTED_EXTERNAL_NAME = "groups_external_persisted";
private final IndexConfig indexConfig;
public static Module module() {
return new CacheModule() {
@Override
protected void configure() {
cache(
GROUPS_WITH_MEMBER_NAME,
Account.Id.class,
new TypeLiteral>() {})
.loader(GroupsWithMemberLoader.class);
cache(
PARENT_GROUPS_NAME,
AccountGroup.UUID.class,
new TypeLiteral>() {})
.loader(ParentGroupsLoader.class);
/**
* Splitting the groups external cache into 2 caches: The first one is in memory, used to
* serve the callers and has a single constant key "EXTERNAL_NAME". The second one is
* persisted, its key represents the groups' state in NoteDb. The in-memory cache is used on
* top of the persisted cache to enhance performance because the cache's value is used on
* every request to Gerrit, potentially many times per request and the key computation can
* become expensive.
*/
cache(EXTERNAL_NAME, String.class, new TypeLiteral>() {})
.loader(AllExternalInMemoryLoader.class);
persist(
PERSISTED_EXTERNAL_NAME,
String.class,
new TypeLiteral>() {})
.diskLimit(-1)
.version(1)
.maximumWeight(0)
.keySerializer(StringCacheSerializer.INSTANCE)
.valueSerializer(ExternalGroupsSerializer.INSTANCE);
bind(GroupIncludeCacheImpl.class);
bind(GroupIncludeCache.class).to(GroupIncludeCacheImpl.class);
}
};
}
private final LoadingCache> groupsWithMember;
private final LoadingCache> parentGroups;
private final LoadingCache> external;
@Inject
GroupIncludeCacheImpl(
@Named(GROUPS_WITH_MEMBER_NAME)
LoadingCache> groupsWithMember,
@Named(PARENT_GROUPS_NAME)
LoadingCache> parentGroups,
@Named(EXTERNAL_NAME) LoadingCache> external,
IndexConfig indexConfig) {
this.groupsWithMember = groupsWithMember;
this.parentGroups = parentGroups;
this.external = external;
this.indexConfig = indexConfig;
}
@Override
public Collection getGroupsWithMember(Account.Id memberId) {
try {
return groupsWithMember.get(memberId);
} catch (ExecutionException e) {
logger.atWarning().withCause(e).log("Cannot load groups containing %s as member", memberId);
return ImmutableSet.of();
}
}
@Override
public Collection parentGroupsOf(AccountGroup.UUID groupId) {
try {
return parentGroups.get(groupId);
} catch (ExecutionException e) {
logger.atWarning().withCause(e).log("Cannot load included groups");
return Collections.emptySet();
}
}
@Override
public Collection parentGroupsOf(Set groupIds) {
try {
Set parents = new HashSet<>();
for (List groupIdsBatch :
Lists.partition(new ArrayList<>(groupIds), indexConfig.maxTerms())) {
parentGroups.getAll(groupIdsBatch).values().forEach(p -> parents.addAll(p));
}
return parents;
} catch (ExecutionException e) {
logger.atWarning().withCause(e).log("Cannot load included groups");
return Collections.emptySet();
}
}
@Override
public void evictGroupsWithMember(Account.Id memberId) {
if (memberId != null) {
logger.atFine().log("Evict groups with member %d", memberId.get());
groupsWithMember.invalidate(memberId);
}
}
@Override
public void evictParentGroupsOf(AccountGroup.UUID groupId) {
if (groupId != null) {
logger.atFine().log("Evict parent groups of %s", groupId.get());
parentGroups.invalidate(groupId);
if (!groupId.isInternalGroup()) {
logger.atFine().log("Evict external group %s", groupId.get());
/**
* No need to invalidate the persistent cache, because this eviction will change the state
* of NoteDb causing the persistent cache's loader to use a new key that doesn't exist in
* its cache.n
*/
external.invalidate(EXTERNAL_NAME);
}
}
}
@Override
public Collection allExternalMembers() {
try {
return external.get(EXTERNAL_NAME);
} catch (ExecutionException e) {
logger.atWarning().withCause(e).log("Cannot load set of non-internal groups");
return ImmutableList.of();
}
}
static class GroupsWithMemberLoader
extends CacheLoader> {
private final RetryHelper retryHelper;
@Inject
GroupsWithMemberLoader(RetryHelper retryHelper) {
this.retryHelper = retryHelper;
}
@Override
public ImmutableSet load(Account.Id memberId) {
try (TraceTimer timer =
TraceContext.newTimer(
"Loading groups with member", Metadata.builder().accountId(memberId.get()).build())) {
return retryHelper
.groupIndexQuery(
"loadGroupWithMember",
q ->
q.byMember(memberId).stream()
.map(InternalGroup::getGroupUUID)
.collect(toImmutableSet()))
.call();
}
}
}
static class ParentGroupsLoader
extends CacheLoader> {
// Be conservative with batching: We don't want to exhaust the number of
// results per page and maximum terms per query. Both are usually 1000+.
private static final int MAX_BATCH_SIZE = 100;
private final RetryHelper retryHelper;
@Inject
ParentGroupsLoader(RetryHelper retryHelper) {
this.retryHelper = retryHelper;
}
@Override
public ImmutableSet load(AccountGroup.UUID key) {
try (TraceTimer timer =
TraceContext.newTimer(
"Loading parent groups", Metadata.builder().groupUuid(key.get()).build())) {
return loadAll(ImmutableList.of(key)).get(key);
}
}
@Override
public Map> loadAll(
Iterable keys) {
int numKeys = Iterables.size(keys);
Map> result =
Maps.newHashMapWithExpectedSize(numKeys);
try (TraceTimer timer = TraceContext.newTimer("Loading " + numKeys + " parent groups")) {
Map> bySubgroups =
retryHelper
.groupIndexQuery("loadParentGroups", q -> q.bySubgroups(ImmutableSet.copyOf(keys)))
.call();
Iterables.partition(keys, MAX_BATCH_SIZE)
.forEach(keyPartition -> result.putAll(bySubgroups));
return result;
}
}
}
static class AllExternalInMemoryLoader
extends CacheLoader> {
private final Cache> persisted;
private final GroupsSnapshotReader snapshotReader;
private final Groups groups;
@Inject
AllExternalInMemoryLoader(
@Named(PERSISTED_EXTERNAL_NAME) Cache> persisted,
GroupsSnapshotReader snapshotReader,
Groups groups) {
this.persisted = persisted;
this.snapshotReader = snapshotReader;
this.groups = groups;
}
@Override
public ImmutableList load(String key) throws Exception {
GroupsSnapshotReader.Snapshot snapshot = snapshotReader.getSnapshot();
return persisted.get(
snapshot.hash(),
() -> {
try (TraceTimer timer = TraceContext.newTimer("Loading all external groups")) {
return groups.getExternalGroups(snapshot.groupsRefs()).collect(toImmutableList());
}
});
}
}
public enum ExternalGroupsSerializer
implements CacheSerializer> {
INSTANCE;
@Override
public byte[] serialize(ImmutableList object) {
AllExternalGroupsProto.Builder allBuilder = AllExternalGroupsProto.newBuilder();
object.stream()
.map(group -> ExternalGroupProto.newBuilder().setGroupUuid(group.get()).build())
.forEach(allBuilder::addExternalGroup);
return Protos.toByteArray(allBuilder.build());
}
@Override
public ImmutableList deserialize(byte[] in) {
return Protos.parseUnchecked(AllExternalGroupsProto.parser(), in).getExternalGroupList()
.stream()
.map(groupProto -> AccountGroup.UUID.parse(groupProto.getGroupUuid()))
.collect(toImmutableList());
}
}
}