com.google.gerrit.server.account.GroupIncludeCacheImpl Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of gerrit-plugin-api Show documentation
API for Gerrit Plugins
The newest version!
// Copyright (C) 2011 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.gerrit.server.account;

import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.collect.ImmutableSet.toImmutableSet;

import com.google.common.cache.Cache;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.entities.Account;
import com.google.gerrit.entities.AccountGroup;
import com.google.gerrit.entities.InternalGroup;
import com.google.gerrit.index.IndexConfig;
import com.google.gerrit.proto.Protos;
import com.google.gerrit.server.cache.CacheModule;
import com.google.gerrit.server.cache.proto.Cache.AllExternalGroupsProto;
import com.google.gerrit.server.cache.proto.Cache.AllExternalGroupsProto.ExternalGroupProto;
import com.google.gerrit.server.cache.serialize.CacheSerializer;
import com.google.gerrit.server.cache.serialize.StringCacheSerializer;
import com.google.gerrit.server.group.db.Groups;
import com.google.gerrit.server.logging.Metadata;
import com.google.gerrit.server.logging.TraceContext;
import com.google.gerrit.server.logging.TraceContext.TraceTimer;
import com.google.gerrit.server.update.RetryHelper;
import com.google.inject.Inject;
import com.google.inject.Module;
import com.google.inject.Singleton;
import com.google.inject.TypeLiteral;
import com.google.inject.name.Named;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;

/** Tracks group inclusions in memory for efficient access. */
@Singleton
public class GroupIncludeCacheImpl implements GroupIncludeCache {
  private static final FluentLogger logger = FluentLogger.forEnclosingClass();

  private static final String PARENT_GROUPS_NAME = "groups_bysubgroup";
  private static final String GROUPS_WITH_MEMBER_NAME = "groups_bymember";
  private static final String EXTERNAL_NAME = "groups_external";
  private static final String PERSISTED_EXTERNAL_NAME = "groups_external_persisted";

  private final IndexConfig indexConfig;

  public static Module module() {
    return new CacheModule() {
      @Override
      protected void configure() {
        cache(
                GROUPS_WITH_MEMBER_NAME,
                Account.Id.class,
                new TypeLiteral>() {})
            .loader(GroupsWithMemberLoader.class);

        cache(
                PARENT_GROUPS_NAME,
                AccountGroup.UUID.class,
                new TypeLiteral>() {})
            .loader(ParentGroupsLoader.class);

        /**
         * Splitting the groups external cache into 2 caches: The first one is in memory, used to
         * serve the callers and has a single constant key "EXTERNAL_NAME". The second one is
         * persisted, its key represents the groups' state in NoteDb. The in-memory cache is used on
         * top of the persisted cache to enhance performance because the cache's value is used on
         * every request to Gerrit, potentially many times per request and the key computation can
         * become expensive.
         */
        cache(EXTERNAL_NAME, String.class, new TypeLiteral>() {})
            .loader(AllExternalInMemoryLoader.class);

        persist(
                PERSISTED_EXTERNAL_NAME,
                String.class,
                new TypeLiteral>() {})
            .diskLimit(-1)
            .version(1)
            .maximumWeight(0)
            .keySerializer(StringCacheSerializer.INSTANCE)
            .valueSerializer(ExternalGroupsSerializer.INSTANCE);

        bind(GroupIncludeCacheImpl.class);
        bind(GroupIncludeCache.class).to(GroupIncludeCacheImpl.class);
      }
    };
  }

  private final LoadingCache> groupsWithMember;
  private final LoadingCache> parentGroups;
  private final LoadingCache> external;

  @Inject
  GroupIncludeCacheImpl(
      @Named(GROUPS_WITH_MEMBER_NAME)
          LoadingCache> groupsWithMember,
      @Named(PARENT_GROUPS_NAME)
          LoadingCache> parentGroups,
      @Named(EXTERNAL_NAME) LoadingCache> external,
      IndexConfig indexConfig) {
    this.groupsWithMember = groupsWithMember;
    this.parentGroups = parentGroups;
    this.external = external;
    this.indexConfig = indexConfig;
  }

  @Override
  public Collection getGroupsWithMember(Account.Id memberId) {
    try {
      return groupsWithMember.get(memberId);
    } catch (ExecutionException e) {
      logger.atWarning().withCause(e).log("Cannot load groups containing %s as member", memberId);
      return ImmutableSet.of();
    }
  }

  @Override
  public Collection parentGroupsOf(AccountGroup.UUID groupId) {
    try {
      return parentGroups.get(groupId);
    } catch (ExecutionException e) {
      logger.atWarning().withCause(e).log("Cannot load included groups");
      return Collections.emptySet();
    }
  }

  @Override
  public Collection parentGroupsOf(Set groupIds) {
    try {
      Set parents = new HashSet<>();
      for (List groupIdsBatch :
          Lists.partition(new ArrayList<>(groupIds), indexConfig.maxTerms())) {
        parentGroups.getAll(groupIdsBatch).values().forEach(p -> parents.addAll(p));
      }
      return parents;
    } catch (ExecutionException e) {
      logger.atWarning().withCause(e).log("Cannot load included groups");
      return Collections.emptySet();
    }
  }

  @Override
  public void evictGroupsWithMember(Account.Id memberId) {
    if (memberId != null) {
      logger.atFine().log("Evict groups with member %d", memberId.get());
      groupsWithMember.invalidate(memberId);
    }
  }

  @Override
  public void evictParentGroupsOf(AccountGroup.UUID groupId) {
    if (groupId != null) {
      logger.atFine().log("Evict parent groups of %s", groupId.get());
      parentGroups.invalidate(groupId);

      if (!groupId.isInternalGroup()) {
        logger.atFine().log("Evict external group %s", groupId.get());
        /**
         * No need to invalidate the persistent cache, because this eviction will change the state
         * of NoteDb causing the persistent cache's loader to use a new key that doesn't exist in
         * its cache.n
         */
        external.invalidate(EXTERNAL_NAME);
      }
    }
  }

  @Override
  public Collection allExternalMembers() {
    try {
      return external.get(EXTERNAL_NAME);
    } catch (ExecutionException e) {
      logger.atWarning().withCause(e).log("Cannot load set of non-internal groups");
      return ImmutableList.of();
    }
  }

  static class GroupsWithMemberLoader
      extends CacheLoader> {
    private final RetryHelper retryHelper;

    @Inject
    GroupsWithMemberLoader(RetryHelper retryHelper) {
      this.retryHelper = retryHelper;
    }

    @Override
    public ImmutableSet load(Account.Id memberId) {
      try (TraceTimer timer =
          TraceContext.newTimer(
              "Loading groups with member", Metadata.builder().accountId(memberId.get()).build())) {
        return retryHelper
            .groupIndexQuery(
                "loadGroupWithMember",
                q ->
                    q.byMember(memberId).stream()
                        .map(InternalGroup::getGroupUUID)
                        .collect(toImmutableSet()))
            .call();
      }
    }
  }

  static class ParentGroupsLoader
      extends CacheLoader> {
    // Be conservative with batching: We don't want to exhaust the number of
    // results per page and maximum terms per query. Both are usually 1000+.
    private static final int MAX_BATCH_SIZE = 100;
    private final RetryHelper retryHelper;

    @Inject
    ParentGroupsLoader(RetryHelper retryHelper) {
      this.retryHelper = retryHelper;
    }

    @Override
    public ImmutableSet load(AccountGroup.UUID key) {
      try (TraceTimer timer =
          TraceContext.newTimer(
              "Loading parent groups", Metadata.builder().groupUuid(key.get()).build())) {
        return loadAll(ImmutableList.of(key)).get(key);
      }
    }

    @Override
    public Map> loadAll(
        Iterable keys) {
      int numKeys = Iterables.size(keys);
      Map> result =
          Maps.newHashMapWithExpectedSize(numKeys);
      try (TraceTimer timer = TraceContext.newTimer("Loading " + numKeys + " parent groups")) {
        Map> bySubgroups =
            retryHelper
                .groupIndexQuery("loadParentGroups", q -> q.bySubgroups(ImmutableSet.copyOf(keys)))
                .call();
        Iterables.partition(keys, MAX_BATCH_SIZE)
            .forEach(keyPartition -> result.putAll(bySubgroups));
        return result;
      }
    }
  }

  static class AllExternalInMemoryLoader
      extends CacheLoader> {
    private final Cache> persisted;
    private final GroupsSnapshotReader snapshotReader;
    private final Groups groups;

    @Inject
    AllExternalInMemoryLoader(
        @Named(PERSISTED_EXTERNAL_NAME) Cache> persisted,
        GroupsSnapshotReader snapshotReader,
        Groups groups) {
      this.persisted = persisted;
      this.snapshotReader = snapshotReader;
      this.groups = groups;
    }

    @Override
    public ImmutableList load(String key) throws Exception {
      GroupsSnapshotReader.Snapshot snapshot = snapshotReader.getSnapshot();
      return persisted.get(
          snapshot.hash(),
          () -> {
            try (TraceTimer timer = TraceContext.newTimer("Loading all external groups")) {
              return groups.getExternalGroups(snapshot.groupsRefs()).collect(toImmutableList());
            }
          });
    }
  }

  public enum ExternalGroupsSerializer
      implements CacheSerializer> {
    INSTANCE;

    @Override
    public byte[] serialize(ImmutableList object) {
      AllExternalGroupsProto.Builder allBuilder = AllExternalGroupsProto.newBuilder();
      object.stream()
          .map(group -> ExternalGroupProto.newBuilder().setGroupUuid(group.get()).build())
          .forEach(allBuilder::addExternalGroup);
      return Protos.toByteArray(allBuilder.build());
    }

    @Override
    public ImmutableList deserialize(byte[] in) {
      return Protos.parseUnchecked(AllExternalGroupsProto.parser(), in)
          .getExternalGroupList()
          .stream()
          .map(groupProto -> AccountGroup.UUID.parse(groupProto.getGroupUuid()))
          .collect(toImmutableList());
    }
  }
}