com.google.gerrit.server.notedb.DeleteZombieCommentsRefs Maven / Gradle / Ivy
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.notedb;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.gerrit.entities.RefNames.REFS_DRAFT_COMMENTS;
import static org.eclipse.jgit.lib.Constants.EMPTY_TREE_ID;
import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import com.google.common.collect.Sets.SetView;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.entities.Account;
import com.google.gerrit.entities.Change;
import com.google.gerrit.entities.HumanComment;
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.git.RefUpdateUtil;
import com.google.gerrit.server.CommentsUtil;
import com.google.gerrit.server.IdentifiedUser;
import com.google.gerrit.server.config.AllUsersName;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.util.time.TimeUtil;
import com.google.inject.assistedinject.Assisted;
import com.google.inject.assistedinject.AssistedInject;
import java.io.IOException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.eclipse.jgit.lib.BatchRefUpdate;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.ReceiveCommand;
/**
* This class can be used to clean zombie draft comments refs. More context in
* https://gerrit-review.googlesource.com/c/gerrit/+/246233
*
* The implementation has two cases for detecting zombie drafts:
*
*
* - An earlier bug in the deletion of draft comments {@code
* refs/draft-comments/$change_id_short/$change_id/$user_id} caused some draft refs to remain
* in Git and not get deleted. These refs point to an empty tree. We delete such refs.
*
- Inspecting all draft-comment refs. Check for each draft if there exists a published comment
* with the same UUID. These comments are called zombie drafts. If the program is run in
* {@link #dryRun} mode, the zombie draft IDs will only be logged for tracking, otherwise they
* will also be deleted.
*
*/
public class DeleteZombieCommentsRefs {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
// Number of refs deleted at once in a batch ref-update.
// Log progress after deleting every CHUNK_SIZE refs
private static final int CHUNK_SIZE = 3000;
private final GitRepositoryManager repoManager;
private final AllUsersName allUsers;
private final int cleanupPercentage;
/**
* Run the logic in dry run mode only. That is, detected zombie drafts will be logged only but not
* deleted. Creators of this class can use {@link Factory#create(int, boolean)} to specify the dry
* run mode. If {@link Factory#create(int)} is used, the dry run mode will be set to its default:
* true.
*/
private final boolean dryRun;
private final Consumer uiConsumer;
@Nullable private final DraftCommentNotes.Factory draftNotesFactory;
@Nullable private final ChangeNotes.Factory changeNotesFactory;
@Nullable private final CommentsUtil commentsUtil;
@Nullable private final ChangeUpdate.Factory changeUpdateFactory;
@Nullable private final IdentifiedUser.GenericFactory userFactory;
public interface Factory {
DeleteZombieCommentsRefs create(int cleanupPercentage);
DeleteZombieCommentsRefs create(int cleanupPercentage, boolean dryRun);
}
@AssistedInject
public DeleteZombieCommentsRefs(
AllUsersName allUsers,
GitRepositoryManager repoManager,
ChangeNotes.Factory changeNotesFactory,
DraftCommentNotes.Factory draftNotesFactory,
CommentsUtil commentsUtil,
ChangeUpdate.Factory changeUpdateFactory,
IdentifiedUser.GenericFactory userFactory,
@Assisted Integer cleanupPercentage) {
this(
allUsers,
repoManager,
cleanupPercentage,
/* dryRun= */ true,
(msg) -> {},
changeNotesFactory,
draftNotesFactory,
commentsUtil,
changeUpdateFactory,
userFactory);
}
@AssistedInject
public DeleteZombieCommentsRefs(
AllUsersName allUsers,
GitRepositoryManager repoManager,
ChangeNotes.Factory changeNotesFactory,
DraftCommentNotes.Factory draftNotesFactory,
CommentsUtil commentsUtil,
ChangeUpdate.Factory changeUpdateFactory,
IdentifiedUser.GenericFactory userFactory,
@Assisted Integer cleanupPercentage,
@Assisted boolean dryRun) {
this(
allUsers,
repoManager,
cleanupPercentage,
dryRun,
(msg) -> {},
changeNotesFactory,
draftNotesFactory,
commentsUtil,
changeUpdateFactory,
userFactory);
}
public DeleteZombieCommentsRefs(
AllUsersName allUsers,
GitRepositoryManager repoManager,
Integer cleanupPercentage,
Consumer uiConsumer) {
this(
allUsers,
repoManager,
cleanupPercentage,
/* dryRun= */ false,
uiConsumer,
null,
null,
null,
null,
null);
}
private DeleteZombieCommentsRefs(
AllUsersName allUsers,
GitRepositoryManager repoManager,
Integer cleanupPercentage,
boolean dryRun,
Consumer uiConsumer,
@Nullable ChangeNotes.Factory changeNotesFactory,
@Nullable DraftCommentNotes.Factory draftNotesFactory,
@Nullable CommentsUtil commentsUtil,
@Nullable ChangeUpdate.Factory changeUpdateFactory,
@Nullable IdentifiedUser.GenericFactory userFactory) {
this.allUsers = allUsers;
this.repoManager = repoManager;
this.cleanupPercentage = (cleanupPercentage == null) ? 100 : cleanupPercentage;
this.dryRun = dryRun;
this.uiConsumer = uiConsumer;
this.draftNotesFactory = draftNotesFactory;
this.changeNotesFactory = changeNotesFactory;
this.commentsUtil = commentsUtil;
this.changeUpdateFactory = changeUpdateFactory;
this.userFactory = userFactory;
}
public void execute() throws IOException {
deleteDraftRefsThatPointToEmptyTree();
if (draftNotesFactory != null) {
deleteDraftCommentsThatAreAlsoPublished();
}
}
private void deleteDraftRefsThatPointToEmptyTree() throws IOException {
try (Repository allUsersRepo = repoManager.openRepository(allUsers)) {
List draftRefs = allUsersRepo.getRefDatabase().getRefsByPrefix(REFS_DRAFT_COMMENTS);
List zombieRefs = filterZombieRefs(allUsersRepo, draftRefs);
logInfo(
String.format(
"Found a total of %d zombie draft refs in %s repo.",
zombieRefs.size(), allUsers.get()));
logInfo(String.format("Cleanup percentage = %d", cleanupPercentage));
zombieRefs =
zombieRefs.stream()
.filter(
ref -> Change.Id.fromAllUsersRef(ref.getName()).get() % 100 < cleanupPercentage)
.collect(toImmutableList());
logInfo(String.format("Number of zombie refs to be cleaned = %d", zombieRefs.size()));
if (dryRun) {
logInfo(
"Running in dry run mode. Skipping deletion of draft refs pointing to an empty tree.");
return;
}
long zombieRefsCnt = zombieRefs.size();
long deletedRefsCnt = 0;
long startTime = System.currentTimeMillis();
for (List refsBatch : Iterables.partition(zombieRefs, CHUNK_SIZE)) {
deleteBatchZombieRefs(allUsersRepo, refsBatch);
long elapsed = (System.currentTimeMillis() - startTime) / 1000;
deletedRefsCnt += refsBatch.size();
logProgress(deletedRefsCnt, zombieRefsCnt, elapsed);
}
}
}
/**
* Iterates over all draft refs in All-Users repository. For each draft ref, checks if there
* exists a published comment with the same UUID and deletes the draft ref if that's the case
* because it is a zombie draft.
*
* @return the number of detected and deleted zombie draft comments.
*/
@VisibleForTesting
public int deleteDraftCommentsThatAreAlsoPublished() throws IOException {
try (Repository allUsersRepo = repoManager.openRepository(allUsers)) {
Timestamp earliestZombieTs = null;
Timestamp latestZombieTs = null;
int numZombies = 0;
List draftRefs = allUsersRepo.getRefDatabase().getRefsByPrefix(REFS_DRAFT_COMMENTS);
// Filter the number of draft refs to be processed according to the cleanup percentage.
draftRefs =
draftRefs.stream()
.filter(
ref -> Change.Id.fromAllUsersRef(ref.getName()).get() % 100 < cleanupPercentage)
.collect(toImmutableList());
Set visitedSet = new HashSet<>();
ImmutableSet changeIds =
draftRefs.stream()
.map(d -> Change.Id.fromAllUsersRef(d.getName()))
.collect(ImmutableSet.toImmutableSet());
Map changeProjectMap = mapChangeIdsToProjects(changeIds);
for (Ref draftRef : draftRefs) {
try {
Change.Id changeId = Change.Id.fromAllUsersRef(draftRef.getName());
Account.Id accountId = Account.Id.fromRef(draftRef.getName());
ChangeUserIDsPair changeUserIDsPair = ChangeUserIDsPair.create(changeId, accountId);
if (!visitedSet.add(changeUserIDsPair)) {
continue;
}
if (!changeProjectMap.containsKey(changeId)) {
logger.atWarning().log(
"Could not find a project associated with change ID %s. Skipping draft ref %s.",
changeId, draftRef.getName());
continue;
}
DraftCommentNotes draftNotes = draftNotesFactory.create(changeId, accountId).load();
ChangeNotes notes =
changeNotesFactory.createChecked(changeProjectMap.get(changeId), changeId);
List drafts = draftNotes.getComments().values().asList();
List published = commentsUtil.publishedHumanCommentsByChange(notes);
Set publishedIds = toUuid(published);
List zombieDrafts =
drafts.stream()
.filter(draft -> publishedIds.contains(draft.key.uuid))
.collect(Collectors.toList());
for (HumanComment zombieDraft : zombieDrafts) {
earliestZombieTs = getEarlierTs(earliestZombieTs, zombieDraft.writtenOn);
latestZombieTs = getLaterTs(latestZombieTs, zombieDraft.writtenOn);
}
zombieDrafts.forEach(
zombieDraft ->
logger.atWarning().log(
"Draft comment with uuid '%s' of change %s, account %s, written on %s,"
+ " is a zombie draft that is already published.",
zombieDraft.key.uuid, changeId, accountId, zombieDraft.writtenOn));
if (!zombieDrafts.isEmpty() && !dryRun) {
deleteZombieComments(accountId, notes, zombieDrafts);
}
numZombies += zombieDrafts.size();
} catch (Exception e) {
logger.atWarning().withCause(e).log("Failed to process ref %s", draftRef.getName());
}
}
if (numZombies > 0) {
logger.atWarning().log(
"Detected %d additional zombie drafts (earliest at %s, latest at %s).",
numZombies, earliestZombieTs, latestZombieTs);
}
return numZombies;
}
}
@AutoValue
abstract static class ChangeUserIDsPair {
abstract Change.Id changeId();
abstract Account.Id accountId();
static ChangeUserIDsPair create(Change.Id changeId, Account.Id accountId) {
return new AutoValue_DeleteZombieCommentsRefs_ChangeUserIDsPair(changeId, accountId);
}
}
/**
* Accepts a list of draft (zombie) comments for the same change and delete them by executing a
* {@link ChangeUpdate} on NoteDb. The update is executed using the user account who created this
* draft.
*/
private void deleteZombieComments(
Account.Id accountId, ChangeNotes changeNotes, List draftsToDelete)
throws IOException {
if (changeUpdateFactory == null || userFactory == null) {
return;
}
ChangeUpdate changeUpdate =
changeUpdateFactory.create(changeNotes, userFactory.create(accountId), TimeUtil.now());
draftsToDelete.forEach(c -> changeUpdate.deleteComment(c));
changeUpdate.commit();
logger.atInfo().log(
"Deleted zombie draft comments with UUIDs %s",
draftsToDelete.stream().map(d -> d.key.uuid).collect(Collectors.toList()));
}
/**
* Map each change ID to its associated project.
*
* When doing a ref scan of draft refs
* "refs/draft-comments/$change_id_short/$change_id/$user_id" we don't know which project this
* draft comment is associated with. The project name is needed to load published comments for the
* change, hence we map each change ID to its project here by scanning through the change meta ref
* of the change ID in all projects.
*/
private Map mapChangeIdsToProjects(
ImmutableSet changeIds) {
Map result = new HashMap<>();
for (Project.NameKey project : repoManager.list()) {
try (Repository repo = repoManager.openRepository(project)) {
SetView unmappedChangeIds = Sets.difference(changeIds, result.keySet());
for (Change.Id changeId : unmappedChangeIds) {
Ref ref = repo.getRefDatabase().exactRef(RefNames.changeMetaRef(changeId));
if (ref != null) {
result.put(changeId, project);
}
}
} catch (Exception e) {
logger.atWarning().withCause(e).log("Failed to open repository for project '%s'.", project);
}
if (changeIds.size() == result.size()) {
// We do not need to scan the remaining repositories
break;
}
}
if (result.size() != changeIds.size()) {
logger.atWarning().log(
"Failed to associate the following change Ids to a project: %s",
Sets.difference(changeIds, result.keySet()));
}
return result;
}
/** Map the list of input comments to their UUIDs. */
private Set toUuid(List in) {
return in.stream().map(c -> c.key.uuid).collect(Collectors.toSet());
}
private Timestamp getEarlierTs(@Nullable Timestamp t1, Timestamp t2) {
if (t1 == null) {
return t2;
}
return t1.before(t2) ? t1 : t2;
}
private Timestamp getLaterTs(@Nullable Timestamp t1, Timestamp t2) {
if (t1 == null) {
return t2;
}
return t1.after(t2) ? t1 : t2;
}
private void deleteBatchZombieRefs(Repository allUsersRepo, List refsBatch)
throws IOException {
List deleteCommands =
refsBatch.stream()
.map(
zombieRef ->
new ReceiveCommand(
zombieRef.getObjectId(), ObjectId.zeroId(), zombieRef.getName()))
.collect(toImmutableList());
BatchRefUpdate bru = allUsersRepo.getRefDatabase().newBatchUpdate();
bru.setAtomic(true);
bru.addCommand(deleteCommands);
RefUpdateUtil.executeChecked(bru, allUsersRepo);
}
private List filterZombieRefs(Repository allUsersRepo, List allDraftRefs)
throws IOException {
List zombieRefs = new ArrayList<>((int) (allDraftRefs.size() * 0.5));
for (Ref ref : allDraftRefs) {
if (isZombieRef(allUsersRepo, ref)) {
zombieRefs.add(ref);
}
}
return zombieRefs;
}
private boolean isZombieRef(Repository allUsersRepo, Ref ref) throws IOException {
return allUsersRepo.parseCommit(ref.getObjectId()).getTree().getId().equals(EMPTY_TREE_ID);
}
private void logInfo(String message) {
logger.atInfo().log("%s", message);
uiConsumer.accept(message);
}
private void logProgress(long deletedRefsCount, long allRefsCount, long elapsed) {
logInfo(
String.format(
"Deleted %d/%d zombie draft refs (%d seconds)",
deletedRefsCount, allRefsCount, elapsed));
}
}