com.google.gerrit.server.DeleteZombieComments Maven / Gradle / Ivy
// Copyright (C) 2023 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Sets;
import com.google.common.flogger.FluentLogger;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.entities.Account;
import com.google.gerrit.entities.Change;
import com.google.gerrit.entities.HumanComment;
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.notedb.ChangeNotes;
import java.io.IOException;
import java.sql.Timestamp;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Consumer;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
/**
* This class can be used to clean zombie draft comments. More context in
* https://gerrit-review.googlesource.com/c/gerrit/+/246233
*
* The implementation has two cases for detecting zombie drafts:
*
*
* - An earlier bug in the deletion of draft comments caused some draft refs to remain empty but
* not get deleted.
*
- Inspecting all draft-comments. Check for each draft if there exists a published comment
* with the same UUID. These comments are called zombie drafts. If the program is run in
* {@link DeleteZombieComments#dryRun} mode, the zombie draft IDs will only be logged for
* tracking, otherwise they will also be deleted.
*
*/
public abstract class DeleteZombieComments implements AutoCloseable {
@AutoValue
abstract static class ChangeUserIDsPair {
abstract Change.Id changeId();
abstract Account.Id accountId();
static ChangeUserIDsPair create(Change.Id changeId, Account.Id accountId) {
return new AutoValue_DeleteZombieComments_ChangeUserIDsPair(changeId, accountId);
}
}
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private final int cleanupPercentage;
protected final boolean dryRun;
@Nullable private final Consumer uiConsumer;
@Nullable private final GitRepositoryManager repoManager;
@Nullable private final DraftCommentsReader draftCommentsReader;
@Nullable private final ChangeNotes.Factory changeNotesFactory;
@Nullable private final CommentsUtil commentsUtil;
private Map changeProjectMap = new HashMap<>();
private Map changeNotes = new HashMap<>();
protected DeleteZombieComments(
Integer cleanupPercentage,
boolean dryRun,
Consumer uiConsumer,
GitRepositoryManager repoManager,
DraftCommentsReader draftCommentsReader,
ChangeNotes.Factory changeNotesFactory,
CommentsUtil commentsUtil) {
this.cleanupPercentage = cleanupPercentage == null ? 100 : cleanupPercentage;
this.dryRun = dryRun;
this.uiConsumer = uiConsumer;
this.repoManager = repoManager;
this.draftCommentsReader = draftCommentsReader;
this.changeNotesFactory = changeNotesFactory;
this.commentsUtil = commentsUtil;
}
/** Deletes all draft comments. Returns the number of zombie draft comments that were deleted. */
@CanIgnoreReturnValue
public int execute() throws IOException {
setup();
ListMultimap alreadyPublished = listDraftCommentsThatAreAlsoPublished();
if (!dryRun) {
deleteZombieDrafts(alreadyPublished);
}
List emptyDrafts = filterByCleanupPercentage(listEmptyDrafts(), "empty");
if (!dryRun) {
deleteEmptyDraftsByKey(emptyDrafts);
} else {
logInfo(
String.format(
"Running in dry run mode. Skipping deletion."
+ "\nStats (with %d cleanup-percentage):"
+ "\nEmpty drafts = %d"
+ "\nAlready published drafts (zombies) = %d",
cleanupPercentage, emptyDrafts.size(), alreadyPublished.size()));
}
return emptyDrafts.size() + alreadyPublished.size();
}
@VisibleForTesting
public abstract void setup() throws IOException;
@Override
public abstract void close() throws IOException;
protected abstract List listAllDrafts() throws IOException;
protected abstract List listEmptyDrafts() throws IOException;
protected abstract void deleteEmptyDraftsByKey(Collection keys) throws IOException;
protected abstract void deleteZombieDrafts(ListMultimap drafts)
throws IOException;
protected abstract Change.Id getChangeId(KeyT key);
protected abstract Account.Id getAccountId(KeyT key);
protected abstract String loggable(KeyT key);
protected ChangeNotes getChangeNotes(Change.Id changeId) {
if (changeNotes.containsKey(changeId)) {
return changeNotes.get(changeId);
}
checkState(
changeProjectMap.containsKey(changeId),
"Cannot get a project associated with change ID " + changeId);
ChangeNotes notes = changeNotesFactory.createChecked(changeProjectMap.get(changeId), changeId);
changeNotes.put(changeId, notes);
return notes;
}
private List filterByCleanupPercentage(List drafts, String reason) {
if (cleanupPercentage >= 100) {
logInfo(
String.format(
"Cleanup percentage = %d" + "\nNumber of drafts to be cleaned for %s = %d",
cleanupPercentage, reason, drafts.size()));
return drafts;
}
ImmutableList res =
drafts.stream()
.filter(key -> getChangeId(key).get() % 100 < cleanupPercentage)
.collect(toImmutableList());
logInfo(
String.format(
"Cleanup percentage = %d"
+ "\nOriginal number of drafts for %s = %d"
+ "\nNumber of drafts to be processed for %s = %d",
cleanupPercentage, reason, drafts.size(), reason, res.size()));
return res;
}
@VisibleForTesting
public ListMultimap listDraftCommentsThatAreAlsoPublished()
throws IOException {
List draftKeys = filterByCleanupPercentage(listAllDrafts(), "all-drafts");
changeProjectMap.putAll(mapChangesWithDraftsToProjects(draftKeys));
ListMultimap zombieDrafts = ArrayListMultimap.create();
Set visitedSet = new HashSet<>();
for (KeyT key : draftKeys) {
try {
Change.Id changeId = getChangeId(key);
Account.Id accountId = getAccountId(key);
ChangeUserIDsPair changeUserIDsPair = ChangeUserIDsPair.create(changeId, accountId);
if (!visitedSet.add(changeUserIDsPair)) {
continue;
}
if (!changeProjectMap.containsKey(changeId)) {
logger.atWarning().log(
"Could not find a project associated with change ID %s. Skipping draft [%s]",
changeId, loggable(key));
continue;
}
List drafts =
draftCommentsReader.getDraftsByChangeAndDraftAuthor(changeId, accountId);
ChangeNotes notes = getChangeNotes(changeId);
List published = commentsUtil.publishedHumanCommentsByChange(notes);
ImmutableSet publishedIds = toUuid(published);
ImmutableList zombieDraftsForChangeAndAuthor =
drafts.stream()
.filter(draft -> publishedIds.contains(draft.key.uuid))
.collect(toImmutableList());
zombieDraftsForChangeAndAuthor.forEach(
zombieDraft ->
logger.atWarning().log(
"Draft comment with uuid '%s' of change %s, account %s, written on %s,"
+ " is a zombie draft that is already published.",
zombieDraft.key.uuid, changeId, accountId, zombieDraft.writtenOn));
zombieDrafts.putAll(key, zombieDraftsForChangeAndAuthor);
} catch (RuntimeException e) {
logger.atWarning().withCause(e).log("Failed to process draft [%s]", loggable(key));
}
}
if (!zombieDrafts.isEmpty()) {
Timestamp earliestZombieTs = null;
Timestamp latestZombieTs = null;
for (HumanComment zombieDraft : zombieDrafts.values()) {
earliestZombieTs = getEarlierTs(earliestZombieTs, zombieDraft.writtenOn);
latestZombieTs = getLaterTs(latestZombieTs, zombieDraft.writtenOn);
}
logger.atWarning().log(
"Detected %d zombie drafts that were already published (earliest at %s, latest at %s).",
zombieDrafts.size(), earliestZombieTs, latestZombieTs);
}
return zombieDrafts;
}
/**
* Map each change ID to its associated project.
*
* When doing a ref scan of draft refs
* "refs/draft-comments/$change_id_short/$change_id/$user_id" we don't know which project this
* draft comment is associated with. The project name is needed to load published comments for the
* change, hence we map each change ID to its project here by scanning through the change meta ref
* of the change ID in all projects.
*/
private Map mapChangesWithDraftsToProjects(List drafts) {
ImmutableSet changeIds =
drafts.stream().map(key -> getChangeId(key)).collect(ImmutableSet.toImmutableSet());
Map result = new HashMap<>();
for (Project.NameKey project : repoManager.list()) {
try (Repository repo = repoManager.openRepository(project)) {
Sets.SetView unmappedChangeIds = Sets.difference(changeIds, result.keySet());
for (Change.Id changeId : unmappedChangeIds) {
Ref ref = repo.getRefDatabase().exactRef(RefNames.changeMetaRef(changeId));
if (ref != null) {
result.put(changeId, project);
}
}
} catch (Exception e) {
logger.atWarning().withCause(e).log("Failed to open repository for project '%s'.", project);
}
if (changeIds.size() == result.size()) {
// We do not need to scan the remaining repositories
break;
}
}
if (result.size() != changeIds.size()) {
logger.atWarning().log(
"Failed to associate the following change Ids to a project: %s",
Sets.difference(changeIds, result.keySet()));
}
return result;
}
protected void logInfo(String message) {
logger.atInfo().log("%s", message);
uiConsumer.accept(message);
}
/** Map the list of input comments to their UUIDs. */
private ImmutableSet toUuid(List in) {
return in.stream().map(c -> c.key.uuid).collect(toImmutableSet());
}
private Timestamp getEarlierTs(@Nullable Timestamp t1, Timestamp t2) {
if (t1 == null) {
return t2;
}
return t1.before(t2) ? t1 : t2;
}
private Timestamp getLaterTs(@Nullable Timestamp t1, Timestamp t2) {
if (t1 == null) {
return t2;
}
return t1.after(t2) ? t1 : t2;
}
}