All Downloads are FREE. Search and download functionalities are using the official Maven repository.

com.google.gerrit.server.notedb.rebuild.ChangeRebuilderImpl Maven / Gradle / Ivy

There is a newer version: 3.10.0-rc4
Show newest version
// Copyright (C) 2014 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.gerrit.server.notedb.rebuild;

import static com.google.common.base.MoreObjects.firstNonNull;
import static com.google.common.base.Preconditions.checkState;
import static com.google.gerrit.reviewdb.client.RefNames.changeMetaRef;
import static com.google.gerrit.server.notedb.ChangeNoteUtil.FOOTER_HASHTAGS;
import static com.google.gerrit.server.notedb.ChangeNoteUtil.FOOTER_PATCH_SET;
import static java.util.Objects.requireNonNull;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toList;

import com.google.common.base.Splitter;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.MultimapBuilder;
import com.google.common.collect.Ordering;
import com.google.common.collect.Sets;
import com.google.common.collect.Table;
import com.google.common.primitives.Ints;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.Change;
import com.google.gerrit.reviewdb.client.ChangeMessage;
import com.google.gerrit.reviewdb.client.Comment;
import com.google.gerrit.reviewdb.client.PatchLineComment;
import com.google.gerrit.reviewdb.client.PatchLineComment.Status;
import com.google.gerrit.reviewdb.client.PatchSet;
import com.google.gerrit.reviewdb.client.PatchSetApproval;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.client.RefNames;
import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.reviewdb.server.ReviewDbUtil;
import com.google.gerrit.server.CommentsUtil;
import com.google.gerrit.server.GerritPersonIdent;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.GerritServerId;
import com.google.gerrit.server.notedb.ChangeBundle;
import com.google.gerrit.server.notedb.ChangeBundleReader;
import com.google.gerrit.server.notedb.ChangeDraftUpdate;
import com.google.gerrit.server.notedb.ChangeNoteUtil;
import com.google.gerrit.server.notedb.ChangeNotes;
import com.google.gerrit.server.notedb.ChangeUpdate;
import com.google.gerrit.server.notedb.NoteDbChangeState;
import com.google.gerrit.server.notedb.NoteDbChangeState.PrimaryStorage;
import com.google.gerrit.server.notedb.NoteDbUpdateManager;
import com.google.gerrit.server.notedb.NoteDbUpdateManager.OpenRepo;
import com.google.gerrit.server.notedb.NoteDbUpdateManager.Result;
import com.google.gerrit.server.notedb.NotesMigration;
import com.google.gerrit.server.notedb.ReviewerStateInternal;
import com.google.gerrit.server.patch.PatchListCache;
import com.google.gerrit.server.project.NoSuchChangeException;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.update.ChainedReceiveCommands;
import com.google.gwtorm.client.Key;
import com.google.gwtorm.server.Access;
import com.google.gwtorm.server.AtomicUpdate;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.SchemaFactory;
import com.google.inject.Inject;
import java.io.IOException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.transport.ReceiveCommand;

public class ChangeRebuilderImpl extends ChangeRebuilder {
  /**
   * The maximum amount of time between the ReviewDb timestamp of the first and last events batched
   * together into a single NoteDb update.
   *
   * 

Used to account for the fact that different records with their own timestamps (e.g. {@link * PatchSetApproval} and {@link ChangeMessage}) historically didn't necessarily use the same * timestamp, and tended to call {@code System.currentTimeMillis()} independently. */ public static final long MAX_WINDOW_MS = SECONDS.toMillis(3); /** * The maximum amount of time between two consecutive events to consider them to be in the same * batch. */ static final long MAX_DELTA_MS = SECONDS.toMillis(1); private final ChangeBundleReader bundleReader; private final ChangeDraftUpdate.Factory draftUpdateFactory; private final ChangeNoteUtil changeNoteUtil; private final ChangeNotes.Factory notesFactory; private final ChangeUpdate.Factory updateFactory; private final CommentsUtil commentsUtil; private final NoteDbUpdateManager.Factory updateManagerFactory; private final NotesMigration migration; private final PatchListCache patchListCache; private final PersonIdent serverIdent; private final ProjectCache projectCache; private final String serverId; private final long skewMs; @Inject ChangeRebuilderImpl( @GerritServerConfig Config cfg, SchemaFactory schemaFactory, ChangeBundleReader bundleReader, ChangeDraftUpdate.Factory draftUpdateFactory, ChangeNoteUtil changeNoteUtil, ChangeNotes.Factory notesFactory, ChangeUpdate.Factory updateFactory, CommentsUtil commentsUtil, NoteDbUpdateManager.Factory updateManagerFactory, NotesMigration migration, PatchListCache patchListCache, @GerritPersonIdent PersonIdent serverIdent, @Nullable ProjectCache projectCache, @GerritServerId String serverId) { super(schemaFactory); this.bundleReader = bundleReader; this.draftUpdateFactory = draftUpdateFactory; this.changeNoteUtil = changeNoteUtil; this.notesFactory = notesFactory; this.updateFactory = updateFactory; this.commentsUtil = commentsUtil; this.updateManagerFactory = updateManagerFactory; this.migration = migration; this.patchListCache = patchListCache; this.serverIdent = serverIdent; this.projectCache = projectCache; this.serverId = serverId; this.skewMs = NoteDbChangeState.getReadOnlySkew(cfg); } @Override public Result rebuild(ReviewDb db, Change.Id changeId) throws IOException, OrmException { return rebuild(db, changeId, true); } @Override public Result rebuildEvenIfReadOnly(ReviewDb db, Change.Id changeId) throws IOException, OrmException { return rebuild(db, changeId, false); } private Result rebuild(ReviewDb db, Change.Id changeId, boolean checkReadOnly) throws IOException, OrmException { db = ReviewDbUtil.unwrapDb(db); // Read change just to get project; this instance is then discarded so we can read a consistent // ChangeBundle inside a transaction. Change change = db.changes().get(changeId); if (change == null) { throw new NoSuchChangeException(changeId); } try (NoteDbUpdateManager manager = updateManagerFactory.create(change.getProject())) { buildUpdates(manager, bundleReader.fromReviewDb(db, changeId)); return execute(db, changeId, manager, checkReadOnly, true); } } @Override public Result rebuild(NoteDbUpdateManager manager, ChangeBundle bundle) throws NoSuchChangeException, IOException, OrmException { Change change = new Change(bundle.getChange()); buildUpdates(manager, bundle); return manager.stageAndApplyDelta(change); } @Override public NoteDbUpdateManager stage(ReviewDb db, Change.Id changeId) throws IOException, OrmException { db = ReviewDbUtil.unwrapDb(db); Change change = checkNoteDbState(ChangeNotes.readOneReviewDbChange(db, changeId)); if (change == null) { throw new NoSuchChangeException(changeId); } NoteDbUpdateManager manager = updateManagerFactory.create(change.getProject()); buildUpdates(manager, bundleReader.fromReviewDb(db, changeId)); manager.stage(); return manager; } @Override public Result execute(ReviewDb db, Change.Id changeId, NoteDbUpdateManager manager) throws OrmException, IOException { return execute(db, changeId, manager, true, true); } public Result execute( ReviewDb db, Change.Id changeId, NoteDbUpdateManager manager, boolean checkReadOnly, boolean executeManager) throws OrmException, IOException { db = ReviewDbUtil.unwrapDb(db); Change change = checkNoteDbState(ChangeNotes.readOneReviewDbChange(db, changeId)); if (change == null) { throw new NoSuchChangeException(changeId); } String oldNoteDbStateStr = change.getNoteDbState(); Result r = manager.stageAndApplyDelta(change); String newNoteDbStateStr = change.getNoteDbState(); if (newNoteDbStateStr == null) { throw new OrmException( String.format( "Rebuilding change %s produced no writes to NoteDb: %s", changeId, bundleReader.fromReviewDb(db, changeId))); } NoteDbChangeState newNoteDbState = requireNonNull(NoteDbChangeState.parse(changeId, newNoteDbStateStr)); try { db.changes() .atomicUpdate( changeId, new AtomicUpdate() { @Override public Change update(Change change) { if (checkReadOnly) { NoteDbChangeState.checkNotReadOnly(change, skewMs); } String currNoteDbStateStr = change.getNoteDbState(); if (Objects.equals(currNoteDbStateStr, newNoteDbStateStr)) { // Another thread completed the same rebuild we were about to. throw new AbortUpdateException(); } else if (!Objects.equals(oldNoteDbStateStr, currNoteDbStateStr)) { // Another thread updated the state to something else. throw new ConflictingUpdateRuntimeException(change, oldNoteDbStateStr); } change.setNoteDbState(newNoteDbStateStr); return change; } }); } catch (ConflictingUpdateRuntimeException e) { // Rethrow as an OrmException so the caller knows to use staged results. Strictly speaking // they are not completely up to date, but result we send to the caller is the same as if this // rebuild had executed before the other thread. throw new ConflictingUpdateException(e); } catch (AbortUpdateException e) { if (newNoteDbState.isUpToDate( manager.getChangeRepo().cmds.getRepoRefCache(), manager.getAllUsersRepo().cmds.getRepoRefCache())) { // If the state in ReviewDb matches NoteDb at this point, it means another thread // successfully completed this rebuild. It's ok to not execute the update in this case, // since the object referenced in the Result was flushed to the repo by whatever thread won // the race. return r; } // If the state doesn't match, that means another thread attempted this rebuild, but // failed. Fall through and try to update the ref again. } if (migration.failChangeWrites()) { // Don't even attempt to execute if read-only, it would fail anyway. But do throw an exception // to the caller so they know to use the staged results instead of reading from the repo. throw new OrmException(NoteDbUpdateManager.CHANGES_READ_ONLY); } if (executeManager) { manager.execute(); } return r; } static Change checkNoteDbState(Change c) throws OrmException { // Can only rebuild a change if its primary storage is ReviewDb. NoteDbChangeState s = NoteDbChangeState.parse(c); if (s != null && s.getPrimaryStorage() != PrimaryStorage.REVIEW_DB) { throw new OrmException(String.format("cannot rebuild change %s with state %s", c.getId(), s)); } return c; } @Override public void buildUpdates(NoteDbUpdateManager manager, ChangeBundle bundle) throws IOException, OrmException { manager.setCheckExpectedState(false).setRefLogMessage("Rebuilding change"); Change change = new Change(bundle.getChange()); if (bundle.getPatchSets().isEmpty()) { throw new NoPatchSetsException(change.getId()); } if (change.getLastUpdatedOn().compareTo(change.getCreatedOn()) < 0) { // A bug in data migration might set created_on to the time of the migration. The // correct timestamps were lost, but we can at least set it so created_on is not after // last_updated_on. // See https://bugs.chromium.org/p/gerrit/issues/detail?id=7397 change.setCreatedOn(change.getLastUpdatedOn()); } // We will rebuild all events, except for draft comments, in buckets based on author and // timestamp. List events = new ArrayList<>(); ListMultimap draftCommentEvents = MultimapBuilder.hashKeys().arrayListValues().build(); events.addAll(getHashtagsEvents(change, manager)); // Delete ref only after hashtags have been read. deleteChangeMetaRef(change, manager.getChangeRepo().cmds); deleteDraftRefs(change, manager.getAllUsersRepo()); Integer minPsNum = getMinPatchSetNum(bundle); TreeMap patchSetEvents = new TreeMap<>(ReviewDbUtil.intKeyOrdering()); for (PatchSet ps : bundle.getPatchSets()) { PatchSetEvent pse = new PatchSetEvent(change, ps, manager.getChangeRepo().rw); patchSetEvents.put(ps.getId(), pse); events.add(pse); for (Comment c : getComments(bundle, serverId, Status.PUBLISHED, ps)) { CommentEvent e = new CommentEvent(c, change, ps, patchListCache); events.add(e.addDep(pse)); } for (Comment c : getComments(bundle, serverId, Status.DRAFT, ps)) { DraftCommentEvent e = new DraftCommentEvent(c, change, ps, patchListCache); draftCommentEvents.put(c.author.getId(), e); } } ensurePatchSetOrder(patchSetEvents); for (PatchSetApproval psa : bundle.getPatchSetApprovals()) { PatchSetEvent pse = patchSetEvents.get(psa.getPatchSetId()); if (pse != null) { events.add(new ApprovalEvent(psa, change.getCreatedOn()).addDep(pse)); } } for (Table.Cell r : bundle.getReviewers().asTable().cellSet()) { events.add(new ReviewerEvent(r, change.getCreatedOn())); } Change noteDbChange = new Change(null, null, null, null, null); for (ChangeMessage msg : bundle.getChangeMessages()) { Event msgEvent = new ChangeMessageEvent(change, noteDbChange, msg, change.getCreatedOn()); if (msg.getPatchSetId() != null) { PatchSetEvent pse = patchSetEvents.get(msg.getPatchSetId()); if (pse == null) { continue; // Ignore events for missing patch sets. } msgEvent.addDep(pse); } events.add(msgEvent); } sortAndFillEvents(change, noteDbChange, bundle.getPatchSets(), events, minPsNum); EventList el = new EventList<>(); for (Event e : events) { if (!el.canAdd(e)) { flushEventsToUpdate(manager, el, change); checkState(el.canAdd(e)); } el.add(e); } flushEventsToUpdate(manager, el, change); EventList plcel = new EventList<>(); for (Account.Id author : draftCommentEvents.keys()) { for (DraftCommentEvent e : Ordering.natural().sortedCopy(draftCommentEvents.get(author))) { if (!plcel.canAdd(e)) { flushEventsToDraftUpdate(manager, plcel, change); checkState(plcel.canAdd(e)); } plcel.add(e); } flushEventsToDraftUpdate(manager, plcel, change); } } private static Integer getMinPatchSetNum(ChangeBundle bundle) { Integer minPsNum = null; for (PatchSet ps : bundle.getPatchSets()) { int n = ps.getId().get(); if (minPsNum == null || n < minPsNum) { minPsNum = n; } } return minPsNum; } private static void ensurePatchSetOrder(TreeMap events) { if (events.isEmpty()) { return; } Iterator it = events.values().iterator(); PatchSetEvent curr = it.next(); while (it.hasNext()) { PatchSetEvent next = it.next(); next.addDep(curr); curr = next; } } private static List getComments( ChangeBundle bundle, String serverId, PatchLineComment.Status status, PatchSet ps) { return bundle .getPatchLineComments() .stream() .filter(c -> c.getPatchSetId().equals(ps.getId()) && c.getStatus() == status) .map(plc -> plc.asComment(serverId)) .sorted(CommentsUtil.COMMENT_ORDER) .collect(toList()); } private void sortAndFillEvents( Change change, Change noteDbChange, ImmutableCollection patchSets, List events, Integer minPsNum) { Event finalUpdates = new FinalUpdatesEvent(change, noteDbChange, patchSets); events.add(finalUpdates); setPostSubmitDeps(events); new EventSorter(events).sort(); // Ensure the first event in the list creates the change, setting the author and any required // footers. Also force the creation time of the first patch set to match the creation time of // the change. Event first = events.get(0); if (first instanceof PatchSetEvent && change.getOwner().equals(first.user)) { first.when = change.getCreatedOn(); ((PatchSetEvent) first).createChange = true; } else { events.add(0, new CreateChangeEvent(change, minPsNum)); } // Final pass to correct some inconsistencies. // // First, fill in any missing patch set IDs using the latest patch set of the change at the time // of the event, because NoteDb can't represent actions with no associated patch set ID. This // workaround is as if a user added a ChangeMessage on the change by replying from the latest // patch set. // // Start with the first patch set that actually exists. If there are no patch sets at all, // minPsNum will be null, so just bail and use 1 as the patch set ID. // // Second, ensure timestamps are nondecreasing, by copying the previous timestamp if this // happens. This assumes that the only way this can happen is due to dependency constraints, and // it is ok to give an event the same timestamp as one of its dependencies. int ps = firstNonNull(minPsNum, 1); for (int i = 0; i < events.size(); i++) { Event e = events.get(i); if (e.psId == null) { e.psId = new PatchSet.Id(change.getId(), ps); } else { ps = Math.max(ps, e.psId.get()); } if (i > 0) { Event p = events.get(i - 1); if (e.when.before(p.when)) { e.when = p.when; } } } } private void setPostSubmitDeps(List events) { Optional submitEvent = Lists.reverse(events).stream().filter(Event::isSubmit).findFirst(); if (submitEvent.isPresent()) { events.stream().filter(Event::isPostSubmitApproval).forEach(e -> e.addDep(submitEvent.get())); } } private void flushEventsToUpdate( NoteDbUpdateManager manager, EventList events, Change change) throws OrmException, IOException { if (events.isEmpty()) { return; } Comparator labelNameComparator; if (projectCache != null) { labelNameComparator = projectCache.get(change.getProject()).getLabelTypes().nameComparator(); } else { // No project cache available, bail and use natural ordering; there's no semantic difference // anyway difference. labelNameComparator = Ordering.natural(); } ChangeUpdate update = updateFactory.create( change, events.getAccountId(), events.getRealAccountId(), newAuthorIdent(events), events.getWhen(), labelNameComparator); update.setAllowWriteToNewRef(true); update.setPatchSetId(events.getPatchSetId()); update.setTag(events.getTag()); for (Event e : events) { e.apply(update); } manager.add(update); events.clear(); } private void flushEventsToDraftUpdate( NoteDbUpdateManager manager, EventList events, Change change) { if (events.isEmpty()) { return; } ChangeDraftUpdate update = draftUpdateFactory.create( change, events.getAccountId(), events.getRealAccountId(), newAuthorIdent(events), events.getWhen()); update.setPatchSetId(events.getPatchSetId()); for (DraftCommentEvent e : events) { e.applyDraft(update); } manager.add(update); events.clear(); } private PersonIdent newAuthorIdent(EventList events) { Account.Id id = events.getAccountId(); if (id == null) { return new PersonIdent(serverIdent, events.getWhen()); } return changeNoteUtil.newIdent(id, events.getWhen(), serverIdent); } private List getHashtagsEvents(Change change, NoteDbUpdateManager manager) throws IOException { String refName = changeMetaRef(change.getId()); Optional old = manager.getChangeRepo().getObjectId(refName); if (!old.isPresent()) { return Collections.emptyList(); } RevWalk rw = manager.getChangeRepo().rw; List events = new ArrayList<>(); rw.reset(); rw.markStart(rw.parseCommit(old.get())); for (RevCommit commit : rw) { Account.Id authorId; try { authorId = changeNoteUtil .getLegacyChangeNoteRead() .parseIdent(commit.getAuthorIdent(), change.getId()); } catch (ConfigInvalidException e) { continue; // Corrupt data, no valid hashtags in this commit. } PatchSet.Id psId = parsePatchSetId(change, commit); Set hashtags = parseHashtags(commit); if (authorId == null || psId == null || hashtags == null) { continue; } Timestamp commitTime = new Timestamp(commit.getCommitterIdent().getWhen().getTime()); events.add(new HashtagsEvent(psId, authorId, commitTime, hashtags, change.getCreatedOn())); } return events; } private Set parseHashtags(RevCommit commit) { List hashtagsLines = commit.getFooterLines(FOOTER_HASHTAGS); if (hashtagsLines.isEmpty() || hashtagsLines.size() > 1) { return null; } if (hashtagsLines.get(0).isEmpty()) { return ImmutableSet.of(); } return Sets.newHashSet(Splitter.on(',').split(hashtagsLines.get(0))); } private PatchSet.Id parsePatchSetId(Change change, RevCommit commit) { List psIdLines = commit.getFooterLines(FOOTER_PATCH_SET); if (psIdLines.size() != 1) { return null; } Integer psId = Ints.tryParse(psIdLines.get(0)); if (psId == null) { return null; } return new PatchSet.Id(change.getId(), psId); } private void deleteChangeMetaRef(Change change, ChainedReceiveCommands cmds) throws IOException { String refName = changeMetaRef(change.getId()); Optional old = cmds.get(refName); if (old.isPresent()) { cmds.add(new ReceiveCommand(old.get(), ObjectId.zeroId(), refName)); } } private void deleteDraftRefs(Change change, OpenRepo allUsersRepo) throws IOException { for (Ref r : allUsersRepo .repo .getRefDatabase() .getRefsByPrefix(RefNames.refsDraftCommentsPrefix(change.getId()))) { allUsersRepo.cmds.add(new ReceiveCommand(r.getObjectId(), ObjectId.zeroId(), r.getName())); } } static void createChange(ChangeUpdate update, Change change) { update.setSubjectForCommit("Create change"); update.setChangeId(change.getKey().get()); update.setBranch(change.getDest().get()); update.setSubject(change.getOriginalSubject()); if (change.getRevertOf() != null) { update.setRevertOf(change.getRevertOf().get()); } } @Override public void rebuildReviewDb(ReviewDb db, Project.NameKey project, Change.Id changeId) throws OrmException { // TODO(dborowitz): Fail fast if changes tables are disabled in ReviewDb. ChangeNotes notes = notesFactory.create(db, project, changeId); ChangeBundle bundle = ChangeBundle.fromNotes(commentsUtil, notes); db = ReviewDbUtil.unwrapDb(db); db.changes().beginTransaction(changeId); try { Change c = db.changes().get(changeId); if (c != null) { PrimaryStorage ps = PrimaryStorage.of(c); switch (ps) { case REVIEW_DB: return; // Nothing to do. case NOTE_DB: break; // Continue and rebuild. default: throw new OrmException("primary storage of " + changeId + " is " + ps); } } else { c = notes.getChange(); } db.changes().upsert(Collections.singleton(c)); putExactlyEntities( db.changeMessages(), db.changeMessages().byChange(c.getId()), bundle.getChangeMessages()); putExactlyEntities(db.patchSets(), db.patchSets().byChange(c.getId()), bundle.getPatchSets()); putExactlyEntities( db.patchSetApprovals(), db.patchSetApprovals().byChange(c.getId()), bundle.getPatchSetApprovals()); putExactlyEntities( db.patchComments(), db.patchComments().byChange(c.getId()), bundle.getPatchLineComments()); db.commit(); } finally { db.rollback(); } } private static > void putExactlyEntities( Access access, Iterable existing, Collection ents) throws OrmException { Set toKeep = access.toMap(ents).keySet(); access.delete( FluentIterable.from(existing).filter(e -> !toKeep.contains(access.primaryKey(e)))); access.upsert(ents); } }





© 2015 - 2024 Weber Informatics LLC | Privacy Policy