com.google.gerrit.server.account.externalids.ExternalIdNotes Maven / Gradle / Ivy
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.server.account.externalids;
import static com.google.common.base.Preconditions.checkState;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toSet;
import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import com.google.common.collect.Streams;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.entities.Account;
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.extensions.registration.DynamicMap;
import com.google.gerrit.git.ObjectIds;
import com.google.gerrit.metrics.Counter0;
import com.google.gerrit.metrics.Description;
import com.google.gerrit.metrics.DisabledMetricMaker;
import com.google.gerrit.metrics.MetricMaker;
import com.google.gerrit.server.account.AccountsUpdate;
import com.google.gerrit.server.config.AllUsersName;
import com.google.gerrit.server.config.AuthConfig;
import com.google.gerrit.server.git.meta.MetaDataUpdate;
import com.google.gerrit.server.git.meta.VersionedMetaData;
import com.google.gerrit.server.index.account.AccountIndexer;
import com.google.gerrit.server.logging.CallerFinder;
import com.google.gerrit.server.update.RetryHelper;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.BlobBasedConfig;
import org.eclipse.jgit.lib.CommitBuilder;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectInserter;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.notes.Note;
import org.eclipse.jgit.notes.NoteMap;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevTree;
import org.eclipse.jgit.revwalk.RevWalk;
/**
* {@link VersionedMetaData} subclass to update external IDs.
*
* This is a low-level API. Read/write of external IDs should be done through {@link
* com.google.gerrit.server.account.AccountsUpdate} or {@link
* com.google.gerrit.server.account.AccountConfig}.
*
*
On load the note map from {@code refs/meta/external-ids} is read, but the external IDs are not
* parsed yet (see {@link #onLoad()}).
*
*
After loading the note map callers can access single or all external IDs. Only now the
* requested external IDs are parsed.
*
*
After loading the note map callers can stage various external ID updates (insert, upsert,
* delete, replace).
*
*
On save the staged external ID updates are performed (see {@link #onSave(CommitBuilder)}).
*
*
After committing the external IDs a cache update can be requested which also reindexes the
* accounts for which external IDs have been updated (see {@link
* ExternalIdNotesLoader#updateExternalIdCacheAndMaybeReindexAccounts(ExternalIdNotes,
* Collection)}).
*/
public class ExternalIdNotes extends VersionedMetaData {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private static final int MAX_NOTE_SZ = 1 << 19;
public abstract static class ExternalIdNotesLoader {
protected final ExternalIdCache externalIdCache;
protected final MetricMaker metricMaker;
protected final AllUsersName allUsersName;
protected final DynamicMap upsertPreprocessors;
protected final ExternalIdFactory externalIdFactory;
protected final AuthConfig authConfig;
protected ExternalIdNotesLoader(
ExternalIdCache externalIdCache,
MetricMaker metricMaker,
AllUsersName allUsersName,
DynamicMap upsertPreprocessors,
ExternalIdFactory externalIdFactory,
AuthConfig authConfig) {
this.externalIdCache = externalIdCache;
this.metricMaker = metricMaker;
this.allUsersName = allUsersName;
this.upsertPreprocessors = upsertPreprocessors;
this.externalIdFactory = externalIdFactory;
this.authConfig = authConfig;
}
/**
* Loads the external ID notes from the current tip of the {@code refs/meta/external-ids}
* branch.
*
* @param allUsersRepo the All-Users repository
*/
public abstract ExternalIdNotes load(Repository allUsersRepo)
throws IOException, ConfigInvalidException;
/**
* Loads the external ID notes from the specified revision of the {@code refs/meta/external-ids}
* branch.
*
* @param allUsersRepo the All-Users repository
* @param rev the revision from which the external ID notes should be loaded, if {@code null}
* the external ID notes are loaded from the current tip, if {@link ObjectId#zeroId()} it's
* assumed that the {@code refs/meta/external-ids} branch doesn't exist and the loaded
* external IDs will be empty
*/
public abstract ExternalIdNotes load(Repository allUsersRepo, @Nullable ObjectId rev)
throws IOException, ConfigInvalidException;
/**
* Updates the external ID cache. Subclasses of type {@link Factory} will also reindex the
* accounts for which external IDs were modified, while subclasses of type {@link
* FactoryNoReindex} will skip this.
*
* Must only be called after committing changes.
*
* @param externalIdNotes the committed updates that should be applied to the cache. This first
* and last element must be the updates commited first and last, respectively.
* @param accountsToSkipForReindex accounts that should not be reindexed. This is to avoid
* double reindexing when updated accounts will already be reindexed by
* ReindexAfterRefUpdate.
*/
public void updateExternalIdCacheAndMaybeReindexAccounts(
ExternalIdNotes externalIdNotes, Collection accountsToSkipForReindex)
throws IOException {
checkState(externalIdNotes.oldRev != null, "no changes committed yet");
// readOnly is ignored here (legacy behavior).
// Aggregate all updates.
ExternalIdCacheUpdates updates = new ExternalIdCacheUpdates();
for (CacheUpdate cacheUpdate : externalIdNotes.cacheUpdates) {
cacheUpdate.execute(updates);
}
// Reindex accounts (if the subclass implements reindexAccount()).
if (!externalIdNotes.noReindex) {
Streams.concat(updates.getAdded().stream(), updates.getRemoved().stream())
.map(ExternalId::accountId)
.filter(i -> !accountsToSkipForReindex.contains(i))
.distinct()
.forEach(this::reindexAccount);
}
// Reset instance state.
externalIdNotes.cacheUpdates.clear();
externalIdNotes.keysToAdd.clear();
externalIdNotes.oldRev = null;
}
protected abstract void reindexAccount(Account.Id id);
}
@Singleton
public static class Factory extends ExternalIdNotesLoader {
private final Provider accountIndexer;
@Inject
Factory(
ExternalIdCache externalIdCache,
Provider accountIndexer,
MetricMaker metricMaker,
AllUsersName allUsersName,
DynamicMap upsertPreprocessors,
ExternalIdFactory externalIdFactory,
AuthConfig authConfig) {
super(
externalIdCache,
metricMaker,
allUsersName,
upsertPreprocessors,
externalIdFactory,
authConfig);
this.accountIndexer = accountIndexer;
}
@Override
public ExternalIdNotes load(Repository allUsersRepo)
throws IOException, ConfigInvalidException {
return new ExternalIdNotes(
metricMaker,
allUsersName,
allUsersRepo,
upsertPreprocessors,
externalIdFactory,
authConfig.isUserNameCaseInsensitiveMigrationMode())
.load();
}
@Override
public ExternalIdNotes load(Repository allUsersRepo, @Nullable ObjectId rev)
throws IOException, ConfigInvalidException {
return new ExternalIdNotes(
metricMaker,
allUsersName,
allUsersRepo,
upsertPreprocessors,
externalIdFactory,
authConfig.isUserNameCaseInsensitiveMigrationMode())
.load(rev);
}
@Override
protected void reindexAccount(Account.Id id) {
accountIndexer.get().index(id);
}
}
@Singleton
public static class FactoryNoReindex extends ExternalIdNotesLoader {
@Inject
FactoryNoReindex(
ExternalIdCache externalIdCache,
MetricMaker metricMaker,
AllUsersName allUsersName,
DynamicMap upsertPreprocessors,
ExternalIdFactory externalIdFactory,
AuthConfig authConfig) {
super(
externalIdCache,
metricMaker,
allUsersName,
upsertPreprocessors,
externalIdFactory,
authConfig);
}
@Override
public ExternalIdNotes load(Repository allUsersRepo)
throws IOException, ConfigInvalidException {
return new ExternalIdNotes(
metricMaker,
allUsersName,
allUsersRepo,
upsertPreprocessors,
externalIdFactory,
authConfig.isUserNameCaseInsensitiveMigrationMode())
.setNoReindex()
.load();
}
@Override
public ExternalIdNotes load(Repository allUsersRepo, @Nullable ObjectId rev)
throws IOException, ConfigInvalidException {
return new ExternalIdNotes(
metricMaker,
allUsersName,
allUsersRepo,
upsertPreprocessors,
externalIdFactory,
authConfig.isUserNameCaseInsensitiveMigrationMode())
.setNoReindex()
.load(rev);
}
@Override
protected void reindexAccount(Account.Id id) {
// Do not reindex.
}
}
/**
* Loads the external ID notes for reading only. The external ID notes are loaded from the
* specified revision of the {@code refs/meta/external-ids} branch.
*
* @param rev the revision from which the external ID notes should be loaded, if {@code null} the
* external ID notes are loaded from the current tip, if {@link ObjectId#zeroId()} it's
* assumed that the {@code refs/meta/external-ids} branch doesn't exist and the loaded
* external IDs will be empty
* @return read-only {@link ExternalIdNotes} instance
*/
public static ExternalIdNotes loadReadOnly(
AllUsersName allUsersName,
Repository allUsersRepo,
@Nullable ObjectId rev,
ExternalIdFactory externalIdFactory,
boolean isUserNameCaseInsensitiveMigrationMode)
throws IOException, ConfigInvalidException {
return new ExternalIdNotes(
new DisabledMetricMaker(),
allUsersName,
allUsersRepo,
DynamicMap.emptyMap(),
externalIdFactory,
isUserNameCaseInsensitiveMigrationMode)
.setReadOnly()
.setNoReindex()
.load(rev);
}
/**
* Loads the external ID notes for updates. The external ID notes are loaded from the current tip
* of the {@code refs/meta/external-ids} branch.
*
* Use this only from init, schema upgrades and tests.
*
*
Metrics are disabled.
*
* @return {@link ExternalIdNotes} instance that doesn't updates caches on save
*/
public static ExternalIdNotes load(
AllUsersName allUsersName,
Repository allUsersRepo,
ExternalIdFactory externalIdFactory,
boolean isUserNameCaseInsensitiveMigrationMode)
throws IOException, ConfigInvalidException {
return new ExternalIdNotes(
new DisabledMetricMaker(),
allUsersName,
allUsersRepo,
DynamicMap.emptyMap(),
externalIdFactory,
isUserNameCaseInsensitiveMigrationMode)
.setNoReindex()
.load();
}
private final AllUsersName allUsersName;
private final Counter0 updateCount;
private final Repository repo;
private final DynamicMap upsertPreprocessors;
private final CallerFinder callerFinder;
private final ExternalIdFactory externalIdFactory;
private NoteMap noteMap;
private ObjectId oldRev;
/** Staged note map updates that should be executed on save. */
private final List noteMapUpdates = new ArrayList<>();
/** Staged cache updates that should be executed after external ID changes have been committed. */
private final List cacheUpdates = new ArrayList<>();
/**
* When performing batch updates (cf. {@link AccountsUpdate#updateBatch(List)} we need to ensure
* the batch does not introduce duplicates. In addition to checking against the status quo in
* {@link #noteMap} (cf. {@link #checkExternalIdKeysDontExist(Collection)}), which is sufficient
* for single updates, we also need to check for duplicates among the batch updates. As the actual
* updates are computed lazily just before applying them, we unfortunately need to track keys
* explicitly here even though they are already implicit in the lambdas that constitute the
* updates.
*/
private final Set keysToAdd = new HashSet<>();
private Runnable afterReadRevision;
private boolean readOnly = false;
private boolean noReindex = false;
private boolean isUserNameCaseInsensitiveMigrationMode = false;
protected final Function defaultNoteIdResolver =
(extId) -> {
ObjectId noteId = extId.key().sha1();
try {
if (isUserNameCaseInsensitiveMigrationMode && !noteMap.contains(noteId)) {
noteId = extId.key().caseSensitiveSha1();
}
} catch (IOException e) {
return noteId;
}
return noteId;
};
private ExternalIdNotes(
MetricMaker metricMaker,
AllUsersName allUsersName,
Repository allUsersRepo,
DynamicMap upsertPreprocessors,
ExternalIdFactory externalIdFactory,
boolean isUserNameCaseInsensitiveMigrationMode) {
this.updateCount =
metricMaker.newCounter(
"notedb/external_id_update_count",
new Description("Total number of external ID updates.").setRate().setUnit("updates"));
this.allUsersName = requireNonNull(allUsersName, "allUsersRepo");
this.repo = requireNonNull(allUsersRepo, "allUsersRepo");
this.upsertPreprocessors = upsertPreprocessors;
this.callerFinder =
CallerFinder.builder()
// 1. callers that come through ExternalIds
.addTarget(ExternalIds.class)
// 2. callers that come through AccountsUpdate
.addTarget(AccountsUpdate.class)
.addIgnoredPackage("com.github.rholder.retry")
.addIgnoredClass(RetryHelper.class)
// 3. direct callers
.addTarget(ExternalIdNotes.class)
.build();
this.externalIdFactory = externalIdFactory;
this.isUserNameCaseInsensitiveMigrationMode = isUserNameCaseInsensitiveMigrationMode;
}
public ExternalIdNotes setAfterReadRevision(Runnable afterReadRevision) {
this.afterReadRevision = afterReadRevision;
return this;
}
private ExternalIdNotes setReadOnly() {
readOnly = true;
return this;
}
private ExternalIdNotes setNoReindex() {
noReindex = true;
return this;
}
public Repository getRepository() {
return repo;
}
@Override
protected String getRefName() {
return RefNames.REFS_EXTERNAL_IDS;
}
/**
* Loads the external ID notes from the current tip of the {@code refs/meta/external-ids} branch.
*
* @return {@link ExternalIdNotes} instance for chaining
*/
private ExternalIdNotes load() throws IOException, ConfigInvalidException {
load(allUsersName, repo);
return this;
}
/**
* Loads the external ID notes from the specified revision of the {@code refs/meta/external-ids}
* branch.
*
* @param rev the revision from which the external ID notes should be loaded, if {@code null} the
* external ID notes are loaded from the current tip, if {@link ObjectId#zeroId()} it's
* assumed that the {@code refs/meta/external-ids} branch doesn't exist and the loaded
* external IDs will be empty
* @return {@link ExternalIdNotes} instance for chaining
*/
ExternalIdNotes load(@Nullable ObjectId rev) throws IOException, ConfigInvalidException {
if (rev == null) {
return load();
}
if (ObjectId.zeroId().equals(rev)) {
load(allUsersName, repo, null);
return this;
}
load(allUsersName, repo, rev);
return this;
}
/**
* Parses and returns the specified external ID.
*
* @param key the key of the external ID
* @return the external ID, {@code Optional.empty()} if it doesn't exist
*/
public Optional get(ExternalId.Key key) throws IOException, ConfigInvalidException {
checkLoaded();
ObjectId noteId = getNoteId(key);
if (noteMap.contains(noteId)) {
try (RevWalk rw = new RevWalk(repo)) {
ObjectId noteDataId = noteMap.get(noteId);
byte[] raw = readNoteData(rw, noteDataId);
return Optional.of(externalIdFactory.parse(noteId.name(), raw, noteDataId));
}
}
return Optional.empty();
}
protected ObjectId getNoteId(ExternalId.Key key) throws IOException {
ObjectId noteId = key.sha1();
if (!noteMap.contains(noteId) && isUserNameCaseInsensitiveMigrationMode) {
noteId = key.caseSensitiveSha1();
}
return noteId;
}
/**
* Parses and returns the specified external IDs.
*
* @param keys the keys of the external IDs
* @return the external IDs
*/
public Set get(Collection keys)
throws IOException, ConfigInvalidException {
checkLoaded();
HashSet externalIds = Sets.newHashSetWithExpectedSize(keys.size());
for (ExternalId.Key key : keys) {
get(key).ifPresent(externalIds::add);
}
return externalIds;
}
/**
* Parses and returns all external IDs.
*
* Invalid external IDs are ignored.
*
* @return all external IDs
*/
public ImmutableSet all() throws IOException {
checkLoaded();
try (RevWalk rw = new RevWalk(repo)) {
ImmutableSet.Builder b = ImmutableSet.builder();
for (Note note : noteMap) {
byte[] raw = readNoteData(rw, note.getData());
try {
b.add(externalIdFactory.parse(note.getName(), raw, note.getData()));
} catch (ConfigInvalidException | RuntimeException e) {
logger.atSevere().withCause(e).log(
"Ignoring invalid external ID note %s", note.getName());
}
}
return b.build();
}
}
NoteMap getNoteMap() {
checkLoaded();
return noteMap;
}
static byte[] readNoteData(RevWalk rw, ObjectId noteDataId) throws IOException {
return rw.getObjectReader().open(noteDataId, OBJ_BLOB).getCachedBytes(MAX_NOTE_SZ);
}
/**
* Inserts a new external ID.
*
* @throws IOException on IO error while checking if external ID already exists
* @throws DuplicateExternalIdKeyException if the external ID already exists
*/
public void insert(ExternalId extId) throws IOException, DuplicateExternalIdKeyException {
insert(Collections.singleton(extId));
}
/**
* Inserts new external IDs.
*
* @throws IOException on IO error while checking if external IDs already exist
* @throws DuplicateExternalIdKeyException if any of the external ID already exists
*/
public void insert(Collection extIds)
throws IOException, DuplicateExternalIdKeyException {
checkLoaded();
checkExternalIdsDontExist(extIds);
Set newExtIds = new HashSet<>();
noteMapUpdates.add(
(rw, n) -> {
for (ExternalId extId : extIds) {
ExternalId insertedExtId = upsert(rw, inserter, noteMap, extId);
preprocessUpsert(insertedExtId);
newExtIds.add(insertedExtId);
}
});
cacheUpdates.add(cu -> cu.add(newExtIds));
incrementalDuplicateDetection(extIds);
}
/**
* Inserts or updates an external ID.
*
* If the external ID already exists, it is overwritten, otherwise it is inserted.
*/
public void upsert(ExternalId extId) throws IOException, ConfigInvalidException {
upsert(Collections.singleton(extId));
}
/**
* Inserts or updates external IDs.
*
*
If any of the external IDs already exists, it is overwritten. New external IDs are inserted.
*/
public void upsert(Collection extIds) throws IOException, ConfigInvalidException {
checkLoaded();
Set removedExtIds = get(ExternalId.Key.from(extIds));
Set updatedExtIds = new HashSet<>();
noteMapUpdates.add(
(rw, n) -> {
for (ExternalId extId : extIds) {
ExternalId updatedExtId = upsert(rw, inserter, noteMap, extId);
preprocessUpsert(updatedExtId);
updatedExtIds.add(updatedExtId);
}
});
cacheUpdates.add(cu -> cu.remove(removedExtIds).add(updatedExtIds));
incrementalDuplicateDetection(extIds);
}
/**
* Deletes an external ID.
*
* @throws IllegalStateException is thrown if there is an existing external ID that has the same
* key, but otherwise doesn't match the specified external ID.
*/
public void delete(ExternalId extId) {
delete(Collections.singleton(extId));
}
/**
* Deletes external IDs.
*
* @throws IllegalStateException is thrown if there is an existing external ID that has the same
* key as any of the external IDs that should be deleted, but otherwise doesn't match the that
* external ID.
*/
public void delete(Collection extIds) {
checkLoaded();
Set removedExtIds = new HashSet<>();
noteMapUpdates.add(
(rw, n) -> {
for (ExternalId extId : extIds) {
remove(rw, noteMap, extId);
removedExtIds.add(extId);
}
});
cacheUpdates.add(cu -> cu.remove(removedExtIds));
}
/**
* Delete an external ID by key.
*
* @throws IllegalStateException is thrown if the external ID does not belong to the specified
* account.
*/
public void delete(Account.Id accountId, ExternalId.Key extIdKey) {
delete(accountId, Collections.singleton(extIdKey));
}
/**
* Delete external IDs by external ID key.
*
* @throws IllegalStateException is thrown if any of the external IDs does not belong to the
* specified account.
*/
public void delete(Account.Id accountId, Collection extIdKeys) {
checkLoaded();
Set removedExtIds = new HashSet<>();
noteMapUpdates.add(
(rw, n) -> {
for (ExternalId.Key extIdKey : extIdKeys) {
ExternalId removedExtId = remove(rw, noteMap, extIdKey, accountId);
removedExtIds.add(removedExtId);
}
});
cacheUpdates.add(cu -> cu.remove(removedExtIds));
}
/**
* Delete external IDs by external ID key.
*
* The external IDs are deleted regardless of which account they belong to.
*/
public void deleteByKeys(Collection extIdKeys) {
checkLoaded();
Set removedExtIds = new HashSet<>();
noteMapUpdates.add(
(rw, n) -> {
for (ExternalId.Key extIdKey : extIdKeys) {
ExternalId extId = remove(rw, noteMap, extIdKey, null);
removedExtIds.add(extId);
}
});
cacheUpdates.add(cu -> cu.remove(removedExtIds));
}
public void replace(
Account.Id accountId, Collection toDelete, Collection toAdd)
throws IOException, DuplicateExternalIdKeyException {
replace(accountId, toDelete, toAdd, defaultNoteIdResolver);
}
/**
* Replaces external IDs for an account by external ID keys.
*
* Deletion of external IDs is done before adding the new external IDs. This means if an
* external ID key is specified for deletion and an external ID with the same key is specified to
* be added, the old external ID with that key is deleted first and then the new external ID is
* added (so the external ID for that key is replaced).
*
* @throws IllegalStateException is thrown if any of the specified external IDs does not belong to
* the specified account.
*/
public void replace(
Account.Id accountId,
Collection toDelete,
Collection toAdd,
Function noteIdResolver)
throws IOException, DuplicateExternalIdKeyException {
checkLoaded();
checkSameAccount(toAdd, accountId);
checkExternalIdKeysDontExist(ExternalId.Key.from(toAdd), toDelete);
Set removedExtIds = new HashSet<>();
Set updatedExtIds = new HashSet<>();
noteMapUpdates.add(
(rw, n) -> {
for (ExternalId.Key extIdKey : toDelete) {
ExternalId removedExtId = remove(rw, noteMap, extIdKey, accountId);
if (removedExtId != null) {
removedExtIds.add(removedExtId);
}
}
for (ExternalId extId : toAdd) {
ExternalId insertedExtId = upsert(rw, inserter, noteMap, extId, noteIdResolver);
preprocessUpsert(insertedExtId);
updatedExtIds.add(insertedExtId);
}
});
cacheUpdates.add(cu -> cu.add(updatedExtIds).remove(removedExtIds));
incrementalDuplicateDetection(toAdd);
}
/**
* Replaces external IDs for an account by external ID keys.
*
* Deletion of external IDs is done before adding the new external IDs. This means if an
* external ID key is specified for deletion and an external ID with the same key is specified to
* be added, the old external ID with that key is deleted first and then the new external ID is
* added (so the external ID for that key is replaced).
*
*
The external IDs are replaced regardless of which account they belong to.
*/
public void replaceByKeys(Collection toDelete, Collection toAdd)
throws IOException, DuplicateExternalIdKeyException {
checkLoaded();
checkExternalIdKeysDontExist(ExternalId.Key.from(toAdd), toDelete);
Set removedExtIds = new HashSet<>();
Set updatedExtIds = new HashSet<>();
noteMapUpdates.add(
(rw, n) -> {
for (ExternalId.Key extIdKey : toDelete) {
ExternalId removedExtId = remove(rw, noteMap, extIdKey, null);
removedExtIds.add(removedExtId);
}
for (ExternalId extId : toAdd) {
ExternalId insertedExtId = upsert(rw, inserter, noteMap, extId);
preprocessUpsert(insertedExtId);
updatedExtIds.add(insertedExtId);
}
});
cacheUpdates.add(cu -> cu.add(updatedExtIds).remove(removedExtIds));
incrementalDuplicateDetection(toAdd);
}
/**
* Replaces an external ID.
*
* @throws IllegalStateException is thrown if the specified external IDs belong to different
* accounts.
*/
public void replace(ExternalId toDelete, ExternalId toAdd)
throws IOException, DuplicateExternalIdKeyException {
replace(Collections.singleton(toDelete), Collections.singleton(toAdd));
}
/**
* Replaces external IDs.
*
* Deletion of external IDs is done before adding the new external IDs. This means if an
* external ID is specified for deletion and an external ID with the same key is specified to be
* added, the old external ID with that key is deleted first and then the new external ID is added
* (so the external ID for that key is replaced).
*
* @throws IllegalStateException is thrown if the specified external IDs belong to different
* accounts.
*/
public void replace(Collection toDelete, Collection toAdd)
throws IOException, DuplicateExternalIdKeyException {
Account.Id accountId = checkSameAccount(Iterables.concat(toDelete, toAdd));
if (accountId == null) {
// toDelete and toAdd are empty -> nothing to do
return;
}
replace(accountId, toDelete.stream().map(ExternalId::key).collect(toSet()), toAdd);
}
/**
* Replaces external IDs.
*
* Deletion of external IDs is done before adding the new external IDs. This means if an
* external ID is specified for deletion and an external ID with the same key is specified to be
* added, the old external ID with that key is deleted first and then the new external ID is added
* (so the external ID for that key is replaced).
*
* @throws IllegalStateException is thrown if the specified external IDs belong to different
* accounts.
*/
public void replace(
Collection toDelete,
Collection toAdd,
Function noteIdResolver)
throws IOException, DuplicateExternalIdKeyException {
Account.Id accountId = checkSameAccount(Iterables.concat(toDelete, toAdd));
if (accountId == null) {
// toDelete and toAdd are empty -> nothing to do
return;
}
replace(
accountId, toDelete.stream().map(ExternalId::key).collect(toSet()), toAdd, noteIdResolver);
}
@Override
protected void onLoad() throws IOException, ConfigInvalidException {
if (revision != null) {
logger.atFine().log(
"Reading external ID note map (caller: %s)", callerFinder.findCallerLazy());
noteMap = NoteMap.read(reader, revision);
} else {
noteMap = NoteMap.newEmptyMap();
}
if (afterReadRevision != null) {
afterReadRevision.run();
}
}
@Override
public RevCommit commit(MetaDataUpdate update) throws IOException {
oldRev = ObjectIds.copyOrZero(revision);
RevCommit commit = super.commit(update);
updateCount.increment();
return commit;
}
@Override
protected boolean onSave(CommitBuilder commit) throws IOException, ConfigInvalidException {
checkState(!readOnly, "Updating external IDs is disabled");
if (noteMapUpdates.isEmpty()) {
return false;
}
logger.atFine().log("Updating external IDs");
if (Strings.isNullOrEmpty(commit.getMessage())) {
commit.setMessage("Update external IDs\n");
}
try (RevWalk rw = new RevWalk(reader)) {
for (NoteMapUpdate noteMapUpdate : noteMapUpdates) {
try {
noteMapUpdate.execute(rw, noteMap);
} catch (DuplicateExternalIdKeyException e) {
throw new IOException(e);
}
}
noteMapUpdates.clear();
RevTree oldTree = revision != null ? rw.parseTree(revision) : null;
ObjectId newTreeId = noteMap.writeTree(inserter);
if (newTreeId.equals(oldTree)) {
return false;
}
commit.setTreeId(newTreeId);
return true;
}
}
/**
* Checks that all specified external IDs belong to the same account.
*
* @return the ID of the account to which all specified external IDs belong.
*/
private static Account.Id checkSameAccount(Iterable extIds) {
return checkSameAccount(extIds, null);
}
/**
* Checks that all specified external IDs belong to specified account. If no account is specified
* it is checked that all specified external IDs belong to the same account.
*
* @return the ID of the account to which all specified external IDs belong.
*/
public static Account.Id checkSameAccount(
Iterable extIds, @Nullable Account.Id accountId) {
for (ExternalId extId : extIds) {
if (accountId == null) {
accountId = extId.accountId();
continue;
}
checkState(
accountId.equals(extId.accountId()),
"external id %s belongs to account %s, but expected account %s",
extId.key().get(),
extId.accountId().get(),
accountId.get());
}
return accountId;
}
private void incrementalDuplicateDetection(Collection externalIds) {
externalIds.stream()
.map(ExternalId::key)
.forEach(
key -> {
if (!keysToAdd.add(key)) {
throw new DuplicateExternalIdKeyException(key);
}
});
}
/**
* Inserts or updates a new external ID and sets it in the note map.
*
* If the external ID already exists, it is overwritten.
*/
private ExternalId upsert(RevWalk rw, ObjectInserter ins, NoteMap noteMap, ExternalId extId)
throws IOException, ConfigInvalidException {
return upsert(rw, ins, noteMap, extId, defaultNoteIdResolver);
}
/**
* Inserts or updates a new external ID and sets it in the note map.
*
*
If the external ID already exists, it is overwritten.
*/
private ExternalId upsert(
RevWalk rw,
ObjectInserter ins,
NoteMap noteMap,
ExternalId extId,
Function noteIdResolver)
throws IOException, ConfigInvalidException {
ObjectId noteId = extId.key().sha1();
Config c = new Config();
ObjectId resolvedNoteId = noteIdResolver.apply(extId);
if (noteMap.contains(resolvedNoteId)) {
noteId = resolvedNoteId;
ObjectId noteDataId = noteMap.get(noteId);
byte[] raw = readNoteData(rw, noteDataId);
try {
c = new BlobBasedConfig(null, raw);
} catch (ConfigInvalidException e) {
throw new ConfigInvalidException(
String.format("Invalid external id config for note %s: %s", noteId, e.getMessage()));
}
}
extId.writeToConfig(c);
byte[] raw = c.toText().getBytes(UTF_8);
ObjectId noteData = ins.insert(OBJ_BLOB, raw);
noteMap.set(noteId, noteData);
return externalIdFactory.create(extId, noteData);
}
/**
* Removes an external ID from the note map.
*
* @throws IllegalStateException is thrown if there is an existing external ID that has the same
* key, but otherwise doesn't match the specified external ID.
*/
private void remove(RevWalk rw, NoteMap noteMap, ExternalId extId)
throws IOException, ConfigInvalidException {
ObjectId noteId = getNoteId(extId.key());
if (!noteMap.contains(noteId)) {
return;
}
ObjectId noteDataId = noteMap.get(noteId);
byte[] raw = readNoteData(rw, noteDataId);
ExternalId actualExtId = externalIdFactory.parse(noteId.name(), raw, noteDataId);
checkState(
extId.equals(actualExtId),
"external id %s should be removed, but it doesn't match the actual external id %s",
extId.toString(),
actualExtId.toString());
noteMap.remove(noteId);
}
/**
* Removes an external ID from the note map by external ID key.
*
* @throws IllegalStateException is thrown if an expected account ID is provided and an external
* ID with the specified key exists, but belongs to another account.
* @return the external ID that was removed, {@code null} if no external ID with the specified key
* exists
*/
@Nullable
private ExternalId remove(
RevWalk rw, NoteMap noteMap, ExternalId.Key extIdKey, Account.Id expectedAccountId)
throws IOException, ConfigInvalidException {
ObjectId noteId = getNoteId(extIdKey);
if (!noteMap.contains(noteId)) {
return null;
}
ObjectId noteDataId = noteMap.get(noteId);
byte[] raw = readNoteData(rw, noteDataId);
ExternalId extId = externalIdFactory.parse(noteId.name(), raw, noteDataId);
if (expectedAccountId != null) {
checkState(
expectedAccountId.equals(extId.accountId()),
"external id %s should be removed for account %s,"
+ " but external id belongs to account %s",
extIdKey.get(),
expectedAccountId.get(),
extId.accountId().get());
}
noteMap.remove(noteId);
return extId;
}
private void checkExternalIdsDontExist(Collection extIds)
throws DuplicateExternalIdKeyException, IOException {
checkExternalIdKeysDontExist(ExternalId.Key.from(extIds));
}
private void checkExternalIdKeysDontExist(
Collection extIdKeysToAdd, Collection extIdKeysToDelete)
throws DuplicateExternalIdKeyException, IOException {
HashSet newKeys = new HashSet<>(extIdKeysToAdd);
newKeys.removeAll(extIdKeysToDelete);
checkExternalIdKeysDontExist(newKeys);
}
private void checkExternalIdKeysDontExist(Collection extIdKeys)
throws IOException, DuplicateExternalIdKeyException {
for (ExternalId.Key extIdKey : extIdKeys) {
if (noteMap.contains(extIdKey.sha1())) {
throw new DuplicateExternalIdKeyException(extIdKey);
}
}
}
private void checkLoaded() {
checkState(noteMap != null, "External IDs not loaded yet");
}
private void preprocessUpsert(ExternalId extId) {
upsertPreprocessors.forEach(p -> p.get().upsert(extId));
}
@FunctionalInterface
private interface NoteMapUpdate {
void execute(RevWalk rw, NoteMap noteMap)
throws IOException, ConfigInvalidException, DuplicateExternalIdKeyException;
}
@FunctionalInterface
private interface CacheUpdate {
void execute(ExternalIdCacheUpdates cacheUpdates) throws IOException;
}
private static class ExternalIdCacheUpdates {
final Set added = new HashSet<>();
final Set removed = new HashSet<>();
ExternalIdCacheUpdates add(Collection extIds) {
this.added.addAll(extIds);
return this;
}
Set getAdded() {
return ImmutableSet.copyOf(added);
}
ExternalIdCacheUpdates remove(Collection extIds) {
this.removed.addAll(extIds);
return this;
}
Set getRemoved() {
return ImmutableSet.copyOf(removed);
}
}
}