io.activej.ot.OTAlgorithms Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of activej-ot Show documentation
Show all versions of activej-ot Show documentation
Implementation of operational transformation technology. Allows building collaborative software systems.
/*
* Copyright (C) 2020 ActiveJ LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.activej.ot;
import io.activej.async.function.AsyncPredicate;
import io.activej.common.ref.Ref;
import io.activej.ot.OTCommitFactory.DiffsWithLevel;
import io.activej.ot.exception.GraphExhaustedException;
import io.activej.ot.exception.OTException;
import io.activej.ot.reducers.AbstractGraphReducer;
import io.activej.ot.reducers.DiffsReducer;
import io.activej.ot.reducers.GraphReducer;
import io.activej.ot.repository.OTRepository;
import io.activej.ot.system.OTSystem;
import io.activej.promise.Promise;
import io.activej.promise.Promises;
import io.activej.promise.SettablePromise;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import java.util.function.Function;
import java.util.function.Predicate;
import static io.activej.async.util.LogUtils.thisMethod;
import static io.activej.async.util.LogUtils.toLogger;
import static io.activej.common.Checks.checkArgument;
import static io.activej.common.Utils.*;
import static io.activej.ot.reducers.GraphReducer.Result.*;
import static io.activej.promise.Promises.toList;
import static java.util.Collections.*;
import static java.util.Comparator.comparingLong;
import static java.util.stream.Collectors.toSet;
public final class OTAlgorithms {
private static final Logger logger = LoggerFactory.getLogger(OTAlgorithms.class);
public static Promise reduce(OTRepository repository, OTSystem system,
Set heads, GraphReducer reducer) {
return toList(heads.stream().map(repository::loadCommit))
.then(headCommits -> {
PriorityQueue> queue = new PriorityQueue<>(reverseOrder(comparingLong(OTCommit::getLevel)));
queue.addAll(headCommits);
reducer.onStart(unmodifiableCollection(queue));
return Promise.ofCallback(cb -> walkGraphImpl(repository, reducer, queue, new HashSet<>(heads), cb));
});
}
private static void walkGraphImpl(OTRepository repository, GraphReducer reducer,
PriorityQueue> queue, Set visited, SettablePromise cb) {
OTCommit commit = queue.peek();
if (commit == null) {
cb.setException(new GraphExhaustedException());
return;
}
reducer.onCommit(commit)
.run((maybeResult, e) -> {
if (e != null) {
cb.setException(e);
return;
}
OTCommit polledCommit = queue.poll();
assert polledCommit == commit;
if (maybeResult.isResume()) {
toList(commit.getParents().keySet().stream().filter(visited::add).map(repository::loadCommit))
.async()
.run((parentCommits, e2) -> {
if (e2 == null) {
queue.addAll(parentCommits);
walkGraphImpl(repository, reducer, queue, visited, cb);
} else {
cb.setException(e2);
}
});
} else if (maybeResult.isSkip()) {
walkGraphImpl(repository, reducer, queue, visited, cb);
} else {
cb.set(maybeResult.get());
}
});
}
public static final class FindResult {
private final int epoch;
private final @NotNull K commit;
private final Set commitParents;
private final long commitLevel;
private final K child;
private final long childLevel;
private final A accumulatedDiffs;
private FindResult(int epoch, @NotNull K commit, Set commitParents, long commitLevel, K child, long childLevel, A accumulatedDiffs) {
this.epoch = epoch;
this.commit = commit;
this.commitParents = commitParents;
this.commitLevel = commitLevel;
this.child = child;
this.childLevel = childLevel;
this.accumulatedDiffs = accumulatedDiffs;
}
public int getEpoch() {
return epoch;
}
public @NotNull K getCommit() {
return commit;
}
public K getChild() {
return child;
}
public Long getChildLevel() {
return childLevel;
}
public Set getCommitParents() {
return commitParents;
}
public long getCommitLevel() {
return commitLevel;
}
public A getAccumulatedDiffs() {
return accumulatedDiffs;
}
@Override
public String toString() {
return "FindResult{" +
"commit=" + commit +
", parents=" + commitParents +
", child=" + child +
", accumulator=" + accumulatedDiffs +
'}';
}
}
public static Promise> findParent(OTRepository repository, OTSystem system,
Set startNodes, DiffsReducer diffsReducer, AsyncPredicate> matchPredicate) {
return reduce(repository, system, startNodes,
new AbstractGraphReducer>(diffsReducer) {
int epoch;
@Override
public void onStart(@NotNull Collection> queue) {
this.epoch = queue.iterator().next().getEpoch();
super.onStart(queue);
}
@Override
protected @NotNull Promise>> tryGetResult(OTCommit commit,
Map> accumulators, Map> headCommits) {
return matchPredicate.test(commit)
.map(matched -> {
if (!matched) return Optional.empty();
Map.Entry someHead = accumulators.get(commit.getId()).entrySet().iterator().next();
return Optional.of(new FindResult<>(
epoch, commit.getId(), commit.getParentIds(), commit.getLevel(),
someHead.getKey(), headCommits.get(someHead.getKey()).getLevel(),
someHead.getValue()
));
});
}
});
}
public static Promise mergeAndPush(OTRepository repository, OTSystem system) {
return repository.getHeads()
.then(heads -> mergeAndPush(repository, system, heads))
.whenComplete(toLogger(logger, thisMethod()));
}
public static Promise mergeAndPush(OTRepository repository, OTSystem system, @NotNull Set heads) {
if (heads.size() == 1) return Promise.of(first(heads)); // nothing to merge
return merge(repository, system, heads)
.then(mergeCommit -> repository.push(mergeCommit)
.map($ -> mergeCommit.getId()))
.whenComplete(toLogger(logger, thisMethod()));
}
public static Promise mergeAndUpdateHeads(OTRepository repository, OTSystem system) {
return repository.getHeads()
.then(heads -> mergeAndUpdateHeads(repository, system, heads));
}
public static Promise mergeAndUpdateHeads(OTRepository repository, OTSystem system, Set heads) {
return mergeAndPush(repository, system, heads)
.then(mergeId -> repository.updateHeads(difference(singleton(mergeId), heads), difference(heads, singleton(mergeId)))
.map($ -> mergeId))
.whenComplete(toLogger(logger, thisMethod()));
}
public static @NotNull Promise> merge(OTRepository repository, OTSystem system, @NotNull Set heads) {
checkArgument(heads.size() >= 2, "Cannot merge less than 2 heads");
return repository.getLevels(heads)
.then(levels ->
reduce(repository, system, heads, new LoadGraphReducer<>(system))
.map(graph -> {
try {
Map> mergeResult = graph.merge(graph.excludeParents(heads));
if (logger.isTraceEnabled()) {
logger.info("{}\n", graph.toGraphViz());
}
return mergeResult;
} catch (OTException e) {
if (logger.isTraceEnabled()) {
logger.error("{}\n", graph.toGraphViz(), e);
}
throw e;
}
})
.then(mergeResult -> repository.createCommit(
keysToMap(heads.stream(), head -> new DiffsWithLevel<>(levels.get(head), mergeResult.get(head))))));
}
public static Promise> findCut(OTRepository repository, OTSystem system, Set startNodes,
Predicate>> matchPredicate) {
return reduce(repository, system, startNodes,
new GraphReducer>() {
private Collection> queue;
@Override
public void onStart(@NotNull Collection> queue) {
this.queue = queue;
}
@Override
public @NotNull Promise>> onCommit(@NotNull OTCommit commit) {
if (matchPredicate.test(queue)) {
return completePromise(queue.stream().map(OTCommit::getId).collect(toSet()));
}
return resumePromise();
}
});
}
public static Promise findAnyCommonParent(OTRepository repository, OTSystem system, Set startCut) {
return reduce(repository, system, startCut, new FindAnyCommonParentReducer<>(DiffsReducer.toVoid()))
.map(Map.Entry::getKey)
.whenComplete(toLogger(logger, thisMethod(), startCut));
}
public static Promise> findAllCommonParents(OTRepository repository, OTSystem system, Set startCut) {
return reduce(repository, system, startCut, new FindAllCommonParentsReducer<>(DiffsReducer.toVoid()))
.map(Map::keySet)
.whenComplete(toLogger(logger, thisMethod(), startCut));
}
public static Promise> diff(OTRepository repository, OTSystem system, K node1, K node2) {
Set startCut = setOf(node1, node2);
return reduce(repository, system, startCut, new FindAnyCommonParentReducer<>(DiffsReducer.toList()))
.map(entry -> {
List diffs1 = entry.getValue().get(node1);
List diffs2 = entry.getValue().get(node2);
return concat(diffs2, system.invert(diffs1));
})
.whenComplete(toLogger(logger, thisMethod(), startCut));
}
public static Promise> excludeParents(OTRepository repository, OTSystem system, Set startNodes) {
checkArgument(!startNodes.isEmpty(), "Start nodes are empty");
if (startNodes.size() == 1) return Promise.of(startNodes);
return reduce(repository, system, startNodes,
new GraphReducer>() {
long minLevel;
final Set nodes = new HashSet<>(startNodes);
@Override
public void onStart(@NotNull Collection> queue) {
//noinspection OptionalGetWithoutIsPresent
minLevel = queue.stream().mapToLong(OTCommit::getLevel).min().getAsLong();
}
@Override
public @NotNull Promise>> onCommit(@NotNull OTCommit commit) {
nodes.removeAll(commit.getParentIds());
if (commit.getLevel() <= minLevel) {
return completePromise(nodes);
}
return resumePromise();
}
})
.whenComplete(toLogger(logger, thisMethod(), startNodes));
}
private static final class FindAnyCommonParentReducer extends AbstractGraphReducer>> {
private FindAnyCommonParentReducer(DiffsReducer diffsReducer) {
super(diffsReducer);
}
@Override
protected @NotNull Promise>>> tryGetResult(OTCommit commit,
Map> accumulators, Map> headCommits) {
return Promise.of(accumulators.entrySet()
.stream()
.filter(entry -> Objects.equals(headCommits.keySet(), entry.getValue().keySet()))
.findAny()
);
}
}
private static final class FindAllCommonParentsReducer extends AbstractGraphReducer>> {
private FindAllCommonParentsReducer(DiffsReducer diffsReducer) {
super(diffsReducer);
}
@Override
protected @NotNull Promise>>> tryGetResult(OTCommit commit, Map> accumulators,
Map> headCommits) {
return Promise.of(
accumulators.values()
.stream()
.map(Map::keySet)
.allMatch(headCommits.keySet()::equals) ? Optional.of(accumulators) : Optional.empty()
);
}
}
public static Promise> reduceEdges(OTRepository repository, OTSystem system, Set heads, K parentNode,
DiffsReducer diffAccumulator) {
return reduce(repository, system, heads, new AbstractGraphReducer>(diffAccumulator) {
@Override
protected @NotNull Promise>> tryGetResult(OTCommit commit, Map> accumulators, Map> headCommits) {
if (accumulators.containsKey(parentNode)) {
Map toHeads = accumulators.get(parentNode);
if (Objects.equals(heads, toHeads.keySet())) {
return Promise.of(Optional.of(toHeads));
}
}
return Promise.of(Optional.empty());
}
});
}
public static Promise> checkout(OTRepository repository, OTSystem system) {
Ref> cachedSnapshotRef = new Ref<>();
return repository.getHeads()
.then(heads ->
findParent(repository, system, heads, DiffsReducer.toVoid(),
commit -> repository.loadSnapshot(commit.getId())
.map(maybeSnapshot -> (cachedSnapshotRef.value = maybeSnapshot.orElse(null)) != null))
.then(findResult -> Promise.of(cachedSnapshotRef.value)))
.whenComplete(toLogger(logger, thisMethod()));
}
public static Promise> checkout(OTRepository repository, OTSystem system, K commitId) {
Ref> cachedSnapshotRef = new Ref<>();
return repository.getHeads()
.then(heads ->
findParent(repository, system, union(heads, singleton(commitId)), DiffsReducer.toVoid(),
commit -> repository.loadSnapshot(commit.getId())
.map(maybeSnapshot -> (cachedSnapshotRef.value = maybeSnapshot.orElse(null)) != null))
.then(findResult -> diff(repository, system, findResult.commit, commitId)
.map(diff -> concat(cachedSnapshotRef.value, diff))))
.whenComplete(toLogger(logger, thisMethod(), commitId));
}
public static Promise saveSnapshot(OTRepository repository, OTSystem system, K revisionId) {
return checkout(repository, system, revisionId)
.then(diffs -> repository.saveSnapshot(revisionId, diffs));
}
private static class LoadGraphReducer implements GraphReducer> {
private final OTSystem system;
private final OTLoadedGraph graph;
private final Map> head2roots = new HashMap<>();
private final Map> root2heads = new HashMap<>();
private LoadGraphReducer(OTSystem system) {
this.system = system;
this.graph = new OTLoadedGraph<>(system);
}
@Override
public void onStart(@NotNull Collection> queue) {
for (OTCommit headCommit : queue) {
K head = headCommit.getId();
head2roots.put(head, new HashSet<>(listOf(head)));
root2heads.put(head, new HashSet<>(listOf(head)));
}
}
@Override
public @NotNull Promise>> onCommit(@NotNull OTCommit commit) {
K node = commit.getId();
Map> parents = commit.getParents();
Set affectedHeads = root2heads.remove(node);
for (K affectedHead : affectedHeads) {
head2roots.get(affectedHead).remove(node);
}
for (K parent : commit.isRoot() ? singleton(node) : parents.keySet()) {
Set parentRoots = graph.findRoots(parent);
for (K affectedHead : affectedHeads) {
head2roots.computeIfAbsent(affectedHead, $ -> new HashSet<>()).addAll(parentRoots);
}
for (K parentRoot : parentRoots) {
root2heads.computeIfAbsent(parentRoot, $ -> new HashSet<>()).addAll(affectedHeads);
}
}
graph.addNode(node, commit.getLevel(), parents);
for (Map.Entry> entry : nonNullElseEmpty(graph.getChildren(node)).entrySet()) {
K child = entry.getKey();
List childDiffs = entry.getValue();
Map> grandChildren = nonNullElseEmpty(graph.getChildren(child));
Map> coParents = nonNullElseEmpty(graph.getParents(child));
if (grandChildren.size() != 1 || coParents.size() != 1) continue;
K grandChild = first(grandChildren.keySet());
List grandChildDiffs = first(grandChildren.values());
graph.addEdge(node, grandChild, system.squash(concat(childDiffs, grandChildDiffs)));
graph.removeNode(child);
}
if (head2roots.keySet()
.stream()
.anyMatch(head -> head2roots.get(head).equals(root2heads.keySet()))) {
return completePromise(graph);
}
return resumePromise();
}
}
public static Promise> loadForMerge(OTRepository repository, OTSystem system, Set heads) {
return reduce(repository, system, heads, new LoadGraphReducer<>(system))
.whenComplete(toLogger(logger, thisMethod(), heads));
}
public static Promise> loadGraph(OTRepository repository, OTSystem system, Set heads, OTLoadedGraph graph) {
return reduce(repository, system, heads,
commit -> {
if (graph.hasChild(commit.getId())) {
return skipPromise();
}
graph.addNode(commit.getId(), commit.getLevel(), commit.getParents());
return resumePromise();
})
.then((v, e) -> {
if (e instanceof GraphExhaustedException) return Promise.of(null);
return Promise.of(v, e);
})
.map($ -> graph)
.whenComplete(toLogger(logger, thisMethod(), heads, graph));
}
public static Promise> loadGraph(OTRepository repository, OTSystem system, Set heads) {
return loadGraph(repository, system, heads, new OTLoadedGraph<>(system));
}
public static Promise> loadGraph(OTRepository repository, OTSystem system, Set heads, Function idToString, Function diffToString) {
return loadGraph(repository, system, heads, new OTLoadedGraph<>(system, idToString, diffToString));
}
public static Promise copy(OTRepository from, OTRepository to) {
return from.getHeads()
.then(heads -> toList(heads.stream().map(from::loadCommit))
.then(commits -> {
PriorityQueue> queue = new PriorityQueue<>(reverseOrder(comparingLong(OTCommit::getLevel)));
queue.addAll(commits);
return Promises.repeat(
() -> {
if (queue.isEmpty()) return Promise.of(false);
OTCommit commit = queue.poll();
return to.hasCommit(commit.getId())
.then(b -> b ?
Promise.complete() :
Promises.all(commit.getParents().keySet().stream()
.map(parentId -> from.loadCommit(parentId)
.whenResult(parent -> !queue.contains(parent), queue::add)))
.then(() -> to.push(commit)))
.map($ -> true);
});
})
.then(() -> to.updateHeads(heads, emptySet())));
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy