All Downloads are FREE. Search and download functionalities are using the official Maven repository.

io.activej.ot.OTAlgorithms Maven / Gradle / Ivy

Go to download

Implementation of operational transformation technology. Allows building collaborative software systems.

There is a newer version: 6.0-rc2
Show newest version
/*
 * Copyright (C) 2020 ActiveJ LLC.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package io.activej.ot;

import io.activej.async.function.AsyncPredicate;
import io.activej.common.ref.Ref;
import io.activej.ot.OTCommitFactory.DiffsWithLevel;
import io.activej.ot.exception.GraphExhaustedException;
import io.activej.ot.exception.OTException;
import io.activej.ot.reducers.AbstractGraphReducer;
import io.activej.ot.reducers.DiffsReducer;
import io.activej.ot.reducers.GraphReducer;
import io.activej.ot.repository.OTRepository;
import io.activej.ot.system.OTSystem;
import io.activej.promise.Promise;
import io.activej.promise.Promises;
import io.activej.promise.SettablePromise;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;
import java.util.function.Function;
import java.util.function.Predicate;

import static io.activej.async.util.LogUtils.thisMethod;
import static io.activej.async.util.LogUtils.toLogger;
import static io.activej.common.Checks.checkArgument;
import static io.activej.common.Utils.*;
import static io.activej.ot.reducers.GraphReducer.Result.*;
import static io.activej.promise.Promises.toList;
import static java.util.Collections.*;
import static java.util.Comparator.comparingLong;
import static java.util.stream.Collectors.toSet;

public final class OTAlgorithms {
	private static final Logger logger = LoggerFactory.getLogger(OTAlgorithms.class);

	public static  Promise reduce(OTRepository repository, OTSystem system,
			Set heads, GraphReducer reducer) {
		return toList(heads.stream().map(repository::loadCommit))
				.then(headCommits -> {
					PriorityQueue> queue = new PriorityQueue<>(reverseOrder(comparingLong(OTCommit::getLevel)));
					queue.addAll(headCommits);
					reducer.onStart(unmodifiableCollection(queue));
					return Promise.ofCallback(cb -> walkGraphImpl(repository, reducer, queue, new HashSet<>(heads), cb));
				});
	}

	private static  void walkGraphImpl(OTRepository repository, GraphReducer reducer,
			PriorityQueue> queue, Set visited, SettablePromise cb) {
		OTCommit commit = queue.peek();
		if (commit == null) {
			cb.setException(new GraphExhaustedException());
			return;
		}
		reducer.onCommit(commit)
				.run((maybeResult, e) -> {
					if (e != null) {
						cb.setException(e);
						return;
					}
					OTCommit polledCommit = queue.poll();
					assert polledCommit == commit;
					if (maybeResult.isResume()) {
						toList(commit.getParents().keySet().stream().filter(visited::add).map(repository::loadCommit))
								.async()
								.run((parentCommits, e2) -> {
									if (e2 == null) {
										queue.addAll(parentCommits);
										walkGraphImpl(repository, reducer, queue, visited, cb);
									} else {
										cb.setException(e2);
									}
								});
					} else if (maybeResult.isSkip()) {
						walkGraphImpl(repository, reducer, queue, visited, cb);
					} else {
						cb.set(maybeResult.get());
					}
				});
	}

	public static final class FindResult {
		private final int epoch;
		private final @NotNull K commit;
		private final Set commitParents;
		private final long commitLevel;
		private final K child;
		private final long childLevel;
		private final A accumulatedDiffs;

		private FindResult(int epoch, @NotNull K commit, Set commitParents, long commitLevel, K child, long childLevel, A accumulatedDiffs) {
			this.epoch = epoch;
			this.commit = commit;
			this.commitParents = commitParents;
			this.commitLevel = commitLevel;
			this.child = child;
			this.childLevel = childLevel;
			this.accumulatedDiffs = accumulatedDiffs;
		}

		public int getEpoch() {
			return epoch;
		}

		public @NotNull K getCommit() {
			return commit;
		}

		public K getChild() {
			return child;
		}

		public Long getChildLevel() {
			return childLevel;
		}

		public Set getCommitParents() {
			return commitParents;
		}

		public long getCommitLevel() {
			return commitLevel;
		}

		public A getAccumulatedDiffs() {
			return accumulatedDiffs;
		}

		@Override
		public String toString() {
			return "FindResult{" +
					"commit=" + commit +
					", parents=" + commitParents +
					", child=" + child +
					", accumulator=" + accumulatedDiffs +
					'}';
		}
	}

	public static  Promise> findParent(OTRepository repository, OTSystem system,
			Set startNodes, DiffsReducer diffsReducer, AsyncPredicate> matchPredicate) {
		return reduce(repository, system, startNodes,
				new AbstractGraphReducer>(diffsReducer) {
					int epoch;

					@Override
					public void onStart(@NotNull Collection> queue) {
						this.epoch = queue.iterator().next().getEpoch();
						super.onStart(queue);
					}

					@Override
					protected @NotNull Promise>> tryGetResult(OTCommit commit,
							Map> accumulators, Map> headCommits) {
						return matchPredicate.test(commit)
								.mapIfElse(matched -> matched,
										$ -> {
											Map.Entry someHead = accumulators.get(commit.getId()).entrySet().iterator().next();
											return Optional.of(new FindResult<>(
													epoch, commit.getId(), commit.getParentIds(), commit.getLevel(),
													someHead.getKey(), headCommits.get(someHead.getKey()).getLevel(),
													someHead.getValue()
											));
										},
										$ -> Optional.empty());
					}
				});
	}

	public static  Promise mergeAndPush(OTRepository repository, OTSystem system) {
		return repository.getHeads()
				.then(heads -> mergeAndPush(repository, system, heads))
				.whenComplete(toLogger(logger, thisMethod()));
	}

	public static  Promise mergeAndPush(OTRepository repository, OTSystem system, @NotNull Set heads) {
		if (heads.size() == 1) return Promise.of(first(heads)); // nothing to merge
		return merge(repository, system, heads)
				.then(mergeCommit -> repository.push(mergeCommit)
						.map($ -> mergeCommit.getId()))
				.whenComplete(toLogger(logger, thisMethod()));
	}

	public static  Promise mergeAndUpdateHeads(OTRepository repository, OTSystem system) {
		return repository.getHeads()
				.then(heads -> mergeAndUpdateHeads(repository, system, heads));
	}

	public static  Promise mergeAndUpdateHeads(OTRepository repository, OTSystem system, Set heads) {
		return mergeAndPush(repository, system, heads)
				.then(mergeId -> repository.updateHeads(difference(singleton(mergeId), heads), difference(heads, singleton(mergeId)))
						.map($ -> mergeId))
				.whenComplete(toLogger(logger, thisMethod()));
	}

	public static  @NotNull Promise> merge(OTRepository repository, OTSystem system, @NotNull Set heads) {
		checkArgument(heads.size() >= 2, "Cannot merge less than 2 heads");
		return repository.getLevels(heads)
				.then(levels ->
						reduce(repository, system, heads, new LoadGraphReducer<>(system))
								.map(graph -> {
									try {
										Map> mergeResult = graph.merge(graph.excludeParents(heads));
										if (logger.isTraceEnabled()) {
											logger.info("{}\n", graph.toGraphViz());
										}
										return mergeResult;
									} catch (OTException e) {
										if (logger.isTraceEnabled()) {
											logger.error("{}\n", graph.toGraphViz(), e);
										}
										throw e;
									}
								})
								.then(mergeResult -> repository.createCommit(
										keysToMap(heads.stream(), head -> new DiffsWithLevel<>(levels.get(head), mergeResult.get(head))))));
	}

	public static  Promise> findCut(OTRepository repository, OTSystem system, Set startNodes,
			Predicate>> matchPredicate) {
		return reduce(repository, system, startNodes,
				new GraphReducer>() {
					private Collection> queue;

					@Override
					public void onStart(@NotNull Collection> queue) {
						this.queue = queue;
					}

					@Override
					public @NotNull Promise>> onCommit(@NotNull OTCommit commit) {
						if (matchPredicate.test(queue)) {
							return completePromise(queue.stream().map(OTCommit::getId).collect(toSet()));
						}
						return resumePromise();
					}
				});
	}

	public static  Promise findAnyCommonParent(OTRepository repository, OTSystem system, Set startCut) {
		return reduce(repository, system, startCut, new FindAnyCommonParentReducer<>(DiffsReducer.toVoid()))
				.map(Map.Entry::getKey)
				.whenComplete(toLogger(logger, thisMethod(), startCut));
	}

	public static  Promise> findAllCommonParents(OTRepository repository, OTSystem system, Set startCut) {
		return reduce(repository, system, startCut, new FindAllCommonParentsReducer<>(DiffsReducer.toVoid()))
				.map(Map::keySet)
				.whenComplete(toLogger(logger, thisMethod(), startCut));
	}

	public static  Promise> diff(OTRepository repository, OTSystem system, K node1, K node2) {
		Set startCut = setOf(node1, node2);
		return reduce(repository, system, startCut, new FindAnyCommonParentReducer<>(DiffsReducer.toList()))
				.map(entry -> {
					List diffs1 = entry.getValue().get(node1);
					List diffs2 = entry.getValue().get(node2);
					return concat(diffs2, system.invert(diffs1));
				})
				.whenComplete(toLogger(logger, thisMethod(), startCut));
	}

	public static  Promise> excludeParents(OTRepository repository, OTSystem system, Set startNodes) {
		checkArgument(!startNodes.isEmpty(), "Start nodes are empty");
		if (startNodes.size() == 1) return Promise.of(startNodes);
		return reduce(repository, system, startNodes,
				new GraphReducer>() {
					long minLevel;
					final Set nodes = new HashSet<>(startNodes);

					@Override
					public void onStart(@NotNull Collection> queue) {
						//noinspection OptionalGetWithoutIsPresent
						minLevel = queue.stream().mapToLong(OTCommit::getLevel).min().getAsLong();
					}

					@Override
					public @NotNull Promise>> onCommit(@NotNull OTCommit commit) {
						nodes.removeAll(commit.getParentIds());
						if (commit.getLevel() <= minLevel) {
							return completePromise(nodes);
						}
						return resumePromise();
					}
				})
				.whenComplete(toLogger(logger, thisMethod(), startNodes));
	}

	private static final class FindAnyCommonParentReducer extends AbstractGraphReducer>> {
		private FindAnyCommonParentReducer(DiffsReducer diffsReducer) {
			super(diffsReducer);
		}

		@Override
		protected @NotNull Promise>>> tryGetResult(OTCommit commit,
				Map> accumulators, Map> headCommits) {
			return Promise.of(accumulators.entrySet()
					.stream()
					.filter(entry -> Objects.equals(headCommits.keySet(), entry.getValue().keySet()))
					.findAny()
			);
		}
	}

	private static final class FindAllCommonParentsReducer extends AbstractGraphReducer>> {
		private FindAllCommonParentsReducer(DiffsReducer diffsReducer) {
			super(diffsReducer);
		}

		@Override
		protected @NotNull Promise>>> tryGetResult(OTCommit commit, Map> accumulators,
				Map> headCommits) {
			return Promise.of(
					accumulators.values()
							.stream()
							.map(Map::keySet)
							.allMatch(headCommits.keySet()::equals) ? Optional.of(accumulators) : Optional.empty()
			);
		}
	}

	public static  Promise> reduceEdges(OTRepository repository, OTSystem system, Set heads, K parentNode,
			DiffsReducer diffAccumulator) {
		return reduce(repository, system, heads, new AbstractGraphReducer>(diffAccumulator) {
			@Override
			protected @NotNull Promise>> tryGetResult(OTCommit commit, Map> accumulators, Map> headCommits) {
				if (accumulators.containsKey(parentNode)) {
					Map toHeads = accumulators.get(parentNode);
					if (Objects.equals(heads, toHeads.keySet())) {
						return Promise.of(Optional.of(toHeads));
					}
				}
				return Promise.of(Optional.empty());
			}
		});
	}

	public static  Promise> checkout(OTRepository repository, OTSystem system) {
		Ref> cachedSnapshotRef = new Ref<>();
		return repository.getHeads()
				.then(heads ->
						findParent(repository, system, heads, DiffsReducer.toVoid(),
								commit -> repository.loadSnapshot(commit.getId())
										.map(maybeSnapshot -> (cachedSnapshotRef.value = maybeSnapshot.orElse(null)) != null))
								.then(findResult -> Promise.of(cachedSnapshotRef.value)))
				.whenComplete(toLogger(logger, thisMethod()));
	}

	public static  Promise> checkout(OTRepository repository, OTSystem system, K commitId) {
		Ref> cachedSnapshotRef = new Ref<>();
		return repository.getHeads()
				.then(heads ->
						findParent(repository, system, union(heads, singleton(commitId)), DiffsReducer.toVoid(),
								commit -> repository.loadSnapshot(commit.getId())
										.map(maybeSnapshot -> (cachedSnapshotRef.value = maybeSnapshot.orElse(null)) != null))
								.then(findResult -> diff(repository, system, findResult.commit, commitId)
										.map(diff -> concat(cachedSnapshotRef.value, diff))))
				.whenComplete(toLogger(logger, thisMethod(), commitId));
	}

	public static  Promise saveSnapshot(OTRepository repository, OTSystem system, K revisionId) {
		return checkout(repository, system, revisionId)
				.then(diffs -> repository.saveSnapshot(revisionId, diffs));
	}

	private static class LoadGraphReducer implements GraphReducer> {
		private final OTSystem system;
		private final OTLoadedGraph graph;
		private final Map> head2roots = new HashMap<>();
		private final Map> root2heads = new HashMap<>();

		private LoadGraphReducer(OTSystem system) {
			this.system = system;
			this.graph = new OTLoadedGraph<>(system);
		}

		@Override
		public void onStart(@NotNull Collection> queue) {
			for (OTCommit headCommit : queue) {
				K head = headCommit.getId();
				head2roots.put(head, new HashSet<>(listOf(head)));
				root2heads.put(head, new HashSet<>(listOf(head)));
			}
		}

		@Override
		public @NotNull Promise>> onCommit(@NotNull OTCommit commit) {
			K node = commit.getId();
			Map> parents = commit.getParents();

			Set affectedHeads = root2heads.remove(node);
			for (K affectedHead : affectedHeads) {
				head2roots.get(affectedHead).remove(node);
			}
			for (K parent : commit.isRoot() ? singleton(node) : parents.keySet()) {
				Set parentRoots = graph.findRoots(parent);
				for (K affectedHead : affectedHeads) {
					head2roots.computeIfAbsent(affectedHead, $ -> new HashSet<>()).addAll(parentRoots);
				}
				for (K parentRoot : parentRoots) {
					root2heads.computeIfAbsent(parentRoot, $ -> new HashSet<>()).addAll(affectedHeads);
				}
			}

			graph.addNode(node, commit.getLevel(), parents);
			for (Map.Entry> entry : nonNullElseEmpty(graph.getChildren(node)).entrySet()) {
				K child = entry.getKey();
				List childDiffs = entry.getValue();
				Map> grandChildren = nonNullElseEmpty(graph.getChildren(child));
				Map> coParents = nonNullElseEmpty(graph.getParents(child));
				if (grandChildren.size() != 1 || coParents.size() != 1) continue;
				K grandChild = first(grandChildren.keySet());
				List grandChildDiffs = first(grandChildren.values());
				graph.addEdge(node, grandChild, system.squash(concat(childDiffs, grandChildDiffs)));
				graph.removeNode(child);
			}

			if (head2roots.keySet()
					.stream()
					.anyMatch(head -> head2roots.get(head).equals(root2heads.keySet()))) {
				return completePromise(graph);
			}
			return resumePromise();
		}
	}

	public static  Promise> loadForMerge(OTRepository repository, OTSystem system, Set heads) {
		return reduce(repository, system, heads, new LoadGraphReducer<>(system))
				.whenComplete(toLogger(logger, thisMethod(), heads));
	}

	public static  Promise> loadGraph(OTRepository repository, OTSystem system, Set heads, OTLoadedGraph graph) {
		return reduce(repository, system, heads,
				commit -> {
					if (graph.hasChild(commit.getId())) {
						return skipPromise();
					}
					graph.addNode(commit.getId(), commit.getLevel(), commit.getParents());
					return resumePromise();
				})
				.then((v, e) -> {
					if (e instanceof GraphExhaustedException) return Promise.of(null);
					return Promise.of(v, e);
				})
				.map($ -> graph)
				.whenComplete(toLogger(logger, thisMethod(), heads, graph));
	}

	public static  Promise> loadGraph(OTRepository repository, OTSystem system, Set heads) {
		return loadGraph(repository, system, heads, new OTLoadedGraph<>(system));
	}

	public static  Promise> loadGraph(OTRepository repository, OTSystem system, Set heads, Function idToString, Function diffToString) {
		return loadGraph(repository, system, heads, new OTLoadedGraph<>(system, idToString, diffToString));
	}

	public static  Promise copy(OTRepository from, OTRepository to) {
		return from.getHeads()
				.then(heads -> toList(heads.stream().map(from::loadCommit))
						.then(commits -> {
							PriorityQueue> queue = new PriorityQueue<>(reverseOrder(comparingLong(OTCommit::getLevel)));
							queue.addAll(commits);
							return Promises.repeat(
									() -> {
										if (queue.isEmpty()) return Promise.of(false);
										OTCommit commit = queue.poll();
										return to.hasCommit(commit.getId())
												.thenIfElse(hasCommit -> hasCommit,
														$ -> Promise.complete(),
														$ -> Promises.all(commit.getParents().keySet().stream()
																		.map(parentId -> from.loadCommit(parentId)
																				.whenResult(parent -> !queue.contains(parent), queue::add)))
																.then(() -> to.push(commit)))
												.map($ -> true);
									});

						})
						.then(() -> to.updateHeads(heads, emptySet())));
	}

}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy