org.elasticsearch.cluster.DiffableUtils Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of elasticsearch Show documentation
Show all versions of elasticsearch Show documentation
Elasticsearch subproject :server
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.cluster;
import org.elasticsearch.Version;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable.Reader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
public final class DiffableUtils {
private DiffableUtils() {}
private static final MapDiff EMPTY = new MapDiff<>(null, null, List.of(), List.of(), List.of()) {
@Override
public Object apply(Object part) {
return part;
}
};
/**
* Returns a map key serializer for String keys
*/
public static KeySerializer getStringKeySerializer() {
return StringKeySerializer.INSTANCE;
}
/**
* Returns a map key serializer for Integer keys. Encodes as Int.
*/
public static KeySerializer getIntKeySerializer() {
return IntKeySerializer.INSTANCE;
}
/**
* Returns a map key serializer for Integer keys. Encodes as VInt.
*/
public static KeySerializer getVIntKeySerializer() {
return VIntKeySerializer.INSTANCE;
}
/**
* Calculates diff between two ImmutableOpenMaps of Diffable objects
*/
public static > MapDiff> diff(
ImmutableOpenMap before,
ImmutableOpenMap after,
KeySerializer keySerializer
) {
assert after != null && before != null;
return before.equals(after)
? emptyDiff()
: ImmutableOpenMapDiff.create(before, after, keySerializer, DiffableValueSerializer.getWriteOnlyInstance());
}
/**
* Calculates diff between two ImmutableOpenMaps of non-diffable objects
*/
public static MapDiff> diff(
ImmutableOpenMap before,
ImmutableOpenMap after,
KeySerializer keySerializer,
ValueSerializer valueSerializer
) {
assert after != null && before != null;
return before.equals(after) ? emptyDiff() : ImmutableOpenMapDiff.create(before, after, keySerializer, valueSerializer);
}
/**
* Calculates diff between two Maps of Diffable objects.
*/
public static > MapDiff> diff(
Map before,
Map after,
KeySerializer keySerializer
) {
assert after != null && before != null;
return before.equals(after)
? emptyDiff()
: JdkMapDiff.create(before, after, keySerializer, DiffableValueSerializer.getWriteOnlyInstance());
}
/**
* Calculates diff between two Maps of non-diffable objects
*/
public static MapDiff> diff(
Map before,
Map after,
KeySerializer keySerializer,
ValueSerializer valueSerializer
) {
assert after != null && before != null;
return before.equals(after) ? emptyDiff() : JdkMapDiff.create(before, after, keySerializer, valueSerializer);
}
@SuppressWarnings("unchecked")
private static MapDiff emptyDiff() {
return (MapDiff) EMPTY;
}
/**
* Loads an object that represents difference between two ImmutableOpenMaps
*/
public static MapDiff> readImmutableOpenMapDiff(
StreamInput in,
KeySerializer keySerializer,
ValueSerializer valueSerializer
) throws IOException {
return diffOrEmpty(new ImmutableOpenMapDiff<>(in, keySerializer, valueSerializer));
}
/**
* Loads an object that represents difference between two Maps of Diffable objects
*/
public static MapDiff> readJdkMapDiff(
StreamInput in,
KeySerializer keySerializer,
ValueSerializer valueSerializer
) throws IOException {
return diffOrEmpty(new JdkMapDiff<>(in, keySerializer, valueSerializer));
}
/**
* Loads an object that represents difference between two ImmutableOpenMaps of Diffable objects using Diffable proto object
*/
public static > MapDiff> readImmutableOpenMapDiff(
StreamInput in,
KeySerializer keySerializer,
DiffableValueReader diffableValueReader
) throws IOException {
return diffOrEmpty(new ImmutableOpenMapDiff<>(in, keySerializer, diffableValueReader));
}
/**
* Loads an object that represents difference between two Maps of Diffable objects using Diffable proto object
*/
public static > MapDiff> readJdkMapDiff(
StreamInput in,
KeySerializer keySerializer,
Reader reader,
Reader> diffReader
) throws IOException {
return diffOrEmpty(new JdkMapDiff<>(in, keySerializer, new DiffableValueReader<>(reader, diffReader)));
}
private static MapDiff diffOrEmpty(MapDiff diff) {
// TODO: refactor map diff reading to avoid having to construct empty diffs before throwing them away here
if (diff.getUpserts().isEmpty() && diff.getDiffs().isEmpty() && diff.getDeletes().isEmpty()) {
return emptyDiff();
}
return diff;
}
/**
* Represents differences between two Maps of (possibly diffable) objects.
*
* @param the diffable object
*/
private static class JdkMapDiff extends MapDiff> {
private JdkMapDiff(
KeySerializer keySerializer,
ValueSerializer valueSerializer,
List deletes,
List>> diffs,
List> upserts
) {
super(keySerializer, valueSerializer, deletes, diffs, upserts);
}
private JdkMapDiff(StreamInput in, KeySerializer keySerializer, ValueSerializer valueSerializer) throws IOException {
super(in, keySerializer, valueSerializer);
}
private static JdkMapDiff create(
Map before,
Map after,
KeySerializer keySerializer,
ValueSerializer valueSerializer
) {
assert after != null && before != null;
int inserts = 0;
var upserts = new ArrayList>();
var diffs = new ArrayList>>();
for (Map.Entry entry : after.entrySet()) {
T previousValue = before.get(entry.getKey());
if (previousValue == null) {
upserts.add(entry);
inserts++;
} else if (entry.getValue().equals(previousValue) == false) {
if (valueSerializer.supportsDiffableValues()) {
diffs.add(Map.entry(entry.getKey(), valueSerializer.diff(entry.getValue(), previousValue)));
} else {
upserts.add(entry);
}
}
}
int expectedDeletes = before.size() + inserts - after.size();
var deletes = new ArrayList(expectedDeletes);
if (expectedDeletes > 0) {
for (K key : before.keySet()) {
if (after.containsKey(key) == false) {
deletes.add(key);
if (--expectedDeletes == 0) {
break;
}
}
}
}
return new JdkMapDiff<>(keySerializer, valueSerializer, deletes, diffs, upserts);
}
@Override
public Map apply(Map map) {
Map builder = new HashMap<>(map);
for (K part : deletes) {
builder.remove(part);
}
for (Map.Entry> diff : diffs) {
builder.put(diff.getKey(), diff.getValue().apply(builder.get(diff.getKey())));
}
for (Map.Entry upsert : upserts) {
builder.put(upsert.getKey(), upsert.getValue());
}
return builder;
}
}
/**
* Represents differences between two ImmutableOpenMap of (possibly diffable) objects
*
* @param the object type
*/
public static class ImmutableOpenMapDiff extends MapDiff> {
private ImmutableOpenMapDiff(
KeySerializer keySerializer,
ValueSerializer valueSerializer,
List deletes,
List>> diffs,
List> upserts
) {
super(keySerializer, valueSerializer, deletes, diffs, upserts);
}
private ImmutableOpenMapDiff(StreamInput in, KeySerializer keySerializer, ValueSerializer valueSerializer)
throws IOException {
super(in, keySerializer, valueSerializer);
}
private static ImmutableOpenMapDiff create(
ImmutableOpenMap before,
ImmutableOpenMap after,
KeySerializer keySerializer,
ValueSerializer valueSerializer
) {
assert after != null && before != null;
int inserts = 0;
var upserts = new ArrayList>();
var diffs = new ArrayList>>();
for (Map.Entry entry : after.entrySet()) {
T beforeValue = before.get(entry.getKey());
if (beforeValue == null) {
upserts.add(entry);
inserts++;
} else if (entry.getValue().equals(beforeValue) == false) {
if (valueSerializer.supportsDiffableValues()) {
diffs.add(Map.entry(entry.getKey(), valueSerializer.diff(entry.getValue(), beforeValue)));
} else {
upserts.add(entry);
}
}
}
int expectedDeletes = before.size() + inserts - after.size();
var deletes = new ArrayList(expectedDeletes);
if (expectedDeletes > 0) {
for (Map.Entry key : before.entrySet()) {
if (after.containsKey(key.getKey()) == false) {
deletes.add(key.getKey());
if (--expectedDeletes == 0) {
break;
}
}
}
}
return new ImmutableOpenMapDiff<>(keySerializer, valueSerializer, deletes, diffs, upserts);
}
@Override
public ImmutableOpenMap apply(ImmutableOpenMap map) {
ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(map);
for (K part : deletes) {
builder.remove(part);
}
for (Map.Entry> diff : diffs) {
builder.put(diff.getKey(), diff.getValue().apply(builder.get(diff.getKey())));
}
for (Map.Entry upsert : upserts) {
builder.put(upsert.getKey(), upsert.getValue());
}
return builder.build();
}
}
/**
* Represents differences between two maps of objects and is used as base class for different map implementations.
*
* Implements serialization. How differences are applied is left to subclasses.
*
* @param the type of map keys
* @param the type of map values
* @param the map implementation type
*/
public abstract static class MapDiff implements Diff {
protected final List deletes;
protected final List>> diffs; // incremental updates
protected final List> upserts; // additions or full updates
protected final KeySerializer keySerializer;
protected final ValueSerializer valueSerializer;
protected MapDiff(
KeySerializer keySerializer,
ValueSerializer valueSerializer,
List deletes,
List>> diffs,
List> upserts
) {
this.keySerializer = keySerializer;
this.valueSerializer = valueSerializer;
this.deletes = deletes;
this.diffs = diffs;
this.upserts = upserts;
}
protected MapDiff(StreamInput in, KeySerializer keySerializer, ValueSerializer valueSerializer) throws IOException {
this.keySerializer = keySerializer;
this.valueSerializer = valueSerializer;
deletes = in.readList(keySerializer::readKey);
int diffsCount = in.readVInt();
diffs = diffsCount == 0 ? List.of() : new ArrayList<>(diffsCount);
for (int i = 0; i < diffsCount; i++) {
K key = keySerializer.readKey(in);
Diff diff = valueSerializer.readDiff(in, key);
diffs.add(Map.entry(key, diff));
}
int upsertsCount = in.readVInt();
upserts = upsertsCount == 0 ? List.of() : new ArrayList<>(upsertsCount);
for (int i = 0; i < upsertsCount; i++) {
K key = keySerializer.readKey(in);
T newValue = valueSerializer.read(in, key);
upserts.add(Map.entry(key, newValue));
}
}
/**
* The keys that, when this diff is applied to a map, should be removed from the map.
*
* @return the list of keys that are deleted
*/
public List getDeletes() {
return deletes;
}
/**
* Map entries that, when this diff is applied to a map, should be
* incrementally updated. The incremental update is represented using
* the {@link Diff} interface.
*
* @return the map entries that are incrementally updated
*/
public List>> getDiffs() {
return diffs;
}
/**
* Map entries that, when this diff is applied to a map, should be
* added to the map or fully replace the previous value.
*
* @return the map entries that are additions or full updates
*/
public List> getUpserts() {
return upserts;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeCollection(deletes, (o, v) -> keySerializer.writeKey(v, o));
Version version = out.getVersion();
// filter out custom states not supported by the other node
int diffCount = 0;
for (Map.Entry> diff : diffs) {
if (valueSerializer.supportsVersion(diff.getValue(), version)) {
diffCount++;
}
}
out.writeVInt(diffCount);
for (Map.Entry> entry : diffs) {
if (valueSerializer.supportsVersion(entry.getValue(), version)) {
keySerializer.writeKey(entry.getKey(), out);
valueSerializer.writeDiff(entry.getValue(), out);
}
}
// filter out custom states not supported by the other node
int upsertsCount = 0;
for (Map.Entry upsert : upserts) {
if (valueSerializer.supportsVersion(upsert.getValue(), version)) {
upsertsCount++;
}
}
out.writeVInt(upsertsCount);
for (Map.Entry entry : upserts) {
if (valueSerializer.supportsVersion(entry.getValue(), version)) {
keySerializer.writeKey(entry.getKey(), out);
valueSerializer.write(entry.getValue(), out);
}
}
}
}
/**
* Provides read and write operations to serialize keys of map
* @param type of key
*/
public interface KeySerializer {
void writeKey(K key, StreamOutput out) throws IOException;
K readKey(StreamInput in) throws IOException;
}
/**
* Serializes String keys of a map
*/
private static final class StringKeySerializer implements KeySerializer {
private static final StringKeySerializer INSTANCE = new StringKeySerializer();
@Override
public void writeKey(String key, StreamOutput out) throws IOException {
out.writeString(key);
}
@Override
public String readKey(StreamInput in) throws IOException {
return in.readString();
}
}
/**
* Serializes Integer keys of a map as an Int
*/
private static final class IntKeySerializer implements KeySerializer {
public static final IntKeySerializer INSTANCE = new IntKeySerializer();
@Override
public void writeKey(Integer key, StreamOutput out) throws IOException {
out.writeInt(key);
}
@Override
public Integer readKey(StreamInput in) throws IOException {
return in.readInt();
}
}
/**
* Serializes Integer keys of a map as a VInt. Requires keys to be positive.
*/
private static final class VIntKeySerializer implements KeySerializer {
public static final IntKeySerializer INSTANCE = new IntKeySerializer();
@Override
public void writeKey(Integer key, StreamOutput out) throws IOException {
if (key < 0) {
throw new IllegalArgumentException("Map key [" + key + "] must be positive");
}
out.writeVInt(key);
}
@Override
public Integer readKey(StreamInput in) throws IOException {
return in.readVInt();
}
}
/**
* Provides read and write operations to serialize map values.
* Reading of values can be made dependent on map key.
*
* Also provides operations to distinguish whether map values are diffable.
*
* Should not be directly implemented, instead implement either
* {@link DiffableValueSerializer} or {@link NonDiffableValueSerializer}.
*
* @param key type of map
* @param value type of map
*/
public interface ValueSerializer {
/**
* Writes value to stream
*/
void write(V value, StreamOutput out) throws IOException;
/**
* Reads value from stream. Reading operation can be made dependent on map key.
*/
V read(StreamInput in, K key) throws IOException;
/**
* Whether this serializer supports diffable values
*/
boolean supportsDiffableValues();
/**
* Whether this serializer supports the version of the output stream
*/
default boolean supportsVersion(Diff value, Version version) {
return true;
}
/**
* Whether this serializer supports the version of the output stream
*/
default boolean supportsVersion(V value, Version version) {
return true;
}
/**
* Computes diff if this serializer supports diffable values
*/
Diff diff(V value, V beforePart);
/**
* Writes value as diff to stream if this serializer supports diffable values
*/
void writeDiff(Diff value, StreamOutput out) throws IOException;
/**
* Reads value as diff from stream if this serializer supports diffable values.
* Reading operation can be made dependent on map key.
*/
Diff readDiff(StreamInput in, K key) throws IOException;
}
/**
* Serializer for Diffable map values. Needs to implement read and readDiff methods.
*
* @param type of map keys
* @param type of map values
*/
public abstract static class DiffableValueSerializer> implements ValueSerializer {
@SuppressWarnings("rawtypes")
private static final DiffableValueSerializer WRITE_ONLY_INSTANCE = new DiffableValueSerializer() {
@Override
public Object read(StreamInput in, Object key) {
throw new UnsupportedOperationException();
}
@Override
public Diff