org.elasticsearch.cluster.DiffableUtils Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of elasticsearch Show documentation
Show all versions of elasticsearch Show documentation
Elasticsearch subproject :server
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.cluster;
import com.carrotsearch.hppc.cursors.IntCursor;
import com.carrotsearch.hppc.cursors.IntObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.elasticsearch.Version;
import org.elasticsearch.common.collect.ImmutableOpenIntMap;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable.Reader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
public final class DiffableUtils {
private DiffableUtils() {
}
/**
* Returns a map key serializer for String keys
*/
public static KeySerializer getStringKeySerializer() {
return StringKeySerializer.INSTANCE;
}
/**
* Returns a map key serializer for Integer keys. Encodes as Int.
*/
public static KeySerializer getIntKeySerializer() {
return IntKeySerializer.INSTANCE;
}
/**
* Returns a map key serializer for Integer keys. Encodes as VInt.
*/
public static KeySerializer getVIntKeySerializer() {
return VIntKeySerializer.INSTANCE;
}
/**
* Calculates diff between two ImmutableOpenMaps of Diffable objects
*/
public static > MapDiff> diff(ImmutableOpenMap before,
ImmutableOpenMap after, KeySerializer keySerializer) {
assert after != null && before != null;
return new ImmutableOpenMapDiff<>(before, after, keySerializer, DiffableValueSerializer.getWriteOnlyInstance());
}
/**
* Calculates diff between two ImmutableOpenMaps of non-diffable objects
*/
public static MapDiff> diff(ImmutableOpenMap before,
ImmutableOpenMap after, KeySerializer keySerializer, ValueSerializer valueSerializer) {
assert after != null && before != null;
return new ImmutableOpenMapDiff<>(before, after, keySerializer, valueSerializer);
}
/**
* Calculates diff between two ImmutableOpenIntMaps of Diffable objects
*/
public static > MapDiff> diff(ImmutableOpenIntMap before,
ImmutableOpenIntMap after, KeySerializer keySerializer) {
assert after != null && before != null;
return new ImmutableOpenIntMapDiff<>(before, after, keySerializer, DiffableValueSerializer.getWriteOnlyInstance());
}
/**
* Calculates diff between two ImmutableOpenIntMaps of non-diffable objects
*/
public static MapDiff> diff(ImmutableOpenIntMap before,
ImmutableOpenIntMap after, KeySerializer keySerializer, ValueSerializer valueSerializer) {
assert after != null && before != null;
return new ImmutableOpenIntMapDiff<>(before, after, keySerializer, valueSerializer);
}
/**
* Calculates diff between two Maps of Diffable objects.
*/
public static > MapDiff> diff(Map before,
Map after, KeySerializer keySerializer) {
assert after != null && before != null;
return new JdkMapDiff<>(before, after, keySerializer, DiffableValueSerializer.getWriteOnlyInstance());
}
/**
* Calculates diff between two Maps of non-diffable objects
*/
public static MapDiff> diff(Map before, Map after, KeySerializer keySerializer,
ValueSerializer valueSerializer) {
assert after != null && before != null;
return new JdkMapDiff<>(before, after, keySerializer, valueSerializer);
}
/**
* Loads an object that represents difference between two ImmutableOpenMaps
*/
public static MapDiff> readImmutableOpenMapDiff(StreamInput in, KeySerializer keySerializer,
ValueSerializer valueSerializer) throws IOException {
return new ImmutableOpenMapDiff<>(in, keySerializer, valueSerializer);
}
/**
* Loads an object that represents difference between two ImmutableOpenMaps
*/
public static MapDiff> readImmutableOpenIntMapDiff(StreamInput in,
KeySerializer keySerializer, ValueSerializer valueSerializer) throws IOException {
return new ImmutableOpenIntMapDiff<>(in, keySerializer, valueSerializer);
}
/**
* Loads an object that represents difference between two Maps of Diffable objects
*/
public static MapDiff> readJdkMapDiff(StreamInput in, KeySerializer keySerializer,
ValueSerializer valueSerializer) throws IOException {
return new JdkMapDiff<>(in, keySerializer, valueSerializer);
}
/**
* Loads an object that represents difference between two ImmutableOpenMaps of Diffable objects using Diffable proto object
*/
public static > MapDiff> readImmutableOpenMapDiff(StreamInput in,
KeySerializer keySerializer, Reader reader, Reader> diffReader) throws IOException {
return new ImmutableOpenMapDiff<>(in, keySerializer, new DiffableValueReader<>(reader, diffReader));
}
/**
* Loads an object that represents difference between two ImmutableOpenIntMaps of Diffable objects using Diffable proto object
*/
public static > MapDiff> readImmutableOpenIntMapDiff(StreamInput in,
KeySerializer keySerializer, Reader reader, Reader> diffReader) throws IOException {
return new ImmutableOpenIntMapDiff<>(in, keySerializer, new DiffableValueReader<>(reader, diffReader));
}
/**
* Loads an object that represents difference between two Maps of Diffable objects using Diffable proto object
*/
public static > MapDiff> readJdkMapDiff(StreamInput in, KeySerializer keySerializer,
Reader reader, Reader> diffReader) throws IOException {
return new JdkMapDiff<>(in, keySerializer, new DiffableValueReader<>(reader, diffReader));
}
/**
* Represents differences between two Maps of (possibly diffable) objects.
*
* @param the diffable object
*/
private static class JdkMapDiff extends MapDiff> {
protected JdkMapDiff(StreamInput in, KeySerializer keySerializer, ValueSerializer valueSerializer) throws IOException {
super(in, keySerializer, valueSerializer);
}
JdkMapDiff(Map before, Map after,
KeySerializer keySerializer, ValueSerializer valueSerializer) {
super(keySerializer, valueSerializer);
assert after != null && before != null;
for (K key : before.keySet()) {
if (!after.containsKey(key)) {
deletes.add(key);
}
}
for (Map.Entry partIter : after.entrySet()) {
T beforePart = before.get(partIter.getKey());
if (beforePart == null) {
upserts.put(partIter.getKey(), partIter.getValue());
} else if (partIter.getValue().equals(beforePart) == false) {
if (valueSerializer.supportsDiffableValues()) {
diffs.put(partIter.getKey(), valueSerializer.diff(partIter.getValue(), beforePart));
} else {
upserts.put(partIter.getKey(), partIter.getValue());
}
}
}
}
@Override
public Map apply(Map map) {
Map builder = new HashMap<>(map);
for (K part : deletes) {
builder.remove(part);
}
for (Map.Entry> diff : diffs.entrySet()) {
builder.put(diff.getKey(), diff.getValue().apply(builder.get(diff.getKey())));
}
for (Map.Entry upsert : upserts.entrySet()) {
builder.put(upsert.getKey(), upsert.getValue());
}
return builder;
}
}
/**
* Represents differences between two ImmutableOpenMap of (possibly diffable) objects
*
* @param the object type
*/
public static class ImmutableOpenMapDiff extends MapDiff> {
protected ImmutableOpenMapDiff(StreamInput in, KeySerializer keySerializer,
ValueSerializer valueSerializer) throws IOException {
super(in, keySerializer, valueSerializer);
}
private ImmutableOpenMapDiff(KeySerializer keySerializer, ValueSerializer valueSerializer,
List deletes, Map> diffs, Map upserts) {
super(keySerializer, valueSerializer, deletes, diffs, upserts);
}
public ImmutableOpenMapDiff(ImmutableOpenMap before, ImmutableOpenMap after,
KeySerializer keySerializer, ValueSerializer valueSerializer) {
super(keySerializer, valueSerializer);
assert after != null && before != null;
for (ObjectCursor key : before.keys()) {
if (!after.containsKey(key.value)) {
deletes.add(key.value);
}
}
for (ObjectObjectCursor partIter : after) {
T beforePart = before.get(partIter.key);
if (beforePart == null) {
upserts.put(partIter.key, partIter.value);
} else if (partIter.value.equals(beforePart) == false) {
if (valueSerializer.supportsDiffableValues()) {
diffs.put(partIter.key, valueSerializer.diff(partIter.value, beforePart));
} else {
upserts.put(partIter.key, partIter.value);
}
}
}
}
/**
* Returns a new diff map with the given key removed, does not modify the invoking instance.
* If the key does not exist in the diff map, the same instance is returned.
*/
public ImmutableOpenMapDiff withKeyRemoved(K key) {
if (this.diffs.containsKey(key) == false && this.upserts.containsKey(key) == false) {
return this;
}
Map> newDiffs = new HashMap<>(this.diffs);
newDiffs.remove(key);
Map newUpserts = new HashMap<>(this.upserts);
newUpserts.remove(key);
return new ImmutableOpenMapDiff<>(this.keySerializer, this.valueSerializer, this.deletes, newDiffs, newUpserts);
}
@Override
public ImmutableOpenMap apply(ImmutableOpenMap map) {
ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder();
builder.putAll(map);
for (K part : deletes) {
builder.remove(part);
}
for (Map.Entry> diff : diffs.entrySet()) {
builder.put(diff.getKey(), diff.getValue().apply(builder.get(diff.getKey())));
}
for (Map.Entry upsert : upserts.entrySet()) {
builder.put(upsert.getKey(), upsert.getValue());
}
return builder.build();
}
}
/**
* Represents differences between two ImmutableOpenIntMap of (possibly diffable) objects
*
* @param the object type
*/
private static class ImmutableOpenIntMapDiff extends MapDiff> {
protected ImmutableOpenIntMapDiff(StreamInput in, KeySerializer keySerializer,
ValueSerializer valueSerializer) throws IOException {
super(in, keySerializer, valueSerializer);
}
ImmutableOpenIntMapDiff(ImmutableOpenIntMap before, ImmutableOpenIntMap after,
KeySerializer keySerializer, ValueSerializer valueSerializer) {
super(keySerializer, valueSerializer);
assert after != null && before != null;
for (IntCursor key : before.keys()) {
if (!after.containsKey(key.value)) {
deletes.add(key.value);
}
}
for (IntObjectCursor partIter : after) {
T beforePart = before.get(partIter.key);
if (beforePart == null) {
upserts.put(partIter.key, partIter.value);
} else if (partIter.value.equals(beforePart) == false) {
if (valueSerializer.supportsDiffableValues()) {
diffs.put(partIter.key, valueSerializer.diff(partIter.value, beforePart));
} else {
upserts.put(partIter.key, partIter.value);
}
}
}
}
@Override
public ImmutableOpenIntMap apply(ImmutableOpenIntMap map) {
ImmutableOpenIntMap.Builder builder = ImmutableOpenIntMap.builder();
builder.putAll(map);
for (Integer part : deletes) {
builder.remove(part);
}
for (Map.Entry> diff : diffs.entrySet()) {
builder.put(diff.getKey(), diff.getValue().apply(builder.get(diff.getKey())));
}
for (Map.Entry upsert : upserts.entrySet()) {
builder.put(upsert.getKey(), upsert.getValue());
}
return builder.build();
}
}
/**
* Represents differences between two maps of objects and is used as base class for different map implementations.
*
* Implements serialization. How differences are applied is left to subclasses.
*
* @param the type of map keys
* @param the type of map values
* @param the map implementation type
*/
public abstract static class MapDiff implements Diff {
protected final List deletes;
protected final Map> diffs; // incremental updates
protected final Map upserts; // additions or full updates
protected final KeySerializer keySerializer;
protected final ValueSerializer valueSerializer;
protected MapDiff(KeySerializer keySerializer, ValueSerializer valueSerializer) {
this.keySerializer = keySerializer;
this.valueSerializer = valueSerializer;
deletes = new ArrayList<>();
diffs = new HashMap<>();
upserts = new HashMap<>();
}
protected MapDiff(KeySerializer keySerializer, ValueSerializer valueSerializer,
List deletes, Map> diffs, Map upserts) {
this.keySerializer = keySerializer;
this.valueSerializer = valueSerializer;
this.deletes = deletes;
this.diffs = diffs;
this.upserts = upserts;
}
protected MapDiff(StreamInput in, KeySerializer keySerializer, ValueSerializer valueSerializer) throws IOException {
this.keySerializer = keySerializer;
this.valueSerializer = valueSerializer;
deletes = new ArrayList<>();
diffs = new HashMap<>();
upserts = new HashMap<>();
int deletesCount = in.readVInt();
for (int i = 0; i < deletesCount; i++) {
deletes.add(keySerializer.readKey(in));
}
int diffsCount = in.readVInt();
for (int i = 0; i < diffsCount; i++) {
K key = keySerializer.readKey(in);
Diff diff = valueSerializer.readDiff(in, key);
diffs.put(key, diff);
}
int upsertsCount = in.readVInt();
for (int i = 0; i < upsertsCount; i++) {
K key = keySerializer.readKey(in);
T newValue = valueSerializer.read(in, key);
upserts.put(key, newValue);
}
}
/**
* The keys that, when this diff is applied to a map, should be removed from the map.
*
* @return the list of keys that are deleted
*/
public List getDeletes() {
return deletes;
}
/**
* Map entries that, when this diff is applied to a map, should be
* incrementally updated. The incremental update is represented using
* the {@link Diff} interface.
*
* @return the map entries that are incrementally updated
*/
public Map> getDiffs() {
return diffs;
}
/**
* Map entries that, when this diff is applied to a map, should be
* added to the map or fully replace the previous value.
*
* @return the map entries that are additions or full updates
*/
public Map getUpserts() {
return upserts;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(deletes.size());
for (K delete : deletes) {
keySerializer.writeKey(delete, out);
}
Version version = out.getVersion();
// filter out custom states not supported by the other node
int diffCount = 0;
for (Diff diff : diffs.values()) {
if(valueSerializer.supportsVersion(diff, version)) {
diffCount++;
}
}
out.writeVInt(diffCount);
for (Map.Entry> entry : diffs.entrySet()) {
if(valueSerializer.supportsVersion(entry.getValue(), version)) {
keySerializer.writeKey(entry.getKey(), out);
valueSerializer.writeDiff(entry.getValue(), out);
}
}
// filter out custom states not supported by the other node
int upsertsCount = 0;
for (T upsert : upserts.values()) {
if(valueSerializer.supportsVersion(upsert, version)) {
upsertsCount++;
}
}
out.writeVInt(upsertsCount);
for (Map.Entry entry : upserts.entrySet()) {
if(valueSerializer.supportsVersion(entry.getValue(), version)) {
keySerializer.writeKey(entry.getKey(), out);
valueSerializer.write(entry.getValue(), out);
}
}
}
}
/**
* Provides read and write operations to serialize keys of map
* @param type of key
*/
public interface KeySerializer {
void writeKey(K key, StreamOutput out) throws IOException;
K readKey(StreamInput in) throws IOException;
}
/**
* Serializes String keys of a map
*/
private static final class StringKeySerializer implements KeySerializer {
private static final StringKeySerializer INSTANCE = new StringKeySerializer();
@Override
public void writeKey(String key, StreamOutput out) throws IOException {
out.writeString(key);
}
@Override
public String readKey(StreamInput in) throws IOException {
return in.readString();
}
}
/**
* Serializes Integer keys of a map as an Int
*/
private static final class IntKeySerializer implements KeySerializer {
public static final IntKeySerializer INSTANCE = new IntKeySerializer();
@Override
public void writeKey(Integer key, StreamOutput out) throws IOException {
out.writeInt(key);
}
@Override
public Integer readKey(StreamInput in) throws IOException {
return in.readInt();
}
}
/**
* Serializes Integer keys of a map as a VInt. Requires keys to be positive.
*/
private static final class VIntKeySerializer implements KeySerializer {
public static final IntKeySerializer INSTANCE = new IntKeySerializer();
@Override
public void writeKey(Integer key, StreamOutput out) throws IOException {
if (key < 0) {
throw new IllegalArgumentException("Map key [" + key + "] must be positive");
}
out.writeVInt(key);
}
@Override
public Integer readKey(StreamInput in) throws IOException {
return in.readVInt();
}
}
/**
* Provides read and write operations to serialize map values.
* Reading of values can be made dependent on map key.
*
* Also provides operations to distinguish whether map values are diffable.
*
* Should not be directly implemented, instead implement either
* {@link DiffableValueSerializer} or {@link NonDiffableValueSerializer}.
*
* @param key type of map
* @param value type of map
*/
public interface ValueSerializer {
/**
* Writes value to stream
*/
void write(V value, StreamOutput out) throws IOException;
/**
* Reads value from stream. Reading operation can be made dependent on map key.
*/
V read(StreamInput in, K key) throws IOException;
/**
* Whether this serializer supports diffable values
*/
boolean supportsDiffableValues();
/**
* Whether this serializer supports the version of the output stream
*/
default boolean supportsVersion(Diff value, Version version) {
return true;
}
/**
* Whether this serializer supports the version of the output stream
*/
default boolean supportsVersion(V value, Version version) {
return true;
}
/**
* Computes diff if this serializer supports diffable values
*/
Diff diff(V value, V beforePart);
/**
* Writes value as diff to stream if this serializer supports diffable values
*/
void writeDiff(Diff value, StreamOutput out) throws IOException;
/**
* Reads value as diff from stream if this serializer supports diffable values.
* Reading operation can be made dependent on map key.
*/
Diff readDiff(StreamInput in, K key) throws IOException;
}
/**
* Serializer for Diffable map values. Needs to implement read and readDiff methods.
*
* @param type of map keys
* @param type of map values
*/
public abstract static class DiffableValueSerializer> implements ValueSerializer {
private static final DiffableValueSerializer WRITE_ONLY_INSTANCE = new DiffableValueSerializer() {
@Override
public Object read(StreamInput in, Object key) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public Diff