org.codelibs.elasticsearch.common.io.stream.StreamInput Maven / Gradle / Ivy
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.codelibs.elasticsearch.common.io.stream;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexFormatTooNewException;
import org.apache.lucene.index.IndexFormatTooOldException;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.codelibs.elasticsearch.ElasticsearchException;
import org.codelibs.elasticsearch.Version;
import org.codelibs.elasticsearch.common.Nullable;
import org.codelibs.elasticsearch.common.Strings;
import org.codelibs.elasticsearch.common.bytes.BytesArray;
import org.codelibs.elasticsearch.common.bytes.BytesReference;
import org.codelibs.elasticsearch.common.geo.GeoPoint;
import org.codelibs.elasticsearch.common.text.Text;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import java.io.ByteArrayInputStream;
import java.io.EOFException;
import java.io.FileNotFoundException;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.AccessDeniedException;
import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.DirectoryNotEmptyException;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.FileSystemException;
import java.nio.file.FileSystemLoopException;
import java.nio.file.NoSuchFileException;
import java.nio.file.NotDirectoryException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.function.IntFunction;
import java.util.function.Supplier;
import static org.codelibs.elasticsearch.ElasticsearchException.readStackTrace;
/**
* A stream from this node to another node. Technically, it can also be streamed to a byte array but that is mostly for testing.
*
* This class's methods are optimized so you can put the methods that read and write a class next to each other and you can scan them
* visually for differences. That means that most variables should be read and written in a single line so even large objects fit both
* reading and writing on the screen. It also means that the methods on this class are named very similarly to {StreamOutput}. Finally
* it means that the "barrier to entry" for adding new methods to this class is relatively low even though it is a shared class with code
* everywhere. That being said, this class deals primarily with {@code List}s rather than Arrays. For the most part calls should adapt to
* lists, either by storing {@code List}s internally or just converting to and from a {@code List} when calling. This comment is repeated
* on {StreamInput}.
*/
public abstract class StreamInput extends InputStream {
private Version version = Version.CURRENT;
/**
* The version of the node on the other side of this stream.
*/
public Version getVersion() {
return this.version;
}
/**
* Set the version of the node on the other side of this stream.
*/
public void setVersion(Version version) {
this.version = version;
}
/**
* Reads and returns a single byte.
*/
public abstract byte readByte() throws IOException;
/**
* Reads a specified number of bytes into an array at the specified offset.
*
* @param b the array to read bytes into
* @param offset the offset in the array to start storing bytes
* @param len the number of bytes to read
*/
public abstract void readBytes(byte[] b, int offset, int len) throws IOException;
/**
* Reads a bytes reference from this stream, might hold an actual reference to the underlying
* bytes of the stream.
*/
public BytesReference readBytesReference() throws IOException {
int length = readArraySize();
return readBytesReference(length);
}
/**
* Reads an optional bytes reference from this stream. It might hold an actual reference to the underlying bytes of the stream. Use this
* only if you must differentiate null from empty. Use {StreamInput#readBytesReference()} and
* {StreamOutput#writeBytesReference(BytesReference)} if you do not.
*/
@Nullable
public BytesReference readOptionalBytesReference() throws IOException {
int length = readVInt() - 1;
if (length < 0) {
return null;
}
return readBytesReference(length);
}
/**
* Reads a bytes reference from this stream, might hold an actual reference to the underlying
* bytes of the stream.
*/
public BytesReference readBytesReference(int length) throws IOException {
if (length == 0) {
return BytesArray.EMPTY;
}
byte[] bytes = new byte[length];
readBytes(bytes, 0, length);
return new BytesArray(bytes, 0, length);
}
public BytesRef readBytesRef() throws IOException {
int length = readArraySize();
return readBytesRef(length);
}
public BytesRef readBytesRef(int length) throws IOException {
if (length == 0) {
return new BytesRef();
}
byte[] bytes = new byte[length];
readBytes(bytes, 0, length);
return new BytesRef(bytes, 0, length);
}
public void readFully(byte[] b) throws IOException {
readBytes(b, 0, b.length);
}
public short readShort() throws IOException {
return (short) (((readByte() & 0xFF) << 8) | (readByte() & 0xFF));
}
/**
* Reads four bytes and returns an int.
*/
public int readInt() throws IOException {
return ((readByte() & 0xFF) << 24) | ((readByte() & 0xFF) << 16)
| ((readByte() & 0xFF) << 8) | (readByte() & 0xFF);
}
/**
* Reads an int stored in variable-length format. Reads between one and
* five bytes. Smaller values take fewer bytes. Negative numbers
* will always use all 5 bytes and are therefore better serialized
* using {#readInt}
*/
public int readVInt() throws IOException {
byte b = readByte();
int i = b & 0x7F;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= (b & 0x7F) << 7;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= (b & 0x7F) << 14;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= (b & 0x7F) << 21;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
assert (b & 0x80) == 0;
return i | ((b & 0x7F) << 28);
}
/**
* Reads eight bytes and returns a long.
*/
public long readLong() throws IOException {
return (((long) readInt()) << 32) | (readInt() & 0xFFFFFFFFL);
}
/**
* Reads a long stored in variable-length format. Reads between one and ten bytes. Smaller values take fewer bytes. Negative numbers
* are encoded in ten bytes so prefer {#readLong()} or {#readZLong()} for negative numbers.
*/
public long readVLong() throws IOException {
byte b = readByte();
long i = b & 0x7FL;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= (b & 0x7FL) << 7;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= (b & 0x7FL) << 14;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= (b & 0x7FL) << 21;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= (b & 0x7FL) << 28;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= (b & 0x7FL) << 35;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= (b & 0x7FL) << 42;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= (b & 0x7FL) << 49;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= ((b & 0x7FL) << 56);
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
if (b != 0 && b != 1) {
throw new IOException("Invalid vlong (" + Integer.toHexString(b) + " << 63) | " + Long.toHexString(i));
}
i |= ((long) b) << 63;
return i;
}
public long readZLong() throws IOException {
long accumulator = 0L;
int i = 0;
long currentByte;
while (((currentByte = readByte()) & 0x80L) != 0) {
accumulator |= (currentByte & 0x7F) << i;
i += 7;
if (i > 63) {
throw new IOException("variable-length stream is too long");
}
}
return BitUtil.zigZagDecode(accumulator | (currentByte << i));
}
@Nullable
public Long readOptionalLong() throws IOException {
if (readBoolean()) {
return readLong();
}
return null;
}
@Nullable
public Text readOptionalText() throws IOException {
int length = readInt();
if (length == -1) {
return null;
}
return new Text(readBytesReference(length));
}
public Text readText() throws IOException {
// use StringAndBytes so we can cache the string if its ever converted to it
int length = readInt();
return new Text(readBytesReference(length));
}
@Nullable
public String readOptionalString() throws IOException {
if (readBoolean()) {
return readString();
}
return null;
}
@Nullable
public Float readOptionalFloat() throws IOException {
if (readBoolean()) {
return readFloat();
}
return null;
}
@Nullable
public Integer readOptionalVInt() throws IOException {
if (readBoolean()) {
return readVInt();
}
return null;
}
// we don't use a CharsRefBuilder since we exactly know the size of the character array up front
// this prevents calling grow for every character since we don't need this
private final CharsRef spare = new CharsRef();
public String readString() throws IOException {
// TODO it would be nice to not call readByte() for every character but we don't know how much to read up-front
// we can make the loop much more complicated but that won't buy us much compared to the bounds checks in readByte()
final int charCount = readArraySize();
if (spare.chars.length < charCount) {
// we don't use ArrayUtils.grow since there is no need to copy the array
spare.chars = new char[ArrayUtil.oversize(charCount, Character.BYTES)];
}
spare.length = charCount;
final char[] buffer = spare.chars;
for (int i = 0; i < charCount; i++) {
final int c = readByte() & 0xff;
switch (c >> 4) {
case 0:
case 1:
case 2:
case 3:
case 4:
case 5:
case 6:
case 7:
buffer[i] = (char) c;
break;
case 12:
case 13:
buffer[i] = ((char) ((c & 0x1F) << 6 | readByte() & 0x3F));
break;
case 14:
buffer[i] = ((char) ((c & 0x0F) << 12 | (readByte() & 0x3F) << 6 | (readByte() & 0x3F) << 0));
break;
default:
new AssertionError("unexpected character: " + c + " hex: " + Integer.toHexString(c));
}
}
return spare.toString();
}
public final float readFloat() throws IOException {
return Float.intBitsToFloat(readInt());
}
public final double readDouble() throws IOException {
return Double.longBitsToDouble(readLong());
}
@Nullable
public final Double readOptionalDouble() throws IOException {
if (readBoolean()) {
return readDouble();
}
return null;
}
/**
* Reads a boolean.
*/
public final boolean readBoolean() throws IOException {
return readBoolean(readByte());
}
private boolean readBoolean(final byte value) {
if (value == 0) {
return false;
} else if (value == 1) {
return true;
} else {
final String message = String.format(Locale.ROOT, "unexpected byte [0x%02x]", value);
throw new IllegalStateException(message);
}
}
@Nullable
public final Boolean readOptionalBoolean() throws IOException {
final byte value = readByte();
if (value == 2) {
return null;
} else {
return readBoolean(value);
}
}
/**
* Closes the stream to further operations.
*/
@Override
public abstract void close() throws IOException;
@Override
public abstract int available() throws IOException;
public String[] readStringArray() throws IOException {
int size = readArraySize();
if (size == 0) {
return Strings.EMPTY_ARRAY;
}
String[] ret = new String[size];
for (int i = 0; i < size; i++) {
ret[i] = readString();
}
return ret;
}
@Nullable
public String[] readOptionalStringArray() throws IOException {
if (readBoolean()) {
return readStringArray();
}
return null;
}
public Map readMap(Writeable.Reader keyReader, Writeable.Reader valueReader) throws IOException {
int size = readArraySize();
Map map = new HashMap<>(size);
for (int i = 0; i < size; i++) {
K key = keyReader.read(this);
V value = valueReader.read(this);
map.put(key, value);
}
return map;
}
/**
* Read a {Map} of {@code K}-type keys to {@code V}-type {List}s.
*
* Map<String, List<String>> map = in.readMapOfLists(StreamInput::readString, StreamInput::readString);
*
*
* @param keyReader The key reader
* @param valueReader The value reader
* @return Never {@code null}.
*/
public Map> readMapOfLists(final Writeable.Reader keyReader, final Writeable.Reader valueReader)
throws IOException {
final int size = readArraySize();
if (size == 0) {
return Collections.emptyMap();
}
final Map> map = new HashMap<>(size);
for (int i = 0; i < size; ++i) {
map.put(keyReader.read(this), readList(valueReader));
}
return map;
}
@Nullable
@SuppressWarnings("unchecked")
public Map readMap() throws IOException {
return (Map) readGenericValue();
}
@SuppressWarnings({"unchecked"})
@Nullable
public Object readGenericValue() throws IOException {
byte type = readByte();
switch (type) {
case -1:
return null;
case 0:
return readString();
case 1:
return readInt();
case 2:
return readLong();
case 3:
return readFloat();
case 4:
return readDouble();
case 5:
return readBoolean();
case 6:
return readByteArray();
case 7:
return readArrayList();
case 8:
return readArray();
case 9:
return readLinkedHashMap();
case 10:
return readHashMap();
case 11:
return readByte();
case 12:
return readDate();
case 13:
return readDateTime();
case 14:
return readBytesReference();
case 15:
return readText();
case 16:
return readShort();
case 17:
return readIntArray();
case 18:
return readLongArray();
case 19:
return readFloatArray();
case 20:
return readDoubleArray();
case 21:
return readBytesRef();
case 22:
return readGeoPoint();
default:
throw new IOException("Can't read unknown type [" + type + "]");
}
}
@SuppressWarnings("unchecked")
private List readArrayList() throws IOException {
int size = readArraySize();
List list = new ArrayList(size);
for (int i = 0; i < size; i++) {
list.add(readGenericValue());
}
return list;
}
private DateTime readDateTime() throws IOException {
final String timeZoneId = readString();
return new DateTime(readLong(), DateTimeZone.forID(timeZoneId));
}
private Object[] readArray() throws IOException {
int size8 = readArraySize();
Object[] list8 = new Object[size8];
for (int i = 0; i < size8; i++) {
list8[i] = readGenericValue();
}
return list8;
}
private Map readLinkedHashMap() throws IOException {
int size9 = readArraySize();
Map map9 = new LinkedHashMap(size9);
for (int i = 0; i < size9; i++) {
map9.put(readString(), readGenericValue());
}
return map9;
}
private Map readHashMap() throws IOException {
int size10 = readArraySize();
Map map10 = new HashMap(size10);
for (int i = 0; i < size10; i++) {
map10.put(readString(), readGenericValue());
}
return map10;
}
private Date readDate() throws IOException {
return new Date(readLong());
}
/**
* Reads a {GeoPoint} from this stream input
*/
public GeoPoint readGeoPoint() throws IOException {
return new GeoPoint(readDouble(), readDouble());
}
/**
* Read a {@linkplain DateTimeZone}.
*/
public DateTimeZone readTimeZone() throws IOException {
return DateTimeZone.forID(readString());
}
/**
* Read an optional {@linkplain DateTimeZone}.
*/
public DateTimeZone readOptionalTimeZone() throws IOException {
if (readBoolean()) {
return DateTimeZone.forID(readString());
}
return null;
}
public int[] readIntArray() throws IOException {
int length = readArraySize();
int[] values = new int[length];
for (int i = 0; i < length; i++) {
values[i] = readInt();
}
return values;
}
public int[] readVIntArray() throws IOException {
int length = readArraySize();
int[] values = new int[length];
for (int i = 0; i < length; i++) {
values[i] = readVInt();
}
return values;
}
public long[] readLongArray() throws IOException {
int length = readArraySize();
long[] values = new long[length];
for (int i = 0; i < length; i++) {
values[i] = readLong();
}
return values;
}
public long[] readVLongArray() throws IOException {
int length = readArraySize();
long[] values = new long[length];
for (int i = 0; i < length; i++) {
values[i] = readVLong();
}
return values;
}
public float[] readFloatArray() throws IOException {
int length = readArraySize();
float[] values = new float[length];
for (int i = 0; i < length; i++) {
values[i] = readFloat();
}
return values;
}
public double[] readDoubleArray() throws IOException {
int length = readArraySize();
double[] values = new double[length];
for (int i = 0; i < length; i++) {
values[i] = readDouble();
}
return values;
}
public byte[] readByteArray() throws IOException {
final int length = readArraySize();
final byte[] bytes = new byte[length];
readBytes(bytes, 0, bytes.length);
return bytes;
}
public T[] readArray(Writeable.Reader reader, IntFunction arraySupplier) throws IOException {
int length = readArraySize();
T[] values = arraySupplier.apply(length);
for (int i = 0; i < length; i++) {
values[i] = reader.read(this);
}
return values;
}
public T[] readOptionalArray(Writeable.Reader reader, IntFunction arraySupplier) throws IOException {
return readBoolean() ? readArray(reader, arraySupplier) : null;
}
/**
* Serializes a potential null value.
*/
@Nullable
public T readOptionalStreamable(Supplier supplier) throws IOException {
if (readBoolean()) {
T streamable = supplier.get();
streamable.readFrom(this);
return streamable;
} else {
return null;
}
}
@Nullable
public T readOptionalWriteable(Writeable.Reader reader) throws IOException {
if (readBoolean()) {
T t = reader.read(this);
if (t == null) {
throw new IOException("Writeable.Reader [" + reader
+ "] returned null which is not allowed and probably means it screwed up the stream.");
}
return t;
} else {
return null;
}
}
public T readException() throws IOException {
if (readBoolean()) {
int key = readVInt();
switch (key) {
case 0:
final int ord = readVInt();
return (T) ElasticsearchException.readException(this, ord);
case 1:
String msg1 = readOptionalString();
String resource1 = readOptionalString();
return (T) readStackTrace(new CorruptIndexException(msg1, resource1, readException()), this);
case 2:
String resource2 = readOptionalString();
int version2 = readInt();
int minVersion2 = readInt();
int maxVersion2 = readInt();
return (T) readStackTrace(new IndexFormatTooNewException(resource2, version2, minVersion2, maxVersion2), this);
case 3:
String resource3 = readOptionalString();
if (readBoolean()) {
int version3 = readInt();
int minVersion3 = readInt();
int maxVersion3 = readInt();
return (T) readStackTrace(new IndexFormatTooOldException(resource3, version3, minVersion3, maxVersion3), this);
} else {
String version3 = readOptionalString();
return (T) readStackTrace(new IndexFormatTooOldException(resource3, version3), this);
}
case 4:
return (T) readStackTrace(new NullPointerException(readOptionalString()), this);
case 5:
return (T) readStackTrace(new NumberFormatException(readOptionalString()), this);
case 6:
return (T) readStackTrace(new IllegalArgumentException(readOptionalString(), readException()), this);
case 7:
return (T) readStackTrace(new AlreadyClosedException(readOptionalString(), readException()), this);
case 8:
return (T) readStackTrace(new EOFException(readOptionalString()), this);
case 9:
return (T) readStackTrace(new SecurityException(readOptionalString(), readException()), this);
case 10:
return (T) readStackTrace(new StringIndexOutOfBoundsException(readOptionalString()), this);
case 11:
return (T) readStackTrace(new ArrayIndexOutOfBoundsException(readOptionalString()), this);
case 12:
return (T) readStackTrace(new FileNotFoundException(readOptionalString()), this);
case 13:
final int subclass = readVInt();
final String file = readOptionalString();
final String other = readOptionalString();
final String reason = readOptionalString();
readOptionalString(); // skip the msg - it's composed from file, other and reason
final Exception exception;
switch (subclass) {
case 0:
exception = new NoSuchFileException(file, other, reason);
break;
case 1:
exception = new NotDirectoryException(file);
break;
case 2:
exception = new DirectoryNotEmptyException(file);
break;
case 3:
exception = new AtomicMoveNotSupportedException(file, other, reason);
break;
case 4:
exception = new FileAlreadyExistsException(file, other, reason);
break;
case 5:
exception = new AccessDeniedException(file, other, reason);
break;
case 6:
exception = new FileSystemLoopException(file);
break;
case 7:
exception = new FileSystemException(file, other, reason);
break;
default:
throw new IllegalStateException("unknown FileSystemException with index " + subclass);
}
return (T) readStackTrace(exception, this);
case 14:
return (T) readStackTrace(new IllegalStateException(readOptionalString(), readException()), this);
case 15:
return (T) readStackTrace(new LockObtainFailedException(readOptionalString(), readException()), this);
case 16:
return (T) readStackTrace(new InterruptedException(readOptionalString()), this);
case 17:
return (T) readStackTrace(new IOException(readOptionalString(), readException()), this);
default:
assert false : "no such exception for id: " + key;
}
}
return null;
}
/**
* Reads a {NamedWriteable} from the current stream, by first reading its name and then looking for
* the corresponding entry in the registry by name, so that the proper object can be read and returned.
* Default implementation throws {UnsupportedOperationException} as StreamInput doesn't hold a registry.
* Use {FilterInputStream} instead which wraps a stream and supports a {NamedWriteableRegistry} too.
*/
@Nullable
public C readNamedWriteable(@SuppressWarnings("unused") Class categoryClass) throws IOException {
throw new UnsupportedOperationException("can't read named writeable from StreamInput");
}
/**
* Reads a {NamedWriteable} from the current stream with the given name. It is assumed that the caller obtained the name
* from other source, so it's not read from the stream. The name is used for looking for
* the corresponding entry in the registry by name, so that the proper object can be read and returned.
* Default implementation throws {UnsupportedOperationException} as StreamInput doesn't hold a registry.
* Use {FilterInputStream} instead which wraps a stream and supports a {NamedWriteableRegistry} too.
*
* Prefer {StreamInput#readNamedWriteable(Class)} and {StreamOutput#writeNamedWriteable(NamedWriteable)} unless you
* have a compelling reason to use this method instead.
*/
@Nullable
public C readNamedWriteable(@SuppressWarnings("unused") Class categoryClass,
@SuppressWarnings("unused") String name) throws IOException {
throw new UnsupportedOperationException("can't read named writeable from StreamInput");
}
/**
* Reads an optional {NamedWriteable}.
*/
@Nullable
public C readOptionalNamedWriteable(Class categoryClass) throws IOException {
if (readBoolean()) {
return readNamedWriteable(categoryClass);
}
return null;
}
/**
* Read a {List} of {Streamable} objects, using the {@code constructor} to instantiate each instance.
*
* This is expected to take the form:
*
* List<MyStreamableClass> list = in.readStreamList(MyStreamableClass::new);
*
*
* @param constructor Streamable instance creator
* @return Never {@code null}.
* @throws IOException if any step fails
*/
public List readStreamableList(Supplier constructor) throws IOException {
int count = readArraySize();
List builder = new ArrayList<>(count);
for (int i=0; i List readList(Writeable.Reader reader) throws IOException {
int count = readArraySize();
List builder = new ArrayList<>(count);
for (int i=0; i List readNamedWriteableList(Class categoryClass) throws IOException {
int count = readArraySize();
List builder = new ArrayList<>(count);
for (int i=0; i ArrayUtil.MAX_ARRAY_LENGTH) {
throw new IllegalStateException("array length must be <= to " + ArrayUtil.MAX_ARRAY_LENGTH + " but was: " + arraySize);
}
if (arraySize < 0) {
throw new NegativeArraySizeException("array size must be positive but was: " + arraySize);
}
// lets do a sanity check that if we are reading an array size that is bigger that the remaining bytes we can safely
// throw an exception instead of allocating the array based on the size. A simple corrutpted byte can make a node go OOM
// if the size is large and for perf reasons we allocate arrays ahead of time
ensureCanReadBytes(arraySize);
return arraySize;
}
/**
* This method throws an {EOFException} if the given number of bytes can not be read from the this stream. This method might
* be a no-op depending on the underlying implementation if the information of the remaining bytes is not present.
*/
protected abstract void ensureCanReadBytes(int length) throws EOFException;
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy