org.elasticsearch.common.io.stream.StreamInput Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of elasticsearch Show documentation
Show all versions of elasticsearch Show documentation
Elasticsearch subproject :server
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.common.io.stream;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexFormatTooNewException;
import org.apache.lucene.index.IndexFormatTooOldException;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.bytes.ReleasableBytesReference;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.util.Maps;
import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
import org.elasticsearch.core.CharArrays;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.core.TimeValue;
import java.io.EOFException;
import java.io.FileNotFoundException;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.file.AccessDeniedException;
import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.DirectoryNotEmptyException;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.FileSystemException;
import java.nio.file.FileSystemLoopException;
import java.nio.file.NoSuchFileException;
import java.nio.file.NotDirectoryException;
import java.time.Instant;
import java.time.LocalTime;
import java.time.OffsetTime;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.function.IntFunction;
import static org.elasticsearch.ElasticsearchException.readStackTrace;
/**
* A stream from this node to another node. Technically, it can also be streamed to a byte array but that is mostly for testing.
*
* This class's methods are optimized so you can put the methods that read and write a class next to each other and you can scan them
* visually for differences. That means that most variables should be read and written in a single line so even large objects fit both
* reading and writing on the screen. It also means that the methods on this class are named very similarly to {@link StreamOutput}. Finally
* it means that the "barrier to entry" for adding new methods to this class is relatively low even though it is a shared class with code
* everywhere. That being said, this class deals primarily with {@code List}s rather than Arrays. For the most part calls should adapt to
* lists, either by storing {@code List}s internally or just converting to and from a {@code List} when calling. This comment is repeated
* on {@link StreamInput}.
*/
public abstract class StreamInput extends InputStream {
private Version version = Version.CURRENT;
/**
* The version of the node on the other side of this stream.
*/
public Version getVersion() {
return this.version;
}
/**
* Set the version of the node on the other side of this stream.
*/
public void setVersion(Version version) {
this.version = version;
}
/**
* Reads and returns a single byte.
*/
public abstract byte readByte() throws IOException;
/**
* Reads a specified number of bytes into an array at the specified offset.
*
* @param b the array to read bytes into
* @param offset the offset in the array to start storing bytes
* @param len the number of bytes to read
*/
public abstract void readBytes(byte[] b, int offset, int len) throws IOException;
/**
* Reads a bytes reference from this stream, copying any bytes read to a new {@code byte[]}. Use {@link #readReleasableBytesReference()}
* when reading large bytes references where possible top avoid needless allocations and copying.
*/
public BytesReference readBytesReference() throws IOException {
int length = readArraySize();
return readBytesReference(length);
}
/**
* Reads a releasable bytes reference from this stream. Unlike {@link #readBytesReference()} the returned bytes reference may reference
* bytes in a pooled buffer and must be explicitly released via {@link ReleasableBytesReference#close()} once no longer used.
* Prefer this method over {@link #readBytesReference()} when reading large bytes references to avoid allocations and copying.
*/
public ReleasableBytesReference readReleasableBytesReference() throws IOException {
return ReleasableBytesReference.wrap(readBytesReference());
}
/**
* Reads an optional bytes reference from this stream. It might hold an actual reference to the underlying bytes of the stream. Use this
* only if you must differentiate null from empty. Use {@link StreamInput#readBytesReference()} and
* {@link StreamOutput#writeBytesReference(BytesReference)} if you do not.
*/
@Nullable
public BytesReference readOptionalBytesReference() throws IOException {
int length = readVInt() - 1;
if (length < 0) {
return null;
}
return readBytesReference(length);
}
/**
* Reads a bytes reference from this stream, might hold an actual reference to the underlying
* bytes of the stream.
*/
public BytesReference readBytesReference(int length) throws IOException {
if (length == 0) {
return BytesArray.EMPTY;
}
byte[] bytes = new byte[length];
readBytes(bytes, 0, length);
return new BytesArray(bytes, 0, length);
}
public BytesRef readBytesRef() throws IOException {
int length = readArraySize();
return readBytesRef(length);
}
public BytesRef readBytesRef(int length) throws IOException {
if (length == 0) {
return new BytesRef();
}
byte[] bytes = new byte[length];
readBytes(bytes, 0, length);
return new BytesRef(bytes, 0, length);
}
public void readFully(byte[] b) throws IOException {
readBytes(b, 0, b.length);
}
public short readShort() throws IOException {
return (short) (((readByte() & 0xFF) << 8) | (readByte() & 0xFF));
}
/**
* Reads four bytes and returns an int.
*/
public int readInt() throws IOException {
return ((readByte() & 0xFF) << 24) | ((readByte() & 0xFF) << 16) | ((readByte() & 0xFF) << 8) | (readByte() & 0xFF);
}
/**
* Reads an optional {@link Integer}.
*/
public Integer readOptionalInt() throws IOException {
if (readBoolean()) {
return readInt();
}
return null;
}
/**
* Reads an int stored in variable-length format. Reads between one and
* five bytes. Smaller values take fewer bytes. Negative numbers
* will always use all 5 bytes and are therefore better serialized
* using {@link #readInt}
*/
public int readVInt() throws IOException {
return readVIntSlow();
}
protected final int readVIntSlow() throws IOException {
byte b = readByte();
int i = b & 0x7F;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= (b & 0x7F) << 7;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= (b & 0x7F) << 14;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= (b & 0x7F) << 21;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
if ((b & 0x80) != 0) {
throwOnBrokenVInt(b, i);
}
return i | ((b & 0x7F) << 28);
}
protected static void throwOnBrokenVInt(byte b, int accumulated) throws IOException {
throw new IOException("Invalid vInt ((" + Integer.toHexString(b) + " & 0x7f) << 28) | " + Integer.toHexString(accumulated));
}
/**
* Reads eight bytes and returns a long.
*/
public long readLong() throws IOException {
return (((long) readInt()) << 32) | (readInt() & 0xFFFFFFFFL);
}
/**
* Reads a long stored in variable-length format. Reads between one and ten bytes. Smaller values take fewer bytes. Negative numbers
* are encoded in ten bytes so prefer {@link #readLong()} or {@link #readZLong()} for negative numbers.
*/
public long readVLong() throws IOException {
return readVLongSlow();
}
protected final long readVLongSlow() throws IOException {
byte b = readByte();
long i = b & 0x7FL;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= (b & 0x7FL) << 7;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= (b & 0x7FL) << 14;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= (b & 0x7FL) << 21;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= (b & 0x7FL) << 28;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= (b & 0x7FL) << 35;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= (b & 0x7FL) << 42;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= (b & 0x7FL) << 49;
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
i |= ((b & 0x7FL) << 56);
if ((b & 0x80) == 0) {
return i;
}
b = readByte();
if (b != 0 && b != 1) {
throwOnBrokenVLong(b, i);
}
i |= ((long) b) << 63;
return i;
}
protected static void throwOnBrokenVLong(byte b, long accumulated) throws IOException {
throw new IOException("Invalid vlong (" + Integer.toHexString(b) + " << 63) | " + Long.toHexString(accumulated));
}
@Nullable
public Long readOptionalVLong() throws IOException {
if (readBoolean()) {
return readVLong();
}
return null;
}
public long readZLong() throws IOException {
long accumulator = 0L;
int i = 0;
long currentByte;
while (((currentByte = readByte()) & 0x80L) != 0) {
accumulator |= (currentByte & 0x7F) << i;
i += 7;
if (i > 63) {
throw new IOException("variable-length stream is too long");
}
}
return BitUtil.zigZagDecode(accumulator | (currentByte << i));
}
@Nullable
public Long readOptionalLong() throws IOException {
if (readBoolean()) {
return readLong();
}
return null;
}
public BigInteger readBigInteger() throws IOException {
return new BigInteger(readString());
}
@Nullable
public Text readOptionalText() throws IOException {
int length = readInt();
if (length == -1) {
return null;
}
return new Text(readBytesReference(length));
}
public Text readText() throws IOException {
// use StringAndBytes so we can cache the string if its ever converted to it
int length = readInt();
return new Text(readBytesReference(length));
}
@Nullable
public String readOptionalString() throws IOException {
if (readBoolean()) {
return readString();
}
return null;
}
@Nullable
public SecureString readOptionalSecureString() throws IOException {
SecureString value = null;
BytesReference bytesRef = readOptionalBytesReference();
if (bytesRef != null) {
byte[] bytes = BytesReference.toBytes(bytesRef);
try {
value = new SecureString(CharArrays.utf8BytesToChars(bytes));
} finally {
Arrays.fill(bytes, (byte) 0);
}
}
return value;
}
@Nullable
public Float readOptionalFloat() throws IOException {
if (readBoolean()) {
return readFloat();
}
return null;
}
@Nullable
public Integer readOptionalVInt() throws IOException {
if (readBoolean()) {
return readVInt();
}
return null;
}
// Maximum char-count to de-serialize via the thread-local CharsRef buffer
private static final int SMALL_STRING_LIMIT = 1024;
// Reusable bytes for deserializing strings
private static final ThreadLocal stringReadBuffer = ThreadLocal.withInitial(() -> new byte[1024]);
// Thread-local buffer for smaller strings
private static final ThreadLocal smallSpare = ThreadLocal.withInitial(() -> new char[SMALL_STRING_LIMIT]);
// Larger buffer used for long strings that can't fit into the thread-local buffer
// We don't use a CharsRefBuilder since we exactly know the size of the character array up front
// this prevents calling grow for every character since we don't need this
private char[] largeSpare;
private char[] ensureLargeSpare(int charCount) {
char[] spare = largeSpare;
if (spare == null || spare.length < charCount) {
// we don't use ArrayUtils.grow since there is no need to copy the array
spare = new char[ArrayUtil.oversize(charCount, Character.BYTES)];
largeSpare = spare;
}
return spare;
}
public String readString() throws IOException {
final int charCount = readArraySize();
final char[] charBuffer = charCount > SMALL_STRING_LIMIT ? ensureLargeSpare(charCount) : smallSpare.get();
int charsOffset = 0;
int offsetByteArray = 0;
int sizeByteArray = 0;
int missingFromPartial = 0;
final byte[] byteBuffer = stringReadBuffer.get();
for (; charsOffset < charCount;) {
final int charsLeft = charCount - charsOffset;
int bufferFree = byteBuffer.length - sizeByteArray;
// Determine the minimum amount of bytes that are left in the string
final int minRemainingBytes;
if (missingFromPartial > 0) {
// One byte for each remaining char except for the already partially read char
minRemainingBytes = missingFromPartial + charsLeft - 1;
missingFromPartial = 0;
} else {
// Each char has at least a single byte
minRemainingBytes = charsLeft;
}
final int toRead;
if (bufferFree < minRemainingBytes) {
// We don't have enough space left in the byte array to read as much as we'd like to so we free up as many bytes in the
// buffer by moving unused bytes that didn't make up a full char in the last iteration to the beginning of the buffer,
// if there are any
if (offsetByteArray > 0) {
sizeByteArray = sizeByteArray - offsetByteArray;
switch (sizeByteArray) { // We only have 0, 1 or 2 => no need to bother with a native call to System#arrayCopy
case 1 -> byteBuffer[0] = byteBuffer[offsetByteArray];
case 2 -> {
byteBuffer[0] = byteBuffer[offsetByteArray];
byteBuffer[1] = byteBuffer[offsetByteArray + 1];
}
}
assert sizeByteArray <= 2 : "We never copy more than 2 bytes here since a char is 3 bytes max";
toRead = Math.min(bufferFree + offsetByteArray, minRemainingBytes);
offsetByteArray = 0;
} else {
toRead = bufferFree;
}
} else {
toRead = minRemainingBytes;
}
readBytes(byteBuffer, sizeByteArray, toRead);
sizeByteArray += toRead;
// As long as we at least have three bytes buffered we don't need to do any bounds checking when getting the next char since we
// read 3 bytes per char/iteration at most
for (; offsetByteArray < sizeByteArray - 2; offsetByteArray++) {
final int c = byteBuffer[offsetByteArray] & 0xff;
switch (c >> 4) {
case 0, 1, 2, 3, 4, 5, 6, 7 -> charBuffer[charsOffset++] = (char) c;
case 12, 13 -> charBuffer[charsOffset++] = (char) ((c & 0x1F) << 6 | byteBuffer[++offsetByteArray] & 0x3F);
case 14 -> charBuffer[charsOffset++] = (char) ((c & 0x0F) << 12 | (byteBuffer[++offsetByteArray] & 0x3F) << 6
| (byteBuffer[++offsetByteArray] & 0x3F));
default -> throwOnBrokenChar(c);
}
}
// try to extract chars from remaining bytes with bounds checks for multi-byte chars
final int bufferedBytesRemaining = sizeByteArray - offsetByteArray;
for (int i = 0; i < bufferedBytesRemaining; i++) {
final int c = byteBuffer[offsetByteArray] & 0xff;
switch (c >> 4) {
case 0, 1, 2, 3, 4, 5, 6, 7 -> {
charBuffer[charsOffset++] = (char) c;
offsetByteArray++;
}
case 12, 13 -> {
missingFromPartial = 2 - (bufferedBytesRemaining - i);
if (missingFromPartial == 0) {
offsetByteArray++;
charBuffer[charsOffset++] = (char) ((c & 0x1F) << 6 | byteBuffer[offsetByteArray++] & 0x3F);
}
++i;
}
case 14 -> {
missingFromPartial = 3 - (bufferedBytesRemaining - i);
++i;
}
default -> throwOnBrokenChar(c);
}
}
}
return new String(charBuffer, 0, charCount);
}
private static void throwOnBrokenChar(int c) throws IOException {
throw new IOException("Invalid string; unexpected character: " + c + " hex: " + Integer.toHexString(c));
}
public SecureString readSecureString() throws IOException {
BytesReference bytesRef = readBytesReference();
byte[] bytes = BytesReference.toBytes(bytesRef);
try {
return new SecureString(CharArrays.utf8BytesToChars(bytes));
} finally {
Arrays.fill(bytes, (byte) 0);
}
}
public final float readFloat() throws IOException {
return Float.intBitsToFloat(readInt());
}
public final double readDouble() throws IOException {
return Double.longBitsToDouble(readLong());
}
@Nullable
public final Double readOptionalDouble() throws IOException {
if (readBoolean()) {
return readDouble();
}
return null;
}
/**
* Reads a boolean.
*/
public final boolean readBoolean() throws IOException {
return readBoolean(readByte());
}
private static boolean readBoolean(final byte value) {
if (value == 0) {
return false;
} else if (value == 1) {
return true;
} else {
final String message = String.format(Locale.ROOT, "unexpected byte [0x%02x]", value);
throw new IllegalStateException(message);
}
}
@Nullable
public final Boolean readOptionalBoolean() throws IOException {
final byte value = readByte();
if (value == 2) {
return null;
} else {
return readBoolean(value);
}
}
/**
* Closes the stream to further operations.
*/
@Override
public abstract void close() throws IOException;
@Override
public abstract int available() throws IOException;
public String[] readStringArray() throws IOException {
int size = readArraySize();
if (size == 0) {
return Strings.EMPTY_ARRAY;
}
String[] ret = new String[size];
for (int i = 0; i < size; i++) {
ret[i] = readString();
}
return ret;
}
@Nullable
public String[] readOptionalStringArray() throws IOException {
if (readBoolean()) {
return readStringArray();
}
return null;
}
/**
* If the returned map contains any entries it will be mutable. If it is empty it might be immutable.
*/
public Map readMap(Writeable.Reader keyReader, Writeable.Reader valueReader) throws IOException {
return readMap(keyReader, valueReader, Maps::newHashMapWithExpectedSize);
}
public Map readOrderedMap(Writeable.Reader keyReader, Writeable.Reader valueReader) throws IOException {
return readMap(keyReader, valueReader, Maps::newLinkedHashMapWithExpectedSize);
}
private Map readMap(Writeable.Reader keyReader, Writeable.Reader valueReader, IntFunction