org.apache.cassandra.io.util.RandomAccessReader Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of cassandra-all Show documentation
Show all versions of cassandra-all Show documentation
A fork of the Apache Cassandra Project ready to embed Elasticsearch.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.io.util;
import java.io.*;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import com.google.common.primitives.Ints;
import com.google.common.util.concurrent.RateLimiter;
import org.apache.cassandra.io.FSReadError;
import org.apache.cassandra.io.compress.BufferType;
import org.apache.cassandra.utils.memory.BufferPool;
public class RandomAccessReader extends RebufferingInputStream implements FileDataInput
{
// The default buffer size when the client doesn't specify it
public static final int DEFAULT_BUFFER_SIZE = 4096;
// The maximum buffer size, we will never buffer more than this size. Further,
// when the limiter is not null, i.e. when throttling is enabled, we read exactly
// this size, since when throttling the intention is to eventually read everything,
// see CASSANDRA-8630
// NOTE: this size is chosen both for historical consistency, as a reasonable upper bound,
// and because our BufferPool currently has a maximum allocation size of this.
public static final int MAX_BUFFER_SIZE = 1 << 16; // 64k
// the IO channel to the file, we do not own a reference to this due to
// performance reasons (CASSANDRA-9379) so it's up to the owner of the RAR to
// ensure that the channel stays open and that it is closed afterwards
protected final ChannelProxy channel;
// optional memory mapped regions for the channel
protected final MmappedRegions regions;
// An optional limiter that will throttle the amount of data we read
protected final RateLimiter limiter;
// the file length, this can be overridden at construction to a value shorter
// than the true length of the file; if so, it acts as an imposed limit on reads,
// required when opening sstables early not to read past the mark
private final long fileLength;
// the buffer size for buffered readers
protected final int bufferSize;
// the buffer type for buffered readers
protected final BufferType bufferType;
// offset from the beginning of the file
protected long bufferOffset;
// offset of the last file mark
protected long markedPointer;
protected RandomAccessReader(Builder builder)
{
super(builder.createBuffer());
this.channel = builder.channel;
this.regions = builder.regions;
this.limiter = builder.limiter;
this.fileLength = builder.overrideLength <= 0 ? builder.channel.size() : builder.overrideLength;
this.bufferSize = builder.bufferSize;
this.bufferType = builder.bufferType;
this.buffer = builder.buffer;
}
protected static ByteBuffer allocateBuffer(int size, BufferType bufferType)
{
return BufferPool.get(size, bufferType).order(ByteOrder.BIG_ENDIAN);
}
protected void releaseBuffer()
{
if (buffer != null)
{
if (regions == null)
BufferPool.put(buffer);
buffer = null;
}
}
/**
* Read data from file starting from current currentOffset to populate buffer.
*/
public void reBuffer()
{
if (isEOF())
return;
if (regions == null)
reBufferStandard();
else
reBufferMmap();
if (limiter != null)
limiter.acquire(buffer.remaining());
assert buffer.order() == ByteOrder.BIG_ENDIAN : "Buffer must have BIG ENDIAN byte ordering";
}
protected void reBufferStandard()
{
bufferOffset += buffer.position();
assert bufferOffset < fileLength;
buffer.clear();
long position = bufferOffset;
long limit = bufferOffset;
long pageAligedPos = position & ~4095;
// Because the buffer capacity is a multiple of the page size, we read less
// the first time and then we should read at page boundaries only,
// unless the user seeks elsewhere
long upperLimit = Math.min(fileLength, pageAligedPos + buffer.capacity());
buffer.limit((int)(upperLimit - position));
while (buffer.hasRemaining() && limit < upperLimit)
{
int n = channel.read(buffer, position);
if (n < 0)
throw new FSReadError(new IOException("Unexpected end of file"), channel.filePath());
position += n;
limit = bufferOffset + buffer.position();
}
buffer.flip();
}
protected void reBufferMmap()
{
long position = bufferOffset + buffer.position();
assert position < fileLength;
MmappedRegions.Region region = regions.floor(position);
bufferOffset = region.bottom();
buffer = region.buffer.duplicate();
buffer.position(Ints.checkedCast(position - bufferOffset));
if (limiter != null && bufferSize < buffer.remaining())
{ // ensure accurate throttling
buffer.limit(buffer.position() + bufferSize);
}
}
@Override
public long getFilePointer()
{
return current();
}
protected long current()
{
return bufferOffset + (buffer == null ? 0 : buffer.position());
}
public String getPath()
{
return channel.filePath();
}
public ChannelProxy getChannel()
{
return channel;
}
@Override
public void reset() throws IOException
{
seek(markedPointer);
}
@Override
public boolean markSupported()
{
return true;
}
public long bytesPastMark()
{
long bytes = current() - markedPointer;
assert bytes >= 0;
return bytes;
}
public DataPosition mark()
{
markedPointer = current();
return new BufferedRandomAccessFileMark(markedPointer);
}
public void reset(DataPosition mark)
{
assert mark instanceof BufferedRandomAccessFileMark;
seek(((BufferedRandomAccessFileMark) mark).pointer);
}
public long bytesPastMark(DataPosition mark)
{
assert mark instanceof BufferedRandomAccessFileMark;
long bytes = current() - ((BufferedRandomAccessFileMark) mark).pointer;
assert bytes >= 0;
return bytes;
}
/**
* @return true if there is no more data to read
*/
public boolean isEOF()
{
return current() == length();
}
public long bytesRemaining()
{
return length() - getFilePointer();
}
@Override
public int available() throws IOException
{
return Ints.saturatedCast(bytesRemaining());
}
@Override
public void close()
{
//make idempotent
if (buffer == null)
return;
bufferOffset += buffer.position();
releaseBuffer();
//For performance reasons we don't keep a reference to the file
//channel so we don't close it
}
@Override
public String toString()
{
return getClass().getSimpleName() + "(filePath='" + channel + "')";
}
/**
* Class to hold a mark to the position of the file
*/
protected static class BufferedRandomAccessFileMark implements DataPosition
{
final long pointer;
public BufferedRandomAccessFileMark(long pointer)
{
this.pointer = pointer;
}
}
@Override
public void seek(long newPosition)
{
if (newPosition < 0)
throw new IllegalArgumentException("new position should not be negative");
if (buffer == null)
throw new IllegalStateException("Attempted to seek in a closed RAR");
if (newPosition >= length()) // it is save to call length() in read-only mode
{
if (newPosition > length())
throw new IllegalArgumentException(String.format("Unable to seek to position %d in %s (%d bytes) in read-only mode",
newPosition, getPath(), length()));
buffer.limit(0);
bufferOffset = newPosition;
return;
}
if (newPosition >= bufferOffset && newPosition < bufferOffset + buffer.limit())
{
buffer.position((int) (newPosition - bufferOffset));
return;
}
// Set current location to newPosition and clear buffer so reBuffer calculates from newPosition
bufferOffset = newPosition;
buffer.clear();
reBuffer();
assert current() == newPosition;
}
/**
* Reads a line of text form the current position in this file. A line is
* represented by zero or more characters followed by {@code '\n'}, {@code
* '\r'}, {@code "\r\n"} or the end of file marker. The string does not
* include the line terminating sequence.
*
* Blocks until a line terminating sequence has been read, the end of the
* file is reached or an exception is thrown.
*
* @return the contents of the line or {@code null} if no characters have
* been read before the end of the file has been reached.
* @throws IOException if this file is closed or another I/O error occurs.
*/
public final String readLine() throws IOException
{
StringBuilder line = new StringBuilder(80); // Typical line length
boolean foundTerminator = false;
long unreadPosition = -1;
while (true)
{
int nextByte = read();
switch (nextByte)
{
case -1:
return line.length() != 0 ? line.toString() : null;
case (byte) '\r':
if (foundTerminator)
{
seek(unreadPosition);
return line.toString();
}
foundTerminator = true;
/* Have to be able to peek ahead one byte */
unreadPosition = getPosition();
break;
case (byte) '\n':
return line.toString();
default:
if (foundTerminator)
{
seek(unreadPosition);
return line.toString();
}
line.append((char) nextByte);
}
}
}
public long length()
{
return fileLength;
}
public long getPosition()
{
return current();
}
public static class Builder
{
// The NIO file channel or an empty channel
public final ChannelProxy channel;
// We override the file length when we open sstables early, so that we do not
// read past the early mark
public long overrideLength;
// The size of the buffer for buffered readers
public int bufferSize;
// The type of the buffer for buffered readers
public BufferType bufferType;
// The buffer
public ByteBuffer buffer;
// The mmap segments for mmap readers
public MmappedRegions regions;
// An optional limiter that will throttle the amount of data we read
public RateLimiter limiter;
public Builder(ChannelProxy channel)
{
this.channel = channel;
this.overrideLength = -1L;
this.bufferSize = DEFAULT_BUFFER_SIZE;
this.bufferType = BufferType.OFF_HEAP;
this.regions = null;
this.limiter = null;
}
/** The buffer size is typically already page aligned but if that is not the case
* make sure that it is a multiple of the page size, 4096. Also limit it to the maximum
* buffer size unless we are throttling, in which case we may as well read the maximum
* directly since the intention is to read the full file, see CASSANDRA-8630.
* */
private void setBufferSize()
{
if (limiter != null)
{
bufferSize = MAX_BUFFER_SIZE;
return;
}
if ((bufferSize & ~4095) != bufferSize)
{ // should already be a page size multiple but if that's not case round it up
bufferSize = (bufferSize + 4095) & ~4095;
}
bufferSize = Math.min(MAX_BUFFER_SIZE, bufferSize);
}
protected ByteBuffer createBuffer()
{
setBufferSize();
buffer = regions == null
? allocateBuffer(bufferSize, bufferType)
: regions.floor(0).buffer.duplicate();
buffer.limit(0);
return buffer;
}
public Builder overrideLength(long overrideLength)
{
this.overrideLength = overrideLength;
return this;
}
public Builder bufferSize(int bufferSize)
{
if (bufferSize <= 0)
throw new IllegalArgumentException("bufferSize must be positive");
this.bufferSize = bufferSize;
return this;
}
public Builder bufferType(BufferType bufferType)
{
this.bufferType = bufferType;
return this;
}
public Builder regions(MmappedRegions regions)
{
this.regions = regions;
return this;
}
public Builder limiter(RateLimiter limiter)
{
this.limiter = limiter;
return this;
}
public RandomAccessReader build()
{
return new RandomAccessReader(this);
}
public RandomAccessReader buildWithChannel()
{
return new RandomAccessReaderWithOwnChannel(this);
}
}
// A wrapper of the RandomAccessReader that closes the channel when done.
// For performance reasons RAR does not increase the reference count of
// a channel but assumes the owner will keep it open and close it,
// see CASSANDRA-9379, this thin class is just for those cases where we do
// not have a shared channel.
public static class RandomAccessReaderWithOwnChannel extends RandomAccessReader
{
protected RandomAccessReaderWithOwnChannel(Builder builder)
{
super(builder);
}
@Override
public void close()
{
try
{
super.close();
}
finally
{
channel.close();
}
}
}
@SuppressWarnings("resource")
public static RandomAccessReader open(File file)
{
return new Builder(new ChannelProxy(file)).buildWithChannel();
}
public static RandomAccessReader open(ChannelProxy channel)
{
return new Builder(channel).build();
}
}