com.kolibrifx.plovercrest.server.internal.MappedBigFile Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of plovercrest-server Show documentation
Show all versions of plovercrest-server Show documentation
Plovercrest server library.
The newest version!
/*
* Copyright (c) 2010-2017, KolibriFX AS. Licensed under the Apache License, version 2.0.
*/
package com.kolibrifx.plovercrest.server.internal;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import com.kolibrifx.plovercrest.client.PlovercrestException;
import com.kolibrifx.plovercrest.client.TableClosedException;
public class MappedBigFile {
// Windows size is set to 1GB, which was measured to be faster than using numbers close to 2GB for unknown
// reasons. (10-20% faster on a MacBook Pro)
public static final long DEFAULT_WINDOW_SIZE = 0x40000000;
// The first time data is written to the file, it will be expanded to this size.
// From then on, it will double until the maximum chunk size is reached.
static final int INITIAL_FILE_SIZE = 1024 * 4;
// The default maximum chunk size. Once the file size reaches this, it will grow linearly instead of
// exponentially.
static final long DEFAULT_MAXIMUM_CHUNK_SIZE = 1024 * 1024 * 64;
private final ArrayList windows;
private boolean closed = false; // synchronized through windows
private final FileChannel channel;
private final long maximumChunkSize;
private final long windowSize;
private final String name;
private final ThreadLocal tempBufferLocal = new ThreadLocal() {
@Override
public ByteBuffer initialValue() {
return ByteBuffer.allocate(8);
}
};
private long fileSize;
private final RandomAccessFile file;
private class FilePosition {
protected long offset;
protected Window window;
protected FilePosition(final long offset) {
this.offset = offset;
this.window = windowAtOffset(offset);
}
protected int remainingInWindow() {
return window.remaining(offset);
}
protected long remainingInFile() {
return fileSize - offset;
}
public void seek(final long offset) {
assert isValidOffset(offset);
this.offset = offset;
if (!window.isWithinRange(offset)) {
window = windowAtOffset(offset);
}
}
public boolean isValidOffset(final long offset) {
return offset >= 0 && offset <= fileSize;
}
public long tell() {
return offset;
}
public long getFileSize() {
return fileSize;
}
public boolean assertInvariants() {
assert window != null : "Window not null";
// Note: offset == window.endOffset() is a legal state, it will cause an IndexOutOfBoundsException
// on the next read, and this is handled.
assert offset >= window.startOffset() && offset <= window.endOffset() : "Offset within range";
// assert (offset & 3) == 0 : "Offset aligned on 4";
return true;
}
}
public class Reader extends FilePosition {
Reader(final long offset) {
super(offset);
}
public long readLong() {
assert assertInvariants();
try {
final long r = window.readLong(offset);
offset += 8;
return r;
} catch (final IndexOutOfBoundsException e) {
final ByteBuffer tempBuffer = tempBufferLocal.get();
tempBuffer.clear();
tempBuffer.limit(8);
readBuffer(tempBuffer);
tempBuffer.flip();
return tempBuffer.getLong();
}
}
public int readInt() {
assert assertInvariants();
try {
final int r = window.readInt(offset);
offset += 4;
return r;
} catch (final IndexOutOfBoundsException e) {
final ByteBuffer tempBuffer = tempBufferLocal.get();
tempBuffer.clear();
tempBuffer.limit(4);
readBuffer(tempBuffer);
tempBuffer.flip();
return tempBuffer.getInt();
}
}
public int peekInt() {
try {
return window.readInt(offset);
} catch (final IndexOutOfBoundsException e) {
final int result = readInt();
seek(offset - 4);
return result;
}
}
public void readBuffer(final ByteBuffer bytes) {
final int wantedLimit = bytes.limit();
final int canWrite = Math.min(remainingInWindow(), bytes.remaining());
bytes.limit(canWrite);
offset += window.readBuffer(offset, bytes);
if (bytes.limit() < wantedLimit) {
bytes.limit(wantedLimit);
window = nextWindow(window);
assert offset == window.startOffset();
offset += window.readBuffer(offset, bytes);
}
assert assertInvariants();
}
}
public class Writer extends FilePosition {
Writer(final long offset) {
super(offset);
}
public long writeBuffer(final ByteBuffer bytes) {
final long oldOffset = offset;
final int wantedLimit = bytes.limit();
if (remainingInFile() < bytes.remaining()) {
expandFile(offset + bytes.remaining());
window.remap();
}
final int canWrite = Math.min(remainingInWindow(), bytes.remaining());
bytes.limit(bytes.position() + canWrite);
offset += window.writeBuffer(offset, bytes);
if (bytes.limit() < wantedLimit) {
window = nextWindow(window);
assert offset == window.startOffset();
bytes.limit(wantedLimit);
offset += window.writeBuffer(offset, bytes);
}
return offset - oldOffset;
}
public long writeLong(final long value) {
if (remainingInWindow() >= 8) {
final long written = window.writeLong(offset, value);
offset += written;
return written;
} else {
final ByteBuffer tempBuffer = tempBufferLocal.get();
tempBuffer.clear();
tempBuffer.putLong(value);
tempBuffer.flip();
return writeBuffer(tempBuffer);
}
}
public long writeInt(final int value) {
if (remainingInWindow() >= 4) {
final long written = window.writeInt(offset, value);
offset += written;
return written;
} else {
final ByteBuffer tempBuffer = tempBufferLocal.get();
tempBuffer.clear();
tempBuffer.putInt(value);
tempBuffer.flip();
return writeBuffer(tempBuffer);
}
}
}
public MappedBigFile(final String name,
final RandomAccessFile file,
final long maximumChunkSize,
final long windowSize) throws IOException {
this.name = name;
this.file = file;
this.channel = file.getChannel();
this.maximumChunkSize = maximumChunkSize;
this.windowSize = windowSize;
windows = new ArrayList();
fileSize = channel.size();
}
public MappedBigFile(final String name, final RandomAccessFile file) throws IOException {
this(name, file, DEFAULT_MAXIMUM_CHUNK_SIZE, DEFAULT_WINDOW_SIZE);
}
long calculateNewFileSize(final long minimumSize) {
long size = (fileSize == 0) ? INITIAL_FILE_SIZE : fileSize;
while (size < minimumSize) {
size += Math.min(size, maximumChunkSize);
}
return size;
}
public void expandFile(final long minimumSize) {
try {
assert minimumSize >= channel.size();
if (minimumSize < channel.size()) {
return;
}
final long newSize = calculateNewFileSize(minimumSize);
file.setLength(newSize);
fileSize = channel.size();
assert fileSize >= minimumSize;
assert fileSize == newSize;
} catch (final IOException e) {
throw new PlovercrestException("Failed to expand data file for table " + name, e);
}
}
public Reader createReader(final long offset) {
return new Reader(offset);
}
public Writer createWriter(final long offset) {
return new Writer(offset);
}
/**
* Forces any changes made to be written to disk.
*/
public void force() {
synchronized (windows) {
if (closed) {
throw new TableClosedException(name);
}
for (final Window window : windows) {
if (window != null) {
window.force();
}
}
}
}
/**
* Forces changes to be written to disk, then frees resources, allowing mapped regions to be
* garbage collected.
*
* @param flush
*/
public void close(final boolean flush) {
synchronized (windows) {
closed = true;
for (final Window window : windows) {
if (window != null) {
window.close(flush);
}
}
windows.clear();
}
}
private Window windowAtIndex(final int index) {
synchronized (windows) {
if (closed) {
throw new TableClosedException(name);
}
while (windows.size() <= index) {
windows.add(null);
}
if (windows.get(index) == null) {
windows.set(index, new Window(name, channel, index * windowSize, windowSize));
}
return windows.get(index);
}
}
private Window windowAtOffset(final long offset) {
final int index = (int) (offset / windowSize);
return windowAtIndex(index);
}
private Window nextWindow(final Window window) {
return windowAtOffset(window.startOffset() + windowSize);
}
public long getFileSize() {
return fileSize;
}
}