com.adobe.internal.io.MemoryMappedByteWriter Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of aem-sdk-api Show documentation
Show all versions of aem-sdk-api Show documentation
The Adobe Experience Manager SDK
/*************************************************************************
*
* File: MemoryMappedByteWriter.java
*
**************************************************************************
*
* ADOBE CONFIDENTIAL
* ___________________
*
* Copyright 2013 Adobe Systems Incorporated
* All Rights Reserved.
*
* NOTICE: All information contained herein is, and remains
* the property of Adobe Systems Incorporated and its suppliers,
* if any. The intellectual and technical concepts contained
* herein are proprietary to Adobe Systems Incorporated and its
* suppliers and are protected by trade secret or copyright law.
* Dissemination of this information or reproduction of this material
* is strictly forbidden unless prior written permission is obtained
* from Adobe Systems Incorporated.
**************************************************************************/
package com.adobe.internal.io;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.MappedByteBuffer;
import java.nio.channels.NonWritableChannelException;
import java.nio.channels.FileChannel.MapMode;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
/**
* An implementation of the ByteWriter interface that provides access to a
* RandomAccessFile based on the memory mapped byte writter of NIO.2 from JDK 7.
* This implementation also uses buffering to limit the number of calls to the methods
* of the {@link java.io.RandomAccessFile RandomAccessFile} class.
* This is because such methods use JNI and any calls to native code from Java are very slow.
*
* This class is not threadsafe. It is not safe to pass an instance of this class
* to multiple threads. It is not safe to pass an instance of this class to multiple users even
* if in the same thread. It is not safe to give the same RandomAccessFile to multiple instances
* of this class.
*/
public final class MemoryMappedByteWriter implements ByteWriter
{
private boolean closed = false;
private long fileLength = -1;
private static final int DEFAULT_BUFFERSIZE = 4096;
private static final int DEFAULT_NUMBERBUFFERS = 4;
private int numberOfBuffers;
private int bufferSize;
private RandomAccessFile file;
private Buffer[] buffers;
private long counter;
private Buffer mru;
// basic performance stats
// to activate uncomment all other blocks marked with "basic performance stats"
// private long singleReadAccess;
// private long bulkReadAccess;
// private long overlapReads;
//
// private long singleWriteAccess;
// private long bulkWriteAccess;
// private long overlapWrites;
private List mmBuffers = new ArrayList();
private long mmBuffersLimit;
private boolean disableMMBuffers;
// We only create a memory mapped file which is greater than the following size.
private static final int MM_THRESHOLD = 400 * 1024;
private static final int MM_RELOAD_CHUNK_SIZE = 256 * 1024;
private static final int MM_SIZE_LIMIT = 64 * 1024 * 1024;
/**
* Class storing the mapped byte buffer, and method which returns current position.
*/
private class MappedBufferReference {
private MappedByteBuffer mappedByteBuffer;
private long base;
private boolean isWritable;
MappedBufferReference(MappedByteBuffer buf, long base, boolean isWritable) {
this.mappedByteBuffer = buf;
this.base = base;
this.isWritable = isWritable;
}
public int getPosition(long position) {
if(this.base <= position && position < (this.base + this.mappedByteBuffer.limit()))
return (int)(position - this.base);
return -1;
}
}
private class Buffer
{
private long base = Long.MAX_VALUE;
private long references;
private boolean isDirty = false;
private int bytesUsed; // number of bytes in the buffer that are used
private byte data[];
Buffer()
{
this.data = new byte[bufferSize];
}
void loadBuffer(long position)
throws IOException
{
// need to align buffer to boundary and then load it
this.base = this.calculateBufferBase(position);
this.references = ++counter;
this.bytesUsed = (int) Math.min(bufferSize, Math.max(fileLength - this.base, 0));
if (this.base >= fileLength)
{
// no need to read if the buffer is beyond the current end of the file
return;
}
long startingBufferPosition = base;
int offset = 0;
int length = this.bytesUsed = (int) Math.min(bufferSize, Math.max(fileLength - this.base, 0));
ArrayList lengthList = new ArrayList();
lengthList.add(length);
lengthList = readFromMappedBuffer(this.base, this.data, startingBufferPosition, offset, lengthList);
length = lengthList.size()==3?0:lengthList.get(0);
// Fall back if mapping is not available
if(length > 0) {
file.seek(startingBufferPosition);
long bytesRead = file.read(this.data, 0, this.bytesUsed);
if ( (bytesRead != this.bytesUsed)
&& !((bytesRead == -1) && (this.bytesUsed == 0)))
{
throw new IOException("Didn't read enough bytes from the file. Expected = " + this.bytesUsed + ", Actual = " + bytesRead);
}
}
}
void flushBuffer()
throws IOException
{
file.seek(this.base);
file.write(this.data, 0, this.bytesUsed);
this.isDirty = false;
}
void resetBuffer()
{
this.base = Long.MAX_VALUE;
}
int buffersRequiredForRequest(long position, int length)
{
long base = calculateBufferBase(position);
long initialBufferSpace = bufferSize - (position - base);
long overage = Math.max(length - initialBufferSpace, 0);
int extraBuffers = (int) Math.ceil(((float) overage) / bufferSize);
return ((length != 0) ? 1 : 0) + extraBuffers;
}
private long calculateBufferBase(long position)
{
return (position / bufferSize) * bufferSize;
}
}
/**
* Create a new RandomAccessFileByteReader with the given
* {@link java.io.RandomAccessFile RandomAccessFile}.
* The {@link java.io.RandomAccessFile RandomAccessFile} given to this ByteWriter belongs
* to it and must not be used after construction of this ByteWriter. It will be closed
* when this ByteWriter is closed.
* @param file the location to read bytes from.
* @param numberOfBuffers the number of buffers to use
* @param bufferSize the size in bytes for the buffers
*/
public MemoryMappedByteWriter(RandomAccessFile file, int numberOfBuffers, int bufferSize)
{
if ((numberOfBuffers < 1) || (bufferSize < 1))
{
throw new IllegalArgumentException("Invalid buffer size or number of buffers.");
}
this.numberOfBuffers = numberOfBuffers;
this.bufferSize = bufferSize;
this.file = file;
this.buffers = new Buffer[this.numberOfBuffers];
for (int i = 0; i < this.numberOfBuffers; i++)
{
this.buffers[i] = new Buffer();
}
this.mru = this.buffers[0];
try {
refreshMappedBuffers(false, MM_THRESHOLD);
} catch (Exception e) {}
}
/**
* Create a new RandomAccessFileByteReader with the given
* {@link java.io.RandomAccessFile RandomAccessFile}.
* {@link java.io.RandomAccessFile RandomAccessFile} given to this ByteReader must not
* be written to during the time it is being used by this or any ByteReader.
* @param file the location to read bytes from.
*/
public MemoryMappedByteWriter(RandomAccessFile file)
{
this(file, DEFAULT_NUMBERBUFFERS, DEFAULT_BUFFERSIZE);
}
/**
* @see com.adobe.internal.io.ByteWriter#write(long, int)
*/
public void write(long position, int b) throws IOException
{
if (this.closed)
{
throw new IOException("ByteReader was closed");
}
// basic performance stats
// this.singleWriteAccess++;
boolean bufFound = false;
if (this.fileLength == -1)
{
this.fileLength = this.file.length();
}
if (position < 0)
{
throw new IOException("Position is less than zero.");
}
// write request is in the most recently used buffer?
if ((position >= this.mru.base) && (position < this.mru.base + this.bufferSize))
{
bufFound = true;
} else {
// is the the write request in any buffer?
for (int bufferIndex = 0; bufferIndex < this.numberOfBuffers; bufferIndex++)
{
Buffer currentBuffer = this.buffers[bufferIndex];
if ( (position >= currentBuffer.base)
&& (position < currentBuffer.base + this.bufferSize))
{
this.mru = currentBuffer;
bufFound = true;
break;
}
}
}
if (!bufFound)
{
try {
byte[] data = new byte[1];
data[0] = (byte) (b & 0xFF);
writeToMappedBuffer(position, data , 0, 0, true);
} catch (Exception e) {
}
// write request isn't in any buffer so load it
this.mru = loadLRU(position);
bufFound = true;
}
this.mru.references = ++this.counter;
this.mru.data[(int) (position - this.mru.base)] = (byte) (b & 0xFF);
this.mru.bytesUsed = (int) Math.max(this.mru.bytesUsed, (position - this.mru.base) + 1);
this.mru.isDirty = true;
this.fileLength = Math.max(this.fileLength, position + 1);
}
/**
* @see com.adobe.internal.io.ByteWriter#write(long, byte[], int, int)
*/
public void write(long position, byte[] b, int offset, int length) throws IOException
{
if (this.closed)
{
throw new IOException("ByteReader was closed");
}
// basic performance stats
// this.bulkWriteAccess++;
boolean bufFound = false;
if (this.fileLength == -1)
{
this.fileLength = this.file.length();
}
if (position < 0)
{
throw new IOException("Position is less than zero.");
}
// check to see if request is within the last used buffer
if ( (position >= this.mru.base)
&& (position + length <= this.mru.base + this.bufferSize))
{
bufFound = true;
} else {
// if it's not then loop over all loaded buffers
for (int bufferIndex = 0; bufferIndex < this.numberOfBuffers; bufferIndex++)
{
Buffer currentBuffer = this.buffers[bufferIndex];
// is start position within a buffer?
if ( (position >= currentBuffer.base)
&& (position + length <= currentBuffer.base + this.bufferSize))
{
this.mru = currentBuffer;
bufFound = true;
break;
}
}
}
// if the request isn't in any of the buffers then load a buffer with that region
if (!bufFound)
{
writeToMappedBuffer(position, b, offset, length, false);
// is the request too big for a buffer?
if ( this.mru.buffersRequiredForRequest(position, length) != 1)
{
// TODO
// If we want to get really smart we can do two things
// 1) check which buffers need flushed (overlap with request)
// 2) split request and put overlapped parts into the buffer
// must flush to avoid stale buffers
this.flush();
file.seek(position);
file.write(b, offset, length);
this.fileLength = this.file.length();
// basic performance stats
// this.overlapWrites++;
return;
} else {
// not too big - load a buffer
this.mru = loadLRU(position);
bufFound = true;
}
}
this.mru.references = ++this.counter;
System.arraycopy(b, offset, this.mru.data, (int) (position - this.mru.base), length);
this.mru.bytesUsed = (int) Math.max(this.mru.bytesUsed, (position - this.mru.base) + length);
this.mru.isDirty = true;
this.fileLength = Math.max(this.fileLength, position + length);
}
/**
* @see com.adobe.internal.io.ByteWriter#length()
*/
public long length() throws IOException
{
if (this.closed)
{
throw new IOException("ByteReader was closed");
}
if(this.fileLength == -1)
{
this.fileLength = this.file.length();
}
return this.fileLength;
}
/**
* @see com.adobe.internal.io.ByteWriter#flush()
*/
public void flush() throws IOException
{
if (this.closed)
{
throw new IOException("ByteReader was closed");
}
for (int bufferIndex = 0; bufferIndex < this.numberOfBuffers; bufferIndex++)
{
if (this.buffers[bufferIndex].isDirty)
{
this.buffers[bufferIndex].flushBuffer();
}
this.buffers[bufferIndex].resetBuffer();
}
}
/**
* @see com.adobe.internal.io.ByteWriter#close()
*/
public void close() throws IOException
{
if (this.closed)
{
return;
}
flush();
this.closed = true;
file.close();
// basic performance stats
// System.out.println("==== RAFByteWriter closing");
// System.out.print("\tSingle Reads = " + this.singleReadAccess);
// System.out.println(", Bulk Reads = " + this.bulkReadAccess);
// if (this.bulkReadAccess != 0 && this.overlapReads != 0)
// {
// float overlapReadPercentage =
// (float) (((this.overlapReads * 10000) / this.bulkReadAccess) / 100.0);
// System.out.println("\tBeyond Buffer Reads = " + this.overlapReads + ", "
// + overlapReadPercentage + "% of bulk reads");
// }
//
// System.out.print("\tSingle Writes = " + this.singleWriteAccess);
// System.out.println(", Bulk Writes = " + this.bulkWriteAccess);
// if (this.bulkWriteAccess != 0 && this.overlapWrites != 0)
// {
// float overlapWritePercentage =
// (float) (((this.overlapWrites * 10000) / this.bulkWriteAccess) / 100.0);
// System.out.println("\tBeyond Buffer Writes = " + this.overlapWrites + ", "
// + overlapWritePercentage + "% of bulk writes");
// }
// System.out.println("");
}
/*** Reader methods **********************************************************/
/**
* @see com.adobe.internal.io.ByteReader#read(long)
*/
public int read(long position)
throws IOException
{
if (this.closed)
{
throw new IOException("ByteReader was closed");
}
// basic performance stats
// this.singleReadAccess++;
boolean bufFound = false;
if (this.fileLength == -1)
{
this.fileLength = file.length();
}
if (position < 0 || position >= this.fileLength)
{
return ByteReader.EOF;
}
// request is in the most recently used buffer?
if ((position >= this.mru.base) && (position < this.mru.base + this.bufferSize))
{
bufFound = true;
} else {
// is the request in any buffer?
for (int bufferIndex = 0; bufferIndex < this.numberOfBuffers; bufferIndex++)
{
Buffer currentBuffer = this.buffers[bufferIndex];
if ( (position >= currentBuffer.base)
&& (position < currentBuffer.base + this.bufferSize))
{
this.mru = currentBuffer;
bufFound = true;
break;
}
}
}
if (!bufFound)
{
// request isn't in any buffer so load it
this.mru = loadLRU(position);
bufFound = true;
}
this.mru.references = ++this.counter;
return this.mru.data[(int) (position - this.mru.base)] & 0xFF;
}
/**
* @see com.adobe.internal.io.ByteReader#read(long, byte[], int, int)
*/
public int read(long position, byte[] b, int offset, int length)
throws IOException
{
if (this.closed)
{
throw new IOException("ByteReader was closed");
}
// basic performance stats
// this.bulkReadAccess++;
boolean bufFound = false;
if (this.fileLength == -1)
{
this.fileLength = this.file.length();
}
if (position < 0 || position >= this.fileLength)
{
return ByteReader.EOF;
}
// limit the length to the length of the file
length = (int) Math.min(length, this.fileLength - position);
// check to see if request is within the last used buffer
if ( (position >= this.mru.base)
&& (position + length <= this.mru.base + this.bufferSize))
{
bufFound = true;
} else {
// if it's not then loop over all loaded buffers
for (int bufferIndex = 0; bufferIndex < this.numberOfBuffers; bufferIndex++)
{
Buffer currentBuffer = this.buffers[bufferIndex];
// is start position within a buffer?
if ( (position >= currentBuffer.base)
&& (position + length <= currentBuffer.base + this.bufferSize))
{
this.mru = currentBuffer;
bufFound = true;
break;
}
}
}
// if the request isn't in any of the buffers then load a buffer with that region
if (!bufFound)
{
// is the request too big for a buffer?
if ( this.mru.buffersRequiredForRequest(position, length) != 1)
{
// TODO
// If we want to get really smart we can do two things
// 1) check which buffers need flushed (overlap with request)
// 2) split request and put overlapped parts into the buffer
// must flush to avoid stale read
this.flush();
int alreadyReadLength = 0;
ArrayList lengthList = new ArrayList();
lengthList.add(length);
lengthList = readFromMappedBuffer(position, b, position, offset, lengthList);
length = lengthList.get(0);
alreadyReadLength = lengthList.get(1);
if(lengthList.size()==3)
return alreadyReadLength + length;
// Fallback if mapping is not available
if(length > 0) {
file.seek(position);
return file.read(b, offset, length);
}
return alreadyReadLength + length;
// basic performance stats
// this.overlapReads++;
} else {
// not too big - load a buffer
this.mru = loadLRU(position);
bufFound = true;
}
}
this.mru.references = ++this.counter;
System.arraycopy(this.mru.data, (int) (position - this.mru.base), b, offset, length);
return length;
}
private Buffer loadLRU(long position)
throws IOException
{
// find the least recently used buffer
Buffer lru = null;
long minCounter = 0x7fffffffffffffffL;
for (int bufferIndex = 0; bufferIndex < this.numberOfBuffers; bufferIndex++)
{
if (this.buffers[bufferIndex].references < minCounter)
{
lru = this.buffers[bufferIndex];
minCounter = lru.references;
}
}
// if it's dirty we need to write it out first
if (lru.isDirty)
{
lru.flushBuffer();
}
lru.loadBuffer(position);
return lru;
}
/**
* Refresh the memory mapped buffer if there is enough unmapped data.
* @param doFlush
* @param threshold
* @throws IOException
*/
private void refreshMappedBuffers(boolean doFlush, int threshold) throws IOException {
if(!disableMMBuffers) {
long extra = length() - mmBuffersLimit;
if(extra > threshold) {
if(doFlush)
flush();
if(mmBuffers.size() > 0) {
MappedBufferReference mbRef = mmBuffers.get(mmBuffers.size() - 1);
int mbRefLimit = mbRef.mappedByteBuffer.limit();
if(mbRefLimit < MM_SIZE_LIMIT - extra) {
mmBuffers.remove(mmBuffers.size() - 1);
mmBuffersLimit -= mbRefLimit;
addMappedBuffer(mbRef.base, mbRefLimit + extra);
} else
addMappedBuffer(mmBuffersLimit, extra);
} else
addMappedBuffer(0, extra);
}
}
}
/**
* Add mapped buffer, in case any exception occurred disable memory mapped buffer.
* @param offset
* @param size
*/
private void addMappedBuffer(long offset, long size) {
for (long i = offset; i < size; i += MM_SIZE_LIMIT) {
long bufSize = (size < i + MM_SIZE_LIMIT) ? size : i + MM_SIZE_LIMIT;
try {
mmBuffers.add(new MappedBufferReference(this.file.getChannel()
.map(MapMode.READ_WRITE, i, bufSize), i, true));
mmBuffersLimit += size;
} catch(NonWritableChannelException ex) {
try {
mmBuffers.add(new MappedBufferReference(this.file.getChannel()
.map(MapMode.READ_ONLY, i, bufSize), i, false));
} catch (IOException e) {
disableMMBuffers = true;
}
mmBuffersLimit += size;
} catch (IOException e) {
// We can still work with the existing buffers
disableMMBuffers = true;
}
}
}
/**
* Write it to the mapped buffer rather than reloading the buffer.
* @param position
* @param data
* @param offset
* @param length
* @param isSingleByte
* @throws IOException
*/
private void writeToMappedBuffer(long position, byte[] data, int offset, int length, boolean isSingleByte) throws IOException {
refreshMappedBuffers(true, MM_RELOAD_CHUNK_SIZE);
if (!disableMMBuffers && position < mmBuffersLimit) {
Iterator iterator = mmBuffers.iterator();
while (iterator.hasNext()) {
MappedBufferReference mbRef = iterator.next();
int bufPosition = mbRef.getPosition(position);
if (mbRef.isWritable && bufPosition > 0) {
mbRef.mappedByteBuffer.position(bufPosition);
if(isSingleByte){
mbRef.mappedByteBuffer.put(data[0]);
}else{
int size = ((bufPosition + length) > mbRef.mappedByteBuffer.limit()) ? (mbRef.mappedByteBuffer.limit()-bufPosition) : length;
mbRef.mappedByteBuffer.put(data, offset, size);
if (size < length) {
position += size;
offset += size;
length -= size;
}
}
return;
}
if (mbRef.base > position)
break;
}
}
}
/**
* Read from the mapped Byte Buffer.
* @param position
* @param data
* @param startingBufferPosition
* @param offset
* @param lengthList
* @return ArrayList
* @throws IOException
*/
private ArrayList readFromMappedBuffer(long position, byte[] data, long startingBufferPosition,
int offset, ArrayList lengthList) throws IOException {
int length = lengthList.get(0);
int alreadyReadLength = 0;
refreshMappedBuffers(false, MM_RELOAD_CHUNK_SIZE);
for (int i = 0; i < mmBuffers.size(); i++) {
MappedBufferReference bufRef = mmBuffers.get(i);
int bufferPosition = bufRef.getPosition(position);
if (bufferPosition >= 0) {
int modifiedLength = Math.min(bufRef.mappedByteBuffer.limit() - bufferPosition,length);
bufRef.mappedByteBuffer.position(bufferPosition);
bufRef.mappedByteBuffer.get(data, offset, modifiedLength);
if (modifiedLength == length){
lengthList.set(0, length);
lengthList.add(1, alreadyReadLength);
lengthList.add(2, alreadyReadLength + length);
return lengthList;
}else {
startingBufferPosition += modifiedLength;
offset += modifiedLength;
length -= modifiedLength;
alreadyReadLength += modifiedLength;
}
}
}
lengthList.set(0, length);
lengthList.add(1, alreadyReadLength);
return lengthList;
}
/**
* @see java.lang.Object#toString()
*/
public String toString()
{
return file.toString();
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy