src.it.unimi.dsi.util.ByteBufferLongBigList Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of dsiutils Show documentation
The DSI utilities are a mishmash of classes accumulated during the last twenty years in projects developed at the DSI (Dipartimento di Scienze dell'Informazione, i.e., Information Sciences Department), now DI (Dipartimento di Informatica, i.e., Informatics Department), of the Universita` degli Studi di Milano.
There is a newer version: 2.7.3
Show newest version
package it.unimi.dsi.util;

/*
 * DSI utilities
 *
 * Copyright (C) 2012-2020 Sebastiano Vigna
 *
 *  This library is free software; you can redistribute it and/or modify it
 *  under the terms of the GNU Lesser General Public License as published by the Free
 *  Software Foundation; either version 3 of the License, or (at your option)
 *  any later version.
 *
 *  This library is distributed in the hope that it will be useful, but
 *  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 *  or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
 *  for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public License
 *  along with this program; if not, see .
 *
 */

import java.io.DataOutput;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.util.Arrays;

import it.unimi.dsi.fastutil.longs.AbstractLongBigList;
import it.unimi.dsi.fastutil.longs.LongBigList;
import it.unimi.dsi.lang.FlyweightPrototype;


/** A bridge between byte {@linkplain ByteBuffer buffers} and {@linkplain LongBigList long big lists}.
 *
 * Java's {@linkplain FileChannel#map(MapMode, long, long) memory-mapping facilities} have
 * the severe limitation of mapping at most {@link Integer#MAX_VALUE} bytes, as they
 * expose the content of a file using a {@link MappedByteBuffer}. This class can {@linkplain #map(FileChannel, ByteOrder, FileChannel.MapMode) expose
 * a file of longs of arbitrary length} as a {@linkplain LongBigList}
 * that is actually based on an array of {@link MappedByteBuffer}s, each mapping
 * a chunk of {@link #CHUNK_SIZE} longs.
 *
 * Instances of this class are not thread safe, but the {@link #copy()} method provides a lightweight
 * duplicate that can be accessed independently by another thread.
 * Only chunks that are actually used will be {@linkplain ByteBuffer#duplicate() duplicated} lazily.
 *
 * @author Sebastiano Vigna
 */

public class ByteBufferLongBigList extends AbstractLongBigList implements FlyweightPrototype  {
	private static int CHUNK_SHIFT = 27;

	/** The size in longs of a chunk created by {@link #map(FileChannel, ByteOrder, FileChannel.MapMode)}. */
	public static final long CHUNK_SIZE = 1L << CHUNK_SHIFT;

	/** The mask used to compute the offset in the chunk in longs. */
	private static final long CHUNK_MASK = CHUNK_SIZE - 1;

	/** The underlying byte buffers. */
	private final ByteBuffer[] byteBuffer;

	/** An array parallel to {@link #byteBuffer} specifying which buffers do not need to be
	 * {@linkplain ByteBuffer#duplicate() duplicated} before being used. */
	private final boolean[] readyToUse;

	/** The number of byte buffers. */
	private final int n;

	/** The overall size in longs. */
	private final long size;

	/** Creates a new byte-buffer long big list from a single {@link ByteBuffer}.
	 *
	 * @param byteBuffer the underlying byte buffer.
	 */
	public ByteBufferLongBigList(final ByteBuffer byteBuffer) {
		this(new ByteBuffer[] { byteBuffer }, byteBuffer.capacity() / 8, new boolean[1]);
	}

	/** Creates a new byte-buffer long big list.
	 *
	 * @param byteBuffer the underlying byte buffers.
	 * @param size the overall number of longs in the underlying byte buffers (i.e., the
	 * sum of the {@linkplain ByteBuffer#capacity() capacities} of the byte buffers divided by eight).
	 * @param readyToUse an array parallel to byteBuffer specifying which buffers do not need to be
	 * {@linkplain ByteBuffer#duplicate() duplicated} before being used (the process will happen lazily); the array
	 * will be used internally by the newly created byte-buffer long big list.
	 */

	protected ByteBufferLongBigList(final ByteBuffer[] byteBuffer, final long size, final boolean[] readyToUse) {
		this.byteBuffer = byteBuffer;
		this.n = byteBuffer.length;
		this.size = size;
		this.readyToUse = readyToUse;

		for(int i = 0; i < n; i++) if (i < n - 1 && byteBuffer[i].capacity() / 8 != CHUNK_SIZE) throw new IllegalArgumentException();
	}

	/** Creates a new byte-buffer long big list by read-only mapping a given file channel using the standard Java (i.e., {@link DataOutput}) byte order ({@link ByteOrder#BIG_ENDIAN}).
	 *
	 * @param fileChannel the file channel that will be mapped.
	 * @return a new read-only byte-buffer long big list over the contents of fileChannel.
	 *
	 * @see #map(FileChannel, ByteOrder, MapMode)
	 */

	public static ByteBufferLongBigList map(final FileChannel fileChannel) throws IOException {
		return map(fileChannel, ByteOrder.BIG_ENDIAN);
	}

	/** Creates a new byte-buffer long big list by read-only mapping a given file channel.
	 *
	 * @param fileChannel the file channel that will be mapped.
	 * @param byteOrder a prescribed byte order.
	 * @return a new read-only byte-buffer long big list over the contents of fileChannel.
	 *
	 * @see #map(FileChannel, ByteOrder, MapMode)
	 */

	public static ByteBufferLongBigList map(final FileChannel fileChannel, final ByteOrder byteOrder) throws IOException {
		return map(fileChannel, byteOrder, MapMode.READ_ONLY);
	}

	/** Creates a new byte-buffer long big list by mapping a given file channel.
	 *
	 * @param fileChannel the file channel that will be mapped.
	 * @param byteOrder a prescribed byte order.
	 * @param mapMode the mapping mode: usually {@link MapMode#READ_ONLY}, but if intend to make the list
	 * {@linkplain #set(long, long) mutable}, you have to pass {@link MapMode#READ_WRITE}.
	 * @return a new byte-buffer long big list over the contents of fileChannel.
	 */

	public static ByteBufferLongBigList map(final FileChannel fileChannel, final ByteOrder byteOrder, final MapMode mapMode) throws IOException {
		final long size = fileChannel.size() / 8;
		final int chunks = (int)((size + (CHUNK_SIZE - 1)) / CHUNK_SIZE);
		final ByteBuffer[] byteBuffer = new ByteBuffer[chunks];
		for(int i = 0; i < chunks; i++) byteBuffer[i] = fileChannel.map(mapMode, i * CHUNK_SIZE * 8, Math.min(CHUNK_SIZE, size - i * CHUNK_SIZE) * 8).order(byteOrder);
		final boolean[] readyToUse = new boolean[chunks];
		Arrays.fill(readyToUse, true);
		return new ByteBufferLongBigList(byteBuffer, size, readyToUse);
	}

	private ByteBuffer byteBuffer(final int n) {
		if (readyToUse[n]) return byteBuffer[n];
		readyToUse[n] = true;
		return byteBuffer[n] = byteBuffer[n].duplicate().order(byteBuffer[n].order());
	}

	@Override
	public ByteBufferLongBigList copy() {
		return new ByteBufferLongBigList(byteBuffer.clone(), size, new boolean[n]);
	}

	@Override
	public long getLong(final long index) {
		return byteBuffer((int)(index >>> CHUNK_SHIFT)).getLong((int)(index & CHUNK_MASK) << 3);
	}

	@Override
	public long set(final long index, final long value) {
		final ByteBuffer b = byteBuffer((int)(index >>> CHUNK_SHIFT));
		final int i = (int)(index & CHUNK_MASK) << 3;
		final long previousValue = b.getLong(i);
		b.putLong(i, value);
		return previousValue;
	}

	@Override
	public long size64() {
		return size;
	}
}