All Downloads are FREE. Search and download functionalities are using the official Maven repository.

io.datakernel.stream.processor.StreamLZ4Compressor Maven / Gradle / Ivy

Go to download

Composable asynchronous/reactive streams with powerful data processing capabilities.

The newest version!
/*
 * Copyright (C) 2015 SoftIndex LLC.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package io.datakernel.stream.processor;

import io.datakernel.bytebuf.ByteBuf;
import io.datakernel.bytebuf.ByteBufPool;
import io.datakernel.eventloop.Eventloop;
import io.datakernel.stream.AbstractStreamTransformer_1_1_Stateless;
import io.datakernel.stream.StreamDataReceiver;
import net.jpountz.lz4.LZ4Compressor;
import net.jpountz.lz4.LZ4Factory;
import net.jpountz.xxhash.StreamingXXHash32;
import net.jpountz.xxhash.XXHashFactory;

public final class StreamLZ4Compressor extends AbstractStreamTransformer_1_1_Stateless implements StreamDataReceiver, StreamLZ4CompressorMBean {
	static final byte[] MAGIC = new byte[]{'L', 'Z', '4', 'B', 'l', 'o', 'c', 'k'};
	static final int MAGIC_LENGTH = MAGIC.length;

	static final int HEADER_LENGTH =
			MAGIC_LENGTH    // magic bytes
					+ 1     // token
					+ 4     // compressed length
					+ 4     // decompressed length
					+ 4;    // checksum

	static final int COMPRESSION_LEVEL_BASE = 10;

	static final int COMPRESSION_METHOD_RAW = 0x10;
	static final int COMPRESSION_METHOD_LZ4 = 0x20;

	static final int DEFAULT_SEED = 0x9747b28c;

	private static final int MIN_BLOCK_SIZE = 64;

	private final LZ4Compressor compressor;
	private final StreamingXXHash32 checksum = XXHashFactory.fastestInstance().newStreamingHash32(DEFAULT_SEED);

	private long jmxBytesInput;
	private long jmxBytesOutput;
	private int jmxBufs;

	/**
	 * Returns new instance of StreamLZ4Compressor without compression.
	 *
	 * @param eventloop event loop in which compressor will run
	 */
	public static StreamLZ4Compressor rawCompressor(Eventloop eventloop) {
		return new StreamLZ4Compressor(eventloop, null);
	}

	/**
	 * Returns new instance of StreamLZ4Compressor with a {@link LZ4Factory#fastCompressor()}.
	 * for data compression.
	 *
	 * @param eventloop event loop in which compressor will run
	 */
	public static StreamLZ4Compressor fastCompressor(Eventloop eventloop) {
		return new StreamLZ4Compressor(eventloop, LZ4Factory.fastestInstance().fastCompressor());
	}

	/**
	 * Returns new instance of StreamLZ4Compressor with a {@link LZ4Factory#highCompressor()}.
	 * for data compression.
	 *
	 * @param eventloop event loop in which compressor will run
	 */
	public static StreamLZ4Compressor highCompressor(Eventloop eventloop) {
		return new StreamLZ4Compressor(eventloop, LZ4Factory.fastestInstance().highCompressor());
	}

	/**
	 * Returns new instance of StreamLZ4Compressor with a {@link LZ4Factory#highCompressor(int)}
	 * for data compression.
	 *
	 * @param eventloop        event loop in which compressor will run
	 * @param compressionLevel compression level in the same manner as the {@link LZ4Factory#highCompressor(int)}
	 */
	public static StreamLZ4Compressor highCompressor(Eventloop eventloop, int compressionLevel) {
		return new StreamLZ4Compressor(eventloop, LZ4Factory.fastestInstance().highCompressor(compressionLevel));
	}

	/**
	 * Returns new instance of this compressor. Large blocks require more memory at compression
	 * and decompression time but should improve the compression ratio.
	 *
	 * @param eventloop  event loop in which compressor will run
	 * @param compressor compressor which will use; can be {@code null} for transmission without compression
	 */
	private StreamLZ4Compressor(Eventloop eventloop, LZ4Compressor compressor) {
		super(eventloop);
		this.compressor = compressor;
	}

	private static int compressionLevel(int blockSize) {
		int compressionLevel = 32 - Integer.numberOfLeadingZeros(blockSize - 1); // ceil of log2
		assert (1 << compressionLevel) >= blockSize;
		assert blockSize * 2 > (1 << compressionLevel);
		compressionLevel = Math.max(0, compressionLevel - COMPRESSION_LEVEL_BASE);
		assert compressionLevel >= 0 && compressionLevel <= 0x0F;
		return compressionLevel;
	}

	private static void writeIntLE(int i, byte[] buf, int off) {
		buf[off++] = (byte) i;
		buf[off++] = (byte) (i >>> 8);
		buf[off++] = (byte) (i >>> 16);
		buf[off] = (byte) (i >>> 24);
	}

	public static ByteBuf compressBlock(LZ4Compressor compressor, StreamingXXHash32 checksum,
	                                    byte[] buffer, int off, int len) {
		int compressionLevel = compressionLevel(len < MIN_BLOCK_SIZE ? MIN_BLOCK_SIZE : len);

		int outputBufMaxSize = HEADER_LENGTH + ((compressor == null) ? len : compressor.maxCompressedLength(len));
		ByteBuf outputBuf = ByteBufPool.allocate(outputBufMaxSize);
		byte[] outputBytes = outputBuf.array();
		System.arraycopy(MAGIC, 0, outputBytes, 0, MAGIC_LENGTH);

		checksum.reset();
		checksum.update(buffer, off, len);
		int check = checksum.getValue();

		int compressedLength = len;
		if (compressor != null) {
			compressedLength = compressor.compress(buffer, off, len, outputBytes, HEADER_LENGTH);
		}

		int compressMethod;
		if (compressor == null || compressedLength >= len) {
			compressMethod = COMPRESSION_METHOD_RAW;
			compressedLength = len;
			System.arraycopy(buffer, off, outputBytes, HEADER_LENGTH, len);
		} else {
			compressMethod = COMPRESSION_METHOD_LZ4;
		}

		outputBytes[MAGIC_LENGTH] = (byte) (compressMethod | compressionLevel);
		writeIntLE(compressedLength, outputBytes, MAGIC_LENGTH + 1);
		writeIntLE(len, outputBytes, MAGIC_LENGTH + 5);
		writeIntLE(check, outputBytes, MAGIC_LENGTH + 9);
		assert MAGIC_LENGTH + 13 == HEADER_LENGTH;

		outputBuf.limit(HEADER_LENGTH + compressedLength);

		return outputBuf;
	}

	public static ByteBuf createEndOfStreamBlock() {
		int compressionLevel = compressionLevel(MIN_BLOCK_SIZE);

		ByteBuf outputBuf = ByteBufPool.allocate(HEADER_LENGTH);
		byte[] outputBytes = outputBuf.array();
		System.arraycopy(MAGIC, 0, outputBytes, 0, MAGIC_LENGTH);

		outputBytes[MAGIC_LENGTH] = (byte) (COMPRESSION_METHOD_RAW | compressionLevel);
		writeIntLE(0, outputBytes, MAGIC_LENGTH + 1);
		writeIntLE(0, outputBytes, MAGIC_LENGTH + 5);
		writeIntLE(0, outputBytes, MAGIC_LENGTH + 9);
		assert MAGIC_LENGTH + 13 == HEADER_LENGTH;

		outputBuf.limit(HEADER_LENGTH);
		return outputBuf;
	}

	@Override
	public void onData(ByteBuf buf) {
		if (status >= END_OF_STREAM)
			return;
		try {
			jmxBufs++;
			jmxBytesInput += buf.remaining();

			ByteBuf outputBuffer = compressBlock(compressor, checksum,
					buf.array(), buf.position(), buf.remaining());
			jmxBytesOutput += outputBuffer.remaining();

			send(outputBuffer);

			buf.recycle();
		} catch (Exception e) {
			onInternalError(e);
		}
	}

	@Override
	public void onEndOfStream() {
		send(createEndOfStreamBlock());
		sendEndOfStream();
	}

	@Override
	public StreamDataReceiver getDataReceiver() {
		return this;
	}

	@Override
	public long getBytesInput() {
		return jmxBytesInput;
	}

	@Override
	public long getBytesOutput() {
		return jmxBytesOutput;
	}

	@Override
	public int getBufs() {
		return jmxBufs;
	}

	@SuppressWarnings("AssertWithSideEffects")
	@Override
	public String toString() {
		return '{' + super.toString() +
				" inBytes:" + jmxBytesInput +
				" outBytes:" + jmxBytesOutput +
				" bufs:" + jmxBufs +
				'}';
	}
}




© 2015 - 2025 Weber Informatics LLC | Privacy Policy