All Downloads are FREE. Search and download functionalities are using the official Maven repository.

cn.nukkit.nbt.stream.PGZIPOutputStream Maven / Gradle / Ivy

There is a newer version: 1.20.40-r1
Show newest version
package cn.nukkit.nbt.stream;

import cn.nukkit.api.PowerNukkitOnly;
import cn.nukkit.api.Since;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntList;

import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.concurrent.*;
import java.util.zip.CRC32;
import java.util.zip.Deflater;
import java.util.zip.DeflaterOutputStream;

/**
 * A multi-threaded version of {@link java.util.zip.GZIPOutputStream}.
 *
 * @author shevek
 */
public class PGZIPOutputStream extends FilterOutputStream {

    private static final ExecutorService EXECUTOR = Executors.newCachedThreadPool(t -> new Thread(t, "PGZIPOutputStream#EXECUTOR"));

    public static ExecutorService getSharedThreadPool() {
        return EXECUTOR;
    }


    // private static final Logger LOG = LoggerFactory.getLogger(PGZIPOutputStream.class);
    private final static int GZIP_MAGIC = 0x8b1f;

    // todo: remove after block guessing is implemented
    // array list that contains the block sizes
    private IntList blockSizes = new IntArrayList();

    private int level = Deflater.BEST_SPEED;
    private int strategy = Deflater.DEFAULT_STRATEGY;

    protected Deflater newDeflater() {
        Deflater def = new Deflater(level, true);
        def.setStrategy(strategy);
        return def;
    }

    public void setStrategy(int strategy) {
        this.strategy = strategy;
    }

    public void setLevel(int level) {
        this.level = level;
    }

    protected static DeflaterOutputStream newDeflaterOutputStream(OutputStream out, Deflater deflater) {
        return new DeflaterOutputStream(out, deflater, 512, true);
    }

    // TODO: Share, daemonize.
    private final ExecutorService executor;
    private final int nthreads;
    private final CRC32 crc = new CRC32();
    private final BlockingQueue> emitQueue;
    private PGZIPBlock block = new PGZIPBlock(this/* 0 */);
    /**
     * Used as a sentinel for 'closed'.
     */
    private int bytesWritten = 0;

    // Master thread only
    public PGZIPOutputStream(OutputStream out, ExecutorService executor, int nthreads) throws IOException {
        super(out);
        this.executor = executor;
        this.nthreads = nthreads;
        this.emitQueue = new ArrayBlockingQueue>(nthreads);
        writeHeader();
    }

    /**
     * Creates a PGZIPOutputStream
     * using {@link PGZIPOutputStream#getSharedThreadPool()}.
     *
     * @param out the eventual output stream for the compressed data.
     * @throws java.io.IOException if it all goes wrong.
     */
    public PGZIPOutputStream(OutputStream out, int nthreads) throws IOException {
        this(out, PGZIPOutputStream.getSharedThreadPool(), nthreads);
    }

    /**
     * Creates a PGZIPOutputStream
     * using {@link PGZIPOutputStream#getSharedThreadPool()}
     * and {@link Runtime#availableProcessors()}.
     *
     * @param out the eventual output stream for the compressed data.
     * @throws java.io.IOException if it all goes wrong.
     */
    public PGZIPOutputStream(OutputStream out) throws IOException {
        this(out, Runtime.getRuntime().availableProcessors());
    }

    /*
     * @see http://www.gzip.org/zlib/rfc-gzip.html#file-format
     */
    private void writeHeader() throws IOException {
        out.write(new byte[]{
                (byte) GZIP_MAGIC, // ID1: Magic number (little-endian short)
                (byte) (GZIP_MAGIC >> 8), // ID2: Magic number (little-endian short)
                Deflater.DEFLATED, // CM: Compression method
                0, // FLG: Flags (byte)
                0, 0, 0, 0, // MTIME: Modification time (int)
                0, // XFL: Extra flags
                3 // OS: Operating system (3 = Linux)
        });
    }

    // Master thread only
    @Override
    public void write(int b) throws IOException {
        byte[] single = new byte[1];
        single[0] = (byte) (b & 0xFF);
        write(single);
    }

    // Master thread only
    @Override
    public void write(byte[] b) throws IOException {
        write(b, 0, b.length);
    }

    // Master thread only
    @Override
    public void write(byte[] b, int off, int len) throws IOException {
        crc.update(b, off, len);
        bytesWritten += len;
        while (len > 0) {
            // assert block.in_length < block.in.length
            int capacity = block.in.length - block.in_length;
            if (len >= capacity) {
                System.arraycopy(b, off, block.in, block.in_length, capacity);
                block.in_length += capacity;   // == block.in.length
                off += capacity;
                len -= capacity;
                submit();
            } else {
                System.arraycopy(b, off, block.in, block.in_length, len);
                block.in_length += len;
                // off += len;
                // len = 0;
                break;
            }
        }
    }

    // Master thread only
    private void submit() throws IOException {
        emitUntil(nthreads - 1);
        emitQueue.add(executor.submit(block));
        block = new PGZIPBlock(this/* block.index + 1 */);
    }

    // Emit If Available - submit always
    // Emit At Least one - submit when executor is full
    // Emit All Remaining - flush(), close()
    // Master thread only
    private void tryEmit() throws IOException, InterruptedException, ExecutionException {
        for (; ; ) {
            Future future = emitQueue.peek();
            // LOG.info("Peeked future " + future);
            if (future == null)
                return;
            if (!future.isDone())
                return;
            // It's an ordered queue. This MUST be the same element as above.
            emitQueue.remove();
            byte[] toWrite = future.get();
            blockSizes.add(toWrite.length);  // todo: remove after block guessing is implemented
            out.write(toWrite);
        }
    }

    // Master thread only

    /**
     * Emits any opportunistically available blocks. Furthermore, emits blocks until the number of executing tasks is less than taskCountAllowed.
     */
    private void emitUntil(int taskCountAllowed) throws IOException {
        try {
            while (emitQueue.size() > taskCountAllowed) {
                // LOG.info("Waiting for taskCount=" + emitQueue.size() + " -> " + taskCountAllowed);
                Future future = emitQueue.remove(); // Valid because emitQueue.size() > 0
                byte[] toWrite = future.get();  // Blocks until this task is done.
                blockSizes.add(toWrite.length);  // todo: remove after block guessing is implemented
                out.write(toWrite);
            }
            // We may have achieved more opportunistically available blocks
            // while waiting for a block above. Let's emit them here.
            tryEmit();
        } catch (ExecutionException e) {
            throw new IOException(e);
        } catch (InterruptedException e) {
            throw new InterruptedIOException();
        }
    }

    // Master thread only
    @Override
    public void flush() throws IOException {
        // LOG.info("Flush: " + block);
        if (block.in_length > 0)
            submit();
        emitUntil(0);
        super.flush();
    }
    
    @PowerNukkitOnly
    @Since("1.4.0.0-PN")
    public void finish() throws IOException {
        if (bytesWritten >= 0) {
            flush();

            newDeflaterOutputStream(out, newDeflater()).finish();

            ByteBuffer buf = ByteBuffer.allocate(8);
            buf.order(ByteOrder.LITTLE_ENDIAN);
            // LOG.info("CRC is " + crc.getValue());
            buf.putInt((int) crc.getValue());
            buf.putInt(bytesWritten);
            out.write(buf.array()); // allocate() guarantees a backing array.
            // LOG.info("trailer is " + Arrays.toString(buf.array()));
        }
    }

    // Master thread only
    @Override
    public void close() throws IOException {
        // LOG.info("Closing: bytesWritten=" + bytesWritten);
        if (bytesWritten >= 0) {
            finish();

            out.flush();
            out.close();

            bytesWritten = Integer.MIN_VALUE;
            // } else {
            // LOG.warn("Already closed.");
        }
    }
}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy