org.apache.hadoop.io.compress.ZStandardCodec Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of hadoop-common Show documentation
Apache Hadoop Common
There is a newer version: 3.4.1
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.io.compress;

import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.compress.zstd.ZStandardCompressor;
import org.apache.hadoop.io.compress.zstd.ZStandardDecompressor;
import org.apache.hadoop.util.NativeCodeLoader;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

import static org.apache.hadoop.fs.CommonConfigurationKeys.IO_COMPRESSION_CODEC_ZSTD_BUFFER_SIZE_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeys.IO_COMPRESSION_CODEC_ZSTD_BUFFER_SIZE_KEY;

/**
 * This class creates zstd compressors/decompressors.
 */
public class ZStandardCodec implements
    Configurable, CompressionCodec, DirectDecompressionCodec  {
  private Configuration conf;

  /**
   * Set the configuration to be used by this object.
   *
   * @param conf the configuration object.
   */
  @Override
  public void setConf(Configuration conf) {
    this.conf = conf;
  }

  /**
   * Return the configuration used by this object.
   *
   * @return the configuration object used by this object.
   */
  @Override
  public Configuration getConf() {
    return conf;
  }

  public static void checkNativeCodeLoaded() {
    if (!NativeCodeLoader.isNativeCodeLoaded() ||
        !NativeCodeLoader.buildSupportsZstd()) {
      throw new RuntimeException("native zStandard library "
          + "not available: this version of libhadoop was built "
          + "without zstd support.");
    }
    if (!ZStandardCompressor.isNativeCodeLoaded()) {
      throw new RuntimeException("native zStandard library not "
          + "available: ZStandardCompressor has not been loaded.");
    }
    if (!ZStandardDecompressor.isNativeCodeLoaded()) {
      throw new RuntimeException("native zStandard library not "
          + "available: ZStandardDecompressor has not been loaded.");
    }
  }

  public static boolean isNativeCodeLoaded() {
    return ZStandardCompressor.isNativeCodeLoaded()
        && ZStandardDecompressor.isNativeCodeLoaded();
  }

  public static String getLibraryName() {
    return ZStandardCompressor.getLibraryName();
  }

  public static int getCompressionLevel(Configuration conf) {
    return conf.getInt(
        CommonConfigurationKeys.IO_COMPRESSION_CODEC_ZSTD_LEVEL_KEY,
        CommonConfigurationKeys.IO_COMPRESSION_CODEC_ZSTD_LEVEL_DEFAULT);
  }

  public static int getCompressionBufferSize(Configuration conf) {
    int bufferSize = getBufferSize(conf);
    return bufferSize == 0 ?
        ZStandardCompressor.getRecommendedBufferSize() :
        bufferSize;
  }

  public static int getDecompressionBufferSize(Configuration conf) {
    int bufferSize = getBufferSize(conf);
    return bufferSize == 0 ?
        ZStandardDecompressor.getRecommendedBufferSize() :
        bufferSize;
  }

  private static int getBufferSize(Configuration conf) {
    return conf.getInt(IO_COMPRESSION_CODEC_ZSTD_BUFFER_SIZE_KEY,
        IO_COMPRESSION_CODEC_ZSTD_BUFFER_SIZE_DEFAULT);
  }

  /**
   * Create a {@link CompressionOutputStream} that will write to the given
   * {@link OutputStream}.
   *
   * @param out the location for the final output stream
   * @return a stream the user can write uncompressed data to have compressed
   * @throws IOException
   */
  @Override
  public CompressionOutputStream createOutputStream(OutputStream out)
      throws IOException {
    return Util.
        createOutputStreamWithCodecPool(this, conf, out);
  }

  /**
   * Create a {@link CompressionOutputStream} that will write to the given
   * {@link OutputStream} with the given {@link Compressor}.
   *
   * @param out        the location for the final output stream
   * @param compressor compressor to use
   * @return a stream the user can write uncompressed data to have compressed
   * @throws IOException
   */
  @Override
  public CompressionOutputStream createOutputStream(OutputStream out,
      Compressor compressor)
      throws IOException {
    checkNativeCodeLoaded();
    return new CompressorStream(out, compressor,
        getCompressionBufferSize(conf));
  }

  /**
   * Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
   *
   * @return the type of compressor needed by this codec.
   */
  @Override
  public Class getCompressorType() {
    checkNativeCodeLoaded();
    return ZStandardCompressor.class;
  }

  /**
   * Create a new {@link Compressor} for use by this {@link CompressionCodec}.
   *
   * @return a new compressor for use by this codec
   */
  @Override
  public Compressor createCompressor() {
    checkNativeCodeLoaded();
    return new ZStandardCompressor(
        getCompressionLevel(conf), getCompressionBufferSize(conf));
  }


  /**
   * Create a {@link CompressionInputStream} that will read from the given
   * input stream.
   *
   * @param in the stream to read compressed bytes from
   * @return a stream to read uncompressed bytes from
   * @throws IOException
   */
  @Override
  public CompressionInputStream createInputStream(InputStream in)
      throws IOException {
    return Util.
        createInputStreamWithCodecPool(this, conf, in);
  }

  /**
   * Create a {@link CompressionInputStream} that will read from the given
   * {@link InputStream} with the given {@link Decompressor}.
   *
   * @param in           the stream to read compressed bytes from
   * @param decompressor decompressor to use
   * @return a stream to read uncompressed bytes from
   * @throws IOException
   */
  @Override
  public CompressionInputStream createInputStream(InputStream in,
                                                  Decompressor decompressor)
      throws IOException {
    checkNativeCodeLoaded();
    return new DecompressorStream(in, decompressor,
        getDecompressionBufferSize(conf));
  }

  /**
   * Get the type of {@link Decompressor} needed by
   * this {@link CompressionCodec}.
   *
   * @return the type of decompressor needed by this codec.
   */
  @Override
  public Class getDecompressorType() {
    checkNativeCodeLoaded();
    return ZStandardDecompressor.class;
  }

  /**
   * Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
   *
   * @return a new decompressor for use by this codec
   */
  @Override
  public Decompressor createDecompressor() {
    checkNativeCodeLoaded();
    return new ZStandardDecompressor(getDecompressionBufferSize(conf));
  }

  /**
   * Get the default filename extension for this kind of compression.
   *
   * @return .zst.
   */
  @Override
  public String getDefaultExtension() {
    return ".zst";
  }

  @Override
  public DirectDecompressor createDirectDecompressor() {
    return new ZStandardDecompressor.ZStandardDirectDecompressor(
        getDecompressionBufferSize(conf)
    );
  }
}