org.apache.hadoop.io.compress.CodecPool Maven / Gradle / Ivy
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in org.apache.hadoop.shaded.com.liance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org.apache.hadoop.shaded.org.licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.shaded.org.apache.hadoop.org.apache.hadoop.shaded.io.org.apache.hadoop.shaded.com.ress;
import java.util.HashSet;
import java.util.HashMap;
import java.util.Set;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.shaded.org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.shaded.org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.shaded.org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.shaded.org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.shaded.org.apache.hadoop.thirdparty.org.apache.hadoop.shaded.com.google.org.apache.hadoop.shaded.com.on.cache.CacheBuilder;
import org.apache.hadoop.shaded.org.apache.hadoop.thirdparty.org.apache.hadoop.shaded.com.google.org.apache.hadoop.shaded.com.on.cache.CacheLoader;
import org.apache.hadoop.shaded.org.apache.hadoop.thirdparty.org.apache.hadoop.shaded.com.google.org.apache.hadoop.shaded.com.on.cache.LoadingCache;
import org.apache.hadoop.shaded.org.slf4j.Logger;
import org.apache.hadoop.shaded.org.slf4j.LoggerFactory;
/**
* A global org.apache.hadoop.shaded.com.ressor/decompressor pool used to save and reuse
* (possibly native) org.apache.hadoop.shaded.com.ression/decompression codecs.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class CodecPool {
private static final Logger LOG = LoggerFactory.getLogger(CodecPool.class);
/**
* A global org.apache.hadoop.shaded.com.ressor pool used to save the expensive
* construction/destruction of (possibly native) decompression codecs.
*/
private static final Map, Set> org.apache.hadoop.shaded.com.ressorPool =
new HashMap, Set>();
/**
* A global decompressor pool used to save the expensive
* construction/destruction of (possibly native) decompression codecs.
*/
private static final Map, Set> decompressorPool =
new HashMap, Set>();
private static LoadingCache, AtomicInteger> createCache(
Class klass) {
return CacheBuilder.newBuilder().build(
new CacheLoader, AtomicInteger>() {
@Override
public AtomicInteger load(Class key) throws Exception {
return new AtomicInteger();
}
});
}
/**
* Map to track the number of leased org.apache.hadoop.shaded.com.ressors
*/
private static final LoadingCache, AtomicInteger> org.apache.hadoop.shaded.com.ressorCounts =
createCache(Compressor.class);
/**
* Map to tracks the number of leased decompressors
*/
private static final LoadingCache, AtomicInteger> decompressorCounts =
createCache(Decompressor.class);
private static T borrow(Map, Set> pool,
Class codecClass) {
T codec = null;
// Check if an appropriate codec is available
Set codecSet;
synchronized (pool) {
codecSet = pool.get(codecClass);
}
if (codecSet != null) {
synchronized (codecSet) {
if (!codecSet.isEmpty()) {
codec = codecSet.iterator().next();
codecSet.remove(codec);
}
}
}
return codec;
}
private static boolean payback(Map, Set> pool, T codec) {
if (codec != null) {
Class codecClass = ReflectionUtils.getClass(codec);
Set codecSet;
synchronized (pool) {
codecSet = pool.get(codecClass);
if (codecSet == null) {
codecSet = new HashSet();
pool.put(codecClass, codecSet);
}
}
synchronized (codecSet) {
return codecSet.add(codec);
}
}
return false;
}
@SuppressWarnings("unchecked")
private static int getLeaseCount(
LoadingCache, AtomicInteger> usageCounts,
Class codecClass) {
return usageCounts.getUnchecked((Class) codecClass).get();
}
private static void updateLeaseCount(
LoadingCache, AtomicInteger> usageCounts, T codec, int delta) {
if (codec != null) {
Class codecClass = ReflectionUtils.getClass(codec);
usageCounts.getUnchecked(codecClass).addAndGet(delta);
}
}
/**
* Get a {@link Compressor} for the given {@link CompressionCodec} from the
* pool or a new one.
*
* @param codec the CompressionCodec for which to get the
* Compressor
* @param conf the Configuration object which contains confs for creating or reinit the org.apache.hadoop.shaded.com.ressor
* @return Compressor for the given
* CompressionCodec from the pool or a new one
*/
public static Compressor getCompressor(CompressionCodec codec, Configuration conf) {
Compressor org.apache.hadoop.shaded.com.ressor = borrow(org.apache.hadoop.shaded.com.ressorPool, codec.getCompressorType());
if (org.apache.hadoop.shaded.com.ressor == null) {
org.apache.hadoop.shaded.com.ressor = codec.createCompressor();
LOG.info("Got brand-new org.apache.hadoop.shaded.com.ressor ["+codec.getDefaultExtension()+"]");
} else {
org.apache.hadoop.shaded.com.ressor.reinit(conf);
if(LOG.isDebugEnabled()) {
LOG.debug("Got recycled org.apache.hadoop.shaded.com.ressor");
}
}
if (org.apache.hadoop.shaded.com.ressor != null &&
!org.apache.hadoop.shaded.com.ressor.getClass().isAnnotationPresent(DoNotPool.class)) {
updateLeaseCount(org.apache.hadoop.shaded.com.ressorCounts, org.apache.hadoop.shaded.com.ressor, 1);
}
return org.apache.hadoop.shaded.com.ressor;
}
public static Compressor getCompressor(CompressionCodec codec) {
return getCompressor(codec, null);
}
/**
* Get a {@link Decompressor} for the given {@link CompressionCodec} from the
* pool or a new one.
*
* @param codec the CompressionCodec for which to get the
* Decompressor
* @return Decompressor for the given
* CompressionCodec the pool or a new one
*/
public static Decompressor getDecompressor(CompressionCodec codec) {
Decompressor decompressor = borrow(decompressorPool, codec.getDecompressorType());
if (decompressor == null) {
decompressor = codec.createDecompressor();
LOG.info("Got brand-new decompressor ["+codec.getDefaultExtension()+"]");
} else {
if(LOG.isDebugEnabled()) {
LOG.debug("Got recycled decompressor");
}
}
if (decompressor != null &&
!decompressor.getClass().isAnnotationPresent(DoNotPool.class)) {
updateLeaseCount(decompressorCounts, decompressor, 1);
}
return decompressor;
}
/**
* Return the {@link Compressor} to the pool.
*
* @param org.apache.hadoop.shaded.com.ressor the Compressor to be returned to the pool
*/
public static void returnCompressor(Compressor org.apache.hadoop.shaded.com.ressor) {
if (org.apache.hadoop.shaded.com.ressor == null) {
return;
}
// if the org.apache.hadoop.shaded.com.ressor can't be reused, don't pool it.
if (org.apache.hadoop.shaded.com.ressor.getClass().isAnnotationPresent(DoNotPool.class)) {
return;
}
org.apache.hadoop.shaded.com.ressor.reset();
if (payback(org.apache.hadoop.shaded.com.ressorPool, org.apache.hadoop.shaded.com.ressor)) {
updateLeaseCount(org.apache.hadoop.shaded.com.ressorCounts, org.apache.hadoop.shaded.com.ressor, -1);
}
}
/**
* Return the {@link Decompressor} to the pool.
*
* @param decompressor the Decompressor to be returned to the
* pool
*/
public static void returnDecompressor(Decompressor decompressor) {
if (decompressor == null) {
return;
}
// if the decompressor can't be reused, don't pool it.
if (decompressor.getClass().isAnnotationPresent(DoNotPool.class)) {
return;
}
decompressor.reset();
if (payback(decompressorPool, decompressor)) {
updateLeaseCount(decompressorCounts, decompressor, -1);
}
}
/**
* Return the number of leased {@link Compressor}s for this
* {@link CompressionCodec}
*/
public static int getLeasedCompressorsCount(CompressionCodec codec) {
return (codec == null) ? 0 : getLeaseCount(org.apache.hadoop.shaded.com.ressorCounts,
codec.getCompressorType());
}
/**
* Return the number of leased {@link Decompressor}s for this
* {@link CompressionCodec}
*/
public static int getLeasedDecompressorsCount(CompressionCodec codec) {
return (codec == null) ? 0 : getLeaseCount(decompressorCounts,
codec.getDecompressorType());
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy