All Downloads are FREE. Search and download functionalities are using the official Maven repository.
Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.apache.tika.parser.pkg.CompressorParser Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.tika.parser.pkg;
import static org.apache.tika.metadata.HttpHeaders.CONTENT_TYPE;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.commons.compress.MemoryLimitException;
import org.apache.commons.compress.compressors.CompressorException;
import org.apache.commons.compress.compressors.CompressorInputStream;
import org.apache.commons.compress.compressors.CompressorStreamFactory;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.compress.compressors.gzip.GzipUtils;
import org.apache.commons.compress.compressors.lzma.LZMACompressorInputStream;
import org.apache.commons.compress.compressors.pack200.Pack200CompressorInputStream;
import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorInputStream;
import org.apache.commons.compress.compressors.snappy.SnappyCompressorInputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
import org.apache.commons.compress.compressors.z.ZCompressorInputStream;
import org.apache.commons.io.input.CloseShieldInputStream;
import org.apache.tika.config.Field;
import org.apache.tika.exception.TikaException;
import org.apache.tika.exception.TikaMemoryLimitException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.AbstractParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.sax.XHTMLContentHandler;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
/**
* Parser for various compression formats.
*/
public class CompressorParser extends AbstractParser {
/** Serial version UID */
private static final long serialVersionUID = 2793565792967222459L;
private static final MediaType BROTLI = MediaType.application("x-brotli");
private static final MediaType LZ4_BLOCK = MediaType.application("x-lz4-block");
private static final MediaType SNAPPY_RAW = MediaType.application("x-snappy-raw");
private static final MediaType BZIP = MediaType.application("x-bzip");
private static final MediaType BZIP2 = MediaType.application("x-bzip2");
private static final MediaType GZIP = MediaType.application("gzip");
private static final MediaType GZIP_ALT = MediaType.application("x-gzip");
private static final MediaType COMPRESS = MediaType.application("x-compress");
private static final MediaType XZ = MediaType.application("x-xz");
private static final MediaType PACK = MediaType.application("x-java-pack200");
private static final MediaType SNAPPY_FRAMED = MediaType.application("x-snappy");
private static final MediaType ZLIB = MediaType.application("zlib");
private static final MediaType LZMA = MediaType.application("x-lzma");
private static final MediaType LZ4_FRAMED = MediaType.application("x-lz4");
private static final MediaType ZSTD = MediaType.application("zstd");
private static final MediaType DEFLATE64= MediaType.application("deflate64");
private static Set SUPPORTED_TYPES;
private static Map MIMES_TO_NAME;
static {
Set TMP_SET = new HashSet<>();
TMP_SET.addAll(
MediaType.set(BZIP, BZIP2, DEFLATE64, GZIP, GZIP_ALT, LZ4_FRAMED, COMPRESS,
XZ, PACK, SNAPPY_FRAMED, ZLIB, LZMA));
try {
Class.forName("org.brotli.dec.BrotliInputStream");
TMP_SET.add(BROTLI);
} catch (NoClassDefFoundError|ClassNotFoundException e) {
//swallow
}
try {
Class.forName("com.github.luben.zstd.ZstdInputStream");
TMP_SET.add(ZSTD);
} catch (NoClassDefFoundError|ClassNotFoundException e) {
//swallow
}
SUPPORTED_TYPES = Collections.unmodifiableSet(TMP_SET);
}
static {
//map the mime type strings to the compressor stream names
Map tmpMimesToName = new HashMap<>();
tmpMimesToName.put(BZIP2.toString(), CompressorStreamFactory.BZIP2);
tmpMimesToName.put(GZIP.toString(), CompressorStreamFactory.GZIP);
tmpMimesToName.put(LZ4_FRAMED.toString(), CompressorStreamFactory.LZ4_FRAMED);
tmpMimesToName.put(LZ4_BLOCK.toString(), CompressorStreamFactory.LZ4_BLOCK);
tmpMimesToName.put(XZ.toString(), CompressorStreamFactory.XZ);
tmpMimesToName.put(PACK.toString(), CompressorStreamFactory.PACK200);
tmpMimesToName.put(SNAPPY_FRAMED.toString(), CompressorStreamFactory.SNAPPY_FRAMED);
tmpMimesToName.put(ZLIB.toString(), CompressorStreamFactory.DEFLATE);
tmpMimesToName.put(COMPRESS.toString(), CompressorStreamFactory.Z);
tmpMimesToName.put(LZMA.toString(), CompressorStreamFactory.LZMA);
tmpMimesToName.put(BROTLI.toString(), CompressorStreamFactory.BROTLI);
tmpMimesToName.put(ZSTD.toString(), CompressorStreamFactory.ZSTANDARD);
MIMES_TO_NAME = Collections.unmodifiableMap(tmpMimesToName);
}
private int memoryLimitInKb = 100000;//100MB
/**
*
* @deprecated use {@link #getMediaType(String)}
* @param stream stream
* @return MediaType
*/
@Deprecated
static MediaType getMediaType(CompressorInputStream stream) {
// TODO Add support for the remaining CompressorInputStream formats:
// LZ4
// LZWInputStream -> UnshrinkingInputStream
if (stream instanceof BZip2CompressorInputStream) {
return BZIP2;
} else if (stream instanceof GzipCompressorInputStream) {
return GZIP;
} else if (stream instanceof XZCompressorInputStream) {
return XZ;
} else if (stream instanceof DeflateCompressorInputStream) {
return ZLIB;
} else if (stream instanceof ZCompressorInputStream) {
return COMPRESS;
} else if (stream instanceof Pack200CompressorInputStream) {
return PACK;
} else if (stream instanceof FramedSnappyCompressorInputStream ||
stream instanceof SnappyCompressorInputStream) {
// TODO Add unit tests for this format
return SNAPPY_FRAMED;
} else if (stream instanceof LZMACompressorInputStream) {
return LZMA;
} else {
return MediaType.OCTET_STREAM;
}
}
static MediaType getMediaType(String name) {
if (CompressorStreamFactory.BROTLI.equals(name)) {
return BROTLI;
} else if (CompressorStreamFactory.LZ4_BLOCK.equals(name)) {
return LZ4_BLOCK;
} else if (CompressorStreamFactory.LZ4_FRAMED.equals(name)) {
return LZ4_FRAMED;
} else if (CompressorStreamFactory.BZIP2.equals(name)) {
return BZIP2;
} else if (CompressorStreamFactory.GZIP.equals(name)) {
return GZIP;
} else if (CompressorStreamFactory.XZ.equals(name)) {
return XZ;
} else if (CompressorStreamFactory.DEFLATE.equals(name)) {
return ZLIB;
} else if (CompressorStreamFactory.Z.equals(name)) {
return COMPRESS;
} else if (CompressorStreamFactory.PACK200.equals(name)) {
return PACK;
} else if (CompressorStreamFactory.SNAPPY_FRAMED.equals(name)) {
return SNAPPY_FRAMED;
} else if (CompressorStreamFactory.SNAPPY_RAW.equals(name)) {
return SNAPPY_RAW;
} else if (CompressorStreamFactory.LZMA.equals(name)) {
return LZMA;
} else if (CompressorStreamFactory.ZSTANDARD.equals(name)) {
return ZSTD;
} else if (CompressorStreamFactory.DEFLATE64.equals(name)) {
return DEFLATE64;
} else {
return MediaType.OCTET_STREAM;
}
}
public Set getSupportedTypes(ParseContext context) {
return SUPPORTED_TYPES;
}
public void parse(
InputStream stream, ContentHandler handler,
Metadata metadata, ParseContext context)
throws IOException, SAXException, TikaException {
// At the end we want to close the compression stream to release
// any associated resources, but the underlying document stream
// should not be closed
if (stream.markSupported()) {
stream = new CloseShieldInputStream(stream);
} else {
// Ensure that the stream supports the mark feature
stream = new BufferedInputStream(new CloseShieldInputStream(stream));
}
CompressorInputStream cis;
try {
CompressorParserOptions options =
context.get(CompressorParserOptions.class, new CompressorParserOptions() {
public boolean decompressConcatenated(Metadata metadata) {
return false;
}
});
CompressorStreamFactory factory =
new CompressorStreamFactory(options.decompressConcatenated(metadata), memoryLimitInKb);
//if we've already identified it via autodetect
//trust that and go with the appropriate name
//to avoid calling CompressorStreamFactory.detect() twice
String name = getStreamName(metadata);
if (name != null) {
cis = factory.createCompressorInputStream(name, stream);
} else {
cis = factory.createCompressorInputStream(stream);
MediaType type = getMediaType(cis);
if (!type.equals(MediaType.OCTET_STREAM)) {
metadata.set(CONTENT_TYPE, type.toString());
}
}
} catch (CompressorException e) {
if (e.getCause() != null && e.getCause() instanceof MemoryLimitException) {
throw new TikaMemoryLimitException(e.getMessage());
}
throw new TikaException("Unable to uncompress document stream", e);
}
XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
xhtml.startDocument();
try {
Metadata entrydata = new Metadata();
String name = metadata.get(Metadata.RESOURCE_NAME_KEY);
if (name != null) {
if (name.endsWith(".tbz")) {
name = name.substring(0, name.length() - 4) + ".tar";
} else if (name.endsWith(".tbz2")) {
name = name.substring(0, name.length() - 5) + ".tar";
} else if (name.endsWith(".bz")) {
name = name.substring(0, name.length() - 3);
} else if (name.endsWith(".bz2")) {
name = name.substring(0, name.length() - 4);
} else if (name.endsWith(".xz")) {
name = name.substring(0, name.length() - 3);
} else if (name.endsWith(".zlib")) {
name = name.substring(0, name.length() - 5);
} else if (name.endsWith(".pack")) {
name = name.substring(0, name.length() - 5);
} else if (name.endsWith(".br")) {
name = name.substring(0, name.length() - 3);
} else if (name.length() > 0) {
name = GzipUtils.getUncompressedFilename(name);
}
entrydata.set(Metadata.RESOURCE_NAME_KEY, name);
}
// Use the delegate parser to parse the compressed document
EmbeddedDocumentExtractor extractor =
EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context);
if (extractor.shouldParseEmbedded(entrydata)) {
extractor.parseEmbedded(cis, xhtml, entrydata, true);
}
} finally {
cis.close();
}
xhtml.endDocument();
}
/**
* @param metadata
* @return CompressorStream name based on the content-type value
* in metadata or null if not found
* ind
*/
private String getStreamName(Metadata metadata) {
String mimeString = metadata.get(Metadata.CONTENT_TYPE);
if (mimeString == null) {
return null;
}
return MIMES_TO_NAME.get(mimeString);
}
@Field
public void setMemoryLimitInKb(int memoryLimitInKb) {
this.memoryLimitInKb = memoryLimitInKb;
}
}
