org.apache.tika.cli.TikaCLI Maven / Gradle / Ivy
The newest version!
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.tika.cli;
import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.Serializable;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.lang.reflect.Field;
import java.net.URI;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeSet;
import java.util.UUID;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.sax.SAXTransformerFactory;
import javax.xml.transform.sax.TransformerHandler;
import javax.xml.transform.stream.StreamResult;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.CloseShieldInputStream;
import org.apache.logging.log4j.Level;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
import org.apache.tika.Tika;
import org.apache.tika.async.cli.TikaAsyncCLI;
import org.apache.tika.batch.BatchProcessDriverCLI;
import org.apache.tika.config.TikaConfig;
import org.apache.tika.config.TikaConfigSerializer;
import org.apache.tika.detect.CompositeDetector;
import org.apache.tika.detect.Detector;
import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.DefaultEmbeddedStreamTranslator;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedStreamTranslator;
import org.apache.tika.fork.ForkParser;
import org.apache.tika.gui.TikaGUI;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.language.detect.LanguageHandler;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.mime.MediaTypeRegistry;
import org.apache.tika.mime.MimeType;
import org.apache.tika.mime.MimeTypeException;
import org.apache.tika.mime.MimeTypes;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.CompositeParser;
import org.apache.tika.parser.DigestingParser;
import org.apache.tika.parser.NetworkParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.ParserDecorator;
import org.apache.tika.parser.PasswordProvider;
import org.apache.tika.parser.RecursiveParserWrapper;
import org.apache.tika.parser.digestutils.CommonsDigester;
import org.apache.tika.parser.pdf.PDFParserConfig;
import org.apache.tika.sax.BasicContentHandlerFactory;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.sax.ContentHandlerFactory;
import org.apache.tika.sax.ExpandedTitleContentHandler;
import org.apache.tika.sax.RecursiveParserWrapperHandler;
import org.apache.tika.sax.WriteOutContentHandler;
import org.apache.tika.sax.boilerpipe.BoilerpipeContentHandler;
import org.apache.tika.serialization.JsonMetadata;
import org.apache.tika.serialization.JsonMetadataList;
import org.apache.tika.xmp.XMPMetadata;
/**
* Simple command line interface for Apache Tika.
*/
public class TikaCLI {
private static final Logger LOG = LoggerFactory.getLogger(TikaCLI.class);
private final int MAX_MARK = 20 * 1024 * 1024;//20MB
private final OutputType NO_OUTPUT = new OutputType() {
@Override
protected ContentHandler getContentHandler(OutputStream output, Metadata metadata) {
return new DefaultHandler();
}
};
private File extractDir = new File(".");
private ParseContext context;
private Detector detector;
private Parser parser;
private TikaConfig config;
private String configFilePath;
private boolean recursiveJSON = false;
private URI networkURI = null;
/**
* Output character encoding, or null for platform default
*/
private String encoding = null;
private final OutputType TEXT = new OutputType() {
@Override
protected ContentHandler getContentHandler(OutputStream output, Metadata metadata) throws Exception {
return new BodyContentHandler(getOutputWriter(output, encoding));
}
};
private final OutputType TEXT_MAIN = new OutputType() {
@Override
protected ContentHandler getContentHandler(OutputStream output, Metadata metadata) throws Exception {
return new BoilerpipeContentHandler(getOutputWriter(output, encoding));
}
};
private final OutputType TEXT_ALL = new OutputType() {
@Override
protected ContentHandler getContentHandler(OutputStream output, Metadata metadata) throws Exception {
return new WriteOutContentHandler(getOutputWriter(output, encoding));
}
};
private final OutputType METADATA = new OutputType() {
@Override
protected ContentHandler getContentHandler(OutputStream output, Metadata metadata) throws Exception {
final PrintWriter writer = new PrintWriter(getOutputWriter(output, encoding));
return new NoDocumentMetHandler(metadata, writer);
}
};
private final OutputType JSON = new OutputType() {
@Override
protected ContentHandler getContentHandler(OutputStream output, Metadata metadata) throws Exception {
final PrintWriter writer = new PrintWriter(getOutputWriter(output, encoding));
return new NoDocumentJSONMetHandler(metadata, writer);
}
};
private final OutputType XMP = new OutputType() {
@Override
protected ContentHandler getContentHandler(OutputStream output, final Metadata metadata) throws Exception {
final PrintWriter writer = new PrintWriter(getOutputWriter(output, encoding));
return new NoDocumentXMPMetaHandler(metadata, writer);
}
};
private final OutputType LANGUAGE = new OutputType() {
@Override
protected ContentHandler getContentHandler(OutputStream output, Metadata metadata) throws Exception {
final PrintWriter writer = new PrintWriter(getOutputWriter(output, encoding));
return new LanguageHandler() {
public void endDocument() {
writer.println(getLanguage().getLanguage());
writer.flush();
}
};
}
};
private final OutputType DETECT = new OutputType() {
@Override
public void process(InputStream stream, OutputStream output, Metadata metadata) throws Exception {
PrintWriter writer = new PrintWriter(getOutputWriter(output, encoding));
writer.println(detector
.detect(stream, metadata)
.toString());
writer.flush();
}
};
/**
* Password for opening encrypted documents, or null.
*/
private String password = System.getenv("TIKA_PASSWORD");
private DigestingParser.Digester digester = null;
private boolean asyncMode = false;
private boolean pipeMode = true;
private boolean fork = false;
private boolean prettyPrint;
private final OutputType XML = new OutputType() {
@Override
protected ContentHandler getContentHandler(OutputStream output, Metadata metadata) throws Exception {
return getTransformerHandler(output, "xml", encoding, prettyPrint);
}
};
private OutputType type = XML;
private final OutputType HTML = new OutputType() {
@Override
protected ContentHandler getContentHandler(OutputStream output, Metadata metadata) throws Exception {
return new ExpandedTitleContentHandler(getTransformerHandler(output, "html", encoding, prettyPrint));
}
};
public TikaCLI() {
context = new ParseContext();
}
public static void main(String[] args) throws Exception {
TikaCLI cli = new TikaCLI();
if (cli.testForHelp(args)) {
cli.usage();
return;
} else if (cli.testForBatch(args)) {
String[] batchArgs = BatchCommandLineBuilder.build(args);
BatchProcessDriverCLI batchDriver = new BatchProcessDriverCLI(batchArgs);
batchDriver.execute();
return;
} else if (cli.testForAsync(args)) {
async(args);
return;
}
if (args.length > 0) {
for (String arg : args) {
cli.process(arg);
}
if (cli.pipeMode) {
cli.process("-");
}
} else {
// Started with no arguments. Wait for up to 0.1s to see if
// we have something waiting in standard input and use the
// pipe mode if we have. If no input is seen, start the GUI.
if (System.in.available() == 0) {
Thread.sleep(100);
}
if (System.in.available() > 0) {
cli.process("-");
} else {
cli.process("--gui");
}
}
}
private static void async(String[] args) throws Exception {
String tikaConfigPath = "";
String config = "--config=";
for (String arg : args) {
if (arg.startsWith(config)) {
tikaConfigPath = arg.substring(config.length());
}
}
TikaAsyncCLI.main(new String[]{tikaConfigPath});
}
/**
* Returns a output writer with the given encoding.
*
* @param output output stream
* @param encoding output encoding,
* or null for the platform default
* @return output writer
* @throws UnsupportedEncodingException if the given encoding is not supported
* @see TIKA-277
*/
private static Writer getOutputWriter(OutputStream output, String encoding) throws UnsupportedEncodingException {
if (encoding != null) {
return new OutputStreamWriter(output, encoding);
} else {
return new OutputStreamWriter(output, UTF_8);
}
}
/**
* Returns a transformer handler that serializes incoming SAX events
* to XHTML or HTML (depending the given method) using the given output
* encoding.
*
* @param output output stream
* @param method "xml" or "html"
* @param encoding output encoding,
* or null for the platform default
* @return {@link System#out} transformer handler
* @throws TransformerConfigurationException if the transformer can not be created
* @see TIKA-277
*/
private static TransformerHandler getTransformerHandler(OutputStream output, String method, String encoding, boolean prettyPrint) throws TransformerConfigurationException {
SAXTransformerFactory factory = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
TransformerHandler handler = factory.newTransformerHandler();
handler
.getTransformer()
.setOutputProperty(OutputKeys.METHOD, method);
handler
.getTransformer()
.setOutputProperty(OutputKeys.INDENT, prettyPrint ? "yes" : "no");
if (encoding != null) {
handler
.getTransformer()
.setOutputProperty(OutputKeys.ENCODING, encoding);
}
handler.setResult(new StreamResult(output));
return handler;
}
private boolean testForAsync(String[] args) {
for (String arg : args) {
if (arg.equals("-a") || arg.equals("--async")) {
return true;
}
}
return false;
}
private void configurePDFExtractSettings() {
if (configFilePath == null && context.get(PDFParserConfig.class) == null) {
PDFParserConfig pdfParserConfig = new PDFParserConfig();
pdfParserConfig.setExtractInlineImages(true);
pdfParserConfig.setParseIncrementalUpdates(true);
String warn = "As a convenience, TikaCLI has turned on extraction of\n" + "inline images and incremental updates for the PDFParser (TIKA-2374 and " + "TIKA-4017).\n" +
"Aside from the -z option, this is not the default behavior\n" + "in Tika generally or in tika-server.";
LOG.info(warn);
context.set(PDFParserConfig.class, pdfParserConfig);
}
}
public void process(String arg) throws Exception {
if (arg.equals("-?") || arg.equals("--help")) {
pipeMode = false;
usage();
} else if (arg.equals("-V") || arg.equals("--version")) {
pipeMode = false;
version();
} else if (arg.equals("-v") || arg.equals("--verbose")) {
org.apache.logging.log4j.core.config.Configurator.setRootLevel(Level.DEBUG);
} else if (arg.equals("-g") || arg.equals("--gui")) {
pipeMode = false;
if (configFilePath != null) {
TikaGUI.main(new String[]{configFilePath});
} else {
TikaGUI.main(new String[0]);
}
} else if (arg.equals("--list-parser") || arg.equals("--list-parsers")) {
pipeMode = false;
displayParsers(false, false);
} else if (arg.equals("--list-detector") || arg.equals("--list-detectors")) {
pipeMode = false;
displayDetectors();
} else if (arg.equals("--list-parser-detail") || arg.equals("--list-parser-details")) {
pipeMode = false;
displayParsers(true, false);
} else if (arg.equals("--list-parser-detail-apt") || arg.equals("--list-parser-details-apt")) {
pipeMode = false;
displayParsers(true, true);
} else if (arg.equals("--list-met-models")) {
pipeMode = false;
displayMetModels();
} else if (arg.equals("--list-supported-types")) {
pipeMode = false;
displaySupportedTypes();
} else if (arg.startsWith("--compare-file-magic=")) {
pipeMode = false;
compareFileMagic(arg.substring("--compare-file-magic=".length()));
} else if (arg.equals("--dump-minimal-config")) {
pipeMode = false;
dumpConfig(TikaConfigSerializer.Mode.MINIMAL);
} else if (arg.equals("--dump-current-config")) {
pipeMode = false;
dumpConfig(TikaConfigSerializer.Mode.CURRENT);
} else if (arg.equals("--dump-static-config")) {
pipeMode = false;
dumpConfig(TikaConfigSerializer.Mode.STATIC);
} else if (arg.equals("--dump-static-full-config")) {
pipeMode = false;
dumpConfig(TikaConfigSerializer.Mode.STATIC_FULL);
} else if (arg.equals("--container-aware") || arg.equals("--container-aware-detector")) {
// ignore, as container-aware detectors are now always used
} else if (arg.equals("-f") || arg.equals("--fork")) {
fork = true;
} else if (arg.equals("-a") || arg.equals("--async")) {
asyncMode = true;
} else if (arg.startsWith("--config=")) {
configFilePath = arg.substring("--config=".length());
} else if (arg.startsWith("--digest=")) {
digester = new CommonsDigester(MAX_MARK, arg.substring("--digest=".length()));
} else if (arg.startsWith("-e")) {
encoding = arg.substring("-e".length());
} else if (arg.startsWith("--encoding=")) {
encoding = arg.substring("--encoding=".length());
} else if (arg.startsWith("-p") && !arg.equals("-p")) {
password = arg.substring("-p".length());
} else if (arg.startsWith("--password=")) {
password = arg.substring("--password=".length());
} else if (arg.equals("-j") || arg.equals("--json")) {
type = JSON;
} else if (arg.equals("-J") || arg.equals("--jsonRecursive")) {
recursiveJSON = true;
} else if (arg.equals("-y") || arg.equals("--xmp")) {
type = XMP;
} else if (arg.equals("-x") || arg.equals("--xml")) {
type = XML;
} else if (arg.equals("-h") || arg.equals("--html")) {
type = HTML;
} else if (arg.equals("-t") || arg.equals("--text")) {
type = TEXT;
} else if (arg.equals("-T") || arg.equals("--text-main")) {
type = TEXT_MAIN;
} else if (arg.equals("-A") || arg.equals("--text-all")) {
type = TEXT_ALL;
} else if (arg.equals("-m") || arg.equals("--metadata")) {
type = METADATA;
} else if (arg.equals("-l") || arg.equals("--language")) {
type = LANGUAGE;
} else if (arg.equals("-d") || arg.equals("--detect")) {
type = DETECT;
} else if (arg.startsWith("--extract-dir=")) {
String dirPath = arg.substring("--extract-dir=".length());
//if the user accidentally doesn't include
//a directory, set the directory to the cwd
if (dirPath.length() == 0) {
dirPath = ".";
}
extractDir = new File(dirPath);
} else if (arg.equals("-z") || arg.equals("--extract")) {
configurePDFExtractSettings();
type = NO_OUTPUT;
context.set(EmbeddedDocumentExtractor.class, new FileEmbeddedDocumentExtractor());
} else if (arg.equals("-r") || arg.equals("--pretty-print")) {
prettyPrint = true;
} else if (arg.equals("-p") || arg.equals("--port") || arg.equals("-s") || arg.equals("--server")) {
throw new IllegalArgumentException("As of Tika 2.0, the server option is no longer supported in tika-app.\n" + "See https://wiki.apache.org/tika/TikaJAXRS for usage.");
} else if (arg.startsWith("-c")) {
networkURI = new URI(arg.substring("-c".length()));
} else if (arg.startsWith("--client=")) {
networkURI = new URI(arg.substring("--client=".length()));
} else {
pipeMode = false;
configure();
if (arg.equals("-")) {
try (InputStream stream = TikaInputStream.get(CloseShieldInputStream.wrap(System.in))) {
type.process(stream, System.out, new Metadata());
}
} else {
URL url;
File file = new File(arg);
if (file.isFile()) {
url = file
.toURI()
.toURL();
} else {
url = new URL(arg);
}
if (recursiveJSON) {
handleRecursiveJson(url, System.out);
} else {
Metadata metadata = new Metadata();
try (InputStream input = TikaInputStream.get(url, metadata)) {
type.process(input, System.out, metadata);
} finally {
System.out.flush();
}
}
}
}
}
private void dumpConfig(TikaConfigSerializer.Mode mode) throws Exception {
configure();
TikaConfig localConfig = (config == null) ? TikaConfig.getDefaultConfig() : config;
TikaConfigSerializer.serialize(localConfig, mode, new OutputStreamWriter(System.out, UTF_8), UTF_8);
}
private void handleRecursiveJson(URL url, OutputStream output) throws IOException, SAXException, TikaException {
Metadata metadata = new Metadata();
RecursiveParserWrapper wrapper = new RecursiveParserWrapper(parser);
RecursiveParserWrapperHandler handler = new RecursiveParserWrapperHandler(getContentHandlerFactory(type), -1, config.getMetadataFilter());
try (InputStream input = TikaInputStream.get(url, metadata)) {
wrapper.parse(input, handler, metadata, context);
}
JsonMetadataList.setPrettyPrinting(prettyPrint);
try (Writer writer = getOutputWriter(output, encoding)) {
JsonMetadataList.toJson(handler.getMetadataList(), writer);
}
}
private ContentHandlerFactory getContentHandlerFactory(OutputType type) {
BasicContentHandlerFactory.HANDLER_TYPE handlerType = BasicContentHandlerFactory.HANDLER_TYPE.IGNORE;
if (type.equals(HTML)) {
handlerType = BasicContentHandlerFactory.HANDLER_TYPE.HTML;
} else if (type.equals(XML)) {
handlerType = BasicContentHandlerFactory.HANDLER_TYPE.XML;
} else if (type.equals(TEXT)) {
handlerType = BasicContentHandlerFactory.HANDLER_TYPE.TEXT;
} else if (type.equals(TEXT_MAIN)) {
handlerType = BasicContentHandlerFactory.HANDLER_TYPE.BODY;
} else if (type.equals(METADATA)) {
handlerType = BasicContentHandlerFactory.HANDLER_TYPE.IGNORE;
}
return new BasicContentHandlerFactory(handlerType, -1);
}
private void usage() {
PrintStream out = System.out;
out.println("usage: java -jar tika-app.jar [option...] [file...]");
out.println();
out.println("Options:");
out.println(" -? or --help Print this usage message");
out.println(" -v or --verbose Print debug level messages");
out.println(" -V or --version Print the Apache Tika version number");
out.println();
out.println(" -g or --gui Start the Apache Tika GUI");
out.println(" -f or --fork Use Fork Mode for out-of-process extraction");
out.println();
out.println(" --config=");
out.println(" TikaConfig file. Must be specified before -g, -s, -f or the dump-x-config !");
out.println(" --dump-minimal-config Print minimal TikaConfig");
out.println(" --dump-current-config Print current TikaConfig");
out.println(" --dump-static-config Print static config");
out.println(" --dump-static-full-config Print static explicit config");
out.println("");
out.println(" -x or --xml Output XHTML content (default)");
out.println(" -h or --html Output HTML content");
out.println(" -t or --text Output plain text content (body)");
out.println(" -T or --text-main Output plain text content (main content only via boilerpipe handler)");
out.println(" -A or --text-all Output all text content");
out.println(" -m or --metadata Output only metadata");
out.println(" -j or --json Output metadata in JSON");
out.println(" -y or --xmp Output metadata in XMP");
out.println(" -J or --jsonRecursive Output metadata and content from all");
out.println(" embedded files (choose content type");
out.println(" with -x, -h, -t or -m; default is -x)");
out.println(" -a or --async Run Tika in async mode; must specify details in a" + " tikaConfig file");
out.println(" -l or --language Output only language");
out.println(" -d or --detect Detect document type");
out.println(" --digest=X Include digest X (md2, md5, sha1,");
out.println(" sha256, sha384, sha512");
out.println(" -eX or --encoding=X Use output encoding X");
out.println(" -pX or --password=X Use document password X");
out.println(" -z or --extract Extract all attachements into current directory");
out.println(" --extract-dir= Specify target directory for -z");
out.println(" -r or --pretty-print For JSON, XML and XHTML outputs, adds newlines and");
out.println(" whitespace, for better readability");
out.println();
out.println(" --list-parsers");
out.println(" List the available document parsers");
out.println(" --list-parser-details");
out.println(" List the available document parsers and their supported mime types");
out.println(" --list-parser-details-apt");
out.println(" List the available document parsers and their supported mime types in apt format.");
out.println(" --list-detectors");
out.println(" List the available document detectors");
out.println(" --list-met-models");
out.println(" List the available metadata models, and their supported keys");
out.println(" --list-supported-types");
out.println(" List all known media types and related information");
out.println();
out.println();
out.println(" --compare-file-magic=");
out.println(" Compares Tika's known media types to the File(1) tool's magic directory");
out.println("Description:");
out.println(" Apache Tika will parse the file(s) specified on the");
out.println(" command line and output the extracted text content");
out.println(" or metadata to standard output.");
out.println();
out.println(" Instead of a file name you can also specify the URL");
out.println(" of a document to be parsed.");
out.println();
out.println(" If no file name or URL is specified (or the special");
out.println(" name \"-\" is used), then the standard input stream");
out.println(" is parsed. If no arguments were given and no input");
out.println(" data is available, the GUI is started instead.");
out.println();
out.println("- GUI mode");
out.println();
out.println(" Use the \"--gui\" (or \"-g\") option to start the");
out.println(" Apache Tika GUI. You can drag and drop files from");
out.println(" a normal file explorer to the GUI window to extract");
out.println(" text content and metadata from the files.");
out.println();
out.println("- Batch mode");
out.println();
out.println(" Simplest method.");
out.println(" Specify two directories as args with no other args:");
out.println(" java -jar tika-app.jar ");
out.println();
out.println("Batch Options:");
out.println(" -i or --inputDir Input directory");
out.println(" -o or --outputDir Output directory");
out.println(" -numConsumers Number of processing threads");
out.println(" -bc Batch config file");
out.println(" -maxRestarts Maximum number of times the ");
out.println(" watchdog process will restart the forked process.");
out.println(" -timeoutThresholdMillis Number of milliseconds allowed to a parse");
out.println(" before the process is terminated and restarted");
out.println(" -fileList List of files to process, with");
out.println(" paths relative to the input directory");
out.println(" -includeFilePat Regular expression to determine which");
out.println(" files to process, e.g. \"(?i)\\.pdf\"");
out.println(" -excludeFilePat Regular expression to determine which");
out.println(" files to avoid processing, e.g. \"(?i)\\.pdf\"");
out.println(" -maxFileSizeBytes Skip files longer than this value");
out.println();
out.println(" Control the type of output with -x, -h, -t and/or -J.");
out.println();
out.println(" To modify forked process jvm args, prepend \"J\" as in:");
out.println(" -JXmx4g or -JDlog4j.configuration=file:log4j.xml.");
}
private void version() {
System.out.println(Tika.getString());
}
private boolean testForHelp(String[] args) {
for (String s : args) {
if (s.equals("-?") || s.equals("--help")) {
return true;
}
}
return false;
}
private boolean testForBatch(String[] args) {
if (args.length == 2 && !args[0].startsWith("-") && !args[1].startsWith("-")) {
Path inputCand = Paths.get(args[0]);
Path outputCand = Paths.get(args[1]);
if (Files.isDirectory(inputCand) && !Files.isRegularFile(outputCand)) {
return true;
}
}
for (String s : args) {
if (s.equals("-inputDir") || s.equals("--inputDir") || s.equals("-i")) {
return true;
}
}
return false;
}
private void configure() throws TikaException, IOException, SAXException {
if (networkURI != null) {
parser = new NetworkParser(networkURI);
config = TikaConfig.getDefaultConfig();
} else {
if (configFilePath != null) {
config = new TikaConfig(new File(configFilePath));
} else {
config = TikaConfig.getDefaultConfig();
}
parser = new AutoDetectParser(config);
if (digester != null) {
parser = new DigestingParser(parser, digester, false);
}
}
detector = config.getDetector();
context.set(Parser.class, parser);
context.set(PasswordProvider.class, new SimplePasswordProvider(password));
}
private void displayMetModels() {
Class[] modelClasses = Metadata.class.getInterfaces();
Arrays.sort(modelClasses, Comparator.comparing(Class::getName));
for (Class modelClass : modelClasses) {
// we don't care about internal Tika met classes
// if we do, then we can take this conditional out
if (!modelClass
.getSimpleName()
.contains("Tika")) {
System.out.println(modelClass.getSimpleName());
Field[] keyFields = modelClass.getFields();
Arrays.sort(keyFields, Comparator.comparing(Field::getName));
for (Field keyField : keyFields) {
System.out.println(" " + keyField.getName());
}
}
}
}
/*
* Displays loaded parsers and their mime types
* If a parser is a composite parser, it will list the
* sub parsers and their mime-types.
*/
private void displayParsers(boolean includeMimeTypes, boolean aptListFormat) throws TikaException, IOException, SAXException {
configure();
displayParser(parser, includeMimeTypes, aptListFormat, 3);
}
private void displayParser(Parser p, boolean includeMimeTypes, boolean apt, int i) {
String decorated = null;
if (p instanceof ParserDecorator) {
ParserDecorator pd = (ParserDecorator) p;
decorated = " (Wrapped by " + pd.getDecorationName() + ")";
p = pd.getWrappedParser();
}
boolean isComposite = (p instanceof CompositeParser);
String name = p
.getClass()
.getName();
if (apt) {
name = name.substring(0, name.lastIndexOf(".") + 1) + "{{{./api/" + name.replace(".", "/") + "}" + name.substring(name.lastIndexOf(".") + 1) + "}}";
} else if (decorated != null) {
name += decorated;
}
if ((apt && !isComposite) || !apt) { // Don't display Composite parsers in the apt output.
System.out.println(indent(i) + ((apt) ? "* " : "") + name + (isComposite ? " (Composite Parser):" : ""));
if (apt) {
System.out.println();
}
if (includeMimeTypes && !isComposite) {
for (MediaType mt : p.getSupportedTypes(context)) {
System.out.println(indent(i + 3) + ((apt) ? "* " : "") + mt);
if (apt) {
System.out.println();
}
}
}
}
if (isComposite) {
Parser[] subParsers = sortParsers(invertMediaTypeMap(((CompositeParser) p).getParsers()));
for (Parser sp : subParsers) {
displayParser(sp, includeMimeTypes, apt, i + ((apt) ? 0 : 3)); // Don't indent for Composites in apt.
}
}
}
/*
* Displays loaded detectors and their mime types
* If a detector is a composite detector, it will list the
* sub detectors.
*/
private void displayDetectors() throws TikaException, IOException, SAXException {
configure();
displayDetector(detector, 0);
}
private void displayDetector(Detector d, int i) {
boolean isComposite = (d instanceof CompositeDetector);
String name = d
.getClass()
.getName();
System.out.println(indent(i) + name + (isComposite ? " (Composite Detector):" : ""));
if (isComposite) {
List subDetectors = ((CompositeDetector) d).getDetectors();
for (Detector sd : subDetectors) {
displayDetector(sd, i + 2);
}
}
}
private String indent(int indent) {
return " ".substring(0, indent);
}
private Parser[] sortParsers(Map> parsers) {
// Get a nicely sorted list of the parsers
Parser[] sortedParsers = parsers
.keySet()
.toArray(new Parser[0]);
Arrays.sort(sortedParsers, (p1, p2) -> {
String name1 = p1
.getClass()
.getName();
String name2 = p2
.getClass()
.getName();
return name1.compareTo(name2);
});
return sortedParsers;
}
private Map> invertMediaTypeMap(Map supported) {
Map> parsers = new HashMap<>();
for (Entry e : supported.entrySet()) {
if (!parsers.containsKey(e.getValue())) {
parsers.put(e.getValue(), new HashSet<>());
}
parsers
.get(e.getValue())
.add(e.getKey());
}
return parsers;
}
/**
* Prints all the known media types, aliases and matching parser classes.
*/
private void displaySupportedTypes() {
AutoDetectParser parser = new AutoDetectParser();
MediaTypeRegistry registry = parser.getMediaTypeRegistry();
Map parsers = parser.getParsers();
for (MediaType type : registry.getTypes()) {
System.out.println(type);
for (MediaType alias : registry.getAliases(type)) {
System.out.println(" alias: " + alias);
}
MediaType supertype = registry.getSupertype(type);
if (supertype != null) {
System.out.println(" supertype: " + supertype);
}
Parser p = parsers.get(type);
if (p != null) {
if (p instanceof CompositeParser) {
p = ((CompositeParser) p)
.getParsers()
.get(type);
}
System.out.println(" parser: " + p
.getClass()
.getName());
}
}
}
/**
* Compares our mime types registry with the File(1) tool's
* directory of (uncompiled) Magic entries.
* (Well, those with mimetypes anyway)
*
* @param magicDir Path to the magic directory
*/
private void compareFileMagic(String magicDir) throws Exception {
Set tikaLacking = new TreeSet<>();
Set tikaNoMagic = new TreeSet<>();
// Plausibility check
File dir = new File(magicDir);
if ((new File(dir, "elf")).exists() && (new File(dir, "mime")).exists() && (new File(dir, "vorbis")).exists()) {
// Looks plausible
} else {
throw new IllegalArgumentException(magicDir + " doesn't seem to hold uncompressed file magic entries");
}
// Find all the mimetypes in the directory
Set fileMimes = new HashSet<>();
for (File mf : dir.listFiles()) {
if (mf.isFile()) {
BufferedReader r = new BufferedReader(new InputStreamReader(new FileInputStream(mf), UTF_8));
String line;
while ((line = r.readLine()) != null) {
if (line.startsWith("!:mime") || line.startsWith("#!:mime")) {
String mime = line
.substring(7)
.trim();
fileMimes.add(mime);
}
}
r.close();
}
}
// See how those compare to the Tika ones
TikaConfig config = TikaConfig.getDefaultConfig();
MimeTypes mimeTypes = config.getMimeRepository();
MediaTypeRegistry registry = config.getMediaTypeRegistry();
for (String mime : fileMimes) {
try {
final MimeType type = mimeTypes.getRegisteredMimeType(mime);
if (type == null) {
// Tika doesn't know about this one
tikaLacking.add(mime);
} else {
// Tika knows about this one!
// Does Tika have magic for it?
boolean hasMagic = type.hasMagic();
// How about the children?
if (!hasMagic) {
for (MediaType child : registry.getChildTypes(type.getType())) {
MimeType childType = mimeTypes.getRegisteredMimeType(child.toString());
if (childType != null && childType.hasMagic()) {
hasMagic = true;
}
}
}
// How about the parents?
MimeType parentType = type;
while (parentType != null && !hasMagic) {
if (parentType.hasMagic()) {
// Has magic, fine
hasMagic = true;
} else {
// Check the parent next
MediaType parent = registry.getSupertype(type.getType());
if (parent == MediaType.APPLICATION_XML || parent == MediaType.TEXT_PLAIN || parent == MediaType.OCTET_STREAM) {
// Stop checking parents if we hit a top level type
parent = null;
}
if (parent != null) {
parentType = mimeTypes.getRegisteredMimeType(parent.toString());
} else {
parentType = null;
}
}
}
if (!hasMagic) {
tikaNoMagic.add(mime);
}
}
} catch (MimeTypeException e) {
// Broken entry in the file magic directory
// Silently skip
}
}
// Check how many tika knows about
int tikaTypes = 0;
int tikaAliases = 0;
for (MediaType type : registry.getTypes()) {
tikaTypes++;
tikaAliases += registry
.getAliases(type)
.size();
}
// Report
System.out.println("Tika knows about " + tikaTypes + " unique mime types");
System.out.println("Tika knows about " + (tikaTypes + tikaAliases) + " mime types including aliases");
System.out.println("The File Magic directory knows about " + fileMimes.size() + " unique mime types");
System.out.println();
System.out.println("The following mime types are known to File but not Tika:");
for (String mime : tikaLacking) {
System.out.println(" " + mime);
}
System.out.println();
System.out.println("The following mime types from File have no Tika magic (but their children might):");
for (String mime : tikaNoMagic) {
System.out.println(" " + mime);
}
}
private static class NoDocumentMetHandler extends DefaultHandler {
protected final Metadata metadata;
protected PrintWriter writer;
private boolean metOutput;
public NoDocumentMetHandler(Metadata metadata, PrintWriter writer) {
this.metadata = metadata;
this.writer = writer;
this.metOutput = false;
}
@Override
public void endDocument() {
String[] names = metadata.names();
Arrays.sort(names);
outputMetadata(names);
writer.flush();
this.metOutput = true;
}
public void outputMetadata(String[] names) {
for (String name : names) {
for (String value : metadata.getValues(name)) {
writer.println(name + ": " + value);
}
}
}
public boolean metOutput() {
return this.metOutput;
}
}
/**
* Outputs the Tika metadata as XMP using the Tika XMP module
*/
private static class NoDocumentXMPMetaHandler extends DefaultHandler {
protected final Metadata metadata;
protected PrintWriter writer;
public NoDocumentXMPMetaHandler(Metadata metadata, PrintWriter writer) {
this.metadata = metadata;
this.writer = writer;
}
@Override
public void endDocument() throws SAXException {
try {
XMPMetadata xmp = new XMPMetadata(metadata);
String result;
result = xmp.toString();
writer.write(result);
writer.flush();
} catch (TikaException e) {
throw new SAXException(e);
}
}
}
private static class SimplePasswordProvider implements PasswordProvider, Serializable {
private final String password;
public SimplePasswordProvider(String password) {
this.password = password;
}
@Override
public String getPassword(Metadata metadata) {
return password;
}
}
private class OutputType {
public void process(InputStream input, OutputStream output, Metadata metadata) throws Exception {
Parser p = parser;
if (fork) {
p = new ForkParser(TikaCLI.class.getClassLoader(), p);
}
ContentHandler handler = getContentHandler(output, metadata);
try {
p.parse(input, handler, metadata, context);
// fix for TIKA-596: if a parser doesn't generate
// XHTML output, the lack of an output document prevents
// metadata from being output: this fixes that
if (handler instanceof NoDocumentMetHandler) {
NoDocumentMetHandler metHandler = (NoDocumentMetHandler) handler;
if (!metHandler.metOutput()) {
metHandler.endDocument();
}
}
} finally {
if (fork) {
((ForkParser) p).close();
}
}
}
protected ContentHandler getContentHandler(OutputStream output, Metadata metadata) throws Exception {
throw new UnsupportedOperationException();
}
}
private class FileEmbeddedDocumentExtractor implements EmbeddedDocumentExtractor {
private final TikaConfig config = TikaConfig.getDefaultConfig();
private final EmbeddedStreamTranslator embeddedStreamTranslator = new DefaultEmbeddedStreamTranslator();
private int count = 0;
public boolean shouldParseEmbedded(Metadata metadata) {
return true;
}
public void parseEmbedded(InputStream inputStream, ContentHandler contentHandler, Metadata metadata, boolean outputHtml) throws SAXException, IOException {
if (!inputStream.markSupported()) {
inputStream = TikaInputStream.get(inputStream);
}
MediaType contentType = detector.detect(inputStream, metadata);
String name = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY);
File outputFile = null;
if (name == null) {
name = "file" + count++;
}
outputFile = getOutputFile(name, metadata, contentType);
File parent = outputFile.getParentFile();
if (!parent.exists()) {
if (!parent.mkdirs()) {
throw new IOException("unable to create directory \"" + parent + "\"");
}
}
System.out.println("Extracting '" + name + "' (" + contentType + ") to " + outputFile);
try (FileOutputStream os = new FileOutputStream(outputFile)) {
if (embeddedStreamTranslator.shouldTranslate(inputStream, metadata)) {
try (InputStream translated = embeddedStreamTranslator.translate(inputStream, metadata)) {
IOUtils.copy(translated, os);
}
} else {
IOUtils.copy(inputStream, os);
}
} catch (Exception e) {
//
// being a CLI program messages should go to the stderr too
//
String msg = String.format(Locale.ROOT, "Ignoring unexpected exception trying to save embedded file %s (%s)", name, e.getMessage());
LOG.warn(msg, e);
}
}
private File getOutputFile(String name, Metadata metadata, MediaType contentType) {
String ext = getExtension(contentType);
if (name.indexOf('.') == -1 && contentType != null) {
name += ext;
}
String relID = metadata.get(TikaCoreProperties.EMBEDDED_RELATIONSHIP_ID);
if (relID != null && !name.startsWith(relID)) {
name = relID + "_" + name;
}
//defensively do this so that we don't get an exception
//from FilenameUtils.normalize
name = name.replaceAll("\u0000", " ");
String normalizedName = FilenameUtils.normalize(name);
if (normalizedName == null) {
normalizedName = FilenameUtils.getName(name);
}
if (normalizedName == null) {
normalizedName = "file" + count++ + ext;
}
//strip off initial C:/ or ~/ or /
int prefixLength = FilenameUtils.getPrefixLength(normalizedName);
if (prefixLength > -1) {
normalizedName = normalizedName.substring(prefixLength);
}
File outputFile = new File(extractDir, normalizedName);
//if file already exists, prepend uuid
if (outputFile.exists()) {
String fileName = FilenameUtils.getName(normalizedName);
outputFile = new File(extractDir, UUID
.randomUUID()
.toString() + "-" + fileName);
}
return outputFile;
}
private String getExtension(MediaType contentType) {
try {
String ext = config
.getMimeRepository()
.forName(contentType.toString())
.getExtension();
if (ext == null) {
return ".bin";
} else {
return ext;
}
} catch (MimeTypeException e) {
e.printStackTrace();
}
return ".bin";
}
}
private class NoDocumentJSONMetHandler extends DefaultHandler {
protected final Metadata metadata;
protected PrintWriter writer;
public NoDocumentJSONMetHandler(Metadata metadata, PrintWriter writer) {
this.metadata = metadata;
this.writer = writer;
}
@Override
public void endDocument() throws SAXException {
try {
JsonMetadata.setPrettyPrinting(prettyPrint);
JsonMetadata.toJson(metadata, writer);
writer.flush();
} catch (IOException e) {
throw new SAXException(e);
}
}
}
}