org.apache.poi.ss.extractor.EmbeddedExtractor Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of apache-poi Show documentation
Show all versions of apache-poi Show documentation
The Apache Commons Codec package contains simple encoder and decoders for
various formats such as Base64 and Hexadecimal. In addition to these
widely used encoders and decoders, the codec package also maintains a
collection of phonetic encoding utilities.
/* ====================================================================
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==================================================================== */
package org.apache.poi.ss.extractor;
import static org.apache.poi.util.StringUtil.endsWithIgnoreCase;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.poi.hpsf.ClassID;
import org.apache.poi.hpsf.ClassIDPredefined;
import org.apache.poi.poifs.filesystem.DirectoryNode;
import org.apache.poi.poifs.filesystem.DocumentInputStream;
import org.apache.poi.poifs.filesystem.Entry;
import org.apache.poi.poifs.filesystem.Ole10Native;
import org.apache.poi.poifs.filesystem.Ole10NativeException;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
import org.apache.poi.ss.usermodel.Drawing;
import org.apache.poi.ss.usermodel.ObjectData;
import org.apache.poi.ss.usermodel.Picture;
import org.apache.poi.ss.usermodel.PictureData;
import org.apache.poi.ss.usermodel.Shape;
import org.apache.poi.ss.usermodel.ShapeContainer;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.util.Beta;
import org.apache.poi.util.IOUtils;
import org.apache.poi.util.LocaleUtil;
/**
* This extractor class tries to identify various embedded documents within Excel files
* and provide them via a common interface, i.e. the EmbeddedData instances
*/
@Beta
public class EmbeddedExtractor implements Iterable {
private static final Logger LOG = LogManager.getLogger(EmbeddedExtractor.class);
//arbitrarily selected; may need to increase
private static final int DEFAULT_MAX_RECORD_LENGTH = 1_000_000;
private static int MAX_RECORD_LENGTH = DEFAULT_MAX_RECORD_LENGTH;
// contentType
private static final String CONTENT_TYPE_BYTES = "binary/octet-stream";
private static final String CONTENT_TYPE_PDF = "application/pdf";
private static final String CONTENT_TYPE_DOC = "application/msword";
private static final String CONTENT_TYPE_XLS = "application/vnd.ms-excel";
/**
* @param length the max record length allowed for EmbeddedExtractor
*/
public static void setMaxRecordLength(int length) {
MAX_RECORD_LENGTH = length;
}
/**
* @return the max record length allowed for EmbeddedExtractor
*/
public static int getMaxRecordLength() {
return MAX_RECORD_LENGTH;
}
/**
* @return the list of known extractors, if you provide custom extractors, override this method
*/
@Override
public Iterator iterator() {
EmbeddedExtractor[] ee = {
new Ole10Extractor(), new PdfExtractor(), new BiffExtractor(), new OOXMLExtractor(), new FsExtractor()
};
return Arrays.asList(ee).iterator();
}
public EmbeddedData extractOne(DirectoryNode src) throws IOException {
for (EmbeddedExtractor ee : this) {
if (ee.canExtract(src)) {
return ee.extract(src);
}
}
return null;
}
public EmbeddedData extractOne(Picture src) throws IOException {
for (EmbeddedExtractor ee : this) {
if (ee.canExtract(src)) {
return ee.extract(src);
}
}
return null;
}
public List extractAll(Sheet sheet) throws IOException {
Drawing patriarch = sheet.getDrawingPatriarch();
if (null == patriarch){
return Collections.emptyList();
}
List embeddings = new ArrayList<>();
extractAll(patriarch, embeddings);
return embeddings;
}
protected void extractAll(ShapeContainer parent, List embeddings) throws IOException {
for (Shape shape : parent) {
EmbeddedData data = null;
if (shape instanceof ObjectData) {
ObjectData od = (ObjectData)shape;
try {
if (od.hasDirectoryEntry()) {
data = extractOne((DirectoryNode)od.getDirectory());
} else {
data = new EmbeddedData(od.getFileName(), od.getObjectData(), od.getContentType());
}
} catch (Exception e) {
LOG.atWarn().withThrowable(e).log("Entry not found / readable - ignoring OLE embedding");
}
} else if (shape instanceof Picture) {
data = extractOne((Picture)shape);
} else if (shape instanceof ShapeContainer) {
extractAll((ShapeContainer)shape, embeddings);
}
if (data == null) {
continue;
}
data.setShape(shape);
String filename = data.getFilename();
String extension = (filename == null || filename.lastIndexOf('.') == -1) ? ".bin" : filename.substring(filename.lastIndexOf('.'));
// try to find an alternative name
if (filename == null || filename.isEmpty() || filename.startsWith("MBD") || filename.startsWith("Root Entry")) {
filename = shape.getShapeName();
if (filename != null) {
filename += extension;
}
}
// default to dummy name
if (filename == null || filename.isEmpty()) {
filename = "picture_" + embeddings.size() + extension;
}
filename = filename.trim();
data.setFilename(filename);
embeddings.add(data);
}
}
public boolean canExtract(DirectoryNode source) {
return false;
}
public boolean canExtract(Picture source) {
return false;
}
protected EmbeddedData extract(DirectoryNode dn) throws IOException {
assert(canExtract(dn));
try (UnsynchronizedByteArrayOutputStream bos = new UnsynchronizedByteArrayOutputStream(20000);
POIFSFileSystem dest = new POIFSFileSystem()) {
copyNodes(dn, dest.getRoot());
// start with a reasonable big size
dest.writeFilesystem(bos);
return new EmbeddedData(dn.getName(), bos.toByteArray(), CONTENT_TYPE_BYTES);
}
}
protected EmbeddedData extract(Picture source) throws IOException {
return null;
}
public static class Ole10Extractor extends EmbeddedExtractor {
@Override
public boolean canExtract(DirectoryNode dn) {
ClassID clsId = dn.getStorageClsid();
return ClassIDPredefined.lookup(clsId) == ClassIDPredefined.OLE_V1_PACKAGE;
}
@Override
public EmbeddedData extract(DirectoryNode dn) throws IOException {
try {
// TODO: inspect the CompObj record for more details, i.e. the content type
Ole10Native ole10 = Ole10Native.createFromEmbeddedOleObject(dn);
return new EmbeddedData(ole10.getFileName(), ole10.getDataBuffer(), CONTENT_TYPE_BYTES);
} catch (Ole10NativeException e) {
throw new IOException(e);
}
}
}
static class PdfExtractor extends EmbeddedExtractor {
@Override
public boolean canExtract(DirectoryNode dn) {
ClassID clsId = dn.getStorageClsid();
return (ClassIDPredefined.PDF.equals(clsId) || dn.hasEntry("CONTENTS"));
}
@Override
public EmbeddedData extract(DirectoryNode dn) throws IOException {
try(UnsynchronizedByteArrayOutputStream bos = new UnsynchronizedByteArrayOutputStream();
InputStream is = dn.createDocumentInputStream("CONTENTS")) {
IOUtils.copy(is, bos);
return new EmbeddedData(dn.getName() + ".pdf", bos.toByteArray(), CONTENT_TYPE_PDF);
}
}
@Override
public boolean canExtract(Picture source) {
PictureData pd = source.getPictureData();
return (pd != null && pd.getPictureType() == Workbook.PICTURE_TYPE_EMF);
}
/**
* Mac Office encodes embedded objects inside the picture, e.g. PDF is part of an EMF.
* If an embedded stream is inside an EMF picture, this method extracts the payload.
*
* @return the embedded data in an EMF picture or null if none is found
*/
@Override
protected EmbeddedData extract(Picture source) throws IOException {
// check for emf+ embedded pdf (poor mans style :( )
// Mac Excel 2011 embeds pdf files with this method.
PictureData pd = source.getPictureData();
if (pd == null || pd.getPictureType() != Workbook.PICTURE_TYPE_EMF) {
return null;
}
// TODO: investigate if this is just an EMF-hack or if other formats are also embedded in EMF
byte[] pictureBytes = pd.getData();
int idxStart = indexOf(pictureBytes, 0, "%PDF-".getBytes(LocaleUtil.CHARSET_1252));
if (idxStart == -1) {
return null;
}
int idxEnd = indexOf(pictureBytes, idxStart, "%%EOF".getBytes(LocaleUtil.CHARSET_1252));
if (idxEnd == -1) {
return null;
}
int pictureBytesLen = idxEnd-idxStart+6;
byte[] pdfBytes = IOUtils.safelyClone(pictureBytes, idxStart, pictureBytesLen, MAX_RECORD_LENGTH);
String filename = source.getShapeName().trim();
if (!endsWithIgnoreCase(filename, ".pdf")) {
filename += ".pdf";
}
return new EmbeddedData(filename, pdfBytes, CONTENT_TYPE_PDF);
}
}
static class OOXMLExtractor extends EmbeddedExtractor {
@Override
public boolean canExtract(DirectoryNode dn) {
return dn.hasEntry("package");
}
@Override
public EmbeddedData extract(DirectoryNode dn) throws IOException {
ClassIDPredefined clsId = ClassIDPredefined.lookup(dn.getStorageClsid());
String contentType = null;
String ext = null;
if (clsId != null) {
contentType = clsId.getContentType();
ext = clsId.getFileExtension();
}
if (contentType == null || ext == null) {
contentType = "application/zip";
ext = ".zip";
}
DocumentInputStream dis = dn.createDocumentInputStream("package");
byte[] data = IOUtils.toByteArray(dis);
dis.close();
return new EmbeddedData(dn.getName()+ext, data, contentType);
}
}
static class BiffExtractor extends EmbeddedExtractor {
@Override
public boolean canExtract(DirectoryNode dn) {
return canExtractExcel(dn) || canExtractWord(dn);
}
protected boolean canExtractExcel(DirectoryNode dn) {
ClassIDPredefined clsId = ClassIDPredefined.lookup(dn.getStorageClsid());
return (ClassIDPredefined.EXCEL_V7 == clsId
|| ClassIDPredefined.EXCEL_V8 == clsId
|| dn.hasEntry("Workbook") /*...*/);
}
protected boolean canExtractWord(DirectoryNode dn) {
ClassIDPredefined clsId = ClassIDPredefined.lookup(dn.getStorageClsid());
return (ClassIDPredefined.WORD_V7 == clsId
|| ClassIDPredefined.WORD_V8 == clsId
|| dn.hasEntry("WordDocument"));
}
@Override
public EmbeddedData extract(DirectoryNode dn) throws IOException {
EmbeddedData ed = super.extract(dn);
if (canExtractExcel(dn)) {
ed.setFilename(dn.getName() + ".xls");
ed.setContentType(CONTENT_TYPE_XLS);
} else if (canExtractWord(dn)) {
ed.setFilename(dn.getName() + ".doc");
ed.setContentType(CONTENT_TYPE_DOC);
}
return ed;
}
}
static class FsExtractor extends EmbeddedExtractor {
@Override
public boolean canExtract(DirectoryNode dn) {
return true;
}
@Override
public EmbeddedData extract(DirectoryNode dn) throws IOException {
EmbeddedData ed = super.extract(dn);
ed.setFilename(dn.getName() + ".ole");
// TODO: read the content type from CombObj stream
return ed;
}
}
protected static void copyNodes(DirectoryNode src, DirectoryNode dest) throws IOException {
for (Entry e : src) {
if (e instanceof DirectoryNode) {
DirectoryNode srcDir = (DirectoryNode)e;
DirectoryNode destDir = (DirectoryNode)dest.createDirectory(srcDir.getName());
destDir.setStorageClsid(srcDir.getStorageClsid());
copyNodes(srcDir, destDir);
} else {
try (InputStream is = src.createDocumentInputStream(e)) {
dest.createDocument(e.getName(), is);
}
}
}
}
/**
* Knuth-Morris-Pratt Algorithm for Pattern Matching
* Finds the first occurrence of the pattern in the text.
*/
private static int indexOf(byte[] data, int offset, byte[] pattern) {
int[] failure = computeFailure(pattern);
int j = 0;
if (data.length == 0) {
return -1;
}
for (int i = offset; i < data.length; i++) {
while (j > 0 && pattern[j] != data[i]) {
j = failure[j - 1];
}
if (pattern[j] == data[i]) { j++; }
if (j == pattern.length) {
return i - pattern.length + 1;
}
}
return -1;
}
/**
* Computes the failure function using a boot-strapping process,
* where the pattern is matched against itself.
*/
private static int[] computeFailure(byte[] pattern) {
int[] failure = new int[pattern.length];
int j = 0;
for (int i = 1; i < pattern.length; i++) {
while (j > 0 && pattern[j] != pattern[i]) {
j = failure[j - 1];
}
if (pattern[j] == pattern[i]) {
j++;
}
failure[i] = j;
}
return failure;
}
}