org.apache.tika.parser.dbf.DBFParser Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.tika.parser.dbf;
import org.apache.tika.detect.EncodingDetector;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.AbstractParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.txt.Icu4jEncodingDetector;
import org.apache.tika.sax.XHTMLContentHandler;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Calendar;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
/**
* This is a Tika wrapper around the DBFReader.
*
* This reads many dbase3 file variants (not DBASE 7, yet!).
*
* It caches the first 10 rows and then runs encoding dectection
* on the "character" cells.
*/
public class DBFParser extends AbstractParser {
private static final int ROWS_TO_BUFFER_FOR_CHARSET_DETECTION = 10;
private static final int MAX_CHARS_FOR_CHARSET_DETECTION = 20000;
private static final Charset DEFAULT_CHARSET = StandardCharsets.ISO_8859_1;
private static final Set SUPPORTED_TYPES =
Collections.singleton(MediaType.application("x-dbf"));
@Override
public Set getSupportedTypes(ParseContext context) {
return SUPPORTED_TYPES;
}
@Override
public void parse(InputStream stream, ContentHandler handler, Metadata metadata,
ParseContext context) throws IOException, SAXException, TikaException {
DBFReader reader = DBFReader.open(stream);
DBFFileHeader header = reader.getHeader();
metadata.set(Metadata.CONTENT_TYPE, header.getVersion().getFullMimeString());
//insert metadata here
Calendar lastModified = header.getLastModified();
if (lastModified != null) {
metadata.set(TikaCoreProperties.MODIFIED, lastModified);
}
//buffer first X rows for charset detection
List firstRows = new LinkedList<>();
DBFRow row = reader.next();
int i = 0;
while (row != null && i++ < ROWS_TO_BUFFER_FOR_CHARSET_DETECTION) {
firstRows.add(row.deepCopy());
row = reader.next();
}
Charset charset = getCharset(firstRows, header);
metadata.set(Metadata.CONTENT_ENCODING, charset.toString());
XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
xhtml.startDocument();
xhtml.startElement("table");
xhtml.startElement("thead");
for (DBFColumnHeader col : header.getCols()) {
xhtml.startElement("th");
xhtml.characters(col.getName(charset));
xhtml.endElement("th");
}
xhtml.endElement("thead");
xhtml.startElement("tbody");
//now write cached rows
while (firstRows.size() > 0) {
DBFRow cachedRow = firstRows.remove(0);
writeRow(cachedRow, charset, xhtml);
}
//now continue with rest
while (row != null) {
writeRow(row, charset, xhtml);
row = reader.next();
}
xhtml.endElement("tbody");
xhtml.endElement("table");
xhtml.endDocument();
}
private Charset getCharset(List firstRows, DBFFileHeader header) throws IOException,
TikaException {
//TODO: potentially use codepage info in the header
Charset charset = DEFAULT_CHARSET;
ByteArrayOutputStream bos = new ByteArrayOutputStream();
for (DBFRow row : firstRows) {
for (DBFCell cell : row.cells) {
if (cell.getColType().equals(DBFColumnHeader.ColType.C)) {
byte[] bytes = cell.getBytes();
bos.write(bytes);
if (bos.size() > MAX_CHARS_FOR_CHARSET_DETECTION) {
break;
}
}
}
}
byte[] bytes = bos.toByteArray();
if (bytes.length > 20) {
EncodingDetector detector = new Icu4jEncodingDetector();
detector.detect(TikaInputStream.get(bytes), new Metadata());
charset = detector.detect(new ByteArrayInputStream(bytes), new Metadata());
}
return charset;
}
private void writeRow(DBFRow row, Charset charset, XHTMLContentHandler xhtml) throws SAXException {
xhtml.startElement("tr");
for (DBFCell cell : row.cells) {
xhtml.startElement("td");
xhtml.characters(cell.getString(charset));
xhtml.endElement("td");
}
xhtml.endElement("tr");
}
}