com.univocity.parsers.csv.CsvFormatDetector Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of univocity-parsers Show documentation
Show all versions of univocity-parsers Show documentation
univocity's open source parsers for processing different text formats using a consistent API
/*******************************************************************************
* Copyright 2015 uniVocity Software Pty Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************************/
package com.univocity.parsers.csv;
import com.univocity.parsers.common.input.*;
import java.util.*;
import java.util.Map.*;
/**
* An {@link InputAnalysisProcess} to detect column delimiters, quotes and quote escapes in a CSV input.
*
* @author uniVocity Software Pty Ltd - [email protected]
*
*/
abstract class CsvFormatDetector implements InputAnalysisProcess {
private final int MAX_ROW_SAMPLES;
private final char comment;
private final char suggestedDelimiter;
private final char normalizedNewLine;
/**
* Builds a new {@code CsvFormatDetector}
* @param maxRowSamples the number of row samples to collect before analyzing the statistics
* @param settings the configuration provided by the user with potential defaults in case the detection is unable to discover the proper column delimiter or quote character.
*/
CsvFormatDetector(int maxRowSamples, CsvParserSettings settings) {
this.MAX_ROW_SAMPLES = maxRowSamples;
suggestedDelimiter = settings.getFormat().getDelimiter();
normalizedNewLine = settings.getFormat().getNormalizedNewline();
comment = settings.getFormat().getComment();
}
@Override
public void execute(char[] characters, int length) {
Set allSymbols = new HashSet();
Map symbols = new HashMap();
Map escape = new HashMap();
List> symbolsPerRow = new ArrayList>();
int doubleQuoteCount = 0;
int singleQuoteCount = 0;
int i;
char inQuote = '\0';
boolean afterNewLine = true;
for (i = 0; i < length; i++) {
char ch = characters[i];
if (afterNewLine && ch == comment) {
while (++i < length) {
ch = characters[i];
if (ch == '\r' || ch == '\n' || ch == normalizedNewLine) {
break;
}
}
continue;
}
if (ch == '"' || ch == '\'') {
if (inQuote == ch) { //closing quotes (potentially)
if (ch == '"') {
doubleQuoteCount++;
} else {
singleQuoteCount++;
}
if (i + 1 < length) {
char next = characters[i + 1];
if (Character.isLetterOrDigit(next) || next <= ' ') { //no special characters after quote, might be escaping
//special character before (potentially) closing quote, might be an escape
char prev = characters[i - 1];
if (!Character.isLetterOrDigit(prev)) {
increment(escape, prev);
}
}
}
inQuote = '\0';
} else {
inQuote = ch;
}
continue;
}
if (inQuote != '\0') { //keep looping until the quote is closed.
continue;
}
afterNewLine = false;
if (!Character.isLetterOrDigit(ch) && (ch == '\t' || ch > ' ')) { //counts all symbols. Skips letters, digits and white spaces (except the tab character)
allSymbols.add(ch);
increment(symbols, ch);
} else if ((ch == '\r' || ch == '\n' || ch == normalizedNewLine) && symbols.size() > 0) { //got a newline and collected some symbols? Good!
afterNewLine = true;
symbolsPerRow.add(symbols);
if (symbolsPerRow.size() == MAX_ROW_SAMPLES) {
break;
}
symbols = new HashMap();
}
}
if (i >= length && symbolsPerRow.size() > 1) { // if got to the end of the buffer, discard last row. It's probably incomplete anyway.
symbolsPerRow.remove(symbolsPerRow.size() - 1);
}
Map sums = new HashMap();
Set toRemove = new HashSet();
//combines the number of symbols found in each row and sums the difference.
for (Map previous : symbolsPerRow) {
for (Map current : symbolsPerRow) {
for (Character symbol : allSymbols) {
Integer previousCount = previous.get(symbol);
Integer currentCount = current.get(symbol);
if (previousCount == null && currentCount == null) { // got a symbol that does not appear in all rows? Discard it.
toRemove.add(symbol);
}
if (previousCount == null || currentCount == null) {
continue;
}
increment(sums, symbol, Math.abs(previousCount - currentCount)); // we expect to always get 0 or close to 0 here, so the symbol occurs in all rows
}
}
}
sums.keySet().removeAll(toRemove);
char delimiter = min(sums, suggestedDelimiter);
char quote = doubleQuoteCount >= singleQuoteCount ? '"' : '\'';
escape.remove(delimiter);
char quoteEscape = max(escape, quote);
apply(delimiter, quote, quoteEscape);
}
/**
* Increments the number associated with a character in a map by 1
* @param map the map of characters and their numbers
* @param symbol the character whose number should be increment
*/
private static void increment(Map map, char symbol) {
increment(map, symbol, 1);
}
/**
* Increments the number associated with a character in a map
* @param map the map of characters and their numbers
* @param symbol the character whose number should be increment
* @param incrementSize the size of the increment
*/
private static void increment(Map map, char symbol, int incrementSize) {
Integer count = map.get(symbol);
if (count == null) {
count = 0;
}
map.put(symbol, count + incrementSize);
}
/**
* Returns the character with the lowest associated number.
* @param map the map of characters and their numbers
* @param defaultChar the default character to return in case the map is empty
* @return the character with the lowest number associated.
*/
private static char min(Map map, char defaultChar) {
return getChar(map, defaultChar, true);
}
/**
* Returns the character with the highest associated number.
* @param map the map of characters and their numbers
* @param defaultChar the default character to return in case the map is empty
* @return the character with the highest number associated.
*/
private static char max(Map map, char defaultChar) {
return getChar(map, defaultChar, false);
}
/**
* Returns the character with the highest or lowest associated number.
* @param map the map of characters and their numbers
* @param defaultChar the default character to return in case the map is empty
* @param min a flag indicating whether to return the character associated with the lowest number in the map.
* If {@code false} then the character associated with the highest number found will be returned.
* @return the character with the highest/lowest number associated.
*/
private static char getChar(Map map, char defaultChar, boolean min) {
int val = min ? Integer.MAX_VALUE : Integer.MIN_VALUE;
for (Entry e : map.entrySet()) {
int sum = e.getValue();
if ((min && sum < val) || (!min && sum > val)) {
val = sum;
defaultChar = e.getKey();
}
}
return defaultChar;
}
/**
* Applies the discovered CSV format elements to the {@link CsvParser}
* @param delimiter the discovered delimiter character
* @param quote the discovered quote character
* @param quoteEscape the discovered quote escape character.
*/
abstract void apply(char delimiter, char quote, char quoteEscape);
}