com.landawn.abacus.util.CSVUtil Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of abacus-common Show documentation
Show all versions of abacus-common Show documentation
A general programming library in Java/Android. It's easy to learn and simple to use with concise and powerful APIs.
/*
* Copyright (c) 2015, Haiyang Li.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.landawn.abacus.util;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.BiConsumer;
import java.util.function.Function;
import java.util.function.Predicate;
import com.landawn.abacus.exception.UncheckedIOException;
import com.landawn.abacus.parser.JSONDeserializationConfig;
import com.landawn.abacus.parser.JSONDeserializationConfig.JDC;
import com.landawn.abacus.parser.JSONParser;
import com.landawn.abacus.parser.ParserFactory;
import com.landawn.abacus.parser.ParserUtil;
import com.landawn.abacus.parser.ParserUtil.BeanInfo;
import com.landawn.abacus.parser.ParserUtil.PropInfo;
import com.landawn.abacus.type.Type;
import com.landawn.abacus.util.Fn.Fnn;
import com.landawn.abacus.util.stream.Stream;
/**
*
* @author Haiyang Li
* @since 0.8
*/
public final class CSVUtil {
private CSVUtil() {
// Utillity class
}
public static final JSONParser jsonParser = ParserFactory.createJSONParser();
static final JSONDeserializationConfig jdc = JDC.create().setElementType(String.class);
static final Splitter lineSplitter = Splitter.with(',').trimResults();
static final CSVParser csvParser = new CSVParser();
public static final Function CSV_HEADER_PARSER = csvParser::parseLineToArray;
public static final BiConsumer CSV_LINE_PARSER = csvParser::parseLineToArray;
public static final Function CSV_HEADER_PARSER_BY_SPLITTER = it -> {
final String[] strs = lineSplitter.splitToArray(it);
int subStrLen = 0;
for (int i = 0, len = strs.length; i < len; i++) {
subStrLen = N.len(strs[i]);
if (subStrLen > 1 && strs[i].charAt(0) == '"' && strs[i].charAt(subStrLen - 1) == '"') {
strs[i] = strs[i].substring(0, subStrLen - 1);
}
}
return strs;
};
public static final BiConsumer CSV_LINE_PARSER_BY_SPLITTER = (strs, it) -> {
lineSplitter.splitToArray(strs, it);
int subStrLen = 0;
for (int i = 0, len = strs.length; i < len; i++) {
subStrLen = N.len(strs[i]);
if (subStrLen > 1 && strs[i].charAt(0) == '"' && strs[i].charAt(subStrLen - 1) == '"') {
strs[i] = strs[i].substring(0, subStrLen - 1);
}
}
};
static final Function CSV_HEADER_PARSER_IN_JSON = line -> jsonParser.readString(String[].class, line, jdc);
static final BiConsumer CSV_LINE_PARSER_IN_JSON = (output, line) -> jsonParser.readString(output, line, jdc);
static final Function defaultCsvHeadereParser = CSV_HEADER_PARSER_IN_JSON;
static final BiConsumer defaultCsvLineParser = CSV_LINE_PARSER_IN_JSON;
static final ThreadLocal> csvHeaderParser_TL = ThreadLocal.withInitial(() -> defaultCsvHeadereParser);
static final ThreadLocal> csvLineParser_TL = ThreadLocal.withInitial(() -> defaultCsvLineParser);
/**
*
*
* @param parser
*/
// TODO should share/use the same parser for line?
public static void setCSVHeaderParser(final Function parser) {
N.checkArgNotNull(parser, "parser");
csvHeaderParser_TL.set(parser);
}
/**
*
*
* @param parser
*/
public static void setCSVLineParser(final BiConsumer parser) {
N.checkArgNotNull(parser, "parser");
csvLineParser_TL.set(parser);
}
/**
*
*/
public static void resetCSVHeaderParser() {
csvHeaderParser_TL.set(defaultCsvHeadereParser);
}
/**
*
*/
public static void resetCSVLineParser() {
csvLineParser_TL.set(defaultCsvLineParser);
}
/**
*
*
* @return
*/
public static Function getCurrentHeaderParser() {
return csvHeaderParser_TL.get();
}
/**
*
*
* @return
*/
public static BiConsumer getCurrentLineParser() {
return csvLineParser_TL.get();
}
/**
*
* @param csvFile
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
public static DataSet loadCSV(final File csvFile) throws UncheckedIOException {
return loadCSV(csvFile, (Collection) null);
}
/**
*
* @param csvFile
* @param selectColumnNames
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
public static DataSet loadCSV(final File csvFile, final Collection selectColumnNames) throws UncheckedIOException {
return loadCSV(csvFile, selectColumnNames, 0, Long.MAX_VALUE);
}
/**
*
* @param csvFile
* @param selectColumnNames
* @param offset
* @param count
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
public static DataSet loadCSV(final File csvFile, final Collection selectColumnNames, final long offset, final long count)
throws UncheckedIOException {
return loadCSV(csvFile, selectColumnNames, offset, count, Fn. alwaysTrue());
}
/**
* Load the data from CSV.
*
* @param
* @param csvFile
* @param selectColumnNames
* @param offset
* @param count
* @param filter
* @return
* @throws UncheckedIOException the unchecked IO exception
* @throws E the e
*/
public static DataSet loadCSV(final File csvFile, final Collection selectColumnNames, final long offset, final long count,
final Throwables.Predicate filter) throws UncheckedIOException, E {
InputStream csvInputStream = null;
try {
csvInputStream = IOUtil.newFileInputStream(csvFile);
return loadCSV(csvInputStream, selectColumnNames, offset, count, filter);
} finally {
IOUtil.closeQuietly(csvInputStream);
}
}
/**
*
* @param csvInputStream
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
public static DataSet loadCSV(final InputStream csvInputStream) throws UncheckedIOException {
return loadCSV(csvInputStream, (Collection) null);
}
/**
*
* @param csvInputStream
* @param selectColumnNames
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
public static DataSet loadCSV(final InputStream csvInputStream, final Collection selectColumnNames) throws UncheckedIOException {
return loadCSV(csvInputStream, selectColumnNames, 0, Long.MAX_VALUE);
}
/**
*
* @param csvInputStream
* @param selectColumnNames
* @param offset
* @param count
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
public static DataSet loadCSV(final InputStream csvInputStream, final Collection selectColumnNames, final long offset, final long count)
throws UncheckedIOException {
return loadCSV(csvInputStream, selectColumnNames, offset, count, Fn. alwaysTrue());
}
/**
* Load the data from CSV.
*
* @param
* @param csvInputStream
* @param selectColumnNames
* @param offset
* @param count
* @param filter
* @return
* @throws UncheckedIOException the unchecked IO exception
* @throws E the e
*/
public static DataSet loadCSV(final InputStream csvInputStream, final Collection selectColumnNames, final long offset,
final long count, final Throwables.Predicate filter) throws UncheckedIOException, E {
final Reader csvReader = new InputStreamReader(csvInputStream);
return loadCSV(csvReader, selectColumnNames, offset, count, filter);
}
/**
*
* @param csvReader
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
public static DataSet loadCSV(final Reader csvReader) throws UncheckedIOException {
return loadCSV(csvReader, (Collection) null);
}
/**
*
* @param csvReader
* @param selectColumnNames
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
public static DataSet loadCSV(final Reader csvReader, final Collection selectColumnNames) throws UncheckedIOException {
return loadCSV(csvReader, selectColumnNames, 0, Long.MAX_VALUE);
}
/**
*
* @param csvReader
* @param selectColumnNames
* @param offset
* @param count
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
public static DataSet loadCSV(final Reader csvReader, final Collection selectColumnNames, long offset, long count) throws UncheckedIOException {
return loadCSV(csvReader, selectColumnNames, offset, count, Fn. alwaysTrue());
}
/**
* Load the data from CSV.
*
* @param
* @param csvReader
* @param selectColumnNames
* @param offset
* @param count
* @param filter
* @return
* @throws UncheckedIOException the unchecked IO exception
* @throws E the e
*/
public static DataSet loadCSV(final Reader csvReader, final Collection selectColumnNames, long offset, long count,
final Throwables.Predicate filter) throws UncheckedIOException, E {
N.checkArgument(offset >= 0 && count >= 0, "'offset'=%s and 'count'=%s can't be negative", offset, count); //NOSONAR
final Function headerParser = csvHeaderParser_TL.get();
final BiConsumer lineParser = csvLineParser_TL.get();
final BufferedReader br = csvReader instanceof BufferedReader ? (BufferedReader) csvReader : Objectory.createBufferedReader(csvReader);
try {
String line = br.readLine();
if (line == null) {
return N.newEmptyDataSet();
}
final String[] titles = headerParser.apply(line);
final int columnCount = titles.length;
final Type[] columnTypes = new Type[columnCount];
final List columnNameList = new ArrayList<>(selectColumnNames == null ? columnCount : selectColumnNames.size());
final List> columnList = new ArrayList<>(selectColumnNames == null ? columnCount : selectColumnNames.size());
final Set selectPropNameSet = selectColumnNames == null ? null : N.newHashSet(selectColumnNames);
for (int i = 0; i < columnCount; i++) {
if (selectPropNameSet == null || selectPropNameSet.remove(titles[i])) {
columnNameList.add(titles[i]);
columnList.add(new ArrayList<>());
columnTypes[i] = N.typeOf(String.class);
}
}
if (selectPropNameSet != null && selectPropNameSet.size() > 0) {
throw new IllegalArgumentException(selectPropNameSet + " are not included in titles: " + N.toString(titles)); //NOSONAR
}
final String[] strs = new String[titles.length];
while (offset-- > 0 && br.readLine() != null) {
// continue;
}
while (count > 0 && (line = br.readLine()) != null) {
lineParser.accept(strs, line);
if (filter != null && !filter.test(strs)) {
continue;
}
for (int i = 0, columnIndex = 0; i < columnCount; i++) {
if (columnTypes[i] != null) {
columnList.get(columnIndex++).add(strs[i]);
}
}
count--;
}
return new RowDataSet(columnNameList, columnList);
} catch (IOException e) {
throw new UncheckedIOException(e);
} finally {
if (br != csvReader) {
Objectory.recycle(br);
}
}
}
/**
*
* @param beanClass
* @param csvFile
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
public static DataSet loadCSV(final Class beanClass, final File csvFile) throws UncheckedIOException {
return loadCSV(beanClass, csvFile, null);
}
/**
*
* @param beanClass
* @param csvFile
* @param selectColumnNames
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
public static DataSet loadCSV(final Class beanClass, final File csvFile, final Collection selectColumnNames) throws UncheckedIOException {
return loadCSV(beanClass, csvFile, selectColumnNames, 0, Long.MAX_VALUE);
}
/**
*
* @param beanClass
* @param csvFile
* @param selectColumnNames
* @param offset
* @param count
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
public static DataSet loadCSV(final Class beanClass, final File csvFile, final Collection selectColumnNames, final long offset, final long count)
throws UncheckedIOException {
return loadCSV(beanClass, csvFile, selectColumnNames, offset, count, Fn. alwaysTrue());
}
/**
* Load the data from CSV.
*
* @param
* @param beanClass
* @param csvFile
* @param selectColumnNames
* @param offset
* @param count
* @param filter
* @return
* @throws UncheckedIOException the unchecked IO exception
* @throws E the e
*/
public static DataSet loadCSV(final Class beanClass, final File csvFile, final Collection selectColumnNames,
final long offset, final long count, final Throwables.Predicate filter) throws UncheckedIOException, E {
InputStream csvInputStream = null;
try {
csvInputStream = IOUtil.newFileInputStream(csvFile);
return loadCSV(beanClass, csvInputStream, selectColumnNames, offset, count, filter);
} finally {
IOUtil.closeQuietly(csvInputStream);
}
}
/**
*
* @param beanClass
* @param csvInputStream
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
public static DataSet loadCSV(final Class beanClass, final InputStream csvInputStream) throws UncheckedIOException {
return loadCSV(beanClass, csvInputStream, null);
}
/**
*
* @param beanClass
* @param csvInputStream
* @param selectColumnNames
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
public static DataSet loadCSV(final Class beanClass, final InputStream csvInputStream, final Collection selectColumnNames)
throws UncheckedIOException {
return loadCSV(beanClass, csvInputStream, selectColumnNames, 0, Long.MAX_VALUE);
}
/**
*
* @param beanClass
* @param csvInputStream
* @param selectColumnNames
* @param offset
* @param count
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
public static DataSet loadCSV(final Class beanClass, final InputStream csvInputStream, final Collection selectColumnNames, final long offset,
final long count) throws UncheckedIOException {
return loadCSV(beanClass, csvInputStream, selectColumnNames, offset, count, Fn. alwaysTrue());
}
/**
* Load the data from CSV.
*
* @param
* @param beanClass
* @param csvInputStream
* @param selectColumnNames
* @param offset
* @param count
* @param filter
* @return
* @throws UncheckedIOException the unchecked IO exception
* @throws E the e
*/
public static DataSet loadCSV(final Class beanClass, final InputStream csvInputStream, final Collection selectColumnNames,
final long offset, final long count, final Throwables.Predicate filter) throws UncheckedIOException, E {
final Reader csvReader = new InputStreamReader(csvInputStream);
return loadCSV(beanClass, csvReader, selectColumnNames, offset, count, filter);
}
/**
*
* @param beanClass
* @param csvReader
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
public static DataSet loadCSV(final Class beanClass, final Reader csvReader) throws UncheckedIOException {
return loadCSV(beanClass, csvReader, null);
}
/**
*
* @param beanClass
* @param csvReader
* @param selectColumnNames
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
public static DataSet loadCSV(final Class beanClass, final Reader csvReader, final Collection selectColumnNames) throws UncheckedIOException {
return loadCSV(beanClass, csvReader, selectColumnNames, 0, Long.MAX_VALUE);
}
/**
*
* @param beanClass
* @param csvReader
* @param selectColumnNames
* @param offset
* @param count
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
public static DataSet loadCSV(final Class beanClass, final Reader csvReader, final Collection selectColumnNames, long offset, long count)
throws UncheckedIOException {
return loadCSV(beanClass, csvReader, selectColumnNames, offset, count, Fn. alwaysTrue());
}
/**
* Load the data from CSV.
*
* @param
* @param beanClass
* @param csvReader
* @param selectColumnNames
* @param offset
* @param count
* @param filter
* @return
* @throws UncheckedIOException the unchecked IO exception
* @throws E the e
*/
public static DataSet loadCSV(final Class beanClass, final Reader csvReader, final Collection selectColumnNames,
long offset, long count, final Throwables.Predicate filter) throws UncheckedIOException, E {
N.checkArgument(offset >= 0 && count >= 0, "'offset'=%s and 'count'=%s can't be negative", offset, count);
final Function headerParser = csvHeaderParser_TL.get();
final BiConsumer lineParser = csvLineParser_TL.get();
final BufferedReader br = csvReader instanceof BufferedReader ? (BufferedReader) csvReader : Objectory.createBufferedReader(csvReader);
final BeanInfo beanInfo = ParserUtil.getBeanInfo(beanClass);
try {
String line = br.readLine();
if (line == null) {
return N.newEmptyDataSet();
}
final String[] titles = headerParser.apply(line);
final int columnCount = titles.length;
final PropInfo[] propInfos = new PropInfo[columnCount];
final List columnNameList = new ArrayList<>(selectColumnNames == null ? columnCount : selectColumnNames.size());
final List> columnList = new ArrayList<>(selectColumnNames == null ? columnCount : selectColumnNames.size());
final Set selectPropNameSet = selectColumnNames == null ? null : N.newHashSet(selectColumnNames);
for (int i = 0; i < columnCount; i++) {
if (selectPropNameSet == null || selectPropNameSet.remove(titles[i])) {
propInfos[i] = beanInfo.getPropInfo(titles[i]);
if (propInfos[i] == null) {
if (selectPropNameSet != null && selectPropNameSet.remove(titles[i])) {
throw new IllegalArgumentException(titles[i] + " is not defined in bean class: " + ClassUtil.getCanonicalClassName(beanClass));
}
} else {
if (selectPropNameSet == null || selectPropNameSet.remove(titles[i]) || selectPropNameSet.remove(propInfos[i].name)) {
columnNameList.add(titles[i]);
columnList.add(new ArrayList<>());
} else {
propInfos[i] = null;
}
}
}
}
if (N.notNullOrEmpty(selectPropNameSet)) {
throw new IllegalArgumentException(selectColumnNames + " are not included in titles: " + N.toString(titles));
}
final String[] strs = new String[titles.length];
while (offset-- > 0 && br.readLine() != null) {
// continue
}
while (count > 0 && (line = br.readLine()) != null) {
lineParser.accept(strs, line);
if (filter != null && !filter.test(strs)) {
continue;
}
for (int i = 0, columnIndex = 0; i < columnCount; i++) {
if (propInfos[i] != null) {
columnList.get(columnIndex++).add(propInfos[i].readPropValue(strs[i]));
}
}
count--;
}
return new RowDataSet(columnNameList, columnList);
} catch (IOException e) {
throw new UncheckedIOException(e);
} finally {
if (br != csvReader) {
Objectory.recycle(br);
}
}
}
/**
*
* @param csvFile
* @param columnTypeMap
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
@SuppressWarnings("rawtypes")
public static DataSet loadCSV(final File csvFile, final Map columnTypeMap) throws UncheckedIOException {
return loadCSV(csvFile, 0, Long.MAX_VALUE, columnTypeMap);
}
/**
*
* @param csvFile
* @param offset
* @param count
* @param columnTypeMap
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
@SuppressWarnings("rawtypes")
public static DataSet loadCSV(final File csvFile, final long offset, final long count, final Map columnTypeMap)
throws UncheckedIOException {
return loadCSV(csvFile, offset, count, Fn. alwaysTrue(), columnTypeMap);
}
/**
* Load the data from CSV.
*
* @param
* @param csvFile
* @param offset
* @param count
* @param filter
* @param columnTypeMap
* @return
* @throws UncheckedIOException the unchecked IO exception
* @throws E the e
*/
@SuppressWarnings("rawtypes")
public static DataSet loadCSV(final File csvFile, final long offset, final long count, final Throwables.Predicate filter,
final Map columnTypeMap) throws UncheckedIOException, E {
InputStream csvInputStream = null;
try {
csvInputStream = IOUtil.newFileInputStream(csvFile);
return loadCSV(csvInputStream, offset, count, filter, columnTypeMap);
} finally {
IOUtil.closeQuietly(csvInputStream);
}
}
/**
*
* @param csvInputStream
* @param columnTypeMap
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
@SuppressWarnings("rawtypes")
public static DataSet loadCSV(final InputStream csvInputStream, final Map columnTypeMap) throws UncheckedIOException {
return loadCSV(csvInputStream, 0, Long.MAX_VALUE, columnTypeMap);
}
/**
*
* @param csvInputStream
* @param offset
* @param count
* @param columnTypeMap
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
@SuppressWarnings("rawtypes")
public static DataSet loadCSV(final InputStream csvInputStream, final long offset, final long count, final Map columnTypeMap)
throws UncheckedIOException {
return loadCSV(csvInputStream, offset, count, Fn. alwaysTrue(), columnTypeMap);
}
/**
* Load the data from CSV.
*
* @param
* @param csvInputStream
* @param offset
* @param count
* @param filter
* @param columnTypeMap
* @return
* @throws UncheckedIOException the unchecked IO exception
* @throws E the e
*/
@SuppressWarnings("rawtypes")
public static DataSet loadCSV(final InputStream csvInputStream, final long offset, final long count,
final Throwables.Predicate filter, final Map columnTypeMap) throws UncheckedIOException, E {
final Reader csvReader = new InputStreamReader(csvInputStream);
return loadCSV(csvReader, offset, count, filter, columnTypeMap);
}
/**
*
* @param csvReader
* @param columnTypeMap
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
@SuppressWarnings("rawtypes")
public static DataSet loadCSV(final Reader csvReader, final Map columnTypeMap) throws UncheckedIOException {
return loadCSV(csvReader, 0, Long.MAX_VALUE, columnTypeMap);
}
/**
*
* @param csvReader
* @param offset
* @param count
* @param columnTypeMap
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
@SuppressWarnings("rawtypes")
public static DataSet loadCSV(final Reader csvReader, long offset, long count, final Map columnTypeMap)
throws UncheckedIOException {
return loadCSV(csvReader, offset, count, Fn. alwaysTrue(), columnTypeMap);
}
/**
* Load the data from CSV.
*
* @param
* @param csvReader
* @param offset
* @param count
* @param filter
* @param columnTypeMap
* @return
* @throws UncheckedIOException the unchecked IO exception
* @throws E the e
*/
@SuppressWarnings("rawtypes")
public static DataSet loadCSV(final Reader csvReader, long offset, long count, final Throwables.Predicate filter,
final Map columnTypeMap) throws UncheckedIOException, E {
N.checkArgument(offset >= 0 && count >= 0, "'offset'=%s and 'count'=%s can't be negative", offset, count);
if (N.isNullOrEmpty(columnTypeMap)) {
throw new IllegalArgumentException("columnTypeMap can't be null or empty");
}
final Function headerParser = csvHeaderParser_TL.get();
final BiConsumer lineParser = csvLineParser_TL.get();
final BufferedReader br = csvReader instanceof BufferedReader ? (BufferedReader) csvReader : Objectory.createBufferedReader(csvReader);
try {
String line = br.readLine();
if (line == null) {
return N.newEmptyDataSet();
}
final String[] titles = headerParser.apply(line);
final int columnCount = titles.length;
final Type[] columnTypes = new Type[columnCount];
final List columnNameList = new ArrayList<>(columnTypeMap.size());
final List> columnList = new ArrayList<>(columnTypeMap.size());
for (int i = 0; i < columnCount; i++) {
if (columnTypeMap.containsKey(titles[i])) {
columnTypes[i] = columnTypeMap.get(titles[i]);
columnNameList.add(titles[i]);
columnList.add(new ArrayList<>());
}
}
if (columnNameList.size() != columnTypeMap.size()) {
final List keys = new ArrayList<>(columnTypeMap.keySet());
keys.removeAll(columnNameList);
throw new IllegalArgumentException(keys + " are not included in titles: " + N.toString(titles));
}
final String[] strs = new String[titles.length];
while (offset-- > 0 && br.readLine() != null) {
// continue
}
while (count > 0 && (line = br.readLine()) != null) {
lineParser.accept(strs, line);
if (filter != null && !filter.test(strs)) {
continue;
}
for (int i = 0, columnIndex = 0; i < columnCount; i++) {
if (columnTypes[i] != null) {
columnList.get(columnIndex++).add(columnTypes[i].valueOf(strs[i]));
}
}
count--;
}
return new RowDataSet(columnNameList, columnList);
} catch (IOException e) {
throw new UncheckedIOException(e);
} finally {
if (br != csvReader) {
Objectory.recycle(br);
}
}
}
/**
*
* @param csvFile
* @param columnTypeList
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
@SuppressWarnings("rawtypes")
public static DataSet loadCSV(final File csvFile, final List columnTypeList) throws UncheckedIOException {
return loadCSV(csvFile, 0, Long.MAX_VALUE, columnTypeList);
}
/**
*
* @param csvFile
* @param offset
* @param count
* @param columnTypeList
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
@SuppressWarnings("rawtypes")
public static DataSet loadCSV(final File csvFile, final long offset, final long count, final List columnTypeList)
throws UncheckedIOException {
return loadCSV(csvFile, offset, count, Fn. alwaysTrue(), columnTypeList);
}
/**
* Load the data from CSV.
*
* @param
* @param csvFile
* @param offset
* @param count
* @param filter
* @param columnTypeList set the column type to null to skip the column in CSV.
* @return
* @throws UncheckedIOException the unchecked IO exception
* @throws E the e
*/
@SuppressWarnings("rawtypes")
public static DataSet loadCSV(final File csvFile, final long offset, final long count, final Throwables.Predicate filter,
final List columnTypeList) throws UncheckedIOException, E {
InputStream csvInputStream = null;
try {
csvInputStream = IOUtil.newFileInputStream(csvFile);
return loadCSV(csvInputStream, offset, count, filter, columnTypeList);
} finally {
IOUtil.closeQuietly(csvInputStream);
}
}
/**
*
* @param csvInputStream
* @param columnTypeList
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
@SuppressWarnings("rawtypes")
public static DataSet loadCSV(final InputStream csvInputStream, final List columnTypeList) throws UncheckedIOException {
return loadCSV(csvInputStream, 0, Long.MAX_VALUE, columnTypeList);
}
/**
*
* @param csvInputStream
* @param offset
* @param count
* @param columnTypeList
* @return
* @throws UncheckedIOException the unchecked IO exception
*/
@SuppressWarnings("rawtypes")
public static DataSet loadCSV(final InputStream csvInputStream, final long offset, final long count, final List columnTypeList)
throws UncheckedIOException {
return loadCSV(csvInputStream, offset, count, Fn. alwaysTrue(), columnTypeList);
}
/**
* Load the data from CSV.
*
* @param
* @param csvInputStream
* @param offset
* @param count
* @param filter
* @param columnTypeList set the column type to null to skip the column in CSV.
* @return
* @throws E the e
*/
@SuppressWarnings("rawtypes")
public static DataSet loadCSV(final InputStream csvInputStream, final long offset, final long count,
final Throwables.Predicate filter, final List columnTypeList) throws E {
final Reader csvReader = new InputStreamReader(csvInputStream);
return loadCSV(csvReader, offset, count, filter, columnTypeList);
}
/**
*
* @param csvReader
* @param columnTypeList
* @return
*/
@SuppressWarnings("rawtypes")
public static DataSet loadCSV(final Reader csvReader, final List columnTypeList) {
return loadCSV(csvReader, 0, Long.MAX_VALUE, columnTypeList);
}
/**
*
* @param csvReader
* @param offset
* @param count
* @param columnTypeList
* @return
*/
@SuppressWarnings("rawtypes")
public static DataSet loadCSV(final Reader csvReader, long offset, long count, final List columnTypeList) {
return loadCSV(csvReader, offset, count, Fn. alwaysTrue(), columnTypeList);
}
/**
* Load the data from CSV.
*
* @param
* @param csvReader
* @param offset
* @param count
* @param filter
* @param columnTypeList set the column type to null to skip the column in CSV.
* @return
* @throws UncheckedIOException the unchecked IO exception
* @throws E the e
*/
@SuppressWarnings("rawtypes")
public static DataSet loadCSV(final Reader csvReader, long offset, long count, final Throwables.Predicate filter,
final List columnTypeList) throws UncheckedIOException, E {
N.checkArgument(offset >= 0 && count >= 0, "'offset'=%s and 'count'=%s can't be negative", offset, count);
if (N.isNullOrEmpty(columnTypeList)) {
throw new IllegalArgumentException("columnTypeList can't be null or empty");
}
final Function headerParser = csvHeaderParser_TL.get();
final BiConsumer lineParser = csvLineParser_TL.get();
final BufferedReader br = csvReader instanceof BufferedReader ? (BufferedReader) csvReader : Objectory.createBufferedReader(csvReader);
final Type[] columnTypes = columnTypeList.toArray(new Type[columnTypeList.size()]);
try {
String line = br.readLine();
if (line == null) {
return N.newEmptyDataSet();
}
final String[] titles = headerParser.apply(line);
final int columnCount = titles.length;
final List columnNameList = new ArrayList<>(columnCount);
final List> columnList = new ArrayList<>();
for (int i = 0; i < columnCount; i++) {
if (columnTypes[i] != null) {
columnNameList.add(titles[i]);
columnList.add(new ArrayList<>());
}
}
final String[] strs = new String[titles.length];
while (offset-- > 0 && br.readLine() != null) {
// continue
}
while (count > 0 && (line = br.readLine()) != null) {
lineParser.accept(strs, line);
if (filter != null && !filter.test(strs)) {
continue;
}
for (int i = 0, columnIndex = 0; i < columnCount; i++) {
if (columnTypes[i] != null) {
columnList.get(columnIndex++).add(columnTypes[i].valueOf(strs[i]));
}
}
count--;
}
return new RowDataSet(columnNameList, columnList);
} catch (IOException e) {
throw new UncheckedIOException(e);
} finally {
if (br != csvReader) {
Objectory.recycle(br);
}
}
}
/**
*
*
* @param
* @param targetType
* @param csvFile
* @return
*/
public static Stream stream(final Class targetType, final File csvFile) {
return stream(targetType, csvFile, (Collection) null);
}
/**
*
*
* @param
* @param targetType
* @param csvFile
* @param selectColumnNames
* @return
*/
public static Stream stream(final Class targetType, final File csvFile, final Collection selectColumnNames) {
return stream(targetType, csvFile, selectColumnNames, 0, Long.MAX_VALUE, Fn.alwaysTrue());
}
/**
*
*
* @param
* @param targetType
* @param csvFile
* @param selectColumnNames
* @param offset
* @param count
* @param filter
* @return
*/
public static Stream stream(final Class targetType, final File csvFile, final Collection selectColumnNames, final long offset,
final long count, final Predicate filter) {
FileReader csvReader = null;
try {
csvReader = IOUtil.newFileReader(csvFile);
return stream(targetType, csvReader, selectColumnNames, offset, count, true, filter);
} catch (Exception e) {
if (csvReader != null) {
IOUtil.closeQuietly(csvReader);
}
throw ExceptionUtil.toRuntimeException(e);
}
}
/**
*
*
* @param
* @param targetType
* @param csvReader
* @param closeReaderWhenStreamIsClosed
* @return
*/
public static Stream stream(final Class targetType, final Reader csvReader, final boolean closeReaderWhenStreamIsClosed) {
return stream(targetType, csvReader, (Collection) null, closeReaderWhenStreamIsClosed);
}
/**
*
*
* @param
* @param targetType
* @param csvReader
* @param selectColumnNames
* @param closeReaderWhenStreamIsClosed
* @return
*/
public static Stream stream(final Class targetType, final Reader csvReader, final Collection selectColumnNames,
final boolean closeReaderWhenStreamIsClosed) {
return stream(targetType, csvReader, selectColumnNames, 0, Long.MAX_VALUE, closeReaderWhenStreamIsClosed, Fn.alwaysTrue());
}
/**
*
*
* @param
* @param targetType
* @param csvReader
* @param selectColumnNames
* @param offset
* @param count
* @param closeReaderWhenStreamIsClosed
* @param filter
* @return
*/
public static Stream stream(final Class targetType, final Reader csvReader, final Collection selectColumnNames,
final long offset, final long count, final boolean closeReaderWhenStreamIsClosed, final Predicate filter) {
return Stream.defer(() -> {
N.checkArgNotNull(targetType, "targetType");
N.checkArgument(offset >= 0 && count >= 0, "'offset'=%s and 'count'=%s can't be negative", offset, count);
final BufferedReader br = csvReader instanceof BufferedReader ? (BufferedReader) csvReader : Objectory.createBufferedReader(csvReader);
boolean noException = false;
try {
final Function headerParser = csvHeaderParser_TL.get();
final BiConsumer lineParser = csvLineParser_TL.get();
String line = br.readLine();
if (line == null) {
noException = true;
return Stream.empty();
}
final String[] titles = headerParser.apply(line);
final boolean isBean = ClassUtil.isBeanClass(targetType);
final BeanInfo beanInfo = isBean ? ParserUtil.getBeanInfo(targetType) : null;
final int columnCount = titles.length;
final String[] resultColumnNames = new String[columnCount];
final Set selectPropNameSet = selectColumnNames == null ? null : N.newHashSet(selectColumnNames);
final PropInfo[] propInfos = isBean ? new PropInfo[columnCount] : null;
int resultColumnCount = 0;
for (int i = 0; i < columnCount; i++) {
if (isBean) {
propInfos[i] = beanInfo.getPropInfo(titles[i]);
if (propInfos[i] == null) {
if (selectPropNameSet != null && selectPropNameSet.remove(titles[i])) {
throw new IllegalArgumentException(titles[i] + " is not defined in bean class: " + ClassUtil.getCanonicalClassName(targetType));
}
} else {
if (selectPropNameSet == null || selectPropNameSet.remove(titles[i]) || selectPropNameSet.remove(propInfos[i].name)) {
resultColumnNames[i] = titles[i];
resultColumnCount++;
} else {
propInfos[i] = null;
}
}
} else {
if (selectPropNameSet == null || selectPropNameSet.remove(titles[i])) {
resultColumnNames[i] = titles[i];
resultColumnCount++;
}
}
}
if (N.notNullOrEmpty(selectPropNameSet)) {
throw new IllegalArgumentException(selectColumnNames + " are not included in titles: " + N.toString(titles));
}
long offsetTmp = offset;
while (offsetTmp-- > 0 && br.readLine() != null) {
// continue
}
final Type type = Type.of(targetType);
final int finalResultColumnCount = resultColumnCount;
com.landawn.abacus.util.function.Function mapper = null;
if (type.isObjectArray()) {
final Class componentType = targetType.getComponentType();
mapper = values -> {
final Object[] result = N.newArray(componentType, finalResultColumnCount);
for (int i = 0, j = 0; i < columnCount; i++) {
if (resultColumnNames[i] != null) {
result[j++] = values[i];
}
}
return (T) result;
};
} else if (type.isCollection()) {
mapper = values -> {
@SuppressWarnings("rawtypes")
final Collection