io.trino.parquet.reader.MetadataReader Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of trino-parquet Show documentation
Show all versions of trino-parquet Show documentation
Trino - Parquet file format support
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.parquet.reader;
import com.google.common.collect.ImmutableList;
import io.airlift.log.Logger;
import io.airlift.slice.Slice;
import io.airlift.slice.Slices;
import io.trino.parquet.ParquetCorruptionException;
import io.trino.parquet.ParquetDataSource;
import io.trino.parquet.ParquetDataSourceId;
import io.trino.parquet.ParquetWriteValidation;
import io.trino.parquet.metadata.BlockMetadata;
import io.trino.parquet.metadata.ColumnChunkMetadata;
import io.trino.parquet.metadata.FileMetadata;
import io.trino.parquet.metadata.ParquetMetadata;
import org.apache.parquet.CorruptStatistics;
import org.apache.parquet.column.statistics.BinaryStatistics;
import org.apache.parquet.format.ColumnChunk;
import org.apache.parquet.format.ColumnMetaData;
import org.apache.parquet.format.Encoding;
import org.apache.parquet.format.FileMetaData;
import org.apache.parquet.format.KeyValue;
import org.apache.parquet.format.RowGroup;
import org.apache.parquet.format.SchemaElement;
import org.apache.parquet.format.Statistics;
import org.apache.parquet.hadoop.metadata.ColumnPath;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.Type.Repetition;
import org.apache.parquet.schema.Types;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import static io.trino.parquet.ParquetMetadataConverter.convertEncodingStats;
import static io.trino.parquet.ParquetMetadataConverter.fromParquetStatistics;
import static io.trino.parquet.ParquetMetadataConverter.getEncoding;
import static io.trino.parquet.ParquetMetadataConverter.getLogicalTypeAnnotation;
import static io.trino.parquet.ParquetMetadataConverter.getPrimitive;
import static io.trino.parquet.ParquetMetadataConverter.toColumnIndexReference;
import static io.trino.parquet.ParquetMetadataConverter.toOffsetIndexReference;
import static io.trino.parquet.ParquetValidationUtils.validateParquet;
import static java.lang.Boolean.FALSE;
import static java.lang.Boolean.TRUE;
import static java.lang.Math.min;
import static java.lang.Math.toIntExact;
import static org.apache.parquet.format.Util.readFileMetaData;
public final class MetadataReader
{
private static final Logger log = Logger.get(MetadataReader.class);
private static final Slice MAGIC = Slices.utf8Slice("PAR1");
private static final int POST_SCRIPT_SIZE = Integer.BYTES + MAGIC.length();
// Typical 1GB files produced by Trino were found to have footer size between 30-40KB
private static final int EXPECTED_FOOTER_SIZE = 48 * 1024;
private MetadataReader() {}
public static ParquetMetadata readFooter(ParquetDataSource dataSource, Optional parquetWriteValidation)
throws IOException
{
// Parquet File Layout:
//
// MAGIC
// variable: Data
// variable: Metadata
// 4 bytes: MetadataLength
// MAGIC
validateParquet(dataSource.getEstimatedSize() >= MAGIC.length() + POST_SCRIPT_SIZE, dataSource.getId(), "%s is not a valid Parquet File", dataSource.getId());
// Read the tail of the file
long estimatedFileSize = dataSource.getEstimatedSize();
long expectedReadSize = min(estimatedFileSize, EXPECTED_FOOTER_SIZE);
Slice buffer = dataSource.readTail(toIntExact(expectedReadSize));
Slice magic = buffer.slice(buffer.length() - MAGIC.length(), MAGIC.length());
validateParquet(MAGIC.equals(magic), dataSource.getId(), "Expected magic number: %s got: %s", MAGIC.toStringUtf8(), magic.toStringUtf8());
int metadataLength = buffer.getInt(buffer.length() - POST_SCRIPT_SIZE);
long metadataIndex = estimatedFileSize - POST_SCRIPT_SIZE - metadataLength;
validateParquet(
metadataIndex >= MAGIC.length() && metadataIndex < estimatedFileSize - POST_SCRIPT_SIZE,
dataSource.getId(),
"Metadata index: %s out of range",
metadataIndex);
int completeFooterSize = metadataLength + POST_SCRIPT_SIZE;
if (completeFooterSize > buffer.length()) {
// initial read was not large enough, so just read again with the correct size
buffer = dataSource.readTail(completeFooterSize);
}
InputStream metadataStream = buffer.slice(buffer.length() - completeFooterSize, metadataLength).getInput();
FileMetaData fileMetaData = readFileMetaData(metadataStream);
ParquetMetadata parquetMetadata = createParquetMetadata(fileMetaData, dataSource.getId());
validateFileMetadata(dataSource.getId(), parquetMetadata.getFileMetaData(), parquetWriteValidation);
return parquetMetadata;
}
public static ParquetMetadata createParquetMetadata(FileMetaData fileMetaData, ParquetDataSourceId dataSourceId)
throws ParquetCorruptionException
{
List schema = fileMetaData.getSchema();
validateParquet(!schema.isEmpty(), dataSourceId, "Schema is empty");
MessageType messageType = readParquetSchema(schema);
List blocks = new ArrayList<>();
List rowGroups = fileMetaData.getRow_groups();
if (rowGroups != null) {
for (RowGroup rowGroup : rowGroups) {
List columns = rowGroup.getColumns();
validateParquet(!columns.isEmpty(), dataSourceId, "No columns in row group: %s", rowGroup);
String filePath = columns.get(0).getFile_path();
ImmutableList.Builder columnMetadataBuilder = ImmutableList.builderWithExpectedSize(columns.size());
for (ColumnChunk columnChunk : columns) {
validateParquet(
(filePath == null && columnChunk.getFile_path() == null)
|| (filePath != null && filePath.equals(columnChunk.getFile_path())),
dataSourceId,
"all column chunks of the same row group must be in the same file");
ColumnMetaData metaData = columnChunk.meta_data;
String[] path = metaData.path_in_schema.stream()
.map(value -> value.toLowerCase(Locale.ENGLISH))
.toArray(String[]::new);
ColumnPath columnPath = ColumnPath.get(path);
PrimitiveType primitiveType = messageType.getType(columnPath.toArray()).asPrimitiveType();
ColumnChunkMetadata column = ColumnChunkMetadata.get(
columnPath,
primitiveType,
CompressionCodecName.fromParquet(metaData.codec),
convertEncodingStats(metaData.encoding_stats),
readEncodings(metaData.encodings),
readStats(Optional.ofNullable(fileMetaData.getCreated_by()), Optional.ofNullable(metaData.statistics), primitiveType),
metaData.data_page_offset,
metaData.dictionary_page_offset,
metaData.num_values,
metaData.total_compressed_size,
metaData.total_uncompressed_size);
column.setColumnIndexReference(toColumnIndexReference(columnChunk));
column.setOffsetIndexReference(toOffsetIndexReference(columnChunk));
column.setBloomFilterOffset(metaData.bloom_filter_offset);
columnMetadataBuilder.add(column);
}
blocks.add(new BlockMetadata(rowGroup.getNum_rows(), columnMetadataBuilder.build()));
}
}
Map keyValueMetaData = new HashMap<>();
List keyValueList = fileMetaData.getKey_value_metadata();
if (keyValueList != null) {
for (KeyValue keyValue : keyValueList) {
keyValueMetaData.put(keyValue.key, keyValue.value);
}
}
FileMetadata parquetFileMetadata = new FileMetadata(
messageType,
keyValueMetaData,
fileMetaData.getCreated_by());
return new ParquetMetadata(parquetFileMetadata, blocks);
}
private static MessageType readParquetSchema(List schema)
{
Iterator schemaIterator = schema.iterator();
SchemaElement rootSchema = schemaIterator.next();
Types.MessageTypeBuilder builder = Types.buildMessage();
readTypeSchema(builder, schemaIterator, rootSchema.getNum_children());
return builder.named(rootSchema.name);
}
private static void readTypeSchema(Types.GroupBuilder builder, Iterator schemaIterator, int typeCount)
{
for (int i = 0; i < typeCount; i++) {
SchemaElement element = schemaIterator.next();
Types.Builder typeBuilder;
if (element.type == null) {
typeBuilder = builder.group(Repetition.valueOf(element.repetition_type.name()));
readTypeSchema((Types.GroupBuilder) typeBuilder, schemaIterator, element.num_children);
}
else {
Types.PrimitiveBuilder primitiveBuilder = builder.primitive(getPrimitive(element.type), Repetition.valueOf(element.repetition_type.name()));
if (element.isSetType_length()) {
primitiveBuilder.length(element.type_length);
}
if (element.isSetPrecision()) {
primitiveBuilder.precision(element.precision);
}
if (element.isSetScale()) {
primitiveBuilder.scale(element.scale);
}
typeBuilder = primitiveBuilder;
}
// Reading of element.logicalType and element.converted_type corresponds to parquet-mr's code at
// https://github.com/apache/parquet-mr/blob/apache-parquet-1.12.0/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java#L1568-L1582
LogicalTypeAnnotation annotationFromLogicalType = null;
if (element.isSetLogicalType()) {
annotationFromLogicalType = getLogicalTypeAnnotation(element.logicalType);
typeBuilder.as(annotationFromLogicalType);
}
if (element.isSetConverted_type()) {
LogicalTypeAnnotation annotationFromConvertedType = getLogicalTypeAnnotation(element.converted_type, element);
if (annotationFromLogicalType != null) {
// Both element.logicalType and element.converted_type set
if (annotationFromLogicalType.toOriginalType() == annotationFromConvertedType.toOriginalType()) {
// element.converted_type matches element.logicalType, even though annotationFromLogicalType may differ from annotationFromConvertedType
// Following parquet-mr behavior, we favor LogicalTypeAnnotation derived from element.logicalType, as potentially containing more information.
}
else {
// Following parquet-mr behavior, issue warning and let converted_type take precedence.
log.warn("Converted type and logical type metadata map to different OriginalType (convertedType: %s, logical type: %s). Using value in converted type.",
element.converted_type, element.logicalType);
// parquet-mr reads only OriginalType from converted_type. We retain full LogicalTypeAnnotation
// 1. for compatibility, as previous Trino reader code would read LogicalTypeAnnotation from element.converted_type and some additional fields.
// 2. so that we override LogicalTypeAnnotation annotation read from element.logicalType in case of mismatch detected.
typeBuilder.as(annotationFromConvertedType);
}
}
else {
// parquet-mr reads only OriginalType from converted_type. We retain full LogicalTypeAnnotation for compatibility, as previous
// Trino reader code would read LogicalTypeAnnotation from element.converted_type and some additional fields.
typeBuilder.as(annotationFromConvertedType);
}
}
if (element.isSetField_id()) {
typeBuilder.id(element.field_id);
}
typeBuilder.named(element.name.toLowerCase(Locale.ENGLISH));
}
}
public static org.apache.parquet.column.statistics.Statistics readStats(Optional fileCreatedBy, Optional statisticsFromFile, PrimitiveType type)
{
Statistics statistics = statisticsFromFile.orElse(null);
org.apache.parquet.column.statistics.Statistics columnStatistics = fromParquetStatistics(fileCreatedBy.orElse(null), statistics, type);
if (isStringType(type)
&& statistics != null
&& !statistics.isSetMin_value() && !statistics.isSetMax_value() // the min,max fields used for UTF8 since Parquet PARQUET-1025
&& statistics.isSetMin() && statistics.isSetMax() // the min,max fields used for UTF8 before Parquet PARQUET-1025
&& columnStatistics.genericGetMin() == null && columnStatistics.genericGetMax() == null
&& !CorruptStatistics.shouldIgnoreStatistics(fileCreatedBy.orElse(null), type.getPrimitiveTypeName())) {
columnStatistics = tryReadOldUtf8Stats(statistics, (BinaryStatistics) columnStatistics);
}
return columnStatistics;
}
private static boolean isStringType(PrimitiveType type)
{
if (type.getLogicalTypeAnnotation() == null) {
return false;
}
return type.getLogicalTypeAnnotation()
.accept(new LogicalTypeAnnotation.LogicalTypeAnnotationVisitor()
{
@Override
public Optional visit(LogicalTypeAnnotation.StringLogicalTypeAnnotation stringLogicalType)
{
return Optional.of(TRUE);
}
})
.orElse(FALSE);
}
private static org.apache.parquet.column.statistics.Statistics tryReadOldUtf8Stats(Statistics statistics, BinaryStatistics columnStatistics)
{
byte[] min = statistics.getMin();
byte[] max = statistics.getMax();
if (Arrays.equals(min, max)) {
// If min=max, then there is single value only
min = min.clone();
max = min;
}
else {
int commonPrefix = commonPrefix(min, max);
// For min we can retain all-ASCII, because this produces a strictly lower value.
int minGoodLength = commonPrefix;
while (minGoodLength < min.length && isAscii(min[minGoodLength])) {
minGoodLength++;
}
// For max we can be sure only of the part matching the min. When they differ, we can consider only one next, and only if both are ASCII
int maxGoodLength = commonPrefix;
if (maxGoodLength < max.length && maxGoodLength < min.length && isAscii(min[maxGoodLength]) && isAscii(max[maxGoodLength])) {
maxGoodLength++;
}
// Incrementing 127 would overflow. Incrementing within non-ASCII can have side-effects.
while (maxGoodLength > 0 && (max[maxGoodLength - 1] == 127 || !isAscii(max[maxGoodLength - 1]))) {
maxGoodLength--;
}
if (maxGoodLength == 0) {
// We can return just min bound, but code downstream likely expects both are present or both are absent.
return columnStatistics;
}
min = Arrays.copyOf(min, minGoodLength);
max = Arrays.copyOf(max, maxGoodLength);
max[maxGoodLength - 1]++;
}
return org.apache.parquet.column.statistics.Statistics
.getBuilderForReading(columnStatistics.type())
.withMin(min)
.withMax(max)
.withNumNulls(!columnStatistics.isNumNullsSet() && statistics.isSetNull_count() ? statistics.getNull_count() : columnStatistics.getNumNulls())
.build();
}
private static boolean isAscii(byte b)
{
return 0 <= b;
}
private static int commonPrefix(byte[] a, byte[] b)
{
int commonPrefixLength = 0;
while (commonPrefixLength < a.length && commonPrefixLength < b.length && a[commonPrefixLength] == b[commonPrefixLength]) {
commonPrefixLength++;
}
return commonPrefixLength;
}
private static Set readEncodings(List encodings)
{
Set columnEncodings = new HashSet<>();
for (Encoding encoding : encodings) {
columnEncodings.add(getEncoding(encoding));
}
return Collections.unmodifiableSet(columnEncodings);
}
private static void validateFileMetadata(ParquetDataSourceId dataSourceId, FileMetadata fileMetaData, Optional parquetWriteValidation)
throws ParquetCorruptionException
{
if (parquetWriteValidation.isEmpty()) {
return;
}
ParquetWriteValidation writeValidation = parquetWriteValidation.get();
writeValidation.validateTimeZone(
dataSourceId,
Optional.ofNullable(fileMetaData.getKeyValueMetaData().get("writer.time.zone")));
writeValidation.validateColumns(dataSourceId, fileMetaData.getSchema());
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy