net.snowflake.ingest.streaming.internal.Flusher Maven / Gradle / Ivy
The newest version!
/*
* Copyright (c) 2022-2024 Snowflake Computing Inc. All rights reserved.
*/
package net.snowflake.ingest.streaming.internal;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import net.snowflake.ingest.utils.Pair;
/**
* Interface to convert {@link ChannelData} buffered in {@link RowBuffer} to the underlying format
* implementation for faster processing.
*
* @param type of column data ({@link ParquetChunkData})
*/
public interface Flusher {
/**
* Serialize buffered rows into the underlying format.
*
* @param fullyQualifiedTableName
* @param channelsDataPerTable buffered rows
* @param filePath file path
* @param chunkStartOffset
* @return {@link SerializationResult}
* @throws IOException
*/
SerializationResult serialize(
List> channelsDataPerTable, String filePath, long chunkStartOffset)
throws IOException;
/** Holds result of the buffered rows conversion: channel metadata and stats. */
class SerializationResult {
final List channelsMetadataList;
final Map columnEpStatsMapCombined;
final long rowCount;
final float chunkEstimatedUncompressedSize;
final ByteArrayOutputStream chunkData;
final Pair chunkMinMaxInsertTimeInMs;
final long extendedMetadataSize;
public SerializationResult(
List channelsMetadataList,
Map columnEpStatsMapCombined,
long rowCount,
float chunkEstimatedUncompressedSize,
ByteArrayOutputStream chunkData,
Pair chunkMinMaxInsertTimeInMs,
long extendedMetadataSize) {
this.channelsMetadataList = channelsMetadataList;
this.columnEpStatsMapCombined = columnEpStatsMapCombined;
this.rowCount = rowCount;
this.chunkEstimatedUncompressedSize = chunkEstimatedUncompressedSize;
this.chunkData = chunkData;
this.chunkMinMaxInsertTimeInMs = chunkMinMaxInsertTimeInMs;
this.extendedMetadataSize = extendedMetadataSize;
}
}
}