io.deltastream.flink.connector.snowflake.sink.SnowflakeSink Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of flink-connector-snowflake Show documentation
An Apache Flink connector for integrating with Snowflake.
The newest version!
/*
 * Copyright (c) 2023 DeltaStream, Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package io.deltastream.flink.connector.snowflake.sink;

import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.connector.sink2.Sink;
import org.apache.flink.api.connector.sink2.SinkWriter;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.StringUtils;

import io.deltastream.flink.connector.snowflake.sink.config.SnowflakeChannelConfig;
import io.deltastream.flink.connector.snowflake.sink.config.SnowflakeWriterConfig;
import io.deltastream.flink.connector.snowflake.sink.context.DefaultSnowflakeSinkContext;
import io.deltastream.flink.connector.snowflake.sink.serialization.SnowflakeRowSerializationSchema;

import java.util.Properties;

/**
 * Flink Sink to produce data into a Snowflake table. The sink supports below delivery guarantees as
 * described by {@link org.apache.flink.connector.base.DeliveryGuarantee}.
 *
 * 
 *   {@link org.apache.flink.connector.base.DeliveryGuarantee#NONE} does not provide any guarantees:
 *   messages may be lost in case of issues on the Snowflake ingest channel and messages may be
 *   duplicated in case of a Flink runtime failure.
 * 
 *
 * 
 *   {@link org.apache.flink.connector.base.DeliveryGuarantee#AT_LEAST_ONCE} the sink will flush
 *   data on a checkpoint to ensure all received events have successfully been committed to the
 *   Snowflake service backend. Ingestion failures are retried to ensure delivery of all received
 *   events at least once, but data may be duplicated when Flink restarts.
 * 
 *
 * @param  type of records that the sink receives to serialize and write to the corresponding
 *     Snowflake table
 * @see SnowflakeSinkBuilder for constructing this sink
 */
@PublicEvolving
public class SnowflakeSink implements Sink {

    private static final long serialVersionUID = 3587917829427569404L;

    private final String appId;
    private final Properties connectionConfigs;
    private final SnowflakeWriterConfig writerConfig;
    private final SnowflakeChannelConfig channelConfig;
    private final SnowflakeRowSerializationSchema serializationSchema;

    public String getAppId() {
        return appId;
    }

    public Properties getConnectionConfigs() {
        return connectionConfigs;
    }

    public SnowflakeWriterConfig getWriterConfig() {
        return writerConfig;
    }

    public SnowflakeChannelConfig getChannelConfig() {
        return channelConfig;
    }

    public SnowflakeRowSerializationSchema getSerializationSchema() {
        return serializationSchema;
    }

    SnowflakeSink(
            final String appId,
            final Properties connectionConfigs,
            final SnowflakeWriterConfig writerConfig,
            final SnowflakeChannelConfig channelConfig,
            final SnowflakeRowSerializationSchema serializationSchema) {

        Preconditions.checkArgument(
                !StringUtils.isNullOrWhitespaceOnly(appId), "An application ID is required");
        this.appId = appId;
        this.connectionConfigs = Preconditions.checkNotNull(connectionConfigs);
        this.writerConfig = Preconditions.checkNotNull(writerConfig);
        this.channelConfig = Preconditions.checkNotNull(channelConfig);
        this.serializationSchema = Preconditions.checkNotNull(serializationSchema);
    }

    public static  SnowflakeSinkBuilder builder() {
        return new SnowflakeSinkBuilder<>();
    }

    @Override
    public SinkWriter createWriter(InitContext initContext) {
        return new SnowflakeSinkWriter<>(
                new DefaultSnowflakeSinkContext(
                        Preconditions.checkNotNull(initContext, "initContext"),
                        this.writerConfig,
                        this.appId),
                this.connectionConfigs,
                this.channelConfig,
                this.serializationSchema);
    }
}