io.axual.connect.plugins.adls.gen2.AdlsGen2SinkConnector Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of adls-gen2-sink Show documentation
Collect the records from topics in an Azure Data Lake Storage Gen2
There is a newer version: 1.2.2
package io.axual.connect.plugins.adls.gen2;

/*-
 * ========================LICENSE_START=================================
 * Azure Data Lake Storage Gen2 Sink Connector for Kafka Connect
 * %%
 * Copyright (C) 2021 Axual B.V.
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License")
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * =========================LICENSE_END==================================
 */

import org.apache.kafka.common.config.ConfigDef;
import org.apache.kafka.connect.connector.Task;
import org.apache.kafka.connect.sink.SinkConnector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import io.axual.connect.plugins.adls.gen2.exceptions.AdlsGen2Exception;
import io.axual.connect.plugins.adls.gen2.storage.StorageProvider;

/**
 * The main sink connector class for loading data from Kafka to an Azure DataLake Storage Gen2 container.
 * This class validates the provided data and checks if a connection can be made.
 * 
 * It will always provide the maximum number of tasks allowed for processing the records from the Kafka topics.
 */
public class AdlsGen2SinkConnector extends SinkConnector {
    private static final Logger LOG = LoggerFactory.getLogger(AdlsGen2SinkConnector.class);

    private final Map connectorProperties;

    public AdlsGen2SinkConnector() {
        // Default constructor required for Connect
        connectorProperties = new HashMap<>();
    }

    // Used for mock injection
    AdlsGen2SinkConnector(Map connectorProperties) {
        this.connectorProperties = connectorProperties;
    }

    @Override
    public void start(Map props) {
        LOG.debug("Starting connector");
        AdlsGen2SinkConfig config = new AdlsGen2SinkConfig(props);
        // Storage Provider creation contains connection setup and container check
        StorageProvider storageProvider = config.getStorageProvider();

        final String baseDirectory = config.getBaseDirectory();
        if (!storageProvider.directoryExists(baseDirectory)) {
            LOG.debug("Creating base directory {}", baseDirectory);
            if (!storageProvider.createDirectory(baseDirectory)) {
                throw new AdlsGen2Exception("Could not create base directory " + baseDirectory);
            }
        }

        connectorProperties.putAll(props);
    }

    @Override
    public Class taskClass() {
        return AdlsGen2SinkTask.class;
    }

    @Override
    public List> taskConfigs(int maxTasks) {
        LOG.debug("Creating task configurations for {} tasks", maxTasks);
        if (connectorProperties.isEmpty()) {
            LOG.error("No connector properties found, return empty list for task configurations");
            return Collections.emptyList();
        }
        List> taskProperties = new ArrayList<>(maxTasks);
        for (int i = 0; i < maxTasks; i++) {
            taskProperties.add(new HashMap<>(connectorProperties));
        }
        return taskProperties;
    }

    @Override
    public void stop() {
        LOG.debug("Stopping connector");
        connectorProperties.clear();
    }

    @Override
    public ConfigDef config() {
        return AdlsGen2SinkConfig.config();
    }

    @Override
    public String version() {
        return AdlsGen2SinkConnectorInfo.getVersion();
    }
}