
io.streamthoughts.kafka.connect.filepulse.source.FilePulseSourceConnector Maven / Gradle / Ivy
/*
* Copyright 2019-2020 StreamThoughts.
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.streamthoughts.kafka.connect.filepulse.source;
import io.streamthoughts.kafka.connect.filepulse.Version;
import io.streamthoughts.kafka.connect.filepulse.config.SourceConnectorConfig;
import io.streamthoughts.kafka.connect.filepulse.config.SourceTaskConfig;
import io.streamthoughts.kafka.connect.filepulse.fs.CompositeFileListFilter;
import io.streamthoughts.kafka.connect.filepulse.fs.DefaultFileSystemMonitor;
import io.streamthoughts.kafka.connect.filepulse.fs.DefaultTaskFileURIProvider;
import io.streamthoughts.kafka.connect.filepulse.fs.DelegateTaskFileURIProvider;
import io.streamthoughts.kafka.connect.filepulse.fs.FileSystemListing;
import io.streamthoughts.kafka.connect.filepulse.fs.FileSystemMonitor;
import io.streamthoughts.kafka.connect.filepulse.state.StateBackingStoreAccess;
import org.apache.kafka.common.config.ConfigDef;
import org.apache.kafka.common.config.ConfigException;
import org.apache.kafka.connect.connector.Task;
import org.apache.kafka.connect.errors.ConnectException;
import org.apache.kafka.connect.source.SourceConnector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static io.streamthoughts.kafka.connect.filepulse.state.KafkaFileObjectStateBackingStoreConfig.TASKS_FILE_STATUS_STORAGE_CONSUMER_ENABLED_CONFIG;
import static io.streamthoughts.kafka.connect.filepulse.state.KafkaFileObjectStateBackingStoreConfig.TASKS_FILE_STATUS_STORAGE_NAME_CONFIG;
/**
* The FilePulseSourceConnector.
*/
public class FilePulseSourceConnector extends SourceConnector {
private static final Logger LOG = LoggerFactory.getLogger(FilePulseSourceConnector.class);
private static final long DEFAULT_MAX_TIMEOUT = 5000;
private static final String CONNECT_NAME_CONFIG = "name";
private Map configProperties;
private final AtomicInteger taskConfigsGeneration = new AtomicInteger(0);
private FileSystemMonitorThread fsMonitorThread;
private SourceConnectorConfig connectorConfig;
private FileSystemMonitor monitor;
private String connectorGroupName;
private TaskPartitioner partitioner;
private StateBackingStoreAccess sharedStore;
private final AtomicBoolean closed = new AtomicBoolean(false);
/**
* {@inheritDoc}
*/
@Override
public String version() {
return Version.getVersion();
}
/**
* {@inheritDoc}
*/
@Override
public void start(final Map props) {
connectorGroupName = props.get(CONNECT_NAME_CONFIG);
LOG.info("Starting FilePulse source connector: {}", connectorGroupName);
try {
configProperties = new HashMap<>(props);
configProperties.put(TASKS_FILE_STATUS_STORAGE_NAME_CONFIG, connectorGroupName);
configProperties.put(TASKS_FILE_STATUS_STORAGE_CONSUMER_ENABLED_CONFIG, "true");
connectorConfig = new SourceConnectorConfig(configProperties);
} catch (ConfigException e) {
throw new ConnectException("Failed to initialize FilePulseSourceConnector due to configuration error", e);
}
try {
sharedStore = new StateBackingStoreAccess(
connectorGroupName,
connectorConfig::getStateBackingStore,
false
);
partitioner = connectorConfig.getTaskPartitioner();
final FileSystemListing> fileSystemListing = connectorConfig.getFileSystemListing();
fileSystemListing.setFilter(new CompositeFileListFilter(connectorConfig.getFileSystemListingFilter()));
monitor = new DefaultFileSystemMonitor(
connectorConfig.allowTasksReconfigurationAfterTimeoutMs(),
fileSystemListing,
connectorConfig.getFsCleanupPolicy(),
connectorConfig.getFsCleanupPolicyPredicate(),
connectorConfig.getSourceOffsetPolicy(),
sharedStore.get().getResource(),
connectorConfig.getTaskFilerOrder()
);
monitor.setFileSystemListingEnabled(!connectorConfig.isFileListingTaskDelegationEnabled());
fsMonitorThread = new FileSystemMonitorThread(context, monitor, connectorConfig.getListingInterval());
fsMonitorThread.setUncaughtExceptionHandler((t, e) -> {
LOG.info("Uncaught error from file system monitoring thread [{}]", t.getName(), e);
context.raiseError(new ConnectException("Unexpected error from FileSystemMonitorThread", e));
});
fsMonitorThread.start();
LOG.info("Started FilePulse source connector: {}", connectorGroupName);
} catch (Exception e) {
closeResources();
throw e;
}
}
/**
* {@inheritDoc}
*/
@Override
public Class extends Task> taskClass() {
return FilePulseSourceTask.class;
}
/**
* {@inheritDoc}
*/
@Override
public List
© 2015 - 2025 Weber Informatics LLC | Privacy Policy