org.apache.kafka.connect.runtime.standalone.StandaloneHerder Maven / Gradle / Ivy
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
package org.apache.kafka.connect.runtime.standalone;
import org.apache.kafka.connect.errors.AlreadyExistsException;
import org.apache.kafka.connect.errors.ConnectException;
import org.apache.kafka.connect.errors.NotFoundException;
import org.apache.kafka.connect.runtime.ConnectorConfig;
import org.apache.kafka.connect.runtime.Herder;
import org.apache.kafka.connect.runtime.HerderConnectorContext;
import org.apache.kafka.connect.runtime.TaskConfig;
import org.apache.kafka.connect.runtime.Worker;
import org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo;
import org.apache.kafka.connect.runtime.rest.entities.TaskInfo;
import org.apache.kafka.connect.util.Callback;
import org.apache.kafka.connect.util.ConnectorTaskId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
/**
* Single process, in-memory "herder". Useful for a standalone Kafka Connect process.
*/
public class StandaloneHerder implements Herder {
private static final Logger log = LoggerFactory.getLogger(StandaloneHerder.class);
private final Worker worker;
private HashMap connectors = new HashMap<>();
public StandaloneHerder(Worker worker) {
this.worker = worker;
}
public synchronized void start() {
log.info("Herder starting");
log.info("Herder started");
}
public synchronized void stop() {
log.info("Herder stopping");
// There's no coordination/hand-off to do here since this is all standalone. Instead, we
// should just clean up the stuff we normally would, i.e. cleanly checkpoint and shutdown all
// the tasks.
for (String connName : new HashSet<>(connectors.keySet())) {
removeConnectorTasks(connName);
try {
worker.stopConnector(connName);
} catch (ConnectException e) {
log.error("Error shutting down connector {}: ", connName, e);
}
}
connectors.clear();
log.info("Herder stopped");
}
@Override
public synchronized void connectors(Callback> callback) {
callback.onCompletion(null, new ArrayList<>(connectors.keySet()));
}
@Override
public synchronized void connectorInfo(String connName, Callback callback) {
ConnectorState state = connectors.get(connName);
if (state == null) {
callback.onCompletion(new NotFoundException("Connector " + connName + " not found"), null);
return;
}
callback.onCompletion(null, createConnectorInfo(state));
}
private ConnectorInfo createConnectorInfo(ConnectorState state) {
if (state == null)
return null;
List taskIds = new ArrayList<>();
for (int i = 0; i < state.taskConfigs.size(); i++)
taskIds.add(new ConnectorTaskId(state.name, i));
return new ConnectorInfo(state.name, state.configOriginals, taskIds);
}
@Override
public void connectorConfig(String connName, final Callback> callback) {
// Subset of connectorInfo, so piggy back on that implementation
connectorInfo(connName, new Callback() {
@Override
public void onCompletion(Throwable error, ConnectorInfo result) {
if (error != null) {
callback.onCompletion(error, null);
return;
}
callback.onCompletion(null, result.config());
}
});
}
@Override
public synchronized void putConnectorConfig(String connName, final Map config,
boolean allowReplace,
final Callback> callback) {
try {
boolean created = false;
if (connectors.containsKey(connName)) {
if (!allowReplace) {
callback.onCompletion(new AlreadyExistsException("Connector " + connName + " already exists"), null);
return;
}
if (config == null) // Deletion, kill tasks as well
removeConnectorTasks(connName);
worker.stopConnector(connName);
if (config == null)
connectors.remove(connName);
} else {
if (config == null) {
// Deletion, must already exist
callback.onCompletion(new NotFoundException("Connector " + connName + " not found", null), null);
return;
}
created = true;
}
if (config != null) {
startConnector(config);
updateConnectorTasks(connName);
}
if (config != null)
callback.onCompletion(null, new Created<>(created, createConnectorInfo(connectors.get(connName))));
else
callback.onCompletion(null, new Created(false, null));
} catch (ConnectException e) {
callback.onCompletion(e, null);
}
}
@Override
public synchronized void requestTaskReconfiguration(String connName) {
if (!worker.connectorNames().contains(connName)) {
log.error("Task that requested reconfiguration does not exist: {}", connName);
return;
}
updateConnectorTasks(connName);
}
@Override
public synchronized void taskConfigs(String connName, Callback> callback) {
ConnectorState state = connectors.get(connName);
if (state == null) {
callback.onCompletion(new NotFoundException("Connector " + connName + " not found", null), null);
return;
}
List result = new ArrayList<>();
for (int i = 0; i < state.taskConfigs.size(); i++) {
TaskInfo info = new TaskInfo(new ConnectorTaskId(connName, i), state.taskConfigs.get(i));
result.add(info);
}
callback.onCompletion(null, result);
}
@Override
public void putTaskConfigs(String connName, List> configs, Callback callback) {
throw new UnsupportedOperationException("Kafka Connect in standalone mode does not support externally setting task configurations.");
}
/**
* Start a connector in the worker and record its state.
* @param connectorProps new connector configuration
* @return the connector name
*/
private String startConnector(Map connectorProps) {
ConnectorConfig connConfig = new ConnectorConfig(connectorProps);
String connName = connConfig.getString(ConnectorConfig.NAME_CONFIG);
ConnectorState state = connectors.get(connName);
worker.addConnector(connConfig, new HerderConnectorContext(this, connName));
if (state == null) {
connectors.put(connName, new ConnectorState(connectorProps, connConfig));
} else {
state.configOriginals = connectorProps;
state.config = connConfig;
}
return connName;
}
private List> recomputeTaskConfigs(String connName) {
ConnectorState state = connectors.get(connName);
return worker.connectorTaskConfigs(connName,
state.config.getInt(ConnectorConfig.TASKS_MAX_CONFIG),
state.config.getList(ConnectorConfig.TOPICS_CONFIG));
}
private void createConnectorTasks(String connName) {
ConnectorState state = connectors.get(connName);
int index = 0;
for (Map taskConfigMap : state.taskConfigs) {
ConnectorTaskId taskId = new ConnectorTaskId(connName, index);
TaskConfig config = new TaskConfig(taskConfigMap);
try {
worker.addTask(taskId, config);
} catch (Throwable e) {
log.error("Failed to add task {}: ", taskId, e);
// Swallow this so we can continue updating the rest of the tasks
// FIXME what's the proper response? Kill all the tasks? Consider this the same as a task
// that died after starting successfully.
}
index++;
}
}
private void removeConnectorTasks(String connName) {
ConnectorState state = connectors.get(connName);
for (int i = 0; i < state.taskConfigs.size(); i++) {
ConnectorTaskId taskId = new ConnectorTaskId(connName, i);
try {
worker.stopTask(taskId);
} catch (ConnectException e) {
log.error("Failed to stop task {}: ", taskId, e);
// Swallow this so we can continue stopping the rest of the tasks
// FIXME: Forcibly kill the task?
}
}
state.taskConfigs = new ArrayList<>();
}
private void updateConnectorTasks(String connName) {
List> newTaskConfigs = recomputeTaskConfigs(connName);
ConnectorState state = connectors.get(connName);
if (!newTaskConfigs.equals(state.taskConfigs)) {
removeConnectorTasks(connName);
state.taskConfigs = newTaskConfigs;
createConnectorTasks(connName);
}
}
private static class ConnectorState {
public String name;
public Map configOriginals;
public ConnectorConfig config;
List> taskConfigs;
public ConnectorState(Map configOriginals, ConnectorConfig config) {
this.name = config.getString(ConnectorConfig.NAME_CONFIG);
this.configOriginals = configOriginals;
this.config = config;
this.taskConfigs = new ArrayList<>();
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy