Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kafka.connect.runtime.distributed;
import org.apache.kafka.common.config.provider.ConfigProvider;
import org.apache.kafka.connect.runtime.WorkerConfigTransformer;
import org.apache.kafka.connect.runtime.TargetState;
import org.apache.kafka.connect.util.ConnectorTaskId;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
/**
* An immutable snapshot of the configuration state of connectors and tasks in a Kafka Connect cluster.
*/
public class ClusterConfigState {
public static final long NO_OFFSET = -1;
public static final ClusterConfigState EMPTY = new ClusterConfigState(
NO_OFFSET,
Collections.emptyMap(),
Collections.>emptyMap(),
Collections.emptyMap(),
Collections.>emptyMap(),
Collections.emptySet());
private final long offset;
private final Map connectorTaskCounts;
private final Map> connectorConfigs;
private final Map connectorTargetStates;
private final Map> taskConfigs;
private final Set inconsistentConnectors;
private final WorkerConfigTransformer configTransformer;
public ClusterConfigState(long offset,
Map connectorTaskCounts,
Map> connectorConfigs,
Map connectorTargetStates,
Map> taskConfigs,
Set inconsistentConnectors) {
this(offset,
connectorTaskCounts,
connectorConfigs,
connectorTargetStates,
taskConfigs,
inconsistentConnectors,
null);
}
public ClusterConfigState(long offset,
Map connectorTaskCounts,
Map> connectorConfigs,
Map connectorTargetStates,
Map> taskConfigs,
Set inconsistentConnectors,
WorkerConfigTransformer configTransformer) {
this.offset = offset;
this.connectorTaskCounts = connectorTaskCounts;
this.connectorConfigs = connectorConfigs;
this.connectorTargetStates = connectorTargetStates;
this.taskConfigs = taskConfigs;
this.inconsistentConnectors = inconsistentConnectors;
this.configTransformer = configTransformer;
}
/**
* Get the last offset read to generate this config state. This offset is not guaranteed to be perfectly consistent
* with the recorded state because some partial updates to task configs may have been read.
* @return the latest config offset
*/
public long offset() {
return offset;
}
/**
* Check whether this snapshot contains configuration for a connector.
* @param connector name of the connector
* @return true if this state contains configuration for the connector, false otherwise
*/
public boolean contains(String connector) {
return connectorConfigs.containsKey(connector);
}
/**
* Get a list of the connectors in this configuration
*/
public Set connectors() {
return connectorConfigs.keySet();
}
/**
* Get the configuration for a connector. The configuration will have been transformed by
* {@link org.apache.kafka.common.config.ConfigTransformer} by having all variable
* references replaced with the current values from external instances of
* {@link ConfigProvider}, and may include secrets.
* @param connector name of the connector
* @return a map containing configuration parameters
*/
public Map connectorConfig(String connector) {
Map configs = connectorConfigs.get(connector);
if (configTransformer != null) {
configs = configTransformer.transform(connector, configs);
}
return configs;
}
public Map rawConnectorConfig(String connector) {
return connectorConfigs.get(connector);
}
/**
* Get the target state of the connector
* @param connector name of the connector
* @return the target state
*/
public TargetState targetState(String connector) {
return connectorTargetStates.get(connector);
}
/**
* Get the configuration for a task. The configuration will have been transformed by
* {@link org.apache.kafka.common.config.ConfigTransformer} by having all variable
* references replaced with the current values from external instances of
* {@link ConfigProvider}, and may include secrets.
* @param task id of the task
* @return a map containing configuration parameters
*/
public Map taskConfig(ConnectorTaskId task) {
Map configs = taskConfigs.get(task);
if (configTransformer != null) {
configs = configTransformer.transform(task.connector(), configs);
}
return configs;
}
public Map rawTaskConfig(ConnectorTaskId task) {
return taskConfigs.get(task);
}
/**
* Get all task configs for a connector. The configurations will have been transformed by
* {@link org.apache.kafka.common.config.ConfigTransformer} by having all variable
* references replaced with the current values from external instances of
* {@link ConfigProvider}, and may include secrets.
* @param connector name of the connector
* @return a list of task configurations
*/
public List> allTaskConfigs(String connector) {
Map> taskConfigs = new TreeMap<>();
for (Map.Entry> taskConfigEntry : this.taskConfigs.entrySet()) {
if (taskConfigEntry.getKey().connector().equals(connector)) {
Map configs = taskConfigEntry.getValue();
if (configTransformer != null) {
configs = configTransformer.transform(connector, configs);
}
taskConfigs.put(taskConfigEntry.getKey().task(), configs);
}
}
return new LinkedList<>(taskConfigs.values());
}
/**
* Get the number of tasks assigned for the given connector.
* @param connectorName name of the connector to look up tasks for
* @return the number of tasks
*/
public int taskCount(String connectorName) {
Integer count = connectorTaskCounts.get(connectorName);
return count == null ? 0 : count;
}
/**
* Get the current set of task IDs for the specified connector.
* @param connectorName the name of the connector to look up task configs for
* @return the current set of connector task IDs
*/
public List tasks(String connectorName) {
if (inconsistentConnectors.contains(connectorName))
return Collections.emptyList();
Integer numTasks = connectorTaskCounts.get(connectorName);
if (numTasks == null)
return Collections.emptyList();
List taskIds = new ArrayList<>();
for (int taskIndex = 0; taskIndex < numTasks; taskIndex++) {
ConnectorTaskId taskId = new ConnectorTaskId(connectorName, taskIndex);
taskIds.add(taskId);
}
return taskIds;
}
/**
* Get the set of connectors which have inconsistent data in this snapshot. These inconsistencies can occur due to
* partially completed writes combined with log compaction.
*
* Connectors in this set will appear in the output of {@link #connectors()} since their connector configuration is
* available, but not in the output of {@link #taskConfig(ConnectorTaskId)} since the task configs are incomplete.
*
* When a worker detects a connector in this state, it should request that the connector regenerate its task
* configurations.
*
* @return the set of inconsistent connectors
*/
public Set inconsistentConnectors() {
return inconsistentConnectors;
}
@Override
public String toString() {
return "ClusterConfigState{" +
"offset=" + offset +
", connectorTaskCounts=" + connectorTaskCounts +
", connectorConfigs=" + connectorConfigs +
", taskConfigs=" + taskConfigs +
", inconsistentConnectors=" + inconsistentConnectors +
'}';
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ClusterConfigState that = (ClusterConfigState) o;
return offset == that.offset &&
Objects.equals(connectorTaskCounts, that.connectorTaskCounts) &&
Objects.equals(connectorConfigs, that.connectorConfigs) &&
Objects.equals(connectorTargetStates, that.connectorTargetStates) &&
Objects.equals(taskConfigs, that.taskConfigs) &&
Objects.equals(inconsistentConnectors, that.inconsistentConnectors) &&
Objects.equals(configTransformer, that.configTransformer);
}
@Override
public int hashCode() {
return Objects.hash(
offset,
connectorTaskCounts,
connectorConfigs,
connectorTargetStates,
taskConfigs,
inconsistentConnectors,
configTransformer);
}
}
