org.elasticsearch.transport.RemoteClusterService Maven / Gradle / Ivy
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.transport;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.OriginalIndices;
import org.elasticsearch.action.support.GroupedActionListener;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodeRole;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.SettingUpgrader;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
import org.elasticsearch.common.util.concurrent.CountDown;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.threadpool.ThreadPool;
import java.io.Closeable;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Stream;
import static org.elasticsearch.common.settings.Setting.boolSetting;
import static org.elasticsearch.common.settings.Setting.enumSetting;
import static org.elasticsearch.common.settings.Setting.timeSetting;
/**
* Basic service for accessing remote clusters via gateway nodes
*/
public final class RemoteClusterService extends RemoteClusterAware implements Closeable {
private final Logger logger = LogManager.getLogger(RemoteClusterService.class);
static {
// remove search.remote.* settings in 8.0.0
assert Version.CURRENT.major < 8;
}
public static final Setting SEARCH_REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING = Setting.positiveTimeSetting(
"search.remote.initial_connect_timeout",
TimeValue.timeValueSeconds(30),
Setting.Property.NodeScope,
Setting.Property.Deprecated
);
/**
* The initial connect timeout for remote cluster connections
*/
public static final Setting REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING = Setting.positiveTimeSetting(
"cluster.remote.initial_connect_timeout",
SEARCH_REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING, // the default needs to be thirty seconds when fallback is removed
TimeValue.timeValueSeconds(0),
Setting.Property.NodeScope
);
public static final Setting SEARCH_REMOTE_NODE_ATTRIBUTE = Setting.simpleString(
"search.remote.node.attr",
Setting.Property.NodeScope,
Setting.Property.Deprecated
);
/**
* The name of a node attribute to select nodes that should be connected to in the remote cluster.
* For instance a node can be configured with {@code node.attr.gateway: true} in order to be eligible as a gateway node between
* clusters. In that case {@code search.remote.node.attr: gateway} can be used to filter out other nodes in the remote cluster.
* The value of the setting is expected to be a boolean, {@code true} for nodes that can become gateways, {@code false} otherwise.
*/
public static final Setting REMOTE_NODE_ATTRIBUTE = Setting.simpleString(
"cluster.remote.node.attr",
SEARCH_REMOTE_NODE_ATTRIBUTE, // no default is needed when fallback is removed, use simple string which gives empty
Setting.Property.NodeScope
);
public static final Setting SEARCH_ENABLE_REMOTE_CLUSTERS = Setting.boolSetting(
"search.remote.connect",
true,
Setting.Property.NodeScope,
Setting.Property.Deprecated
);
/**
* If true connecting to remote clusters is supported on this node. If false this node will not establish
* connections to any remote clusters configured. Search requests executed against this node (where this node is the coordinating node)
* will fail if remote cluster syntax is used as an index pattern. The default is true
*/
public static final Setting ENABLE_REMOTE_CLUSTERS = Setting.boolSetting(
"cluster.remote.connect",
SEARCH_ENABLE_REMOTE_CLUSTERS, // the default needs to be true when fallback is removed
Setting.Property.Deprecated,
Setting.Property.NodeScope
);
public static final Setting.AffixSetting SEARCH_REMOTE_CLUSTER_SKIP_UNAVAILABLE = Setting.affixKeySetting(
"search.remote.",
"skip_unavailable",
key -> boolSetting(key, false, Setting.Property.Deprecated, Setting.Property.Dynamic, Setting.Property.NodeScope)
);
public static final SettingUpgrader SEARCH_REMOTE_CLUSTER_SKIP_UNAVAILABLE_UPGRADER = new SettingUpgrader() {
@Override
public Setting getSetting() {
return SEARCH_REMOTE_CLUSTER_SKIP_UNAVAILABLE;
}
@Override
public String getKey(final String key) {
return key.replaceFirst("^search", "cluster");
}
};
public static final Setting.AffixSetting REMOTE_CLUSTER_SKIP_UNAVAILABLE = Setting.affixKeySetting(
"cluster.remote.",
"skip_unavailable",
(ns, key) -> boolSetting(
key,
// the default needs to be false when fallback is removed
"_na_".equals(key)
? SEARCH_REMOTE_CLUSTER_SKIP_UNAVAILABLE.getConcreteSettingForNamespace(key)
: SEARCH_REMOTE_CLUSTER_SKIP_UNAVAILABLE.getConcreteSetting(key.replaceAll("^cluster", "search")),
new RemoteConnectionEnabled<>(ns, key),
Setting.Property.Dynamic,
Setting.Property.NodeScope
)
);
public static final Setting.AffixSetting REMOTE_CLUSTER_PING_SCHEDULE = Setting.affixKeySetting(
"cluster.remote.",
"transport.ping_schedule",
(ns, key) -> timeSetting(
key,
TransportSettings.PING_SCHEDULE,
new RemoteConnectionEnabled<>(ns, key),
Setting.Property.Dynamic,
Setting.Property.NodeScope
)
);
public static final Setting.AffixSetting REMOTE_CLUSTER_COMPRESS = Setting.affixKeySetting(
"cluster.remote.",
"transport.compress",
(ns, key) -> enumSetting(
Compression.Enabled.class,
key,
TransportSettings.TRANSPORT_COMPRESS,
new RemoteConnectionEnabled<>(ns, key),
Setting.Property.Dynamic,
Setting.Property.NodeScope
)
);
public static final Setting.AffixSetting REMOTE_CLUSTER_COMPRESSION_SCHEME = Setting.affixKeySetting(
"cluster.remote.",
"transport.compression_scheme",
(ns, key) -> enumSetting(
Compression.Scheme.class,
key,
TransportSettings.TRANSPORT_COMPRESSION_SCHEME,
new RemoteConnectionEnabled<>(ns, key),
Setting.Property.Dynamic,
Setting.Property.NodeScope
)
);
private final boolean enabled;
public boolean isEnabled() {
return enabled;
}
private final TransportService transportService;
private final Map remoteClusters = ConcurrentCollections.newConcurrentMap();
RemoteClusterService(Settings settings, TransportService transportService) {
super(settings);
this.enabled = DiscoveryNode.isRemoteClusterClient(settings);
this.transportService = transportService;
}
public DiscoveryNode getLocalNode() {
return transportService.getLocalNode();
}
/**
* Returns true if at least one remote cluster is configured
*/
public boolean isCrossClusterSearchEnabled() {
return remoteClusters.isEmpty() == false;
}
boolean isRemoteNodeConnected(final String remoteCluster, final DiscoveryNode node) {
return remoteClusters.get(remoteCluster).isNodeConnected(node);
}
public Map groupIndices(IndicesOptions indicesOptions, String[] indices, Predicate indexExists) {
Map originalIndicesMap = new HashMap<>();
if (isCrossClusterSearchEnabled()) {
final Map> groupedIndices = groupClusterIndices(getRemoteClusterNames(), indices, indexExists);
if (groupedIndices.isEmpty()) {
// search on _all in the local cluster if neither local indices nor remote indices were specified
originalIndicesMap.put(LOCAL_CLUSTER_GROUP_KEY, new OriginalIndices(Strings.EMPTY_ARRAY, indicesOptions));
} else {
for (Map.Entry> entry : groupedIndices.entrySet()) {
String clusterAlias = entry.getKey();
List originalIndices = entry.getValue();
originalIndicesMap.put(clusterAlias, new OriginalIndices(originalIndices.toArray(new String[0]), indicesOptions));
}
}
} else {
originalIndicesMap.put(LOCAL_CLUSTER_GROUP_KEY, new OriginalIndices(indices, indicesOptions));
}
return originalIndicesMap;
}
/**
* Returns true iff the given cluster is configured as a remote cluster. Otherwise false
*/
boolean isRemoteClusterRegistered(String clusterName) {
return remoteClusters.containsKey(clusterName);
}
/**
* Returns the registered remote cluster names.
*/
public Set getRegisteredRemoteClusterNames() {
// remoteClusters is unmodifiable so its key set will be unmodifiable too
return remoteClusters.keySet();
}
/**
* Returns a connection to the given node on the given remote cluster
*
* @throws IllegalArgumentException if the remote cluster is unknown
*/
public Transport.Connection getConnection(DiscoveryNode node, String cluster) {
return getRemoteClusterConnection(cluster).getConnection(node);
}
/**
* Ensures that the given cluster alias is connected. If the cluster is connected this operation
* will invoke the listener immediately.
*/
void ensureConnected(String clusterAlias, ActionListener listener) {
final RemoteClusterConnection remoteClusterConnection;
try {
remoteClusterConnection = getRemoteClusterConnection(clusterAlias);
} catch (NoSuchRemoteClusterException e) {
listener.onFailure(e);
return;
}
remoteClusterConnection.ensureConnected(listener);
}
/**
* Returns whether the cluster identified by the provided alias is configured to be skipped when unavailable
*/
public boolean isSkipUnavailable(String clusterAlias) {
return getRemoteClusterConnection(clusterAlias).isSkipUnavailable();
}
public Transport.Connection getConnection(String cluster) {
return getRemoteClusterConnection(cluster).getConnection();
}
RemoteClusterConnection getRemoteClusterConnection(String cluster) {
if (enabled == false) {
throw new IllegalArgumentException(
"this node does not have the " + DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE.roleName() + " role"
);
}
RemoteClusterConnection connection = remoteClusters.get(cluster);
if (connection == null) {
throw new NoSuchRemoteClusterException(cluster);
}
return connection;
}
Set getRemoteClusterNames() {
return this.remoteClusters.keySet();
}
@Override
public void listenForUpdates(ClusterSettings clusterSettings) {
super.listenForUpdates(clusterSettings);
clusterSettings.addAffixUpdateConsumer(REMOTE_CLUSTER_SKIP_UNAVAILABLE, this::updateSkipUnavailable, (alias, value) -> {});
clusterSettings.addAffixUpdateConsumer(SEARCH_REMOTE_CLUSTER_SKIP_UNAVAILABLE, this::updateSkipUnavailable, (alias, value) -> {});
}
private synchronized void updateSkipUnavailable(String clusterAlias, Boolean skipUnavailable) {
RemoteClusterConnection remote = this.remoteClusters.get(clusterAlias);
if (remote != null) {
remote.setSkipUnavailable(skipUnavailable);
}
}
@Override
protected void updateRemoteCluster(String clusterAlias, Settings settings) {
CountDownLatch latch = new CountDownLatch(1);
updateRemoteCluster(clusterAlias, settings, ActionListener.wrap(latch::countDown));
try {
// Wait 10 seconds for a connections. We must use a latch instead of a future because we
// are on the cluster state thread and our custom future implementation will throw an
// assertion.
if (latch.await(10, TimeUnit.SECONDS) == false) {
logger.warn("failed to connect to new remote cluster {} within {}", clusterAlias, TimeValue.timeValueSeconds(10));
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
/**
* This method updates the list of remote clusters. It's intended to be used as an update consumer on the settings infrastructure
*
* @param clusterAlias a cluster alias to discovery node mapping representing the remote clusters seeds nodes
* @param newSettings the updated settings for the remote connection
* @param listener a listener invoked once every configured cluster has been connected to
*/
synchronized void updateRemoteCluster(String clusterAlias, Settings newSettings, ActionListener listener) {
if (LOCAL_CLUSTER_GROUP_KEY.equals(clusterAlias)) {
throw new IllegalArgumentException("remote clusters must not have the empty string as its key");
}
RemoteClusterConnection remote = this.remoteClusters.get(clusterAlias);
if (RemoteConnectionStrategy.isConnectionEnabled(clusterAlias, newSettings) == false) {
try {
IOUtils.close(remote);
} catch (IOException e) {
logger.warn("failed to close remote cluster connections for cluster: " + clusterAlias, e);
}
remoteClusters.remove(clusterAlias);
listener.onResponse(null);
return;
}
if (remote == null) {
// this is a new cluster we have to add a new representation
Settings finalSettings = Settings.builder().put(this.settings, false).put(newSettings, false).build();
remote = new RemoteClusterConnection(finalSettings, clusterAlias, transportService);
remoteClusters.put(clusterAlias, remote);
remote.ensureConnected(listener);
} else if (remote.shouldRebuildConnection(newSettings)) {
// Changes to connection configuration. Must tear down existing connection
try {
IOUtils.close(remote);
} catch (IOException e) {
logger.warn("failed to close remote cluster connections for cluster: " + clusterAlias, e);
}
remoteClusters.remove(clusterAlias);
Settings finalSettings = Settings.builder().put(this.settings, false).put(newSettings, false).build();
remote = new RemoteClusterConnection(finalSettings, clusterAlias, transportService);
remoteClusters.put(clusterAlias, remote);
remote.ensureConnected(listener);
} else {
// No changes to connection configuration.
listener.onResponse(null);
}
}
/**
* Connects to all remote clusters in a blocking fashion. This should be called on node startup to establish an initial connection
* to all configured seed nodes.
*/
void initializeRemoteClusters() {
final TimeValue timeValue = REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING.get(settings);
final PlainActionFuture> future = new PlainActionFuture<>();
Set enabledClusters = RemoteClusterAware.getEnabledRemoteClusters(settings);
if (enabledClusters.isEmpty()) {
return;
}
GroupedActionListener listener = new GroupedActionListener<>(future, enabledClusters.size());
for (String clusterAlias : enabledClusters) {
updateRemoteCluster(clusterAlias, settings, listener);
}
if (enabledClusters.isEmpty()) {
future.onResponse(null);
}
try {
future.get(timeValue.millis(), TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
} catch (TimeoutException ex) {
logger.warn("failed to connect to remote clusters within {}", timeValue.toString());
} catch (Exception e) {
throw new IllegalStateException("failed to connect to remote clusters", e);
}
}
@Override
public void close() throws IOException {
IOUtils.close(remoteClusters.values());
}
public Stream getRemoteConnectionInfos() {
return remoteClusters.values().stream().map(RemoteClusterConnection::getConnectionInfo);
}
/**
* Collects all nodes of the given clusters and returns / passes a (clusterAlias, nodeId) to {@link DiscoveryNode}
* function on success.
*/
public void collectNodes(Set clusters, ActionListener> listener) {
if (enabled == false) {
throw new IllegalArgumentException(
"this node does not have the " + DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE.roleName() + " role"
);
}
for (String cluster : clusters) {
if (this.remoteClusters.containsKey(cluster) == false) {
listener.onFailure(new NoSuchRemoteClusterException(cluster));
return;
}
}
final Map> clusterMap = new HashMap<>();
CountDown countDown = new CountDown(clusters.size());
Function nullFunction = s -> null;
for (final String cluster : clusters) {
RemoteClusterConnection connection = this.remoteClusters.get(cluster);
connection.collectNodes(new ActionListener>() {
@Override
public void onResponse(Function nodeLookup) {
synchronized (clusterMap) {
clusterMap.put(cluster, nodeLookup);
}
if (countDown.countDown()) {
listener.onResponse((clusterAlias, nodeId) -> clusterMap.getOrDefault(clusterAlias, nullFunction).apply(nodeId));
}
}
@Override
public void onFailure(Exception e) {
if (countDown.fastForward()) { // we need to check if it's true since we could have multiple failures
listener.onFailure(e);
}
}
});
}
}
/**
* Returns a client to the remote cluster if the given cluster alias exists.
*
* @param threadPool the {@link ThreadPool} for the client
* @param clusterAlias the cluster alias the remote cluster is registered under
* @param ensureConnected whether requests should wait for a connection attempt when there isn't a connection available
* @throws IllegalArgumentException if the given clusterAlias doesn't exist
*/
public Client getRemoteClusterClient(ThreadPool threadPool, String clusterAlias, boolean ensureConnected) {
if (transportService.getRemoteClusterService().isEnabled() == false) {
throw new IllegalArgumentException(
"this node does not have the " + DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE.roleName() + " role"
);
}
if (transportService.getRemoteClusterService().getRemoteClusterNames().contains(clusterAlias) == false) {
throw new NoSuchRemoteClusterException(clusterAlias);
}
return new RemoteClusterAwareClient(settings, threadPool, transportService, clusterAlias, ensureConnected);
}
/**
* Returns a client to the remote cluster if the given cluster alias exists.
*
* @param threadPool the {@link ThreadPool} for the client
* @param clusterAlias the cluster alias the remote cluster is registered under
* @throws IllegalArgumentException if the given clusterAlias doesn't exist
*/
public Client getRemoteClusterClient(ThreadPool threadPool, String clusterAlias) {
return getRemoteClusterClient(
threadPool,
clusterAlias,
transportService.getRemoteClusterService().isSkipUnavailable(clusterAlias) == false
);
}
Collection getConnections() {
return remoteClusters.values();
}
private static class RemoteConnectionEnabled implements Setting.Validator {
private final String clusterAlias;
private final String key;
private RemoteConnectionEnabled(String clusterAlias, String key) {
this.clusterAlias = clusterAlias;
this.key = key;
}
@Override
public void validate(T value) {}
@Override
public void validate(T value, Map, Object> settings, boolean isPresent) {
if (isPresent && RemoteConnectionStrategy.isConnectionEnabled(clusterAlias, settings) == false) {
throw new IllegalArgumentException("Cannot configure setting [" + key + "] if remote cluster is not enabled.");
}
}
@Override
public Iterator> settings() {
return Stream.concat(
Stream.of(RemoteConnectionStrategy.REMOTE_CONNECTION_MODE.getConcreteSettingForNamespace(clusterAlias)),
settingsStream()
).iterator();
}
private Stream> settingsStream() {
return Arrays.stream(RemoteConnectionStrategy.ConnectionStrategy.values())
.flatMap(strategy -> strategy.getEnablementSettings().get())
.map(as -> as.getConcreteSettingForNamespace(clusterAlias));
}
};
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy