io.streamnative.pulsar.handlers.kop.proxy.KafkaProxyExtension Maven / Gradle / Ivy
/**
* Copyright (c) 2019 - 2024 StreamNative, Inc.. All Rights Reserved.
*/
/**
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.streamnative.pulsar.handlers.kop.proxy;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.google.common.annotations.VisibleForTesting;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.socket.SocketChannel;
import io.prometheus.client.Counter;
import io.prometheus.client.Gauge;
import io.streamnative.pulsar.handlers.kop.EndPoint;
import io.streamnative.pulsar.handlers.kop.KafkaServiceConfiguration;
import io.streamnative.pulsar.handlers.kop.KopVersion;
import io.streamnative.pulsar.handlers.kop.utils.ConfigurationUtils;
import io.streamnative.pulsar.handlers.kop.utils.ssl.SSLUtils;
import java.net.InetSocketAddress;
import java.time.Duration;
import java.util.Map;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.security.auth.SecurityProtocol;
import org.apache.pulsar.proxy.extensions.ProxyExtension;
import org.apache.pulsar.proxy.server.ProxyConfiguration;
import org.apache.pulsar.proxy.server.ProxyService;
@Slf4j
public class KafkaProxyExtension implements ProxyExtension {
public static final String ACTIVE_CONNECTIONS_METRICS = "ksn_proxy_active_connections";
public static final String NEW_CONNECTIONS_METRICS = "ksn_proxy_new_connections";
public static final String OPS_COUNTER_METRICS = "ksn_proxy_binary_ops";
public static final String BYTES_COUNTER_METRICS = "ksn_proxy_binary_bytes";
static final Gauge ACTIVE_CONNECTIONS = Gauge
.build(ACTIVE_CONNECTIONS_METRICS, "Number of connections currently active in the ksn proxy").create()
.register();
static final Counter NEW_CONNECTIONS = Counter
.build(NEW_CONNECTIONS_METRICS, "Counter of connections being opened in the ksn proxy").create()
.register();
static final Counter OPS_COUNTER = Counter
.build(OPS_COUNTER_METRICS, "Counter of ksn proxy operations").create().register();
static final Counter BYTES_COUNTER = Counter
.build(BYTES_COUNTER_METRICS, "Counter of ksn proxy bytes").create().register();
private final Cache leaderCache = Caffeine.newBuilder()
.maximumSize(10000)
.expireAfterWrite(Duration.ofMinutes(5))
.build();
private KafkaProxyConfiguration config;
private ConnectionFactory connectionFactory;
// Only the SSL related configs are reused
private KafkaServiceConfiguration sslConfig;
private Map> channelInitializers;
@VisibleForTesting
Map> getChannelInitializers() {
return channelInitializers;
}
@Override
public String extensionName() {
return "kafka";
}
@Override
public boolean accept(String extension) {
return extensionName().equals(extension);
}
@Override
public void initialize(ProxyConfiguration conf) {
this.config = new KafkaProxyConfiguration(conf.getProperties());
this.config.setBrokerProxyConnectTimeoutMs(conf.getBrokerProxyConnectTimeoutMs());
this.sslConfig = ConfigurationUtils.create(conf.getProperties(), KafkaServiceConfiguration.class);
log.info("KafkaProxyExtension loaded config: {}", this.config);
}
@Override
public void start(ProxyService service) {
log.info("Starting KafkaProxyExtension, kop version is: '{}'", KopVersion.getVersion());
log.info("Git Revision {}", KopVersion.getGitSha());
log.info("Built by {} on {} at {}",
KopVersion.getBuildUser(),
KopVersion.getBuildHost(),
KopVersion.getBuildTime());
this.connectionFactory = new ConnectionFactory(config, service.getWorkerGroup(),
service.getDnsAddressResolverGroup());
}
@Override
public Map> newChannelInitializers() {
final var listeners = EndPoint.parseListeners(config.getKafkaListeners());
final var advertisedListeners = EndPoint.parseListeners(config.getKafkaAdvertisedListeners());
if (!listeners.keySet().equals(advertisedListeners.keySet())) {
throw new RuntimeException("Listener names do not match between kafkaListeners \""
+ config.getKafkaListeners() + "\" and kafkaAdvertisedListeners \""
+ config.getKafkaAdvertisedListeners() + "\"");
}
channelInitializers = listeners.entrySet().stream().collect(Collectors.toMap(
e -> e.getValue().getInetAddress(),
e -> {
final var advertisedEndPoint = advertisedListeners.get(e.getKey());
final var securityProtocol = advertisedEndPoint.getSecurityProtocol();
final var sslContextFactory = (securityProtocol.equals(SecurityProtocol.SSL)
|| securityProtocol.equals(SecurityProtocol.SASL_SSL)
) ? SSLUtils.createSslContextFactory(sslConfig) : null;
return new KafkaProxyChannelInitializer(advertisedEndPoint, connectionFactory, sslContextFactory,
leaderCache);
}
));
return channelInitializers;
}
@Override
public void close() {
log.info("KafkaProxyExtension closed.");
}
}