io.streamnative.pulsar.handlers.kop.proxy.KafkaProxyExtension Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of pulsar-kafka-proxy Show documentation
Kafka on Pulsar Proxy
The newest version!
/**
 * Copyright (c) 2019 - 2024 StreamNative, Inc.. All Rights Reserved.
 */
/**
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.streamnative.pulsar.handlers.kop.proxy;

import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.google.common.annotations.VisibleForTesting;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.socket.SocketChannel;
import io.prometheus.client.Counter;
import io.prometheus.client.Gauge;
import io.streamnative.pulsar.handlers.kop.EndPoint;
import io.streamnative.pulsar.handlers.kop.KafkaServiceConfiguration;
import io.streamnative.pulsar.handlers.kop.KopVersion;
import io.streamnative.pulsar.handlers.kop.utils.ConfigurationUtils;
import io.streamnative.pulsar.handlers.kop.utils.ssl.SSLUtils;
import java.net.InetSocketAddress;
import java.time.Duration;
import java.util.Map;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.security.auth.SecurityProtocol;
import org.apache.pulsar.proxy.extensions.ProxyExtension;
import org.apache.pulsar.proxy.server.ProxyConfiguration;
import org.apache.pulsar.proxy.server.ProxyService;

@Slf4j
public class KafkaProxyExtension implements ProxyExtension {

    public static final String ACTIVE_CONNECTIONS_METRICS = "ksn_proxy_active_connections";
    public static final String NEW_CONNECTIONS_METRICS = "ksn_proxy_new_connections";
    public static final String OPS_COUNTER_METRICS = "ksn_proxy_binary_ops";
    public static final String BYTES_COUNTER_METRICS = "ksn_proxy_binary_bytes";

    static final Gauge ACTIVE_CONNECTIONS = Gauge
        .build(ACTIVE_CONNECTIONS_METRICS, "Number of connections currently active in the ksn proxy").create()
        .register();

    static final Counter NEW_CONNECTIONS = Counter
        .build(NEW_CONNECTIONS_METRICS, "Counter of connections being opened in the ksn proxy").create()
        .register();

    static final Counter OPS_COUNTER = Counter
        .build(OPS_COUNTER_METRICS, "Counter of ksn proxy operations").create().register();

    static final Counter BYTES_COUNTER = Counter
        .build(BYTES_COUNTER_METRICS, "Counter of ksn proxy bytes").create().register();


    private final Cache leaderCache = Caffeine.newBuilder()
        .maximumSize(10000)
        .expireAfterWrite(Duration.ofMinutes(5))
        .build();
    private KafkaProxyConfiguration config;
    private ConnectionFactory connectionFactory;
    // Only the SSL related configs are reused
    private KafkaServiceConfiguration sslConfig;
    private Map> channelInitializers;

    @VisibleForTesting
    Map> getChannelInitializers() {
        return channelInitializers;
    }

    @Override
    public String extensionName() {
        return "kafka";
    }

    @Override
    public boolean accept(String extension) {
        return extensionName().equals(extension);
    }

    @Override
    public void initialize(ProxyConfiguration conf) {
        this.config = new KafkaProxyConfiguration(conf.getProperties());
        this.config.setBrokerProxyConnectTimeoutMs(conf.getBrokerProxyConnectTimeoutMs());
        this.sslConfig = ConfigurationUtils.create(conf.getProperties(), KafkaServiceConfiguration.class);
        log.info("KafkaProxyExtension loaded config: {}", this.config);
    }

    @Override
    public void start(ProxyService service) {
        log.info("Starting KafkaProxyExtension, kop version is: '{}'", KopVersion.getVersion());
        log.info("Git Revision {}", KopVersion.getGitSha());
        log.info("Built by {} on {} at {}",
                KopVersion.getBuildUser(),
                KopVersion.getBuildHost(),
                KopVersion.getBuildTime());
        this.connectionFactory = new ConnectionFactory(config, service.getWorkerGroup(),
                service.getDnsAddressResolverGroup());
    }

    @Override
    public Map> newChannelInitializers() {
        final var listeners = EndPoint.parseListeners(config.getKafkaListeners());
        final var advertisedListeners = EndPoint.parseListeners(config.getKafkaAdvertisedListeners());
        if (!listeners.keySet().equals(advertisedListeners.keySet())) {
            throw new RuntimeException("Listener names do not match between kafkaListeners \""
                    + config.getKafkaListeners() + "\" and kafkaAdvertisedListeners \""
                    + config.getKafkaAdvertisedListeners() + "\"");
        }
        channelInitializers = listeners.entrySet().stream().collect(Collectors.toMap(
                e -> e.getValue().getInetAddress(),
                e -> {
                    final var advertisedEndPoint = advertisedListeners.get(e.getKey());
                    final var securityProtocol = advertisedEndPoint.getSecurityProtocol();
                    final var sslContextFactory = (securityProtocol.equals(SecurityProtocol.SSL)
                            || securityProtocol.equals(SecurityProtocol.SASL_SSL)
                    ) ? SSLUtils.createSslContextFactory(sslConfig) : null;
                    return new KafkaProxyChannelInitializer(advertisedEndPoint, connectionFactory, sslContextFactory,
                        leaderCache);
                }
        ));
        return channelInitializers;
    }

    @Override
    public void close() {
        log.info("KafkaProxyExtension closed.");
    }
}