com.ibm.cp4waiops.connectors.sdk.CloudEventProducer Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of connectors-sdk Show documentation
A developer SDK for creating connectors for CP4WAIOps.
There is a newer version: 2.2.9
package com.ibm.cp4waiops.connectors.sdk;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Level;
import java.util.logging.Logger;

import com.ibm.aiops.connectors.bridge.ConnectorBridgeGrpc.ConnectorBridgeStub;

import io.cloudevents.CloudEvent;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;

public class CloudEventProducer implements Runnable {
    private static final Logger logger = Logger.getLogger(CloudEventProducer.class.getName());

    private static final int DEFAULT_FIX_PERIOD_MS = 60 * 1000;

    private ConnectorBridgeStub _bridgeStub;

    // Exposing these queues in case the users need more debugging or throttling
    // _emitQueue contains all cloud events
    public BlockingQueue _emitQueue;

    // If the cloud event fails to be sent from _emitQueue, add to the unverified list in _channels for that channel
    public HashMap _channels;
    private AtomicReference> _targetedChannels;

    private int _messageSizeLimit;
    private final long _fixPeriodMS;
    private long _lastFixedAt;
    private boolean _noTopicWarningLogged;

    private MeterRegistry _meterRegistry;

    // Dropped events come from a topic not being sent in the cloud event or the topic is not one
    // that the connection has permissions to produce messages on
    private Counter _droppedEvents;

    CloudEventProducer(ConnectorBridgeStub stub, BlockingQueue emitQueue, int messageSizeLimit,
            MeterRegistry meterRegistry) {
        this(stub, emitQueue, messageSizeLimit, DEFAULT_FIX_PERIOD_MS, meterRegistry);
    }

    CloudEventProducer(ConnectorBridgeStub stub, BlockingQueue emitQueue, int messageSizeLimit,
            int fixPeriod, MeterRegistry meterRegistry) {
        _bridgeStub = stub;
        _emitQueue = emitQueue;
        _channels = new HashMap<>();
        _targetedChannels = new AtomicReference<>(new HashSet<>());
        _messageSizeLimit = messageSizeLimit;
        _fixPeriodMS = fixPeriod;
        _lastFixedAt = System.nanoTime();
        _noTopicWarningLogged = false;
        _meterRegistry = meterRegistry;
        _droppedEvents = _meterRegistry.counter("connector.sdk.producer.badevents");

        logger.log(Level.INFO, "CloudEventProducer created");
    }

    void setAllowedChannels(List names) {
        _targetedChannels.set(new HashSet<>(names));
    }

    double getDroppedCount() {
        return _droppedEvents.count();
    }

    @Override
    public void run() {
        boolean uninterrupted = true;
        while (uninterrupted) {
            try {
                cycle();
            } catch (InterruptedException error) {
                logger.log(Level.WARNING, "run() interrupted: terminating produce loop");
                closeAllChannelsNotInSet(new HashSet<>());
                uninterrupted = false;
                Thread.currentThread().interrupt();
            }
        }
        logger.log(Level.WARNING, "run() interrupted: closing all channels");
        for (String name : _channels.keySet()) {
            _channels.get(name).close();
        }
    }

    private void cycle() throws InterruptedException {
        Set allowedTopics = _targetedChannels.get();
        // Fix broken streams
        if (System.nanoTime() - _lastFixedAt > _fixPeriodMS) {
            _lastFixedAt = System.nanoTime();
            closeAllChannelsNotInSet(allowedTopics);
            fixBrokenProducerChannels();
        }
        // Ensure topics have been setup
        if (allowedTopics.isEmpty()) {
            if (!_noTopicWarningLogged) {
                logger.log(Level.INFO,
                        "no produce topics setup, queued events will not be sent until topics have been configured");
                _noTopicWarningLogged = true;
            }
            final long WAIT_TIME_MS = 2000;
            Thread.sleep(WAIT_TIME_MS);
            return;
        }
        // Poll event and send
        logger.log(Level.FINER, "cycle() current emit queue size: " + _emitQueue.size());
        CloudEvent event = _emitQueue.poll(6, TimeUnit.SECONDS);
        if (event != null) {
            processEvent(allowedTopics, event);
        }
    }

    private void processEvent(Set allowedTopics, CloudEvent event) {
        Object extension = event.getExtension(Connector.TOPIC_CE_EXTENSION_NAME);
        if (!(extension instanceof String)) {
            logger.log(Level.SEVERE, "dropping event with no topic specified: type=" + event.getType());
            _droppedEvents.increment();
            return;
        }
        String channelName = String.class.cast(extension);
        if (!allowedTopics.contains(channelName)) {
            logger.log(Level.WARNING,
                    "dropping event sent to unconfigured topic: topic=" + channelName + ",type=" + event.getType());
            _droppedEvents.increment();
            return;
        }
        GRPCCloudEventProduceChannel channel = _channels.get(channelName);
        if (channel == null) {
            channel = new GRPCCloudEventProduceChannel(channelName, _bridgeStub, _messageSizeLimit, _meterRegistry);
            _channels.put(channelName, channel);
        }
        logger.log(Level.FINER,
                "processEvent() producing to channel: id=" + event.getId() + ",channelName=" + channelName);
        channel.produce(event);
    }

    private void closeAllChannelsNotInSet(Set names) {
        ArrayList toRemove = new ArrayList<>();
        for (var entry : _channels.entrySet()) {
            if (names.contains(entry.getKey())) {
                continue;
            }
            logger.log(Level.INFO, "channel no longer requested, closing: name=" + entry.getKey());
            entry.getValue().close();
            toRemove.add(entry.getKey());
        }
        for (String name : toRemove) {
            _channels.remove(name);
        }
    }

    private void fixBrokenProducerChannels() {
        for (String name : _channels.keySet()) {
            _channels.get(name).fixIfBroken();
        }
    }

    public double getUnverifiedCount() {
        double total = 0;
        for (var entry : _channels.entrySet()) {
            total += entry.getValue().getUnverifiedCount();
        }
        return total;
    }
}