io.kestra.plugin.pulsar.Reader Maven / Gradle / Ivy

Go to download
package io.kestra.plugin.pulsar;

import io.kestra.core.models.annotations.Example;
import io.kestra.core.models.annotations.Plugin;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.runners.RunContext;
import io.kestra.core.utils.Rethrow;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.ToString;
import lombok.experimental.SuperBuilder;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.PulsarClient;

import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.List;
import java.util.concurrent.TimeUnit;

@SuperBuilder
@ToString
@EqualsAndHashCode
@Getter
@NoArgsConstructor
@Schema(
    title = "Read messages from Pulsar topic(s) without subscription"
)
@Plugin(
    examples = {
        @Example(
            code = {
                "uri: pulsar://localhost:26650",
                "topic: test_kestra",
                "deserializer: JSON",
            }
        )
    }
)
public class Reader extends AbstractReader {
    @Schema(
        title = "The initial reader positioning can be set at specific timestamp by providing total rollback duration.",
        description = "So, broker can find a latest message that was published before given duration. eg: " +
            "rollbackDuration in minute = 5 suggests broker to find message which was published 5 mins back and " +
            "set the initial position on that messageId."
    )
    @PluginProperty(dynamic = true)
    private Duration since;

    @Schema(
        title = "Position the reader on a particular message.",
        description = "The first message read will be the one immediately *after* the specified message\n" +
            "If no `since` or `messageId` are provide, we start at the beginning of the topic."
    )
    @PluginProperty(dynamic = true)
    private String messageId;

    @Override
    public AbstractReader.Output run(RunContext runContext) throws Exception {
        try (PulsarClient client = PulsarService.client(this, runContext)) {
            org.apache.pulsar.client.api.ReaderBuilder readerBuilder = client.newReader()
                .topics(this.topics(runContext));

            if (this.since != null) {
                readerBuilder.startMessageFromRollbackDuration(this.since.getNano(), TimeUnit.NANOSECONDS);
            } else if (this.messageId != null) {
                readerBuilder.startMessageId(MessageId.fromByteArray(runContext.render(this.messageId).getBytes(StandardCharsets.UTF_8)));
            } else {
                readerBuilder.startMessageId(MessageId.earliest);
            }

            try (org.apache.pulsar.client.api.Reader reader = readerBuilder.create()) {
                return this.read(
                    runContext,
                    Rethrow.throwSupplier(() -> {
                        Message message = reader.readNext(this.getPollDuration().getNano(), TimeUnit.NANOSECONDS);

                        if (message == null) {
                            return List.of();
                        } else {
                            return List.of(message);
                        }
                    })
                );
            }
        }
    }
}