io.numaproj.numaflow.sessionreducer.Service Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of numaflow-java Show documentation
SDK to implement Numaflow Source or User Defined Functions or Sinks in Java.
The newest version!
package io.numaproj.numaflow.sessionreducer;

import akka.actor.ActorRef;
import akka.actor.ActorSystem;
import akka.actor.AllDeadLetters;
import akka.pattern.Patterns;
import akka.util.Timeout;
import com.google.protobuf.Empty;
import io.grpc.Status;
import io.grpc.stub.StreamObserver;
import io.numaproj.numaflow.sessionreduce.v1.SessionReduceGrpc;
import io.numaproj.numaflow.sessionreduce.v1.Sessionreduce;
import io.numaproj.numaflow.sessionreducer.model.SessionReducer;
import io.numaproj.numaflow.sessionreducer.model.SessionReducerFactory;
import lombok.extern.slf4j.Slf4j;
import scala.concurrent.Await;
import scala.concurrent.Future;
import scala.concurrent.duration.Duration;

import java.util.concurrent.CompletableFuture;

import static io.numaproj.numaflow.reduce.v1.ReduceGrpc.getReduceFnMethod;

@Slf4j
class Service extends SessionReduceGrpc.SessionReduceImplBase {
    public static final ActorSystem sessionReduceActorSystem = ActorSystem.create("sessionreduce");

    private final SessionReducerFactory sessionReducerFactory;

    public Service(SessionReducerFactory sessionReducerFactory) {
        this.sessionReducerFactory = sessionReducerFactory;
    }

    static void handleFailure(
            CompletableFuture failureFuture,
            StreamObserver responseObserver) {
        new Thread(() -> {
            try {
                failureFuture.get();
            } catch (Exception e) {
                e.printStackTrace();
                var status = Status.UNKNOWN.withDescription(e.getMessage()).withCause(e);
                responseObserver.onError(status.asException());
            }
        }).start();
    }

    /**
     * Streams input data to the session reducer functions and returns the result.
     */
    @Override
    public StreamObserver sessionReduceFn(final StreamObserver responseObserver) {
        if (this.sessionReducerFactory == null) {
            return io.grpc.stub.ServerCalls.asyncUnimplementedStreamingCall(
                    getReduceFnMethod(),
                    responseObserver);
        }

        CompletableFuture failureFuture = new CompletableFuture<>();

        // create a shutdown actor that listens to exceptions.
        ActorRef shutdownActorRef = sessionReduceActorSystem.
                actorOf(ShutdownActor.props(failureFuture));

        // subscribe for dead letters
        sessionReduceActorSystem.getEventStream().subscribe(shutdownActorRef, AllDeadLetters.class);

        handleFailure(failureFuture, responseObserver);

        // create an output actor that ensures synchronized delivery of reduce responses.
        ActorRef outputActor = sessionReduceActorSystem.actorOf(OutputActor.props(responseObserver));
        /*
            create a supervisor actor which assign the tasks to child actors.
            we create a child actor for every unique set of keys in a window.
        */
        ActorRef supervisorActor = sessionReduceActorSystem
                .actorOf(SupervisorActor.props(
                        sessionReducerFactory,
                        shutdownActorRef,
                        outputActor));

        return new StreamObserver<>() {
            @Override
            public void onNext(Sessionreduce.SessionReduceRequest sessionReduceRequest) {
                // send the message to parent actor, which takes care of distribution.
                if (!supervisorActor.isTerminated()) {
                    // if the operation is a MERGE, make it a blocking call.
                    if (sessionReduceRequest.getOperation().getEvent()
                            == Sessionreduce.SessionReduceRequest.WindowOperation.Event.MERGE) {
                        // on GO SDK side, we wait forever until the MERGE operation is done.
                        // on Java side, since the Await function requires a timeout, we are setting it to 1h
                        // for now, which is long enough for us to determine the system is hanging.
                        // If a MERGE took more than 1h, the system will panic.
                        Timeout timeout = new Timeout(Duration.create(1, "hour"));
                        try {
                            // ask the supervisor to process a merge request.
                            Future