com.google.cloud.pubsublite.beam.PubsubLiteSink Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of pubsublite-beam-io Show documentation
Beam IO for Google Cloud Pub/Sub Lite
There is a newer version: 0.23.0
/*
 * Copyright 2020 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.cloud.pubsublite.beam;

import com.google.api.core.ApiFuture;
import com.google.api.core.ApiFutureCallback;
import com.google.api.core.ApiFutures;
import com.google.api.core.ApiService.Listener;
import com.google.api.core.ApiService.State;
import com.google.api.gax.rpc.ApiException;
import com.google.cloud.pubsublite.Message;
import com.google.cloud.pubsublite.MessageMetadata;
import com.google.cloud.pubsublite.beam.PublisherOrError.Kind;
import com.google.cloud.pubsublite.internal.CheckedApiException;
import com.google.cloud.pubsublite.internal.ExtractStatus;
import com.google.cloud.pubsublite.internal.Publisher;
import com.google.cloud.pubsublite.internal.wire.SystemExecutors;
import com.google.cloud.pubsublite.proto.PubSubMessage;
import com.google.errorprone.annotations.concurrent.GuardedBy;
import java.util.ArrayDeque;
import java.util.Deque;
import java.util.function.Consumer;
import org.apache.beam.sdk.transforms.DoFn;

/** A sink which publishes messages to Pub/Sub Lite. */
@SuppressWarnings({
  "nullness" // TODO(https://issues.apache.org/jira/browse/BEAM-10402)
})
class PubsubLiteSink extends DoFn {
  private final PublisherOptions options;

  @GuardedBy("this")
  private transient PublisherOrError publisherOrError;

  // Whenever outstanding is decremented, notify() must be called.
  @GuardedBy("this")
  private transient int outstanding;

  @GuardedBy("this")
  private transient Deque errorsSinceLastFinish;

  PubsubLiteSink(PublisherOptions options) {
    this.options = options;
  }

  @Setup
  public void setup() throws ApiException {
    Publisher publisher;
    if (options.usesCache()) {
      publisher = PerServerPublisherCache.PUBLISHER_CACHE.get(options);
    } else {
      publisher = Publishers.newPublisher(options);
    }
    synchronized (this) {
      outstanding = 0;
      errorsSinceLastFinish = new ArrayDeque<>();
      publisherOrError = PublisherOrError.ofPublisher(publisher);
    }
    // cannot declare in inner class since 'this' means something different.
    Consumer onFailure =
        t -> {
          synchronized (this) {
            publisherOrError = PublisherOrError.ofError(ExtractStatus.toCanonical(t));
          }
        };
    publisher.addListener(
        new Listener() {
          @Override
          public void failed(State s, Throwable t) {
            onFailure.accept(t);
          }
        },
        SystemExecutors.getFuturesExecutor());
    if (!options.usesCache()) {
      publisher.startAsync();
    }
  }

  private synchronized void decrementOutstanding() {
    --outstanding;
    notify();
  }

  @ProcessElement
  public synchronized void processElement(@Element PubSubMessage message)
      throws CheckedApiException {
    ++outstanding;
    if (publisherOrError.getKind() == Kind.ERROR) {
      throw publisherOrError.error();
    }
    ApiFuture future =
        publisherOrError.publisher().publish(Message.fromProto(message));
    // cannot declare in inner class since 'this' means something different.
    Consumer onFailure =
        t -> {
          synchronized (this) {
            decrementOutstanding();
            errorsSinceLastFinish.push(ExtractStatus.toCanonical(t));
          }
        };
    ApiFutures.addCallback(
        future,
        new ApiFutureCallback() {
          @Override
          public void onSuccess(MessageMetadata MessageMetadata) {
            decrementOutstanding();
          }

          @Override
          public void onFailure(Throwable t) {
            onFailure.accept(t);
          }
        },
        SystemExecutors.getFuturesExecutor());
  }

  // Intentionally don't flush on bundle finish to allow multi-sink client reuse.
  @FinishBundle
  public synchronized void finishBundle() throws CheckedApiException, InterruptedException {
    while (outstanding > 0) {
      wait();
    }
    if (!errorsSinceLastFinish.isEmpty()) {
      CheckedApiException canonical = errorsSinceLastFinish.pop();
      while (!errorsSinceLastFinish.isEmpty()) {
        canonical.addSuppressed(errorsSinceLastFinish.pop());
      }
      throw canonical;
    }
    if (publisherOrError.getKind() == Kind.ERROR) {
      throw publisherOrError.error();
    }
  }
}