com.google.cloud.pubsublite.beam.UuidDeduplicationOptions Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of pubsublite-beam-io Show documentation
Beam IO for Google Cloud Pub/Sub Lite
There is a newer version: 0.23.0
/*
 * Copyright 2020 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.cloud.pubsublite.beam;

import static com.google.cloud.pubsublite.internal.UncheckedApiPreconditions.checkArgument;

import com.google.auto.value.AutoValue;
import com.google.cloud.pubsublite.proto.SequencedMessage;
import com.google.protobuf.ByteString;
import java.io.Serializable;
import java.util.List;
import org.apache.beam.sdk.state.TimeDomain;
import org.apache.beam.sdk.transforms.Deduplicate;
import org.apache.beam.sdk.transforms.SerializableFunction;
import org.joda.time.Duration;

/** Options for deduplicating Pub/Sub Lite messages based on the UUID they were published with. */
@AutoValue
@SuppressWarnings({
  "nullness" // TODO(https://issues.apache.org/jira/browse/BEAM-10402)
})
public abstract class UuidDeduplicationOptions implements Serializable {
  private static final long serialVersionUID = 9837489720893L;

  public static final SerializableFunction DEFAULT_UUID_EXTRACTOR =
      message -> {
        checkArgument(
            message.getMessage().getAttributesMap().containsKey(Uuid.DEFAULT_ATTRIBUTE),
            "Uuid attribute missing.");
        List attributes =
            message.getMessage().getAttributesMap().get(Uuid.DEFAULT_ATTRIBUTE).getValuesList();
        checkArgument(attributes.size() == 1, "Duplicate Uuid attribute values exist.");
        return Uuid.of(attributes.get(0));
      };

  public static final int DEFAULT_HASH_PARTITIONS = 10000;

  public static final TimeDomain DEFAULT_TIME_DOMAIN = TimeDomain.EVENT_TIME;
  public static final Duration DEFAULT_DEDUPLICATE_DURATION = Duration.standardDays(1);

  // All parameters are optional.
  public abstract SerializableFunction uuidExtractor();

  public abstract Deduplicate.KeyedValues deduplicate();

  @SuppressWarnings("CheckReturnValue")
  public static Builder newBuilder() {
    Builder builder = new AutoValue_UuidDeduplicationOptions.Builder();
    builder.setUuidExtractor(DEFAULT_UUID_EXTRACTOR);
    builder.setDeduplicate(
        Deduplicate.keyedValues()
            .withTimeDomain(DEFAULT_TIME_DOMAIN)
            .withDuration(DEFAULT_DEDUPLICATE_DURATION));
    return builder;
  }

  @AutoValue.Builder
  public abstract static class Builder {
    public abstract Builder setUuidExtractor(
        SerializableFunction uuidExtractor);

    /**
     * Set the deduplication transform.
     *
     * {@code
     * UuidDeduplicationOptions.Builder builder = UuidDeduplicationOptions.newBuilder();
     * builder.setDeduplicate(Deduplicate.keyedValues()
     *     .withTimeDomain(TimeDomain.PROCESSING_TIME));
     * }
     */
    public abstract Builder setDeduplicate(
        Deduplicate.KeyedValues deduplicate);

    public abstract UuidDeduplicationOptions build();
  }
}