• Home
• com.google.cloud.dataflow
• google-cloud-dataflow-java-sdk-all
• 1.5.0
• source code
• WindowedValue.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

com.google.cloud.dataflow.sdk.util.WindowedValue Maven / Gradle / Ivy

/*******************************************************************************
 * Copyright (C) 2015 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 ******************************************************************************/

package com.google.cloud.dataflow.sdk.util;

import static com.google.cloud.dataflow.sdk.util.Structs.addBoolean;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;

import com.google.cloud.dataflow.sdk.coders.Coder;
import com.google.cloud.dataflow.sdk.coders.CoderException;
import com.google.cloud.dataflow.sdk.coders.CollectionCoder;
import com.google.cloud.dataflow.sdk.coders.InstantCoder;
import com.google.cloud.dataflow.sdk.coders.StandardCoder;
import com.google.cloud.dataflow.sdk.transforms.windowing.BoundedWindow;
import com.google.cloud.dataflow.sdk.transforms.windowing.GlobalWindow;
import com.google.cloud.dataflow.sdk.transforms.windowing.PaneInfo;
import com.google.cloud.dataflow.sdk.transforms.windowing.PaneInfo.PaneInfoCoder;
import com.google.cloud.dataflow.sdk.util.common.ElementByteSizeObserver;
import com.google.common.base.MoreObjects;
import com.google.common.base.Preconditions;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;

import org.joda.time.Instant;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;

/**
 * An immutable triple of value, timestamp, and windows.
 *
 * @param  the type of the value
 */
public abstract class WindowedValue {

  protected final T value;
  protected final PaneInfo pane;

  /**
   * Returns a {@code WindowedValue} with the given value, timestamp,
   * and windows.
   */
  public static  WindowedValue of(
      T value,
      Instant timestamp,
      Collection windows,
      PaneInfo pane) {
    Preconditions.checkNotNull(pane);

    if (windows.size() == 0 && BoundedWindow.TIMESTAMP_MIN_VALUE.equals(timestamp)) {
      return valueInEmptyWindows(value, pane);
    } else if (windows.size() == 1) {
      return of(value, timestamp, windows.iterator().next(), pane);
    } else {
      return new TimestampedValueInMultipleWindows<>(value, timestamp, windows, pane);
    }
  }

  /**
   * Returns a {@code WindowedValue} with the given value, timestamp, and window.
   */
  public static  WindowedValue of(
      T value,
      Instant timestamp,
      BoundedWindow window,
      PaneInfo pane) {
    Preconditions.checkNotNull(pane);

    boolean isGlobal = GlobalWindow.INSTANCE.equals(window);
    if (isGlobal && BoundedWindow.TIMESTAMP_MIN_VALUE.equals(timestamp)) {
      return valueInGlobalWindow(value, pane);
    } else if (isGlobal) {
      return new TimestampedValueInGlobalWindow<>(value, timestamp, pane);
    } else {
      return new TimestampedValueInSingleWindow<>(value, timestamp, window, pane);
    }
  }

  /**
   * Returns a {@code WindowedValue} with the given value in the {@link GlobalWindow} using the
   * default timestamp and pane.
   */
  public static  WindowedValue valueInGlobalWindow(T value) {
    return new ValueInGlobalWindow<>(value, PaneInfo.NO_FIRING);
  }

  /**
   * Returns a {@code WindowedValue} with the given value in the {@link GlobalWindow} using the
   * default timestamp and the specified pane.
   */
  public static  WindowedValue valueInGlobalWindow(T value, PaneInfo pane) {
    return new ValueInGlobalWindow<>(value, pane);
  }

  /**
   * Returns a {@code WindowedValue} with the given value and timestamp,
   * {@code GlobalWindow} and default pane.
   */
  public static  WindowedValue timestampedValueInGlobalWindow(T value, Instant timestamp) {
    if (BoundedWindow.TIMESTAMP_MIN_VALUE.equals(timestamp)) {
      return valueInGlobalWindow(value);
    } else {
      return new TimestampedValueInGlobalWindow<>(value, timestamp, PaneInfo.NO_FIRING);
    }
  }

  /**
   * Returns a {@code WindowedValue} with the given value in no windows, and the default timestamp
   * and pane.
   */
  public static  WindowedValue valueInEmptyWindows(T value) {
    return new ValueInEmptyWindows(value, PaneInfo.NO_FIRING);
  }

  /**
   * Returns a {@code WindowedValue} with the given value in no windows, and the default timestamp
   * and the specified pane.
   */
  public static  WindowedValue valueInEmptyWindows(T value, PaneInfo pane) {
    return new ValueInEmptyWindows(value, pane);
  }

  private WindowedValue(T value, PaneInfo pane) {
    this.value = value;
    this.pane = checkNotNull(pane);
  }

  /**
   * Returns a new {@code WindowedValue} that is a copy of this one, but with a different value,
   * which may have a new type {@code NewT}.
   */
  public abstract  WindowedValue withValue(NewT value);

  /**
   * Returns the value of this {@code WindowedValue}.
   */
  public T getValue() {
    return value;
  }

  /**
   * Returns the timestamp of this {@code WindowedValue}.
   */
  public abstract Instant getTimestamp();

  /**
   * Returns the windows of this {@code WindowedValue}.
   */
  public abstract Collection getWindows();

  /**
   * Returns the pane of this {@code WindowedValue} in its window.
   */
  public PaneInfo getPane() {
    return pane;
  }

  @Override
  public abstract boolean equals(Object o);

  @Override
  public abstract int hashCode();

  @Override
  public abstract String toString();

  private static final Collection GLOBAL_WINDOWS =
      Collections.singletonList(GlobalWindow.INSTANCE);

  /**
   * The abstract superclass of WindowedValue representations where
   * timestamp == MIN.
   */
  private abstract static class MinTimestampWindowedValue
      extends WindowedValue {
    public MinTimestampWindowedValue(T value, PaneInfo pane) {
      super(value, pane);
    }

    @Override
    public Instant getTimestamp() {
      return BoundedWindow.TIMESTAMP_MIN_VALUE;
    }
  }

  /**
   * The representation of a WindowedValue where timestamp == MIN and
   * windows == {GlobalWindow}.
   */
  private static class ValueInGlobalWindow
      extends MinTimestampWindowedValue {
    public ValueInGlobalWindow(T value, PaneInfo pane) {
      super(value, pane);
    }

    @Override
    public  WindowedValue withValue(NewT value) {
      return new ValueInGlobalWindow<>(value, pane);
    }

    @Override
    public Collection getWindows() {
      return GLOBAL_WINDOWS;
    }

    @Override
    public boolean equals(Object o) {
      if (o instanceof ValueInGlobalWindow) {
        ValueInGlobalWindow that = (ValueInGlobalWindow) o;
        return Objects.equals(that.pane, this.pane)
            && Objects.equals(that.value, this.value);
      } else {
        return false;
      }
    }

    @Override
    public int hashCode() {
      return Objects.hash(value, pane);
    }

    @Override
    public String toString() {
      return MoreObjects.toStringHelper(getClass())
          .add("value", value)
          .add("pane", pane)
          .toString();
    }
  }

  /**
   * The representation of a WindowedValue where timestamp == MIN and
   * windows == {}.
   */
  private static class ValueInEmptyWindows
      extends MinTimestampWindowedValue {
    public ValueInEmptyWindows(T value, PaneInfo pane) {
      super(value, pane);
    }

    @Override
    public  WindowedValue withValue(NewT value) {
      return new ValueInEmptyWindows<>(value, pane);
    }

    @Override
    public Collection getWindows() {
      return Collections.emptyList();
    }

    @Override
    public boolean equals(Object o) {
      if (o instanceof ValueInEmptyWindows) {
        ValueInEmptyWindows that = (ValueInEmptyWindows) o;
        return Objects.equals(that.pane, this.pane)
            && Objects.equals(that.value, this.value);
      } else {
        return false;
      }
    }

    @Override
    public int hashCode() {
      return Objects.hash(value, pane);
    }

    @Override
    public String toString() {
      return MoreObjects.toStringHelper(getClass())
          .add("value", value)
          .add("pane", pane)
          .toString();
    }
  }

  /**
   * The abstract superclass of WindowedValue representations where
   * timestamp is arbitrary.
   */
  private abstract static class TimestampedWindowedValue
      extends WindowedValue {
    protected final Instant timestamp;

    public TimestampedWindowedValue(T value,
                                    Instant timestamp,
                                    PaneInfo pane) {
      super(value, pane);
      this.timestamp = checkNotNull(timestamp);
    }

    @Override
    public Instant getTimestamp() {
      return timestamp;
    }
  }

  /**
   * The representation of a WindowedValue where timestamp {@code >}
   * MIN and windows == {GlobalWindow}.
   */
  private static class TimestampedValueInGlobalWindow
      extends TimestampedWindowedValue {
    public TimestampedValueInGlobalWindow(T value,
                                          Instant timestamp,
                                          PaneInfo pane) {
      super(value, timestamp, pane);
    }

    @Override
    public  WindowedValue withValue(NewT value) {
      return new TimestampedValueInGlobalWindow<>(value, timestamp, pane);
    }

    @Override
    public Collection getWindows() {
      return GLOBAL_WINDOWS;
    }

    @Override
    public boolean equals(Object o) {
      if (o instanceof TimestampedValueInGlobalWindow) {
        TimestampedValueInGlobalWindow that =
            (TimestampedValueInGlobalWindow) o;
        return this.timestamp.isEqual(that.timestamp) // don't compare chronology objects
            && Objects.equals(that.pane, this.pane)
            && Objects.equals(that.value, this.value);
      } else {
        return false;
      }
    }

    @Override
    public int hashCode() {
      return Objects.hash(value, pane, timestamp.getMillis());
    }

    @Override
    public String toString() {
      return MoreObjects.toStringHelper(getClass())
          .add("value", value)
          .add("timestamp", timestamp)
          .add("pane", pane)
          .toString();
    }
  }

  /**
   * The representation of a WindowedValue where timestamp is arbitrary and
   * windows == a single non-Global window.
   */
  private static class TimestampedValueInSingleWindow
      extends TimestampedWindowedValue {
    private final BoundedWindow window;

    public TimestampedValueInSingleWindow(T value,
                                          Instant timestamp,
                                          BoundedWindow window,
                                          PaneInfo pane) {
      super(value, timestamp, pane);
      this.window = checkNotNull(window);
    }

    @Override
    public  WindowedValue withValue(NewT value) {
      return new TimestampedValueInSingleWindow<>(value, timestamp, window, pane);
    }

    @Override
    public Collection getWindows() {
      return Collections.singletonList(window);
    }

    @Override
    public boolean equals(Object o) {
      if (o instanceof TimestampedValueInSingleWindow) {
        TimestampedValueInSingleWindow that =
            (TimestampedValueInSingleWindow) o;
        return Objects.equals(that.value, this.value)
            && this.timestamp.isEqual(that.timestamp) // don't compare chronology objects
            && Objects.equals(that.pane, this.pane)
            && Objects.equals(that.window, this.window);
      } else {
        return false;
      }
    }

    @Override
    public int hashCode() {
      return Objects.hash(value, timestamp.getMillis(), pane, window);
    }

    @Override
    public String toString() {
      return MoreObjects.toStringHelper(getClass())
          .add("value", value)
          .add("timestamp", timestamp)
          .add("window", window)
          .add("pane", pane)
          .toString();
    }
  }

  /**
   * The representation of a WindowedValue, excluding the special
   * cases captured above.
   */
  private static class TimestampedValueInMultipleWindows
      extends TimestampedWindowedValue {
    private Collection windows;

    public TimestampedValueInMultipleWindows(
        T value,
        Instant timestamp,
        Collection windows,
        PaneInfo pane) {
      super(value, timestamp, pane);
      this.windows = checkNotNull(windows);
    }

    @Override
    public  WindowedValue withValue(NewT value) {
      return new TimestampedValueInMultipleWindows<>(value, timestamp, windows, pane);
    }

    @Override
    public Collection getWindows() {
      return windows;
    }

    @Override
    public boolean equals(Object o) {
      if (o instanceof TimestampedValueInMultipleWindows) {
        TimestampedValueInMultipleWindows that =
            (TimestampedValueInMultipleWindows) o;
        if (this.timestamp.isEqual(that.timestamp) // don't compare chronology objects
            && Objects.equals(that.value, this.value)
            && Objects.equals(that.pane, this.pane)) {
          ensureWindowsAreASet();
          that.ensureWindowsAreASet();
          return that.windows.equals(this.windows);
        }
      }
      return false;
    }

    @Override
    public int hashCode() {
      ensureWindowsAreASet();
      return Objects.hash(value, timestamp.getMillis(), pane, windows);
    }

    @Override
    public String toString() {
      return MoreObjects.toStringHelper(getClass())
          .add("value", value)
          .add("timestamp", timestamp)
          .add("windows", windows)
          .add("pane", pane)
          .toString();
    }

    private void ensureWindowsAreASet() {
      if (!(windows instanceof Set)) {
        windows = new LinkedHashSet<>(windows);
      }
    }
  }


  /////////////////////////////////////////////////////////////////////////////

  /**
   * Returns the {@code Coder} to use for a {@code WindowedValue},
   * using the given valueCoder and windowCoder.
   */
  public static  FullWindowedValueCoder getFullCoder(
      Coder valueCoder,
      Coder windowCoder) {
    return FullWindowedValueCoder.of(valueCoder, windowCoder);
  }

  /**
   * Returns the {@code ValueOnlyCoder} from the given valueCoder.
   */
  public static  ValueOnlyWindowedValueCoder getValueOnlyCoder(Coder valueCoder) {
    return ValueOnlyWindowedValueCoder.of(valueCoder);
  }

  /**
   * Abstract class for {@code WindowedValue} coder.
   */
  public abstract static class WindowedValueCoder
      extends StandardCoder> {
    final Coder valueCoder;

    WindowedValueCoder(Coder valueCoder) {
      this.valueCoder = checkNotNull(valueCoder);
    }

    /**
     * Returns the value coder.
     */
    public Coder getValueCoder() {
      return valueCoder;
    }

    /**
     * Returns a new {@code WindowedValueCoder} that is a copy of this one,
     * but with a different value coder.
     */
    public abstract  WindowedValueCoder withValueCoder(Coder valueCoder);
  }

  /**
   * Coder for {@code WindowedValue}.
   */
  public static class FullWindowedValueCoder extends WindowedValueCoder {
    private final Coder windowCoder;
    // Precompute and cache the coder for a list of windows.
    private final Coder> windowsCoder;

    public static  FullWindowedValueCoder of(
        Coder valueCoder,
        Coder windowCoder) {
      return new FullWindowedValueCoder<>(valueCoder, windowCoder);
    }

    @JsonCreator
    public static FullWindowedValueCoder of(
        @JsonProperty(PropertyNames.COMPONENT_ENCODINGS)
        List> components) {
      checkArgument(components.size() == 2,
                    "Expecting 2 components, got " + components.size());
      @SuppressWarnings("unchecked")
      Coder window = (Coder) components.get(1);
      return of(components.get(0), window);
    }

    FullWindowedValueCoder(Coder valueCoder,
                           Coder windowCoder) {
      super(valueCoder);
      this.windowCoder = checkNotNull(windowCoder);
      // It's not possible to statically type-check correct use of the
      // windowCoder (we have to ensure externally that we only get
      // windows of the class handled by windowCoder), so type
      // windowsCoder in a way that makes encode() and decode() work
      // right, and cast the window type away here.
      @SuppressWarnings({"unchecked", "rawtypes"})
      Coder> collectionCoder =
          (Coder) CollectionCoder.of(this.windowCoder);
      this.windowsCoder = collectionCoder;
    }

    public Coder getWindowCoder() {
      return windowCoder;
    }

    public Coder> getWindowsCoder() {
      return windowsCoder;
    }

    @Override
    public  WindowedValueCoder withValueCoder(Coder valueCoder) {
      return new FullWindowedValueCoder<>(valueCoder, windowCoder);
    }

    @Override
    public void encode(WindowedValue windowedElem,
                       OutputStream outStream,
                       Context context)
        throws CoderException, IOException {
      Context nestedContext = context.nested();
      valueCoder.encode(windowedElem.getValue(), outStream, nestedContext);
      InstantCoder.of().encode(
          windowedElem.getTimestamp(), outStream, nestedContext);
      windowsCoder.encode(windowedElem.getWindows(), outStream, nestedContext);
      PaneInfoCoder.INSTANCE.encode(windowedElem.getPane(), outStream, context);
    }

    @Override
    public WindowedValue decode(InputStream inStream, Context context)
        throws CoderException, IOException {
      Context nestedContext = context.nested();
      T value = valueCoder.decode(inStream, nestedContext);
      Instant timestamp = InstantCoder.of().decode(inStream, nestedContext);
      Collection windows =
          windowsCoder.decode(inStream, nestedContext);
      PaneInfo pane = PaneInfoCoder.INSTANCE.decode(inStream, nestedContext);
      return WindowedValue.of(value, timestamp, windows, pane);
    }

    @Override
    public void verifyDeterministic() throws NonDeterministicException {
      verifyDeterministic(
          "FullWindowedValueCoder requires a deterministic valueCoder",
          valueCoder);
      verifyDeterministic(
          "FullWindowedValueCoder requires a deterministic windowCoder",
          windowCoder);
    }

    @Override
    public void registerByteSizeObserver(WindowedValue value,
                                         ElementByteSizeObserver observer,
                                         Context context) throws Exception {
      valueCoder.registerByteSizeObserver(value.getValue(), observer, context);
      InstantCoder.of().registerByteSizeObserver(value.getTimestamp(), observer, context);
      windowsCoder.registerByteSizeObserver(value.getWindows(), observer, context);
    }

    @Override
    public CloudObject asCloudObject() {
      CloudObject result = super.asCloudObject();
      addBoolean(result, PropertyNames.IS_WRAPPER, true);
      return result;
    }

    @Override
    public List> getCoderArguments() {
      return null;
    }

    @Override
    public List> getComponents() {
      return Arrays.>asList(valueCoder, windowCoder);
    }
  }

  /**
   * Coder for {@code WindowedValue}.
   *
   * 
A {@code ValueOnlyWindowedValueCoder} only encodes and decodes the value. It drops
   * timestamp and windows for encoding, and uses defaults timestamp, and windows for decoding.
   */
  public static class ValueOnlyWindowedValueCoder extends WindowedValueCoder {
    public static  ValueOnlyWindowedValueCoder of(
        Coder valueCoder) {
      return new ValueOnlyWindowedValueCoder<>(valueCoder);
    }

    @JsonCreator
    public static ValueOnlyWindowedValueCoder of(
        @JsonProperty(PropertyNames.COMPONENT_ENCODINGS)
        List> components) {
      checkArgument(components.size() == 1, "Expecting 1 component, got " + components.size());
      return of(components.get(0));
    }

    ValueOnlyWindowedValueCoder(Coder valueCoder) {
      super(valueCoder);
    }

    @Override
    public  WindowedValueCoder withValueCoder(Coder valueCoder) {
      return new ValueOnlyWindowedValueCoder<>(valueCoder);
    }

    @Override
    public void encode(WindowedValue windowedElem, OutputStream outStream, Context context)
        throws CoderException, IOException {
      valueCoder.encode(windowedElem.getValue(), outStream, context);
    }

    @Override
    public WindowedValue decode(InputStream inStream, Context context)
        throws CoderException, IOException {
      T value = valueCoder.decode(inStream, context);
      return WindowedValue.valueInGlobalWindow(value);
    }

    @Override
    public void verifyDeterministic() throws NonDeterministicException {
      verifyDeterministic(
          "ValueOnlyWindowedValueCoder requires a deterministic valueCoder",
          valueCoder);
    }

    @Override
    public void registerByteSizeObserver(
        WindowedValue value, ElementByteSizeObserver observer, Context context)
        throws Exception {
      valueCoder.registerByteSizeObserver(value.getValue(), observer, context);
    }

    @Override
    public CloudObject asCloudObject() {
      CloudObject result = super.asCloudObject();
      addBoolean(result, PropertyNames.IS_WRAPPER, true);
      return result;
    }

    @Override
    public List> getCoderArguments() {
      return Arrays.>asList(valueCoder);
    }
  }
}