com.google.cloud.dataflow.sdk.testing.PaneExtractors Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of google-cloud-dataflow-java-sdk-all Show documentation
Google Cloud Dataflow Java SDK provides a simple, Java-based interface for processing virtually any size data using Google cloud resources. This artifact includes entire Dataflow Java SDK.
There is a newer version: 2.5.0
Show newest version
/*
 * Copyright (C) 2015 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.cloud.dataflow.sdk.testing;

import static com.google.common.base.Preconditions.checkState;

import com.google.cloud.dataflow.sdk.transforms.PTransform;
import com.google.cloud.dataflow.sdk.transforms.SimpleFunction;
import com.google.cloud.dataflow.sdk.transforms.windowing.PaneInfo;
import com.google.cloud.dataflow.sdk.transforms.windowing.PaneInfo.Timing;
import com.google.cloud.dataflow.sdk.util.WindowedValue;
import com.google.cloud.dataflow.sdk.values.PCollection;
import com.google.cloud.dataflow.sdk.values.TypeDescriptor;

import java.util.ArrayList;
import java.util.List;

/**
 * {@link PTransform PTransforms} which take an {@link Iterable} of {@link WindowedValue
 * WindowedValues} and outputs an {@link Iterable} of all values in the specified pane, dropping the
 * {@link WindowedValue} metadata.
 *
 * Although all of the method signatures return SimpleFunction, users should ensure to set the
 * coder of any output {@link PCollection}, as appropriate {@link TypeDescriptor TypeDescriptors}
 * cannot be obtained when the extractor is created.
 */
final class PaneExtractors {
  private PaneExtractors() {
  }

  static  SimpleFunction>, Iterable> onlyPane() {
    return new ExtractOnlyPane<>();
  }

  static  SimpleFunction>, Iterable> onTimePane() {
    return new ExtractOnTimePane<>();
  }

  static  SimpleFunction>, Iterable> finalPane() {
    return new ExtractFinalPane<>();
  }

  static  SimpleFunction>, Iterable> nonLatePanes() {
    return new ExtractNonLatePanes<>();
  }

  static  SimpleFunction>, Iterable> earlyPanes() {
    return new ExtractEarlyPanes<>();
  }

  static  SimpleFunction>, Iterable> allPanes() {
    return new ExtractAllPanes<>();
  }

  private static class ExtractOnlyPane
      extends SimpleFunction>, Iterable> {
    @Override
    public Iterable apply(Iterable> input) {
      List outputs = new ArrayList<>();
      for (WindowedValue value : input) {
        checkState(value.getPane().isFirst() && value.getPane().isLast(),
            "Expected elements to be produced by a trigger that fires at most once, but got"
                + "a value in a pane that is %s. Actual Pane Info: %s",
            value.getPane().isFirst() ? "not the last pane" : "not the first pane",
            value.getPane());
        outputs.add(value.getValue());
      }
      return outputs;
    }
  }


  private static class ExtractOnTimePane
      extends SimpleFunction>, Iterable> {
    @Override
    public Iterable apply(Iterable> input) {
      List outputs = new ArrayList<>();
      for (WindowedValue value : input) {
        if (value.getPane().getTiming().equals(Timing.ON_TIME)) {
          outputs.add(value.getValue());
        }
      }
      return outputs;
    }
  }


  private static class ExtractFinalPane
      extends SimpleFunction>, Iterable> {
    @Override
    public Iterable apply(Iterable> input) {
      List outputs = new ArrayList<>();
      for (WindowedValue value : input) {
        if (value.getPane().isLast()) {
          outputs.add(value.getValue());
        }
      }
      return outputs;
    }
  }


  private static class ExtractAllPanes
      extends SimpleFunction>, Iterable> {
    @Override
    public Iterable apply(Iterable> input) {
      List outputs = new ArrayList<>();
      for (WindowedValue value : input) {
        outputs.add(value.getValue());
      }
      return outputs;
    }
  }


  private static class ExtractNonLatePanes
      extends SimpleFunction>, Iterable> {
    @Override
    public Iterable apply(Iterable> input) {
      List outputs = new ArrayList<>();
      for (WindowedValue value : input) {
        if (value.getPane().getTiming() != PaneInfo.Timing.LATE) {
          outputs.add(value.getValue());
        }
      }
      return outputs;
    }
  }

  private static class ExtractEarlyPanes
      extends SimpleFunction>, Iterable> {
    @Override
    public Iterable apply(Iterable> input) {
      List outputs = new ArrayList<>();
      for (WindowedValue value : input) {
        if (value.getPane().getTiming() == PaneInfo.Timing.EARLY) {
          outputs.add(value.getValue());
        }
      }
      return outputs;
    }
  }
}