com.google.cloud.dataflow.sdk.transforms.CombineWithContext Maven / Gradle / Ivy
Show all versions of google-cloud-dataflow-java-sdk-all Show documentation
/*
* Copyright (C) 2015 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package com.google.cloud.dataflow.sdk.transforms;
import com.google.cloud.dataflow.sdk.coders.CannotProvideCoderException;
import com.google.cloud.dataflow.sdk.coders.Coder;
import com.google.cloud.dataflow.sdk.coders.CoderRegistry;
import com.google.cloud.dataflow.sdk.options.PipelineOptions;
import com.google.cloud.dataflow.sdk.transforms.Combine.CombineFn;
import com.google.cloud.dataflow.sdk.transforms.Combine.KeyedCombineFn;
import com.google.cloud.dataflow.sdk.transforms.display.DisplayData;
import com.google.cloud.dataflow.sdk.values.PCollectionView;
/**
* This class contains combine functions that have access to {@code PipelineOptions} and side inputs
* through {@code CombineWithContext.Context}.
*
* {@link CombineFnWithContext} and {@link KeyedCombineFnWithContext} are for users to extend.
*/
public class CombineWithContext {
/**
* Information accessible to all methods in {@code CombineFnWithContext}
* and {@code KeyedCombineFnWithContext}.
*/
public abstract static class Context {
/**
* Returns the {@code PipelineOptions} specified with the
* {@link com.google.cloud.dataflow.sdk.runners.PipelineRunner}
* invoking this {@code KeyedCombineFn}.
*/
public abstract PipelineOptions getPipelineOptions();
/**
* Returns the value of the side input for the window corresponding to the
* window of the main input element.
*/
public abstract T sideInput(PCollectionView view);
}
/**
* An internal interface for signaling that a {@code GloballyCombineFn}
* or a {@code PerKeyCombineFn} needs to access {@code CombineWithContext.Context}.
*
* For internal use only.
*/
public interface RequiresContextInternal {}
/**
* A combine function that has access to {@code PipelineOptions} and side inputs through
* {@code CombineWithContext.Context}.
*
*
See the equivalent {@link CombineFn} for details about combine functions.
*/
public abstract static class CombineFnWithContext
extends CombineFnBase.AbstractGlobalCombineFn
implements RequiresContextInternal {
/**
* Returns a new, mutable accumulator value, representing the accumulation of zero input values.
*
* It is equivalent to {@link CombineFn#createAccumulator}, but it has additional access to
* {@code CombineWithContext.Context}.
*/
public abstract AccumT createAccumulator(Context c);
/**
* Adds the given input value to the given accumulator, returning the
* new accumulator value.
*
*
It is equivalent to {@link CombineFn#addInput}, but it has additional access to
* {@code CombineWithContext.Context}.
*/
public abstract AccumT addInput(AccumT accumulator, InputT input, Context c);
/**
* Returns an accumulator representing the accumulation of all the
* input values accumulated in the merging accumulators.
*
*
It is equivalent to {@link CombineFn#mergeAccumulators}, but it has additional access to
* {@code CombineWithContext.Context}.
*/
public abstract AccumT mergeAccumulators(Iterable accumulators, Context c);
/**
* Returns the output value that is the result of combining all
* the input values represented by the given accumulator.
*
* It is equivalent to {@link CombineFn#extractOutput}, but it has additional access to
* {@code CombineWithContext.Context}.
*/
public abstract OutputT extractOutput(AccumT accumulator, Context c);
/**
* Returns an accumulator that represents the same logical value as the
* input accumulator, but may have a more compact representation.
*
*
It is equivalent to {@link CombineFn#compact}, but it has additional access to
* {@code CombineWithContext.Context}.
*/
public AccumT compact(AccumT accumulator, Context c) {
return accumulator;
}
@Override
public OutputT defaultValue() {
throw new UnsupportedOperationException(
"Override this function to provide the default value.");
}
@SuppressWarnings({"unchecked", "rawtypes"})
@Override
public KeyedCombineFnWithContext asKeyedFn() {
// The key, an object, is never even looked at.
return new KeyedCombineFnWithContext() {
@Override
public AccumT createAccumulator(K key, Context c) {
return CombineFnWithContext.this.createAccumulator(c);
}
@Override
public AccumT addInput(K key, AccumT accumulator, InputT input, Context c) {
return CombineFnWithContext.this.addInput(accumulator, input, c);
}
@Override
public AccumT mergeAccumulators(K key, Iterable accumulators, Context c) {
return CombineFnWithContext.this.mergeAccumulators(accumulators, c);
}
@Override
public OutputT extractOutput(K key, AccumT accumulator, Context c) {
return CombineFnWithContext.this.extractOutput(accumulator, c);
}
@Override
public AccumT compact(K key, AccumT accumulator, Context c) {
return CombineFnWithContext.this.compact(accumulator, c);
}
@Override
public Coder getAccumulatorCoder(CoderRegistry registry, Coder keyCoder,
Coder inputCoder) throws CannotProvideCoderException {
return CombineFnWithContext.this.getAccumulatorCoder(registry, inputCoder);
}
@Override
public Coder getDefaultOutputCoder(CoderRegistry registry, Coder keyCoder,
Coder inputCoder) throws CannotProvideCoderException {
return CombineFnWithContext.this.getDefaultOutputCoder(registry, inputCoder);
}
@Override
public CombineFnWithContext forKey(K key, Coder keyCoder) {
return CombineFnWithContext.this;
}
@Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
CombineFnWithContext.this.populateDisplayData(builder);
}
};
}
}
/**
* A keyed combine function that has access to {@code PipelineOptions} and side inputs through
* {@code CombineWithContext.Context}.
*
* See the equivalent {@link KeyedCombineFn} for details about keyed combine functions.
*/
public abstract static class KeyedCombineFnWithContext
extends CombineFnBase.AbstractPerKeyCombineFn
implements RequiresContextInternal {
/**
* Returns a new, mutable accumulator value representing the accumulation of zero input values.
*
* It is equivalent to {@link KeyedCombineFn#createAccumulator},
* but it has additional access to {@code CombineWithContext.Context}.
*/
public abstract AccumT createAccumulator(K key, Context c);
/**
* Adds the given input value to the given accumulator, returning the new accumulator value.
*
*
It is equivalent to {@link KeyedCombineFn#addInput}, but it has additional access to
* {@code CombineWithContext.Context}.
*/
public abstract AccumT addInput(K key, AccumT accumulator, InputT value, Context c);
/**
* Returns an accumulator representing the accumulation of all the
* input values accumulated in the merging accumulators.
*
*
It is equivalent to {@link KeyedCombineFn#mergeAccumulators},
* but it has additional access to {@code CombineWithContext.Context}..
*/
public abstract AccumT mergeAccumulators(K key, Iterable accumulators, Context c);
/**
* Returns the output value that is the result of combining all
* the input values represented by the given accumulator.
*
* It is equivalent to {@link KeyedCombineFn#extractOutput}, but it has additional access to
* {@code CombineWithContext.Context}.
*/
public abstract OutputT extractOutput(K key, AccumT accumulator, Context c);
/**
* Returns an accumulator that represents the same logical value as the
* input accumulator, but may have a more compact representation.
*
*
It is equivalent to {@link KeyedCombineFn#compact}, but it has additional access to
* {@code CombineWithContext.Context}.
*/
public AccumT compact(K key, AccumT accumulator, Context c) {
return accumulator;
}
/**
* Applies this {@code KeyedCombineFnWithContext} to a key and a collection
* of input values to produce a combined output value.
*/
public OutputT apply(K key, Iterable inputs, Context c) {
AccumT accum = createAccumulator(key, c);
for (InputT input : inputs) {
accum = addInput(key, accum, input, c);
}
return extractOutput(key, accum, c);
}
@Override
public CombineFnWithContext forKey(
final K key, final Coder keyCoder) {
return new CombineFnWithContext() {
@Override
public AccumT createAccumulator(Context c) {
return KeyedCombineFnWithContext.this.createAccumulator(key, c);
}
@Override
public AccumT addInput(AccumT accumulator, InputT input, Context c) {
return KeyedCombineFnWithContext.this.addInput(key, accumulator, input, c);
}
@Override
public AccumT mergeAccumulators(Iterable accumulators, Context c) {
return KeyedCombineFnWithContext.this.mergeAccumulators(key, accumulators, c);
}
@Override
public OutputT extractOutput(AccumT accumulator, Context c) {
return KeyedCombineFnWithContext.this.extractOutput(key, accumulator, c);
}
@Override
public Coder getAccumulatorCoder(CoderRegistry registry, Coder inputCoder)
throws CannotProvideCoderException {
return KeyedCombineFnWithContext.this.getAccumulatorCoder(registry, keyCoder, inputCoder);
}
@Override
public Coder getDefaultOutputCoder(
CoderRegistry registry, Coder inputCoder) throws CannotProvideCoderException {
return KeyedCombineFnWithContext.this.getDefaultOutputCoder(
registry, keyCoder, inputCoder);
}
};
}
}
}