org.apache.apex.malhar.stream.api.WindowedStream Maven / Gradle / Ivy

Go to download
/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.apex.malhar.stream.api;

import java.util.List;

import org.joda.time.Duration;

import org.apache.apex.malhar.lib.window.Accumulation;
import org.apache.apex.malhar.lib.window.TriggerOption;
import org.apache.apex.malhar.lib.window.Tuple;
import org.apache.apex.malhar.lib.window.accumulation.FoldFn;
import org.apache.apex.malhar.lib.window.accumulation.ReduceFn;
import org.apache.apex.malhar.lib.window.impl.KeyedWindowedOperatorImpl;
import org.apache.apex.malhar.lib.window.impl.WindowedOperatorImpl;
import org.apache.apex.malhar.stream.api.function.Function;
import org.apache.hadoop.classification.InterfaceStability;

import com.datatorrent.lib.util.KeyValPair;

/**
 * 
 * A stream with windowed transformation
 * 
 * 
 * Transformation types:
 * 

 * Combine
 * Group
 * Keyed Combine
 * Keyed Group
 * Join
 * CoGroup
 * 
 * 
 * 
 * Features supported with windowed transformation 
 * 

 * Watermark - Ingestion time watermark / logical tuple watermark
 * Early Triggers - How frequent to emit real-time partial result
 * Late Triggers - When to emit updated result with tuple comes after watermark
 * Customizable Trigger Behaviour - What to do when fires a trigger
 * Spool window state -  In-Memory window state can be spooled to disk if it is full
 * 3 different accumulation models: ignore, accumulation, accumulation + delta
 * Window support: Non-Mergeable window(fix window, sliding window), Mergeable window(session window) base on 3 different tuple time
 * Different tuple time support: event time, system time, ingestion time
 * 
 * 
 *
 * @param  Output tuple type
 *
 * @since 3.4.0
 */
@InterfaceStability.Evolving
public interface WindowedStream extends ApexStream
{

  /**
   * Count of all tuples
   * @return new stream of Integer
   */
  >> STREAM count(Option... opts);

  /**
   * Count tuples by the key

   * @param name name of the operator
   * @param convertToKeyValue The function convert plain tuple to k,v pair
   * @return new stream of Key Value Pair
   */
  >>> STREAM countByKey(Function.ToKeyValue convertToKeyValue, Option... opts);

  /**
   * Return top N tuples by the selected key
   * @param N how many tuples you want to keep
   * @param name name of the operator
   * @param convertToKeyVal The function convert plain tuple to k,v pair
   * @return new stream of Key and top N tuple of the key
   */
  >>>> STREAM topByKey(int N, Function.ToKeyValue convertToKeyVal, Option... opts);

  /**
   * Return top N tuples of all tuples in the window
   * @param N
   * @param name name of the operator
   * @return new stream of topN
   */
  >>> STREAM top(int N, Option... opts);

  /**
   * Add {@link KeyedWindowedOperatorImpl} with specified {@link Accumulation} 

   * Accumulate tuples by some key within the window definition in this stream
   * Also give a name to the accumulation
   * @param accumulation Accumulation function you want to do
   * @param convertToKeyVal The function convert plain tuple to k,v pair
   * @param  The type of the key used to group tuples
   * @param  The type of value you want to do accumulation on
   * @param  The output type for each given key that you want to accumulate the value to
   * @param  The type of accumulation you want to keep (it can be in memory or on disk)
   * @param  return type
   * @return
   */
  >>> STREAM accumulateByKey(Accumulation accumulation,
      Function.ToKeyValue convertToKeyVal, Option... opts);

  /**
   * Add {@link WindowedOperatorImpl} with specified {@link Accumulation} 

   * Accumulate tuples by some key within the window definition in this stream
   * Also give a name to the accumulation
   * @param accumulation Accumulation function you want to do
   * @param  The output type that you want to accumulate the value to
   * @param  The type of accumulation you want to keep (it can be in memory or on disk)
   * @param  return type
   * @return
   */
  >> STREAM accumulate(Accumulation accumulation, Option... opts);

  /**
   * Add {@link WindowedOperatorImpl} with specified {@link ReduceFn} 

   * Do reduce transformation

   * @param reduce reduce function
   * @param  return type
   * @return new stream of same type
   */
  >> STREAM reduce(ReduceFn reduce, Option... opts);

  /**
   * Add {@link KeyedWindowedOperatorImpl} with specified {@link ReduceFn} 

   * Reduce transformation by selected key 

   * Add an operator to the DAG which merge tuple t1, t2 to new tuple by key
   * @param reduce reduce function
   * @param convertToKeyVal The function convert plain tuple to k,v pair
   * @param  The type of key you want to group tuples by
   * @param  The type of value extract from tuple T
   * @param  return type
   * @return new stream of key value pair
   */
  >>> STREAM reduceByKey(ReduceFn reduce, Function.ToKeyValue convertToKeyVal, Option... opts);


  /**
   * Add {@link WindowedOperatorImpl} with specified {@link FoldFn} 

   * Fold transformation 

   * @param fold fold function
   * @param  output type of fold function
   * @param  return type
   * @return
   */
  >> STREAM fold(FoldFn fold, Option... opts);

  /**
   * Add {@link KeyedWindowedOperatorImpl} with specified {@link FoldFn} 

   * Fold transformation by key 

   * @param fold fold function
   * @param  Result type
   * @return new stream of type O
   */
  >>> STREAM foldByKey(FoldFn fold, Function.ToKeyValue convertToKeyVal, Option... opts);


  /**
   * Return tuples for each key for each window
   * @param 
   * @param 
   * @param 
   * @return
   */
  >>> STREAM groupByKey(Function.ToKeyValue convertToKeyVal, Option... opts);

  /**
   * Return tuples for each window
   * @param 
   * @return
   */
  >> STREAM group();

  /**
   * Reset the trigger settings for next transforms
   * @param triggerOption
   * @param 
   */
  > STREAM resetTrigger(TriggerOption triggerOption);

  /**
   * Reset the allowedLateness settings for next transforms
   * @param allowedLateness
   * @param 
   */
  > STREAM resetAllowedLateness(Duration allowedLateness);

}