com.twitter.heron.api.windowing.triggers.WatermarkTimeTriggerPolicy Maven / Gradle / Ivy

Go to download
//  Copyright 2017 Twitter. All rights reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
package com.twitter.heron.api.windowing.triggers;

import java.util.logging.Logger;

import com.twitter.heron.api.windowing.DefaultEvictionContext;
import com.twitter.heron.api.windowing.Event;
import com.twitter.heron.api.windowing.EvictionPolicy;
import com.twitter.heron.api.windowing.TriggerHandler;
import com.twitter.heron.api.windowing.TriggerPolicy;
import com.twitter.heron.api.windowing.WindowManager;

/**
 * Handles watermark events and triggers {@link TriggerHandler#onTrigger()} for each window
 * interval that has events to be processed up to the watermark ts.
 */
public class WatermarkTimeTriggerPolicy implements TriggerPolicy {
  private static final Logger LOG = Logger.getLogger(WatermarkTimeTriggerPolicy.class.getName());
  private final long slidingIntervalMs;
  private final TriggerHandler handler;
  private final EvictionPolicy evictionPolicy;
  private final WindowManager windowManager;
  private volatile long nextWindowEndTs;
  private boolean started;

  public WatermarkTimeTriggerPolicy(long slidingIntervalMs, TriggerHandler handler,
                                    EvictionPolicy evictionPolicy, WindowManager
                                        windowManager) {
    this.slidingIntervalMs = slidingIntervalMs;
    this.handler = handler;
    this.evictionPolicy = evictionPolicy;
    this.windowManager = windowManager;
    this.started = false;
  }

  @Override
  public void track(Event event) {
    if (started && event.isWatermark()) {
      handleWaterMarkEvent(event);
    }
  }

  @Override
  public void reset() {
    // NOOP
  }

  @Override
  public void start() {
    started = true;
  }

  @Override
  public void shutdown() {
    // NOOP
  }

  /**
   * Invokes the trigger all pending windows up to the
   * watermark timestamp. The end ts of the window is set
   * in the eviction policy context so that the events falling
   * within that window can be processed.
   */
  private void handleWaterMarkEvent(Event event) {
    long watermarkTs = event.getTimestamp();
    long windowEndTs = nextWindowEndTs;
    LOG.fine(String.format("Window end ts %d Watermark ts %d", windowEndTs, watermarkTs));
    while (windowEndTs <= watermarkTs) {
      long currentCount = windowManager.getEventCount(windowEndTs);
      evictionPolicy.setContext(new DefaultEvictionContext(windowEndTs, currentCount));
      if (handler.onTrigger()) {
        windowEndTs += slidingIntervalMs;
      } else {
                /*
                 * No events were found in the previous window interval.
                 * Scan through the events in the queue to find the next
                 * window intervals based on event ts.
                 */
        long ts = getNextAlignedWindowTs(windowEndTs, watermarkTs);
        LOG.fine(String.format("Next aligned window end ts %d", ts));
        if (ts == Long.MAX_VALUE) {
          LOG.fine(String.format("No events to process between %d and watermark ts %d",
              windowEndTs, watermarkTs));
          break;
        }
        windowEndTs = ts;
      }
    }
    nextWindowEndTs = windowEndTs;
  }

  /**
   * Computes the next window by scanning the events in the window and
   * finds the next aligned window between the startTs and endTs. Return the end ts
   * of the next aligned window, i.e. the ts when the window should fire.
   *
   * @param startTs the start timestamp (excluding)
   * @param endTs the end timestamp (including)
   * @return the aligned window end ts for the next window or Long.MAX_VALUE if there
   * are no more events to be processed.
   */
  private long getNextAlignedWindowTs(long startTs, long endTs) {
    long nextTs = windowManager.getEarliestEventTs(startTs, endTs);
    if (nextTs == Long.MAX_VALUE || (nextTs % slidingIntervalMs == 0)) {
      return nextTs;
    }
    return nextTs + (slidingIntervalMs - (nextTs % slidingIntervalMs));
  }

  @Override
  public String toString() {
    return "WatermarkTimeTriggerPolicy{" + "slidingIntervalMs=" + slidingIntervalMs
        + ", nextWindowEndTs=" + nextWindowEndTs + ", started=" + started + '}';
  }
}