All Downloads are FREE. Search and download functionalities are using the official Maven repository.
  • Log In
  • Register

    • com.datatorrent.stram.engine.GenericNode Maven / Gradle / Ivy

    There is a newer version: 3.7.0
    Show newest version
    /**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package com.datatorrent.stram.engine;

import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.Map;
import java.util.Map.Entry;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.commons.lang.UnhandledException;

import com.google.common.base.Throwables;

import com.datatorrent.api.Operator;
import com.datatorrent.api.Operator.IdleTimeHandler;
import com.datatorrent.api.Operator.InputPort;
import com.datatorrent.api.Operator.ProcessingMode;
import com.datatorrent.api.Operator.ShutdownException;
import com.datatorrent.api.Sink;
import com.datatorrent.api.annotation.Stateless;
import com.datatorrent.bufferserver.packet.MessageType;
import com.datatorrent.bufferserver.util.Codec;
import com.datatorrent.stram.api.StreamingContainerUmbilicalProtocol.ContainerStats;
import com.datatorrent.stram.debug.TappedReservoir;
import com.datatorrent.stram.plan.logical.LogicalPlan;
import com.datatorrent.stram.plan.logical.Operators;
import com.datatorrent.stram.tuple.ResetWindowTuple;
import com.datatorrent.stram.tuple.Tuple;

/**
 * The base class for node implementation
    
 * 
    
 * Implements the base interface {@link com.datatorrent.stram.engine.Node}
    
 * 
    
 * This is the basic functional block of the DAG. It is responsible for the following
    
 * It emits and consumes tuples
    
 * Upon window boundary it does house cleaning, state sync up etc
    
 * Interacts with Stram with a heartbeat protocol
    
 * 
    
 *
 * @since 0.3.2
 */
public class GenericNode extends Node
{
  protected final HashMap inputs = new HashMap<>();
  protected ArrayList deferredInputConnections = new ArrayList<>();

  @Override
  @SuppressWarnings("unchecked")
  public void addSinks(Map> sinks)
  {
    for (Entry> e : sinks.entrySet()) {
      SweepableReservoir original = inputs.get(e.getKey());
      if (original instanceof TappedReservoir) {
        TappedReservoir tr = (TappedReservoir)original;
        tr.add(e.getValue());
      } else if (original != null) {
        TappedReservoir tr = new TappedReservoir(original, e.getValue());
        inputs.put(e.getKey(), tr);
      }
    }

    super.addSinks(sinks);
  }

  @Override
  public void removeSinks(Map> sinks)
  {
    for (Entry> e : sinks.entrySet()) {
      SweepableReservoir reservoir = inputs.get(e.getKey());
      if (reservoir instanceof TappedReservoir) {
        TappedReservoir tr = (TappedReservoir)reservoir;
        tr.remove(e.getValue());
        if (tr.getSinks().length == 0) {
          tr.reservoir.setSink(tr.setSink(null));
          inputs.put(e.getKey(), tr.reservoir);
        }
      }
    }

    super.removeSinks(sinks);
  }

  public GenericNode(Operator operator, OperatorContext context)
  {
    super(operator, context);
  }

  @SuppressWarnings("unchecked")
  public InputPort getInputPort(String port)
  {
    return (InputPort)descriptor.inputPorts.get(port).component;
  }

  @Override
  public void connectInputPort(String port, final SweepableReservoir reservoir)
  {
    if (reservoir == null) {
      throw new IllegalArgumentException("Reservoir cannot be null for port '" + port + "' on operator '" + operator + "'");
    }

    InputPort inputPort = getInputPort(port);
    if (inputPort == null) {
      throw new IllegalArgumentException("Port '" + port + "' does not exist on operator '" + operator + "'");
    }

    if (inputs.containsKey(port)) {
      deferredInputConnections.add(new DeferredInputConnection(port, reservoir));
    } else {
      inputPort.setConnected(true);
      inputs.put(port, reservoir);
      reservoir.setSink(inputPort.getSink());
    }
  }

  /**
   * @param endWindowTuple the value of endWindowTuple
   */
  protected void processEndWindow(Tuple endWindowTuple)
  {
    if (++applicationWindowCount == APPLICATION_WINDOW_COUNT) {
      insideWindow = false;
      operator.endWindow();
      applicationWindowCount = 0;
    }
    endWindowEmitTime = System.currentTimeMillis();

    if (endWindowTuple == null) {
      emitEndWindow();
    } else {
      for (int s = sinks.length; s-- > 0; ) {
        sinks[s].put(endWindowTuple);
      }
      controlTupleCount++;
    }

    if (doCheckpoint) {
      dagCheckpointOffsetCount = (dagCheckpointOffsetCount + 1) % DAG_CHECKPOINT_WINDOW_COUNT;
    }

    if (++checkpointWindowCount == CHECKPOINT_WINDOW_COUNT) {
      checkpointWindowCount = 0;
      if (doCheckpoint) {
        checkpoint(currentWindowId);
        lastCheckpointWindowId = currentWindowId;
        doCheckpoint = false;
      } else if (PROCESSING_MODE == ProcessingMode.EXACTLY_ONCE) {
        checkpoint(currentWindowId);
        lastCheckpointWindowId = currentWindowId;
      }
    }

    ContainerStats.OperatorStats stats = new ContainerStats.OperatorStats();
    reportStats(stats, currentWindowId);
    if (!insideWindow) {
      stats.metrics = collectMetrics();
    }
    handleRequests(currentWindowId);
  }


  class TupleTracker
  {
    final Tuple tuple;
    SweepableReservoir[] ports;

    TupleTracker(Tuple base, int count)
    {
      tuple = base;
      ports = new SweepableReservoir[count];
    }

  }

  boolean insideWindow;
  boolean doCheckpoint;
  long lastCheckpointWindowId = Stateless.WINDOW_ID;

  @Override
  public void activate()
  {
    super.activate();
    insideWindow = applicationWindowCount != 0;
  }

  private boolean isInputPortConnectedToDelayOperator(String portName)
  {
    Operators.PortContextPair> pcPair = descriptor.inputPorts.get(portName);
    if (pcPair == null || pcPair.context == null) {
      return false;
    }
    return pcPair.context.getValue(LogicalPlan.IS_CONNECTED_TO_DELAY_OPERATOR);
  }

  /**
   * Originally this method was defined in an attempt to implement the interface Runnable.
   *
   * Note that activate does not return as long as there is useful workload for the node.
   */
  @Override
  @SuppressWarnings({"SleepWhileInLoop", "UseSpecificCatch", "BroadCatchBlock", "TooBroadCatch"})
  public final void run()
  {
    doCheckpoint = false;

    final long maxSpinMillis = context.getValue(OperatorContext.SPIN_MILLIS);
    long spinMillis = 0;
    final boolean handleIdleTime = operator instanceof IdleTimeHandler;
    int totalQueues = inputs.size();
    int regularQueues = totalQueues;
    // regularQueues is the number of queues that are not connected to a DelayOperator
    for (String portName : inputs.keySet()) {
      if (isInputPortConnectedToDelayOperator(portName)) {
        regularQueues--;
      }
    }

    ArrayList> activeQueues = new ArrayList<>();
    activeQueues.addAll(inputs.entrySet());

    int expectingBeginWindow = activeQueues.size();
    int receivedEndWindow = 0;
    long firstWindowId = -1;

    calculateNextCheckpointWindow();

    TupleTracker tracker;
    LinkedList resetTupleTracker = new LinkedList<>();
    try {
      do {
        Iterator> buffers = activeQueues.iterator();
      activequeue:
        while (buffers.hasNext()) {
          Map.Entry activePortEntry = buffers.next();
          SweepableReservoir activePort = activePortEntry.getValue();
          Tuple t = activePort.sweep();
          if (t != null) {
            spinMillis = 0;
            boolean delay = (operator instanceof Operator.DelayOperator);
            long windowAhead = 0;
            if (delay) {
              windowAhead = WindowGenerator.getAheadWindowId(t.getWindowId(), firstWindowMillis, windowWidthMillis, 1);
            }
            switch (t.getType()) {
              case BEGIN_WINDOW:
                if (expectingBeginWindow == totalQueues) {
                  // This is the first begin window tuple among all ports
                  if (isInputPortConnectedToDelayOperator(activePortEntry.getKey())) {
                    // We need to wait for the first BEGIN_WINDOW from a port not connected to DelayOperator before
                    // we can do anything with it, because otherwise if a CHECKPOINT tuple arrives from
                    // upstream after the BEGIN_WINDOW tuple for the next window from the delay operator, it would end
                    // up checkpointing in the middle of the window.  This code is assuming we have at least one
                    // input port that is not connected to a DelayOperator, and we might have to change this later.
                    // In the future, this condition will not be needed if we get rid of the CHECKPOINT tuple.
                    continue;
                  }
                  activePort.remove();
                  expectingBeginWindow--;
                  receivedEndWindow = 0;
                  currentWindowId = t.getWindowId();
                  if (delay) {
                    if (WindowGenerator.getBaseSecondsFromWindowId(windowAhead) > t.getBaseSeconds()) {
                      // Buffer server code strips out the base seconds from BEGIN_WINDOW and END_WINDOW tuples for
                      // serialization optimization.  That's why we need a reset window here to tell the buffer
                      // server we are having a new baseSeconds now.
                      Tuple resetWindowTuple = new ResetWindowTuple(windowAhead);
                      for (int s = sinks.length; s-- > 0; ) {
                        sinks[s].put(resetWindowTuple);
                      }
                      controlTupleCount++;
                    }
                    t.setWindowId(windowAhead);
                  }
                  for (int s = sinks.length; s-- > 0; ) {
                    sinks[s].put(t);
                  }
                  controlTupleCount++;

                  context.setWindowsFromCheckpoint(nextCheckpointWindowCount--);

                  if (applicationWindowCount == 0) {
                    insideWindow = true;
                    operator.beginWindow(currentWindowId);
                  }
                } else if (t.getWindowId() == currentWindowId) {
                  activePort.remove();
                  expectingBeginWindow--;
                } else {
                  buffers.remove();
                  String port = activePortEntry.getKey();
                  if (PROCESSING_MODE == ProcessingMode.AT_MOST_ONCE) {
                    if (t.getWindowId() < currentWindowId) {
                      /*
                       * we need to fast forward this stream till we find the current
                       * window or the window which is bigger than the current window.
                       */

                      /* lets move the current reservoir in the background */
                      Sink sink = activePort.setSink(Sink.BLACKHOLE);
                      deferredInputConnections.add(0, new DeferredInputConnection(port, activePort));

                      /* replace it with the reservoir which blocks the tuples in the past */
                      WindowIdActivatedReservoir wiar = new WindowIdActivatedReservoir(port, activePort, currentWindowId);
                      wiar.setSink(sink);
                      inputs.put(port, wiar);
                      activeQueues.add(new AbstractMap.SimpleEntry(port, wiar));
                      break activequeue;
                    } else {
                      expectingBeginWindow--;
                      if (++receivedEndWindow == totalQueues) {
                        processEndWindow(null);
                        activeQueues.addAll(inputs.entrySet());
                        expectingBeginWindow = activeQueues.size();
                        break activequeue;
                      }
                    }
                  } else {
                    logger.error("Catastrophic Error: Out of sequence {} tuple {} on port {} while expecting {}", t.getType(), Codec.getStringWindowId(t.getWindowId()), port, Codec.getStringWindowId(currentWindowId));
                    System.exit(2);
                  }
                }
                break;

              case END_WINDOW:
                buffers.remove();
                if (t.getWindowId() == currentWindowId) {
                  activePort.remove();
                  endWindowDequeueTimes.put(activePort, System.currentTimeMillis());
                  if (++receivedEndWindow == totalQueues) {
                    assert (activeQueues.isEmpty());
                    if (delay) {
                      t.setWindowId(windowAhead);
                    }
                    processEndWindow(t);
                    activeQueues.addAll(inputs.entrySet());
                    expectingBeginWindow = activeQueues.size();
                    break activequeue;
                  }
                }
                break;

              case CHECKPOINT:
                activePort.remove();
                long checkpointWindow = t.getWindowId();
                if (lastCheckpointWindowId < checkpointWindow) {
                  dagCheckpointOffsetCount = 0;
                  if (PROCESSING_MODE == ProcessingMode.EXACTLY_ONCE) {
                    lastCheckpointWindowId = checkpointWindow;
                  } else if (!doCheckpoint) {
                    if (checkpointWindowCount == 0) {
                      checkpoint(checkpointWindow);
                      lastCheckpointWindowId = checkpointWindow;
                    } else {
                      doCheckpoint = true;
                    }
                  }
                  if (!delay) {
                    for (int s = sinks.length; s-- > 0; ) {
                      sinks[s].put(t);
                    }
                    controlTupleCount++;
                  }
                }
                break;

              case RESET_WINDOW:
                /**
                 * we will receive tuples which are equal to the number of input streams.
                 */
                activePort.remove();
                if (isInputPortConnectedToDelayOperator(activePortEntry.getKey())) {
                  break; // breaking out of the switch/case
                }

                buffers.remove();
                int baseSeconds = t.getBaseSeconds();
                tracker = null;
                for (Iterator trackerIterator = resetTupleTracker.iterator(); trackerIterator.hasNext(); ) {
                  tracker = trackerIterator.next();
                  if (tracker.tuple.getBaseSeconds() == baseSeconds) {
                    break;
                  }
                }

                if (tracker == null) {
                  tracker = new TupleTracker(t, regularQueues);
                  resetTupleTracker.add(tracker);
                }
                int trackerIndex = 0;
                while (trackerIndex < tracker.ports.length) {
                  if (tracker.ports[trackerIndex] == null) {
                    tracker.ports[trackerIndex++] = activePort;
                    break;
                  } else if (tracker.ports[trackerIndex] == activePort) {
                    break;
                  }

                  trackerIndex++;
                }

                if (trackerIndex == regularQueues) {
                  Iterator trackerIterator = resetTupleTracker.iterator();
                  while (trackerIterator.hasNext()) {
                    if (trackerIterator.next().tuple.getBaseSeconds() <= baseSeconds) {
                      trackerIterator.remove();
                    }
                  }
                  if (!delay) {
                    for (int s = sinks.length; s-- > 0; ) {
                      sinks[s].put(t);
                    }
                    controlTupleCount++;
                  }
                  if (!activeQueues.isEmpty()) {
                    // make sure they are all queues from DelayOperator
                    for (Map.Entry entry : activeQueues) {
                      if (!isInputPortConnectedToDelayOperator(entry.getKey())) {
                        assert (false);
                      }
                    }
                    activeQueues.clear();
                  }
                  activeQueues.addAll(inputs.entrySet());
                  expectingBeginWindow = activeQueues.size();

                  if (firstWindowId == -1) {
                    if (delay) {
                      for (int s = sinks.length; s-- > 0; ) {
                        sinks[s].put(t);
                      }
                      controlTupleCount++;
                      // if it's a DelayOperator and this is the first RESET_WINDOW (start) or END_STREAM
                      // (recovery), fabricate the first window
                      fabricateFirstWindow((Operator.DelayOperator)operator, windowAhead);
                    }
                    firstWindowId = t.getWindowId();
                  }
                  break activequeue;
                }
                break;

              case END_STREAM:
                activePort.remove();
                buffers.remove();
                if (firstWindowId == -1) {
                  // this is for recovery from a checkpoint for DelayOperator
                  if (delay) {
                    // if it's a DelayOperator and this is the first RESET_WINDOW (start) or END_STREAM (recovery),
                    // fabricate the first window
                    fabricateFirstWindow((Operator.DelayOperator)operator, windowAhead);
                  }
                  firstWindowId = t.getWindowId();
                }
                for (Iterator> it = inputs.entrySet().iterator(); it.hasNext(); ) {
                  Entry e = it.next();
                  if (e.getValue() == activePort) {
                    if (!descriptor.inputPorts.isEmpty()) {
                      descriptor.inputPorts.get(e.getKey()).component.setConnected(false);
                    }
                    it.remove();

                    /* check the deferred connection list for any new port that should be connected here */
                    Iterator dici = deferredInputConnections.iterator();
                    while (dici.hasNext()) {
                      DeferredInputConnection dic = dici.next();
                      if (e.getKey().equals(dic.portname)) {
                        connectInputPort(dic.portname, dic.reservoir);
                        dici.remove();
                        activeQueues.add(new AbstractMap.SimpleEntry<>(dic.portname, dic.reservoir));
                        break activequeue;
                      }
                    }

                    break;
                  }
                }

                /**
                 * We are not going to receive begin window on this ever!
                 */
                expectingBeginWindow--;

                /**
                 * Since one of the operators we care about it gone, we should relook at our ports.
                 * We need to make sure that the END_STREAM comes outside of the window.
                 */
                regularQueues--;
                totalQueues--;

                boolean break_activequeue = false;
                if (regularQueues == 0) {
                  alive = false;
                  break_activequeue = true;
                } else if (activeQueues.isEmpty()) {
                  assert (!inputs.isEmpty());
                  processEndWindow(null);
                  activeQueues.addAll(inputs.entrySet());
                  expectingBeginWindow = activeQueues.size();
                  break_activequeue = true;
                }

                /**
                 * also make sure that we update the reset tuple tracker if this stream had delivered any reset tuples.
                 * Check all the reset buffers to see if current input port has already delivered reset tuple. If it has
                 * then we are waiting for something else to deliver the reset tuple, so just clear current reservoir
                 * from the list of tracked reservoirs. If the current input port has not delivered the reset tuple, and
                 * it's the only one which has not, then we consider it delivered and release the reset tuple downstream.
                 */
                Tuple tuple = null;
                for (Iterator trackerIterator = resetTupleTracker.iterator(); trackerIterator.hasNext(); ) {
                  tracker = trackerIterator.next();

                  trackerIndex = 0;
                  while (trackerIndex < tracker.ports.length) {
                    if (tracker.ports[trackerIndex] == activePort) {
                      SweepableReservoir[] ports = new SweepableReservoir[regularQueues];
                      System.arraycopy(tracker.ports, 0, ports, 0, trackerIndex);
                      if (trackerIndex < regularQueues) {
                        System.arraycopy(tracker.ports, trackerIndex + 1, ports, trackerIndex, tracker.ports.length - trackerIndex - 1);
                      }
                      tracker.ports = ports;
                      break;
                    } else if (tracker.ports[trackerIndex] == null) {
                      if (trackerIndex == regularQueues) { /* regularQueues is already adjusted above */
                        if (tuple == null || tuple.getBaseSeconds() < tracker.tuple.getBaseSeconds()) {
                          tuple = tracker.tuple;
                        }

                        trackerIterator.remove();
                      }
                      break;
                    } else {
                      tracker.ports = Arrays.copyOf(tracker.ports, regularQueues);
                    }

                    trackerIndex++;
                  }
                }

                /*
                 * Since we were waiting for a reset tuple on this stream, we should not any longer.
                 */
                if (tuple != null && !delay) {
                  for (int s = sinks.length; s-- > 0; ) {
                    sinks[s].put(tuple);
                  }
                  controlTupleCount++;
                }

                if (break_activequeue) {
                  break activequeue;
                }
                break;

              default:
                throw new UnhandledException("Unrecognized Control Tuple", new IllegalArgumentException(t.toString()));
            }
          }
        }

        if (activeQueues.isEmpty() && alive) {
          logger.error("Catastrophic Error: Invalid State - the operator blocked forever!");
          System.exit(2);
        } else {
          boolean need2sleep = true;
          for (Map.Entry cb : activeQueues) {
            need2sleep = cb.getValue().isEmpty();
            if (!need2sleep) {
              spinMillis = 0;
              break;
            }
          }

          if (need2sleep) {
            if (handleIdleTime && insideWindow) {
              ((IdleTimeHandler)operator).handleIdleTime();
            } else {
              Thread.sleep(spinMillis);
              spinMillis = Math.min(maxSpinMillis, spinMillis + 1);
            }
          }
        }
      } while (alive);
    } catch (ShutdownException se) {
      logger.debug("Shutdown requested by the operator when alive = {}.", alive);
      alive = false;
    } catch (Throwable cause) {
      synchronized (this) {
        if (alive) {
          throw Throwables.propagate(cause);
        }
      }

      Throwable rootCause = cause;
      while (rootCause != null) {
        if (rootCause instanceof InterruptedException) {
          break;
        }
        rootCause = rootCause.getCause();
      }

      if (rootCause == null) {
        throw Throwables.propagate(cause);
      } else {
        logger.debug("Ignoring InterruptedException after shutdown", cause);
      }
    }

    /**
     * TODO: If shutdown and inside window provide alternate way of notifying the operator in such ways
     * TODO: as using a listener callback
     */
    if (insideWindow && !shutdown) {
      operator.endWindow();
      endWindowEmitTime = System.currentTimeMillis();
      if (++applicationWindowCount == APPLICATION_WINDOW_COUNT) {
        applicationWindowCount = 0;
      }

      if (++checkpointWindowCount == CHECKPOINT_WINDOW_COUNT) {
        checkpointWindowCount = 0;
        if (doCheckpoint || PROCESSING_MODE == ProcessingMode.EXACTLY_ONCE) {
          checkpoint(currentWindowId);
        }
      }

      ContainerStats.OperatorStats stats = new ContainerStats.OperatorStats();
      fixEndWindowDequeueTimesBeforeDeactivate();
      reportStats(stats, currentWindowId);
      stats.metrics = collectMetrics();
      handleRequests(currentWindowId);
    }

  }

  private void fabricateFirstWindow(Operator.DelayOperator delayOperator, long windowAhead)
  {
    Tuple beginWindowTuple = new Tuple(MessageType.BEGIN_WINDOW, windowAhead);
    Tuple endWindowTuple = new Tuple(MessageType.END_WINDOW, windowAhead);
    for (Sink sink : outputs.values()) {
      sink.put(beginWindowTuple);
    }
    controlTupleCount++;
    delayOperator.firstWindow();
    for (Sink sink : outputs.values()) {
      sink.put(endWindowTuple);
    }
    controlTupleCount++;
  }

  /**
   * End window dequeue times may not have been saved for all the input ports during deactivate,
   * so save them for reporting. SPOI-1324.
   */
  private void fixEndWindowDequeueTimesBeforeDeactivate()
  {
    long endWindowDequeueTime = System.currentTimeMillis();
    for (SweepableReservoir sr : inputs.values()) {
      if (endWindowDequeueTimes.get(sr) == null) {
        endWindowDequeueTimes.put(sr, endWindowDequeueTime);
      }
    }
  }

  @Override
  protected void reportStats(ContainerStats.OperatorStats stats, long windowId)
  {
    ArrayList ipstats = new ArrayList<>();
    for (Entry e : inputs.entrySet()) {
      SweepableReservoir ar = e.getValue();
      ContainerStats.OperatorStats.PortStats portStats = new ContainerStats.OperatorStats.PortStats(e.getKey());
      portStats.queueSize = ar.size(DATA_TUPLE_AWARE);
      portStats.tupleCount = ar.getCount(true);
      portStats.endWindowTimestamp = endWindowDequeueTimes.get(e.getValue());
      ipstats.add(portStats);
    }
    stats.inputPorts = ipstats;
    super.reportStats(stats, windowId);
  }

  protected class DeferredInputConnection
  {
    String portname;
    SweepableReservoir reservoir;

    DeferredInputConnection(String portname, SweepableReservoir reservoir)
    {
      this.portname = portname;
      this.reservoir = reservoir;
    }

  }

  private static final Logger logger = LoggerFactory.getLogger(GenericNode.class);
}
    
    
    
    

    




    
    
    © 2015 - 2025 Weber Informatics LLC | Privacy Policy