co.cask.tigon.sql.flowlet.AbstractInputFlowlet Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of tigon-sql Show documentation
There is a newer version: 0.2.1
/*
 * Copyright © 2014 Cask Data, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package co.cask.tigon.sql.flowlet;

import co.cask.tigon.api.ResourceSpecification;
import co.cask.tigon.api.annotation.Tick;
import co.cask.tigon.api.flow.flowlet.AbstractFlowlet;
import co.cask.tigon.api.flow.flowlet.FailurePolicy;
import co.cask.tigon.api.flow.flowlet.FailureReason;
import co.cask.tigon.api.flow.flowlet.FlowletContext;
import co.cask.tigon.api.flow.flowlet.FlowletSpecification;
import co.cask.tigon.api.flow.flowlet.InputContext;
import co.cask.tigon.api.metrics.Metrics;
import co.cask.tigon.sql.conf.Constants;
import co.cask.tigon.sql.internal.DefaultInputFlowletConfigurer;
import co.cask.tigon.sql.internal.HealthInspector;
import co.cask.tigon.sql.internal.InputFlowletService;
import co.cask.tigon.sql.internal.LocalInputFlowletConfiguration;
import co.cask.tigon.sql.internal.MetricsRecorder;
import co.cask.tigon.sql.internal.ProcessMonitor;
import co.cask.tigon.sql.io.GDATDecoder;
import co.cask.tigon.sql.io.MethodsDriver;
import co.cask.tigon.sql.util.MetaInformationParser;
import com.google.common.base.Stopwatch;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.io.Files;
import org.apache.commons.io.FileUtils;
import org.apache.twill.common.Cancellable;
import org.apache.twill.common.Services;
import org.apache.twill.filesystem.LocalLocationFactory;
import org.apache.twill.filesystem.Location;
import org.apache.twill.filesystem.LocationFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;

/**
 * Abstract class to implement InputFlowlet.
 */
public abstract class AbstractInputFlowlet extends AbstractFlowlet implements ProcessMonitor {
  private static final Logger LOG = LoggerFactory.getLogger(AbstractInputFlowlet.class);
  private File tmpFolder;
  private Metrics metrics;
  private InputFlowletConfigurer configurer;
  private InputFlowletService inputFlowletService;
  private HealthInspector healthInspector;
  private MetricsRecorder metricsRecorder;
  private MethodsDriver methodsDriver;
  private GDATRecordQueue recordQueue;
  private Stopwatch stopwatch;
  private int retryCounter;
  private Map dataIngestionPortsMap;
  private List portsAnnouncementList;

  // Default values for runnable configurables.
  private FailurePolicy failurePolicy = FailurePolicy.RETRY;
  private Map arguments = Maps.newHashMap();
  private ResourceSpecification resourceSpecification = ResourceSpecification.BASIC;

  /**
   * Override this method to configure the InputFlowlet.
   */
  public abstract void create();

  public final void create(InputFlowletConfigurer configurer) {
    this.configurer = configurer;
    create();
  }

  @Override
  public final FlowletSpecification configure() {
    return FlowletSpecification.Builder.with()
      .setName(getName())
      .setDescription(getDescription())
      .setMaxInstances(1)
      .setFailurePolicy(getFailurePolicy())
      .withArguments(getArguments())
      .withResources(getResourceSpecification())
      .build();
  }

  /**
   * Set the name for the InputFlowlet.
   * @param name Name of the InputFlowlet.
   */
  protected void setName(String name) {
    configurer.setName(name);
  }

  /**
   * Set the description for the InputFlowlet.
   * @param description Description of the InputFlowlet.
   */
  protected void setDescription(String description) {
    configurer.setDescription(description);
  }

  /**
   * Add a GDAT Input source to the InputFlowlet.
   * @param name Name of the Input Source.
   * @param schema Schema of the Input Source.
   */
  protected void addGDATInput(String name, StreamSchema schema) {
    configurer.addGDATInput(name, schema);
  }

  /**
   * Add a JSON Input source to the InputFlowlet.
   * @param name Name of the Input Source.
   * @param schema Schema of the Input Source.
   */
  protected void addJSONInput(String name, StreamSchema schema) {
    configurer.addJSONInput(name, schema);
  }

  /**
   * Add a Query query to the InputFlowlet.
   * @param sqlOutName Query Query Name (also the name of the Output Stream generated by the Query).
   * @param sql Query query.
   */
  protected void addQuery(String sqlOutName, String sql) {
    configurer.addQuery(sqlOutName, sql);
  }

  /**
   * Set the arguments for the InputFlowlet.
   * @param arguments for the flowlet.
   */
  protected void setArguments(Map arguments) {
    this.arguments = arguments;
  }

  /**
   * Set the {@link ResourceSpecification} for the InputFlowlet.
   * @param resourceSpecification for the flowlet.
   */
  protected void setResourceSpecification(ResourceSpecification resourceSpecification) {
    this.resourceSpecification = resourceSpecification;
  }

  /**
   * Set the {@link FailurePolicy} for the InputFlowlet. Defaults to {@code FailurePolicy.RETRY}.
   * @param failurePolicy of the InputFlowlet.
   */
  protected void setFailurePolicy(FailurePolicy failurePolicy) {
    this.failurePolicy = failurePolicy;
  }

  private FailurePolicy getFailurePolicy() {
    return failurePolicy;
  }

  private ResourceSpecification getResourceSpecification() {
    return resourceSpecification;
  }

  private Map getArguments() {
    return arguments;
  }

  /**
   * This method initializes all the components required to setup the SQL Compiler environment.
   */
  @Override
  public final void initialize(FlowletContext ctx) throws Exception {
    super.initialize(ctx);
    portsAnnouncementList = Lists.newArrayList();
    DefaultInputFlowletConfigurer configurer = new DefaultInputFlowletConfigurer(this);
    create(configurer);
    InputFlowletSpecification spec = configurer.createInputFlowletSpec();

    dataIngestionPortsMap = Maps.newHashMap();
    int httpPort = 0;
    if (ctx.getRuntimeArguments().get(Constants.HTTP_PORT) != null) {
      httpPort = Integer.parseInt(ctx.getRuntimeArguments().get(Constants.HTTP_PORT));
    }
    dataIngestionPortsMap.put(Constants.HTTP_PORT, httpPort);
    for (String inputName : spec.getInputSchemas().keySet()) {
      int tcpPort = 0;
      if (ctx.getRuntimeArguments().get(Constants.TCP_INGESTION_PORT_PREFIX + inputName) != null) {
        tcpPort = Integer.parseInt(ctx.getRuntimeArguments().get(Constants.TCP_INGESTION_PORT_PREFIX + inputName));
      }
      dataIngestionPortsMap.put(Constants.TCP_INGESTION_PORT_PREFIX + inputName, tcpPort);
    }

    // Setup temporary directory structure
    tmpFolder = Files.createTempDir();
    LocationFactory locationFactory = new LocalLocationFactory(tmpFolder);

    Location baseDir = locationFactory.create("baseDir");
    baseDir.mkdirs();

    InputFlowletConfiguration inputFlowletConfiguration = new LocalInputFlowletConfiguration(baseDir, spec);
    Location binDir = inputFlowletConfiguration.createStreamEngineProcesses();

    healthInspector = new HealthInspector(this);
    metricsRecorder = new MetricsRecorder(metrics);

    //Initiating AbstractInputFlowlet Components
    recordQueue = new GDATRecordQueue();

    //Initiating Netty TCP I/O ports
    inputFlowletService = new InputFlowletService(binDir, spec, healthInspector, metricsRecorder, recordQueue,
                                                  dataIngestionPortsMap, this);
    inputFlowletService.startAndWait();

    //Starting health monitor service
    healthInspector.startAndWait();

    //Initializing methodsDriver
    Map schemaMap = MetaInformationParser.getSchemaMap(new File(binDir.toURI()));
    methodsDriver = new MethodsDriver(this, schemaMap);

    //Initialize stopwatch and retry counter
    stopwatch = new Stopwatch();
    retryCounter = 0;
  }

  /**
   * This process method consumes the records queued in dataManager and invokes the associated "process" methods for
   * each output query
   */
  @Tick(delay = 200L, unit = TimeUnit.MILLISECONDS)
  protected void processGDATRecords() throws InvocationTargetException, IllegalAccessException {
    stopwatch.reset();
    stopwatch.start();
    while (!recordQueue.isEmpty()) {
      // Time since start of processing in Seconds
      long elapsedTime = stopwatch.elapsedTime(TimeUnit.SECONDS);
      if (elapsedTime >= Constants.TICKER_TIMEOUT) {
        break;
      }
      Map.Entry record = recordQueue.getNext();
      methodsDriver.invokeMethods(record.getKey(), record.getValue());
    }
    stopwatch.stop();
  }

  @Override
  public void onSuccess(@Nullable Object input, @Nullable InputContext inputContext) {
    //Return if the calling method is not a @Tick method
    if (input != null) {
      return;
    }
    recordQueue.commit();
    super.onSuccess(input, inputContext);
    retryCounter = 0;
  }

  @Override
  public FailurePolicy onFailure(@Nullable Object input, @Nullable InputContext inputContext, FailureReason reason) {
    //Return if the calling method is not a @Tick method
    if (input != null) {
      return FailurePolicy.IGNORE;
    }
    retryCounter++;
    if (retryCounter >= Constants.MAX_RETRY) {
      LOG.info("Transaction failure exceeded maximum number of retries. Dropping uncommitted data records");
      recordQueue.commit();
      return FailurePolicy.IGNORE;
    }
    LOG.info("Transaction Failed. Retrying operation");
    recordQueue.rollback();
    return FailurePolicy.RETRY;
  }

  @Override
  public void destroy() {
    try {
      FileUtils.deleteDirectory(tmpFolder);
    } catch (IOException e) {
      LOG.warn("Failed to delete {}", tmpFolder.toURI().toString());
    }
    for (Cancellable portAnnouncement : portsAnnouncementList) {
      portAnnouncement.cancel();
    }
    Services.chainStop(healthInspector, inputFlowletService);
    super.destroy();
  }

  /**
   * ProcessMonitor callback function
   * Restarts SQL Compiler processes
   */
  @Override
  public void notifyFailure(Set errorProcessNames) {
    LOG.info("Missing Pings From : " + errorProcessNames.toString());
    healthInspector.stopAndWait();
    healthInspector = new HealthInspector(this);
    inputFlowletService.restartService(healthInspector);
    healthInspector.startAndWait();
  }

  @Override
  public void announceReady() {
    FlowletContext ctx = getContext();
    for (String key : dataIngestionPortsMap.keySet()) {
      portsAnnouncementList.add(ctx.announce(key, inputFlowletService.getDataPort(key)));
      LOG.info("Announced Data Port {} - {}", key, inputFlowletService.getDataPort(key));
    }
  }
}