com.streamsets.pipeline.sdk.SourceRunner Maven / Gradle / Ivy

Go to download
/*
 * Copyright (c) 2021 StreamSets Inc.
 */
package com.streamsets.pipeline.sdk;

import com.streamsets.datacollector.main.RuntimeInfo;
import com.streamsets.pipeline.api.BatchMaker;
import com.streamsets.pipeline.api.DeliveryGuarantee;
import com.streamsets.pipeline.api.ExecutionMode;
import com.streamsets.pipeline.api.OffsetCommitter;
import com.streamsets.pipeline.api.OnRecordError;
import com.streamsets.pipeline.api.Source;
import com.streamsets.pipeline.api.StageException;
import com.streamsets.pipeline.api.StageType;
import com.streamsets.pipeline.api.impl.Utils;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.List;
import java.util.Map;

public class SourceRunner extends StageRunner {
  private static final Logger LOG = LoggerFactory.getLogger(SourceRunner.class);

  public SourceRunner(
    Class sourceClass,
    Source source,
    Map configuration,
    List outputLanes,
    boolean isPreview,
    OnRecordError onRecordError,
    Map constants,
    Map stageSdcConf,
    ExecutionMode executionMode,
    DeliveryGuarantee deliveryGuarantee,
    String resourcesDir,
    RuntimeInfo runtimeInfo,
    List services
  ) {
    super(
      sourceClass,
      source,
      StageType.SOURCE,
      configuration,
      outputLanes,
      isPreview,
      onRecordError,
      constants,
      stageSdcConf,
      executionMode,
      deliveryGuarantee,
      resourcesDir,
      runtimeInfo,
      services
    );
  }

  public SourceRunner(
    Class sourceClass,
    Map configuration,
    List outputLanes,
    boolean isPreview,
    OnRecordError onRecordError,
    Map constants,
    Map stageSdcConf,
    ExecutionMode executionMode,
    DeliveryGuarantee deliveryGuarantee,
    String resourcesDir,
    RuntimeInfo runtimeInfo,
    List services
  ) {
    super(
      sourceClass,
      StageType.SOURCE,
      configuration,
      outputLanes,
      isPreview,
      onRecordError,
      constants,
      stageSdcConf,
      executionMode,
      deliveryGuarantee,
      resourcesDir,
      runtimeInfo,
      services
    );
  }

  public Output runProduce(String lastOffset, int maxBatchSize) throws StageException {
    try {
      LOG.debug("Stage '{}' produce starts", getInfo().getInstanceName());
      ensureStatus(Status.INITIALIZED);
      BatchMakerImpl batchMaker = new BatchMakerImpl(((Source.Context) getContext()).getOutputLanes());
      String newOffset = getStage().produce(lastOffset, maxBatchSize, batchMaker);
      if (getStage() instanceof OffsetCommitter) {
        ((OffsetCommitter)getStage()).commit(newOffset);
      }
      return new Output(Source.POLL_SOURCE_OFFSET_KEY, newOffset, batchMaker.getOutput());
    } finally {
      LOG.debug("Stage '{}' produce ends", getInfo().getInstanceName());
    }
  }

  public static class Builder extends StageRunner.Builder {

    public Builder(Class sourceClass,  Source source) {
      super((Class)sourceClass, source);
    }

    @SuppressWarnings("unchecked")
    public Builder(Class sourceClass) {
      super((Class) sourceClass);
    }

    @Override
    public SourceRunner build() {
      Utils.checkState(!outputLanes.isEmpty(), "A Source must have at least one output stream");
      if (stage != null) {
        return new SourceRunner(
          stageClass,
          stage,
          configs,
          outputLanes,
          isPreview,
          onRecordError,
          constants,
          stageSdcConf,
          executionMode,
          deliveryGuarantee,
          resourcesDir,
          runtimeInfo,
          services
        );
      } else {
        return new SourceRunner(
          stageClass,
          configs,
          outputLanes,
          isPreview,
          onRecordError,
          constants,
          stageSdcConf,
          executionMode,
          deliveryGuarantee,
          resourcesDir,
          runtimeInfo,
          services
        );
      }
    }
  }

  public static BatchMaker createTestBatchMaker(String... outputLanes) {
    return StageRunner.createTestBatchMaker(outputLanes);
  }

  public static Output getOutput(BatchMaker batchMaker) {
    return StageRunner.getOutput(batchMaker);
  }

}