org.apache.hadoop.hive.llap.tez.Converters Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of hive-llap-common Show documentation
There is a newer version: 2.3.9_arenadata3
/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hive.llap.tez;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import com.google.protobuf.ByteString;
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto;
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GroupInputSpecProto;
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto;
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryIdentifierProto;
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SignableVertexSpec;
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateProto;
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.tez.common.TezCommonUtils;
import org.apache.tez.dag.api.EntityDescriptor;
import org.apache.tez.dag.api.InputDescriptor;
import org.apache.tez.dag.api.OutputDescriptor;
import org.apache.tez.dag.api.ProcessorDescriptor;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.UserPayload;
import org.apache.tez.dag.api.event.VertexState;
import org.apache.tez.dag.records.TezDAGID;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.runtime.api.TaskContext;
import org.apache.tez.runtime.api.impl.GroupInputSpec;
import org.apache.tez.runtime.api.impl.InputSpec;
import org.apache.tez.runtime.api.impl.OutputSpec;
import org.apache.tez.runtime.api.impl.TaskSpec;

public class Converters {

  public static TaskSpec getTaskSpecfromProto(SignableVertexSpec vectorProto,
      int fragmentNum, int attemptNum, TezTaskAttemptID attemptId) {
    TezTaskAttemptID taskAttemptID = attemptId != null ? attemptId
        : createTaskAttemptId(vectorProto.getQueryIdentifier(), vectorProto.getVertexIndex(),
        fragmentNum, attemptNum);

    ProcessorDescriptor processorDescriptor = null;
    if (vectorProto.hasProcessorDescriptor()) {
      processorDescriptor = convertProcessorDescriptorFromProto(
          vectorProto.getProcessorDescriptor());
    }

    List inputSpecList = new ArrayList(vectorProto.getInputSpecsCount());
    if (vectorProto.getInputSpecsCount() > 0) {
      for (IOSpecProto inputSpecProto : vectorProto.getInputSpecsList()) {
        inputSpecList.add(getInputSpecFromProto(inputSpecProto));
      }
    }

    List outputSpecList =
        new ArrayList(vectorProto.getOutputSpecsCount());
    if (vectorProto.getOutputSpecsCount() > 0) {
      for (IOSpecProto outputSpecProto : vectorProto.getOutputSpecsList()) {
        outputSpecList.add(getOutputSpecFromProto(outputSpecProto));
      }
    }

    List groupInputSpecs =
        new ArrayList(vectorProto.getGroupedInputSpecsCount());
    if (vectorProto.getGroupedInputSpecsCount() > 0) {
      for (GroupInputSpecProto groupInputSpecProto : vectorProto.getGroupedInputSpecsList()) {
        groupInputSpecs.add(getGroupInputSpecFromProto(groupInputSpecProto));
      }
    }

    TaskSpec taskSpec =
        new TaskSpec(taskAttemptID, vectorProto.getDagName(), vectorProto.getVertexName(),
            vectorProto.getVertexParallelism(), processorDescriptor, inputSpecList,
            outputSpecList, groupInputSpecs);
    return taskSpec;
  }

  public static TezTaskAttemptID createTaskAttemptId(
      QueryIdentifierProto queryIdProto, int vertexIndex, int fragmentNum, int attemptNum) {
    // Come ride the API roller-coaster!
    return TezTaskAttemptID.getInstance(
            TezTaskID.getInstance(
                TezVertexID.getInstance(
                    TezDAGID.getInstance(
                        ConverterUtils.toApplicationId(
                            queryIdProto.getApplicationIdString()),
                        queryIdProto.getDagIndex()),
                    vertexIndex),
                fragmentNum),
            attemptNum);
  }

  public static TezTaskAttemptID createTaskAttemptId(TaskContext ctx) {
    // Come ride the API roller-coaster #2! The best part is that ctx has TezTaskAttemptID inside.
    return TezTaskAttemptID.getInstance(
            TezTaskID.getInstance(
                TezVertexID.getInstance(
                    TezDAGID.getInstance(
                        ctx.getApplicationId(),
                    ctx.getDagIdentifier()),
                ctx.getTaskVertexIndex()),
            ctx.getTaskIndex()),
        ctx.getTaskAttemptNumber());
  }

  public static SignableVertexSpec.Builder constructSignableVertexSpec(TaskSpec taskSpec,
                                                                       QueryIdentifierProto queryIdentifierProto,
                                                                       String tokenIdentifier,
                                                                       String user,
                                                                       String hiveQueryIdString) {
    TezTaskAttemptID tId = taskSpec.getTaskAttemptID();

    SignableVertexSpec.Builder builder = SignableVertexSpec.newBuilder();
    builder.setQueryIdentifier(queryIdentifierProto);
    builder.setHiveQueryId(hiveQueryIdString);
    builder.setVertexIndex(tId.getTaskID().getVertexID().getId());
    builder.setDagName(taskSpec.getDAGName());
    builder.setVertexName(taskSpec.getVertexName());
    builder.setVertexParallelism(taskSpec.getVertexParallelism());
    builder.setTokenIdentifier(tokenIdentifier);
    builder.setUser(user);

    if (taskSpec.getProcessorDescriptor() != null) {
      builder.setProcessorDescriptor(
          convertToProto(taskSpec.getProcessorDescriptor()));
    }

    if (taskSpec.getInputs() != null && !taskSpec.getInputs().isEmpty()) {
      for (InputSpec inputSpec : taskSpec.getInputs()) {
        builder.addInputSpecs(convertInputSpecToProto(inputSpec));
      }
    }

    if (taskSpec.getOutputs() != null && !taskSpec.getOutputs().isEmpty()) {
      for (OutputSpec outputSpec : taskSpec.getOutputs()) {
        builder.addOutputSpecs(convertOutputSpecToProto(outputSpec));
      }
    }

    if (taskSpec.getGroupInputs() != null && !taskSpec.getGroupInputs().isEmpty()) {
      for (GroupInputSpec groupInputSpec : taskSpec.getGroupInputs()) {
        builder.addGroupedInputSpecs(convertGroupInputSpecToProto(groupInputSpec));

      }
    }
    return builder;
  }

  private static ProcessorDescriptor convertProcessorDescriptorFromProto(
      EntityDescriptorProto proto) {
    String className = proto.getClassName();
    UserPayload payload = convertPayloadFromProto(proto);
    ProcessorDescriptor pd = ProcessorDescriptor.create(className);
    setUserPayload(pd, payload);
    return pd;
  }

  private static EntityDescriptorProto convertToProto(
      EntityDescriptor descriptor) {
    EntityDescriptorProto.Builder builder = EntityDescriptorProto
        .newBuilder();
    builder.setClassName(descriptor.getClassName());

    UserPayload userPayload = descriptor.getUserPayload();
    if (userPayload != null) {
      UserPayloadProto.Builder payloadBuilder = UserPayloadProto.newBuilder();
      if (userPayload.hasPayload()) {
        payloadBuilder.setUserPayload(ByteString.copyFrom(userPayload.getPayload()));
        payloadBuilder.setVersion(userPayload.getVersion());
      }
      builder.setUserPayload(payloadBuilder.build());
    }
    if (descriptor.getHistoryText() != null) {
      try {
        builder.setHistoryText(TezCommonUtils.compressByteArrayToByteString(
            descriptor.getHistoryText().getBytes("UTF-8")));
      } catch (IOException e) {
        throw new TezUncheckedException(e);
      }
    }
    return builder.build();
  }

  private static InputSpec getInputSpecFromProto(IOSpecProto inputSpecProto) {
    InputDescriptor inputDescriptor = null;
    if (inputSpecProto.hasIoDescriptor()) {
      inputDescriptor =
          convertInputDescriptorFromProto(inputSpecProto.getIoDescriptor());
    }
    InputSpec inputSpec = new InputSpec(inputSpecProto.getConnectedVertexName(), inputDescriptor,
        inputSpecProto.getPhysicalEdgeCount());
    return inputSpec;
  }

  private static InputDescriptor convertInputDescriptorFromProto(
      EntityDescriptorProto proto) {
    String className = proto.getClassName();
    UserPayload payload = convertPayloadFromProto(proto);
    InputDescriptor id = InputDescriptor.create(className);
    setUserPayload(id, payload);
    return id;
  }

  private static OutputDescriptor convertOutputDescriptorFromProto(
      EntityDescriptorProto proto) {
    String className = proto.getClassName();
    UserPayload payload = convertPayloadFromProto(proto);
    OutputDescriptor od = OutputDescriptor.create(className);
    setUserPayload(od, payload);
    return od;
  }

  private static IOSpecProto convertInputSpecToProto(InputSpec inputSpec) {
    IOSpecProto.Builder builder = IOSpecProto.newBuilder();
    if (inputSpec.getSourceVertexName() != null) {
      builder.setConnectedVertexName(inputSpec.getSourceVertexName());
    }
    if (inputSpec.getInputDescriptor() != null) {
      builder.setIoDescriptor(convertToProto(inputSpec.getInputDescriptor()));
    }
    builder.setPhysicalEdgeCount(inputSpec.getPhysicalEdgeCount());
    return builder.build();
  }

  private static OutputSpec getOutputSpecFromProto(IOSpecProto outputSpecProto) {
    OutputDescriptor outputDescriptor = null;
    if (outputSpecProto.hasIoDescriptor()) {
      outputDescriptor =
          convertOutputDescriptorFromProto(outputSpecProto.getIoDescriptor());
    }
    OutputSpec outputSpec =
        new OutputSpec(outputSpecProto.getConnectedVertexName(), outputDescriptor,
            outputSpecProto.getPhysicalEdgeCount());
    return outputSpec;
  }

  public static IOSpecProto convertOutputSpecToProto(OutputSpec outputSpec) {
    IOSpecProto.Builder builder = IOSpecProto.newBuilder();
    if (outputSpec.getDestinationVertexName() != null) {
      builder.setConnectedVertexName(outputSpec.getDestinationVertexName());
    }
    if (outputSpec.getOutputDescriptor() != null) {
      builder.setIoDescriptor(convertToProto(outputSpec.getOutputDescriptor()));
    }
    builder.setPhysicalEdgeCount(outputSpec.getPhysicalEdgeCount());
    return builder.build();
  }

  private static GroupInputSpec getGroupInputSpecFromProto(GroupInputSpecProto groupInputSpecProto) {
    GroupInputSpec groupSpec = new GroupInputSpec(groupInputSpecProto.getGroupName(),
        groupInputSpecProto.getGroupVerticesList(),
        convertInputDescriptorFromProto(groupInputSpecProto.getMergedInputDescriptor()));
    return groupSpec;
  }

  private static GroupInputSpecProto convertGroupInputSpecToProto(GroupInputSpec groupInputSpec) {
    GroupInputSpecProto.Builder builder = GroupInputSpecProto.newBuilder();
    builder.setGroupName(groupInputSpec.getGroupName());
    builder.addAllGroupVertices(groupInputSpec.getGroupVertices());
    builder.setMergedInputDescriptor(convertToProto(groupInputSpec.getMergedInputDescriptor()));
    return builder.build();
  }


  private static void setUserPayload(EntityDescriptor entity, UserPayload payload) {
    if (payload != null) {
      entity.setUserPayload(payload);
    }
  }

  private static UserPayload convertPayloadFromProto(
      EntityDescriptorProto proto) {
    UserPayload userPayload = null;
    if (proto.hasUserPayload()) {
      if (proto.getUserPayload().hasUserPayload()) {
        userPayload =
            UserPayload.create(proto.getUserPayload().getUserPayload().asReadOnlyByteBuffer(), proto.getUserPayload().getVersion());
      } else {
        userPayload = UserPayload.create(null);
      }
    }
    return userPayload;
  }

  public static SourceStateProto fromVertexState(VertexState state) {
    switch (state) {
      case SUCCEEDED:
        return SourceStateProto.S_SUCCEEDED;
      case RUNNING:
        return SourceStateProto.S_RUNNING;
      default:
        throw new RuntimeException("Unexpected state: " + state);
    }
  }
}