io.cdap.cdap.internal.app.deploy.pipeline.MetadataWriterStage Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of cdap-app-fabric Show documentation
The newest version!
/*
 * Copyright © 2016-2019 Cask Data, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package io.cdap.cdap.internal.app.deploy.pipeline;

import com.google.common.reflect.TypeToken;
import io.cdap.cdap.api.ProgramSpecification;
import io.cdap.cdap.api.app.ApplicationSpecification;
import io.cdap.cdap.api.metadata.Metadata;
import io.cdap.cdap.api.metadata.MetadataScope;
import io.cdap.cdap.data2.metadata.system.AppSystemMetadataWriter;
import io.cdap.cdap.data2.metadata.system.ProgramSystemMetadataWriter;
import io.cdap.cdap.data2.metadata.writer.MetadataServiceClient;
import io.cdap.cdap.pipeline.AbstractStage;
import io.cdap.cdap.proto.ProgramType;
import io.cdap.cdap.proto.id.ApplicationId;
import io.cdap.cdap.proto.id.ProgramId;
import io.cdap.cdap.spi.metadata.MetadataMutation;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

/**
 * Stage to write metadata for an application.
 */
public class MetadataWriterStage extends AbstractStage {

  private final MetadataServiceClient metadataServiceClient;
  private String creationTime;

  public MetadataWriterStage(MetadataServiceClient metadataServiceClient) {
    super(TypeToken.of(ApplicationWithPrograms.class));
    this.metadataServiceClient = metadataServiceClient;
  }

  @Override
  public void process(ApplicationWithPrograms input) {
    // use current time as creation time for app and all programs
    creationTime = String.valueOf(System.currentTimeMillis());

    // add system metadata for apps
    ApplicationId appId = input.getApplicationId();
    ApplicationSpecification appSpec = input.getSpecification();
    List mutations = new ArrayList<>();
    // get the system metadata and combine it with our own system metadata, metadata creation mutation will
    // completely override the previous metadata in the scope, so we have to combine these together
    Metadata systemAppMetadata = input.getMetadata().get(MetadataScope.SYSTEM);

    mutations.add(
        new AppSystemMetadataWriter(metadataServiceClient, appId, appSpec,
            input.getApplicationClass(), creationTime,
            systemAppMetadata)
            .getMetadataMutation());

    // collect system metadata for programs
    collectProgramSystemMetadata(appId, ProgramType.MAPREDUCE, appSpec.getMapReduce().values(),
        mutations);
    collectProgramSystemMetadata(appId, ProgramType.SERVICE, appSpec.getServices().values(),
        mutations);
    collectProgramSystemMetadata(appId, ProgramType.SPARK, appSpec.getSpark().values(), mutations);
    collectProgramSystemMetadata(appId, ProgramType.WORKER, appSpec.getWorkers().values(),
        mutations);
    collectProgramSystemMetadata(appId, ProgramType.WORKFLOW, appSpec.getWorkflows().values(),
        mutations);

    // add the rest user defined metadata
    Metadata userAppMetadata = input.getMetadata().get(MetadataScope.USER);
    if (userAppMetadata != null
        && (!userAppMetadata.getProperties().isEmpty() || !userAppMetadata.getTags().isEmpty())) {
      mutations.add(new MetadataMutation.Create(
          appId.toMetadataEntity(),
          new io.cdap.cdap.spi.metadata.Metadata(MetadataScope.USER, userAppMetadata.getTags(),
              userAppMetadata.getProperties()), Collections.emptyMap()));
    }

    // write all metadata
    metadataServiceClient.batch(mutations);

    // Emit input to the next stage
    emit(input);
  }

  private void collectProgramSystemMetadata(ApplicationId appId, ProgramType programType,
      Iterable specs,
      List mutations) {
    for (ProgramSpecification spec : specs) {
      ProgramId programId = appId.program(programType, spec.getName());
      mutations.add(
          new ProgramSystemMetadataWriter(metadataServiceClient, programId, spec,
              creationTime).getMetadataMutation());
    }
  }
}