io.cdap.cdap.internal.app.runtime.worker.WorkerProgramRunner Maven / Gradle / Ivy
The newest version!
/*
* Copyright © 2015-2019 Cask Data, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package io.cdap.cdap.internal.app.runtime.worker;
import com.google.common.base.Preconditions;
import com.google.common.io.Closeables;
import com.google.inject.Inject;
import io.cdap.cdap.api.app.ApplicationSpecification;
import io.cdap.cdap.api.metadata.MetadataReader;
import io.cdap.cdap.api.metrics.MetricsCollectionService;
import io.cdap.cdap.api.security.store.SecureStore;
import io.cdap.cdap.api.security.store.SecureStoreManager;
import io.cdap.cdap.api.worker.Worker;
import io.cdap.cdap.api.worker.WorkerSpecification;
import io.cdap.cdap.app.program.Program;
import io.cdap.cdap.app.runtime.ProgramController;
import io.cdap.cdap.app.runtime.ProgramOptions;
import io.cdap.cdap.app.runtime.ProgramRunner;
import io.cdap.cdap.common.conf.CConfiguration;
import io.cdap.cdap.common.internal.remote.RemoteClientFactory;
import io.cdap.cdap.common.namespace.NamespaceQueryAdmin;
import io.cdap.cdap.data.ProgramContextAware;
import io.cdap.cdap.data2.dataset2.DatasetFramework;
import io.cdap.cdap.data2.metadata.writer.FieldLineageWriter;
import io.cdap.cdap.data2.metadata.writer.MetadataPublisher;
import io.cdap.cdap.internal.app.runtime.AbstractProgramRunnerWithPlugin;
import io.cdap.cdap.internal.app.runtime.AppStateStoreProvider;
import io.cdap.cdap.internal.app.runtime.BasicProgramContext;
import io.cdap.cdap.internal.app.runtime.ProgramControllerServiceAdapter;
import io.cdap.cdap.internal.app.runtime.ProgramOptionConstants;
import io.cdap.cdap.internal.app.runtime.ProgramRunners;
import io.cdap.cdap.internal.app.runtime.plugin.PluginInstantiator;
import io.cdap.cdap.messaging.spi.MessagingService;
import io.cdap.cdap.proto.ProgramType;
import io.cdap.cdap.proto.id.ProgramId;
import io.cdap.cdap.proto.id.ProgramRunId;
import java.util.Collections;
import org.apache.tephra.TransactionSystemClient;
import org.apache.twill.api.RunId;
import org.apache.twill.common.Threads;
import org.apache.twill.discovery.DiscoveryServiceClient;
/**
* A {@link ProgramRunner} that runs a {@link Worker}.
*/
public class WorkerProgramRunner extends AbstractProgramRunnerWithPlugin {
private final CConfiguration cConf;
private final MetricsCollectionService metricsCollectionService;
private final DatasetFramework datasetFramework;
private final DiscoveryServiceClient discoveryServiceClient;
private final TransactionSystemClient txClient;
private final SecureStore secureStore;
private final SecureStoreManager secureStoreManager;
private final MessagingService messagingService;
private final MetadataReader metadataReader;
private final MetadataPublisher metadataPublisher;
private final NamespaceQueryAdmin namespaceQueryAdmin;
private final FieldLineageWriter fieldLineageWriter;
private final RemoteClientFactory remoteClientFactory;
private final AppStateStoreProvider appStateStoreProvider;
@Inject
public WorkerProgramRunner(CConfiguration cConf,
MetricsCollectionService metricsCollectionService,
DatasetFramework datasetFramework, DiscoveryServiceClient discoveryServiceClient,
TransactionSystemClient txClient,
SecureStore secureStore, SecureStoreManager secureStoreManager,
MessagingService messagingService, MetadataReader metadataReader,
MetadataPublisher metadataPublisher, NamespaceQueryAdmin namespaceQueryAdmin,
FieldLineageWriter fieldLineageWriter, RemoteClientFactory remoteClientFactory,
AppStateStoreProvider appStateStoreProvider) {
super(cConf);
this.cConf = cConf;
this.metricsCollectionService = metricsCollectionService;
this.datasetFramework = datasetFramework;
this.discoveryServiceClient = discoveryServiceClient;
this.txClient = txClient;
this.secureStore = secureStore;
this.secureStoreManager = secureStoreManager;
this.messagingService = messagingService;
this.metadataReader = metadataReader;
this.metadataPublisher = metadataPublisher;
this.namespaceQueryAdmin = namespaceQueryAdmin;
this.fieldLineageWriter = fieldLineageWriter;
this.remoteClientFactory = remoteClientFactory;
this.appStateStoreProvider = appStateStoreProvider;
}
@Override
public ProgramController run(Program program, ProgramOptions options) {
ApplicationSpecification appSpec = program.getApplicationSpecification();
Preconditions.checkNotNull(appSpec, "Missing application specification.");
int instanceId = Integer.parseInt(
options.getArguments().getOption(ProgramOptionConstants.INSTANCE_ID, "-1"));
Preconditions.checkArgument(instanceId >= 0, "Missing instance Id");
int instanceCount = Integer.parseInt(
options.getArguments().getOption(ProgramOptionConstants.INSTANCES, "0"));
Preconditions.checkArgument(instanceCount > 0, "Invalid or missing instance count");
RunId runId = ProgramRunners.getRunId(options);
ProgramType programType = program.getType();
Preconditions.checkNotNull(programType, "Missing processor type.");
Preconditions.checkArgument(programType == ProgramType.WORKER,
"Only Worker process type is supported.");
WorkerSpecification workerSpec = appSpec.getWorkers().get(program.getName());
Preconditions.checkArgument(workerSpec != null, "Missing Worker specification for %s",
program.getId());
String instances = options.getArguments().getOption(ProgramOptionConstants.INSTANCES,
String.valueOf(workerSpec.getInstances()));
WorkerSpecification newWorkerSpec = new WorkerSpecification(workerSpec.getClassName(),
workerSpec.getName(),
workerSpec.getDescription(), workerSpec.getProperties(),
workerSpec.getDatasets(), workerSpec.getResources(),
Integer.parseInt(instances), workerSpec.getPlugins());
// Setup dataset framework context, if required
if (datasetFramework instanceof ProgramContextAware) {
ProgramId programId = program.getId();
((ProgramContextAware) datasetFramework).setContext(
new BasicProgramContext(programId.run(runId)));
}
final PluginInstantiator pluginInstantiator = createPluginInstantiator(options,
program.getClassLoader());
try {
BasicWorkerContext context = new BasicWorkerContext(newWorkerSpec, program, options,
cConf, instanceId, instanceCount,
metricsCollectionService, datasetFramework, txClient,
discoveryServiceClient,
pluginInstantiator, secureStore, secureStoreManager,
messagingService, metadataReader, metadataPublisher,
namespaceQueryAdmin, fieldLineageWriter, remoteClientFactory,
appStateStoreProvider);
WorkerDriver worker = new WorkerDriver(program, newWorkerSpec, context);
// Add a service listener to make sure the plugin instantiator is closed when the worker driver finished.
worker.addListener(createRuntimeServiceListener(Collections.singleton(pluginInstantiator)),
Threads.SAME_THREAD_EXECUTOR);
ProgramController controller = new WorkerControllerServiceAdapter(worker,
program.getId().run(runId));
worker.start();
return controller;
} catch (Throwable t) {
Closeables.closeQuietly(pluginInstantiator);
throw t;
}
}
private static final class WorkerControllerServiceAdapter extends
ProgramControllerServiceAdapter {
private final WorkerDriver workerDriver;
WorkerControllerServiceAdapter(WorkerDriver workerDriver, ProgramRunId programRunId) {
super(workerDriver, programRunId);
this.workerDriver = workerDriver;
}
@Override
protected void doCommand(String name, Object value) throws Exception {
super.doCommand(name, value);
if (!ProgramOptionConstants.INSTANCES.equals(name) || !(value instanceof Integer)) {
return;
}
workerDriver.setInstanceCount((Integer) value);
}
}
}