org.apache.reef.client.DriverConfiguration Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of reef-common Show documentation
There is a newer version: 0.16.0
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.reef.client;

import org.apache.reef.annotations.Provided;
import org.apache.reef.annotations.audience.ClientSide;
import org.apache.reef.annotations.audience.Public;
import org.apache.reef.driver.ProgressProvider;
import org.apache.reef.driver.context.ActiveContext;
import org.apache.reef.driver.context.ClosedContext;
import org.apache.reef.driver.context.ContextMessage;
import org.apache.reef.driver.context.FailedContext;
import org.apache.reef.driver.evaluator.AllocatedEvaluator;
import org.apache.reef.driver.evaluator.CompletedEvaluator;
import org.apache.reef.driver.evaluator.FailedEvaluator;
import org.apache.reef.driver.parameters.*;
import org.apache.reef.driver.task.*;
import org.apache.reef.runtime.common.driver.DriverRuntimeConfiguration;
import org.apache.reef.runtime.common.driver.parameters.EvaluatorIdlenessThreadPoolSize;
import org.apache.reef.runtime.common.driver.parameters.EvaluatorIdlenessWaitInMilliseconds;
import org.apache.reef.tang.formats.*;
import org.apache.reef.wake.EventHandler;
import org.apache.reef.wake.time.Clock;
import org.apache.reef.wake.time.event.StartTime;
import org.apache.reef.wake.time.event.StopTime;

/**
 * A ConfigurationModule for Drivers.
 */
@ClientSide
@Public
@Provided
public final class DriverConfiguration extends ConfigurationModuleBuilder {

  /**
   * Identifies the driver and therefore the JOB. Expect to see this e.g. on YARN's dashboard.
   */
  public static final OptionalParameter DRIVER_IDENTIFIER = new OptionalParameter<>();

  /**
   * The amount of memory to be allocated for the Driver. This is the size of the AM container in YARN.
   */
  public static final OptionalParameter DRIVER_MEMORY = new OptionalParameter<>();

  /**
   * Files to be made available on the Driver and all Evaluators.
   */
  public static final OptionalParameter GLOBAL_FILES = new OptionalParameter<>();

  /**
   * Libraries to be made available on the Driver and all Evaluators.
   */
  public static final OptionalParameter GLOBAL_LIBRARIES = new OptionalParameter<>();

  /**
   * Files to be made available on the Driver only.
   */
  public static final OptionalParameter LOCAL_FILES = new OptionalParameter<>();

  /**
   * Libraries to be made available on the Driver only.
   */
  public static final OptionalParameter LOCAL_LIBRARIES = new OptionalParameter<>();

  /**
   * Job submission directory to be used by driver. This is the folder on the DFS used to stage the files
   * for the Driver and subsequently for the Evaluators. It will be created if it doesn't exist yet.
   * If this is set by the user, user must make sure its uniqueness across different jobs.
   */
  public static final OptionalParameter DRIVER_JOB_SUBMISSION_DIRECTORY = new OptionalParameter<>();

  /**
   * The event handler invoked right after the driver boots up.
   */
  public static final RequiredImpl> ON_DRIVER_STARTED = new RequiredImpl<>();

  /**
   * The event handler invoked right before the driver shuts down. Defaults to ignore.
   */
  public static final OptionalImpl> ON_DRIVER_STOP = new OptionalImpl<>();

  // ***** EVALUATOR HANDLER BINDINGS:

  /**
   * Event handler for allocated evaluators. Defaults to returning the evaluator if not bound.
   */
  public static final OptionalImpl> ON_EVALUATOR_ALLOCATED = new OptionalImpl<>();

  /**
   * Event handler for completed evaluators. Defaults to logging if not bound.
   */
  public static final OptionalImpl> ON_EVALUATOR_COMPLETED = new OptionalImpl<>();

  /**
   * Event handler for failed evaluators. Defaults to job failure if not bound.
   */
  public static final OptionalImpl> ON_EVALUATOR_FAILED = new OptionalImpl<>();

  // ***** TASK HANDLER BINDINGS:

  /**
   * Event handler for task messages. Defaults to logging if not bound.
   */
  public static final OptionalImpl> ON_TASK_MESSAGE = new OptionalImpl<>();

  /**
   * Event handler for completed tasks. Defaults to closing the context the task ran on if not bound.
   */
  public static final OptionalImpl> ON_TASK_COMPLETED = new OptionalImpl<>();

  /**
   * Event handler for failed tasks. Defaults to job failure if not bound.
   */
  public static final OptionalImpl> ON_TASK_FAILED = new OptionalImpl<>();

  /**
   * Event handler for running tasks. Defaults to logging if not bound.
   */
  public static final OptionalImpl> ON_TASK_RUNNING = new OptionalImpl<>();

  /**
   * Event handler for suspended tasks. Defaults to job failure if not bound. Rationale: many jobs don't support
   * task suspension. Hence, this parameter should be optional. The only sane default is to crash the job, then.
   */
  public static final OptionalImpl> ON_TASK_SUSPENDED = new OptionalImpl<>();

  // ***** CLIENT HANDLER BINDINGS:

  /**
   * Event handler for client messages. Defaults to logging if not bound.
   */
  public static final OptionalImpl> ON_CLIENT_MESSAGE = new OptionalImpl<>();

  /**
   * Event handler for close messages sent by the client. Defaults to job failure if not bound.
   */
  public static final OptionalImpl> ON_CLIENT_CLOSED = new OptionalImpl<>();

  /**
   * Event handler for close messages sent by the client. Defaults to job failure if not bound.
   */
  public static final OptionalImpl> ON_CLIENT_CLOSED_MESSAGE = new OptionalImpl<>();

  // ***** CONTEXT HANDLER BINDINGS:

  /**
   * Event handler for active context. Defaults to closing the context if not bound.
   */
  public static final OptionalImpl> ON_CONTEXT_ACTIVE = new OptionalImpl<>();

  /**
   * Event handler for closed context. Defaults to logging if not bound.
   */
  public static final OptionalImpl> ON_CONTEXT_CLOSED = new OptionalImpl<>();

  /**
   * Event handler for closed context. Defaults to job failure if not bound.
   */
  public static final OptionalImpl> ON_CONTEXT_FAILED = new OptionalImpl<>();

  /**
   * Event handler for context messages. Defaults to logging if not bound.
   */
  public static final OptionalImpl> ON_CONTEXT_MESSAGE = new OptionalImpl<>();

  // ***** MISC

  /**
   * Progress provider. See {@link ProgressProvider}.
   */
  public static final OptionalImpl PROGRESS_PROVIDER = new OptionalImpl<>();

  /**
   * Number of threads allocated per evaluator to dispatch events from this Evaluator.
   */
  public static final OptionalParameter EVALUATOR_DISPATCHER_THREADS = new OptionalParameter<>();

  /**
   * The number of submissions that the resource manager will attempt to submit the application. Defaults to 1.
   */
  public static final OptionalParameter MAX_APPLICATION_SUBMISSIONS = new OptionalParameter<>();

  /**
   * The number of Threads in a Driver to verify the completion of Evaluators.
   * Used by {@link org.apache.reef.runtime.common.driver.evaluator.EvaluatorIdlenessThreadPool}.
   */
  public static final OptionalParameter EVALUATOR_IDLENESS_THREAD_POOL_SIZE = new OptionalParameter<>();

  /**
   * The number of Threads in a Driver to verify the completion of Evaluators.
   * Used by {@link org.apache.reef.runtime.common.driver.evaluator.EvaluatorIdlenessThreadPool}.
   */
  public static final OptionalParameter EVALUATOR_IDLENESS_WAIT_IN_MS = new OptionalParameter<>();

  /**
   * ConfigurationModule to fill out to get a legal Driver Configuration.
   */
  public static final ConfigurationModule CONF = new DriverConfiguration().merge(DriverRuntimeConfiguration.CONF)
      .bindNamedParameter(DriverIdentifier.class, DRIVER_IDENTIFIER)
      .bindNamedParameter(DriverMemory.class, DRIVER_MEMORY)
      .bindNamedParameter(DriverJobSubmissionDirectory.class, DRIVER_JOB_SUBMISSION_DIRECTORY)
      .bindNamedParameter(MaxApplicationSubmissions.class, MAX_APPLICATION_SUBMISSIONS)
      .bindSetEntry(JobGlobalFiles.class, GLOBAL_FILES)
      .bindSetEntry(JobGlobalLibraries.class, GLOBAL_LIBRARIES)
      .bindSetEntry(DriverLocalFiles.class, LOCAL_FILES)
      .bindSetEntry(DriverLocalLibraries.class, LOCAL_LIBRARIES)

          // Driver start/stop handlers
      .bindSetEntry(DriverStartHandler.class, ON_DRIVER_STARTED)
      .bindSetEntry(Clock.StartHandler.class, org.apache.reef.runtime.common.driver.DriverStartHandler.class)
      .bindSetEntry(Clock.StopHandler.class, ON_DRIVER_STOP)

          // Evaluator handlers
      .bindSetEntry(EvaluatorAllocatedHandlers.class, ON_EVALUATOR_ALLOCATED)
      .bindSetEntry(EvaluatorCompletedHandlers.class, ON_EVALUATOR_COMPLETED)
      .bindSetEntry(EvaluatorFailedHandlers.class, ON_EVALUATOR_FAILED)

          // Task handlers
      .bindSetEntry(TaskRunningHandlers.class, ON_TASK_RUNNING)
      .bindSetEntry(TaskFailedHandlers.class, ON_TASK_FAILED)
      .bindSetEntry(TaskMessageHandlers.class, ON_TASK_MESSAGE)
      .bindSetEntry(TaskCompletedHandlers.class, ON_TASK_COMPLETED)
      .bindSetEntry(TaskSuspendedHandlers.class, ON_TASK_SUSPENDED)

          // Context handlers
      .bindSetEntry(ContextActiveHandlers.class, ON_CONTEXT_ACTIVE)
      .bindSetEntry(ContextClosedHandlers.class, ON_CONTEXT_CLOSED)
      .bindSetEntry(ContextMessageHandlers.class, ON_CONTEXT_MESSAGE)
      .bindSetEntry(ContextFailedHandlers.class, ON_CONTEXT_FAILED)

          // Client handlers
      .bindSetEntry(ClientMessageHandlers.class, ON_CLIENT_MESSAGE)
      .bindSetEntry(ClientCloseHandlers.class, ON_CLIENT_CLOSED)
      .bindSetEntry(ClientCloseWithMessageHandlers.class, ON_CLIENT_CLOSED_MESSAGE)

          // Various parameters
      .bindNamedParameter(EvaluatorDispatcherThreads.class, EVALUATOR_DISPATCHER_THREADS)
      .bindNamedParameter(EvaluatorIdlenessThreadPoolSize.class, EVALUATOR_IDLENESS_THREAD_POOL_SIZE)
      .bindNamedParameter(EvaluatorIdlenessWaitInMilliseconds.class, EVALUATOR_IDLENESS_WAIT_IN_MS)
      .bindImplementation(ProgressProvider.class, PROGRESS_PROVIDER)
      .build();
}