org.apache.hadoop.yarn.client.api.YarnClient Maven / Gradle / Ivy
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.client.api;
import java.io.IOException;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest;
import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse;
import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse;
import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@InterfaceAudience.Public
@InterfaceStability.Stable
public abstract class YarnClient extends AbstractService {
/**
* Create a new instance of YarnClient.
*/
@Public
public static YarnClient createYarnClient() {
YarnClient client = new YarnClientImpl();
return client;
}
@Private
protected YarnClient(String name) {
super(name);
}
/**
*
* Obtain a {@link YarnClientApplication} for a new application,
* which in turn contains the {@link ApplicationSubmissionContext} and
* {@link org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse}
* objects.
*
*
* @return {@link YarnClientApplication} built for a new application
* @throws YarnException
* @throws IOException
*/
public abstract YarnClientApplication createApplication()
throws YarnException, IOException;
/**
*
* Submit a new application to YARN. It is a blocking call - it
* will not return {@link ApplicationId} until the submitted application is
* submitted successfully and accepted by the ResourceManager.
*
*
*
* Users should provide an {@link ApplicationId} as part of the parameter
* {@link ApplicationSubmissionContext} when submitting a new application,
* otherwise it will throw the {@link ApplicationIdNotProvidedException}.
*
*
* This internally calls {@link ApplicationClientProtocol#submitApplication
* (SubmitApplicationRequest)}, and after that, it internally invokes
* {@link ApplicationClientProtocol#getApplicationReport
* (GetApplicationReportRequest)} and waits till it can make sure that the
* application gets properly submitted. If RM fails over or RM restart
* happens before ResourceManager saves the application's state,
* {@link ApplicationClientProtocol
* #getApplicationReport(GetApplicationReportRequest)} will throw
* the {@link ApplicationNotFoundException}. This API automatically resubmits
* the application with the same {@link ApplicationSubmissionContext} when it
* catches the {@link ApplicationNotFoundException}
*
* @param appContext
* {@link ApplicationSubmissionContext} containing all the details
* needed to submit a new application
* @return {@link ApplicationId} of the accepted application
* @throws YarnException
* @throws IOException
* @see #createApplication()
*/
public abstract ApplicationId submitApplication(
ApplicationSubmissionContext appContext) throws YarnException,
IOException;
/**
*
* Kill an application identified by given ID.
*
*
* @param applicationId
* {@link ApplicationId} of the application that needs to be killed
* @throws YarnException
* in case of errors or if YARN rejects the request due to
* access-control restrictions.
* @throws IOException
* @see #getQueueAclsInfo()
*/
public abstract void killApplication(ApplicationId applicationId) throws YarnException,
IOException;
/**
*
* Get a report of the given Application.
*
*
*
* In secure mode, YARN verifies access to the application, queue
* etc. before accepting the request.
*
*
*
* If the user does not have VIEW_APP access then the following
* fields in the report will be set to stubbed values:
*
* - host - set to "N/A"
* - RPC port - set to -1
* - client token - set to "N/A"
* - diagnostics - set to "N/A"
* - tracking URL - set to "N/A"
* - original tracking URL - set to "N/A"
* - resource usage report - all values are -1
*
*
* @param appId
* {@link ApplicationId} of the application that needs a report
* @return application report
* @throws YarnException
* @throws IOException
*/
public abstract ApplicationReport getApplicationReport(ApplicationId appId)
throws YarnException, IOException;
/**
* Get the AMRM token of the application.
*
* The AMRM token is required for AM to RM scheduling operations. For
* managed Application Masters Yarn takes care of injecting it. For unmanaged
* Applications Masters, the token must be obtained via this method and set
* in the {@link org.apache.hadoop.security.UserGroupInformation} of the
* current user.
*
* The AMRM token will be returned only if all the following conditions are
* met:
*
* - the requester is the owner of the ApplicationMaster
* - the application master is an unmanaged ApplicationMaster
* - the application master is in ACCEPTED state
*
* Else this method returns NULL.
*
* @param appId {@link ApplicationId} of the application to get the AMRM token
* @return the AMRM token if available
* @throws YarnException
* @throws IOException
*/
public abstract org.apache.hadoop.security.token.Token
getAMRMToken(ApplicationId appId) throws YarnException, IOException;
/**
*
* Get a report (ApplicationReport) of all Applications in the cluster.
*
*
*
* If the user does not have VIEW_APP access for an application
* then the corresponding report will be filtered as described in
* {@link #getApplicationReport(ApplicationId)}.
*
*
* @return a list of reports of all running applications
* @throws YarnException
* @throws IOException
*/
public abstract List getApplications()
throws YarnException, IOException;
/**
*
* Get a report (ApplicationReport) of Applications
* matching the given application types in the cluster.
*
*
*
* If the user does not have VIEW_APP access for an application
* then the corresponding report will be filtered as described in
* {@link #getApplicationReport(ApplicationId)}.
*
*
* @param applicationTypes
* @return a list of reports of applications
* @throws YarnException
* @throws IOException
*/
public abstract List getApplications(
Set applicationTypes) throws YarnException, IOException;
/**
*
* Get a report (ApplicationReport) of Applications matching the given
* application states in the cluster.
*
*
*
* If the user does not have VIEW_APP access for an application
* then the corresponding report will be filtered as described in
* {@link #getApplicationReport(ApplicationId)}.
*
*
* @param applicationStates
* @return a list of reports of applications
* @throws YarnException
* @throws IOException
*/
public abstract List
getApplications(EnumSet applicationStates)
throws YarnException, IOException;
/**
*
* Get a report (ApplicationReport) of Applications matching the given
* application types and application states in the cluster.
*
*
*
* If the user does not have VIEW_APP access for an application
* then the corresponding report will be filtered as described in
* {@link #getApplicationReport(ApplicationId)}.
*
*
* @param applicationTypes
* @param applicationStates
* @return a list of reports of applications
* @throws YarnException
* @throws IOException
*/
public abstract List getApplications(
Set applicationTypes,
EnumSet applicationStates) throws YarnException,
IOException;
/**
*
* Get metrics ({@link YarnClusterMetrics}) about the cluster.
*
*
* @return cluster metrics
* @throws YarnException
* @throws IOException
*/
public abstract YarnClusterMetrics getYarnClusterMetrics() throws YarnException,
IOException;
/**
*
* Get a report of nodes ({@link NodeReport}) in the cluster.
*
*
* @param states The {@link NodeState}s to filter on. If no filter states are
* given, nodes in all states will be returned.
* @return A list of node reports
* @throws YarnException
* @throws IOException
*/
public abstract List getNodeReports(NodeState... states)
throws YarnException, IOException;
/**
*
* Get a delegation token so as to be able to talk to YARN using those tokens.
*
* @param renewer
* Address of the renewer who can renew these tokens when needed by
* securely talking to YARN.
* @return a delegation token ({@link Token}) that can be used to
* talk to YARN
* @throws YarnException
* @throws IOException
*/
public abstract Token getRMDelegationToken(Text renewer)
throws YarnException, IOException;
/**
*
* Get information ({@link QueueInfo}) about a given queue.
*
*
* @param queueName
* Name of the queue whose information is needed
* @return queue information
* @throws YarnException
* in case of errors or if YARN rejects the request due to
* access-control restrictions.
* @throws IOException
*/
public abstract QueueInfo getQueueInfo(String queueName) throws YarnException,
IOException;
/**
*
* Get information ({@link QueueInfo}) about all queues, recursively if there
* is a hierarchy
*
*
* @return a list of queue-information for all queues
* @throws YarnException
* @throws IOException
*/
public abstract List getAllQueues() throws YarnException, IOException;
/**
*
* Get information ({@link QueueInfo}) about top level queues.
*
*
* @return a list of queue-information for all the top-level queues
* @throws YarnException
* @throws IOException
*/
public abstract List getRootQueueInfos() throws YarnException, IOException;
/**
*
* Get information ({@link QueueInfo}) about all the immediate children queues
* of the given queue
*
*
* @param parent
* Name of the queue whose child-queues' information is needed
* @return a list of queue-information for all queues who are direct children
* of the given parent queue.
* @throws YarnException
* @throws IOException
*/
public abstract List getChildQueueInfos(String parent) throws YarnException,
IOException;
/**
*
* Get information about acls for current user on all the
* existing queues.
*
*
* @return a list of queue acls ({@link QueueUserACLInfo}) for
* current user
* @throws YarnException
* @throws IOException
*/
public abstract List getQueueAclsInfo() throws YarnException,
IOException;
/**
*
* Get a report of the given ApplicationAttempt.
*
*
*
* In secure mode, YARN verifies access to the application, queue
* etc. before accepting the request.
*
*
* @param applicationAttemptId
* {@link ApplicationAttemptId} of the application attempt that needs
* a report
* @return application attempt report
* @throws YarnException
* @throws ApplicationAttemptNotFoundException if application attempt
* not found
* @throws IOException
*/
public abstract ApplicationAttemptReport getApplicationAttemptReport(
ApplicationAttemptId applicationAttemptId) throws YarnException, IOException;
/**
*
* Get a report of all (ApplicationAttempts) of Application in the cluster.
*
*
* @param applicationId
* @return a list of reports for all application attempts for specified
* application.
* @throws YarnException
* @throws IOException
*/
public abstract List getApplicationAttempts(
ApplicationId applicationId) throws YarnException, IOException;
/**
*
* Get a report of the given Container.
*
*
*
* In secure mode, YARN verifies access to the application, queue
* etc. before accepting the request.
*
*
* @param containerId
* {@link ContainerId} of the container that needs a report
* @return container report
* @throws YarnException
* @throws ContainerNotFoundException if container not found.
* @throws IOException
*/
public abstract ContainerReport getContainerReport(ContainerId containerId)
throws YarnException, IOException;
/**
*
* Get a report of all (Containers) of ApplicationAttempt in the cluster.
*
*
* @param applicationAttemptId
* @return a list of reports of all containers for specified application
* attempts
* @throws YarnException
* @throws IOException
*/
public abstract List getContainers(
ApplicationAttemptId applicationAttemptId) throws YarnException,
IOException;
/**
*
* Attempts to move the given application to the given queue.
*
*
* @param appId
* Application to move.
* @param queue
* Queue to place it in to.
* @throws YarnException
* @throws IOException
*/
public abstract void moveApplicationAcrossQueues(ApplicationId appId,
String queue) throws YarnException, IOException;
/**
*
* The interface used by clients to submit a new reservation to the
* {@code ResourceManager}.
*
*
*
* The client packages all details of its request in a
* {@link ReservationSubmissionRequest} object. This contains information
* about the amount of capacity, temporal constraints, and gang needs.
* Furthermore, the reservation might be composed of multiple stages, with
* ordering dependencies among them.
*
*
*
* In order to respond, a new admission control component in the
* {@code ResourceManager} performs an analysis of the resources that have
* been committed over the period of time the user is requesting, verify that
* the user requests can be fulfilled, and that it respect a sharing policy
* (e.g., {@code CapacityOverTimePolicy}). Once it has positively determined
* that the ReservationRequest is satisfiable the {@code ResourceManager}
* answers with a {@link ReservationSubmissionResponse} that includes a
* {@link ReservationId}. Upon failure to find a valid allocation the response
* is an exception with the message detailing the reason of failure.
*
*
*
* The semantics guarantees that the {@link ReservationId} returned,
* corresponds to a valid reservation existing in the time-range request by
* the user. The amount of capacity dedicated to such reservation can vary
* overtime, depending of the allocation that has been determined. But it is
* guaranteed to satisfy all the constraint expressed by the user in the
* {@link ReservationDefinition}
*
*
* @param request request to submit a new Reservation
* @return response contains the {@link ReservationId} on accepting the
* submission
* @throws YarnException if the reservation cannot be created successfully
* @throws IOException
*
*/
@Public
@Unstable
public abstract ReservationSubmissionResponse submitReservation(
ReservationSubmissionRequest request) throws YarnException, IOException;
/**
*
* The interface used by clients to update an existing Reservation. This is
* referred to as a re-negotiation process, in which a user that has
* previously submitted a Reservation.
*
*
*
* The allocation is attempted by virtually substituting all previous
* allocations related to this Reservation with new ones, that satisfy the new
* {@link ReservationDefinition}. Upon success the previous allocation is
* atomically substituted by the new one, and on failure (i.e., if the system
* cannot find a valid allocation for the updated request), the previous
* allocation remains valid.
*
*
* @param request to update an existing Reservation (the
* {@link ReservationUpdateRequest} should refer to an existing valid
* {@link ReservationId})
* @return response empty on successfully updating the existing reservation
* @throws YarnException if the request is invalid or reservation cannot be
* updated successfully
* @throws IOException
*
*/
@Public
@Unstable
public abstract ReservationUpdateResponse updateReservation(
ReservationUpdateRequest request) throws YarnException, IOException;
/**
*
* The interface used by clients to remove an existing Reservation.
*
*
* @param request to remove an existing Reservation (the
* {@link ReservationDeleteRequest} should refer to an existing valid
* {@link ReservationId})
* @return response empty on successfully deleting the existing reservation
* @throws YarnException if the request is invalid or reservation cannot be
* deleted successfully
* @throws IOException
*
*/
@Public
@Unstable
public abstract ReservationDeleteResponse deleteReservation(
ReservationDeleteRequest request) throws YarnException, IOException;
/**
*
* The interface used by client to get node to labels mappings in existing cluster
*
*
* @return node to labels mappings
* @throws YarnException
* @throws IOException
*/
@Public
@Unstable
public abstract Map> getNodeToLabels()
throws YarnException, IOException;
/**
*
* The interface used by client to get labels to nodes mapping
* in existing cluster
*
*
* @return node to labels mappings
* @throws YarnException
* @throws IOException
*/
@Public
@Unstable
public abstract Map> getLabelsToNodes()
throws YarnException, IOException;
/**
*
* The interface used by client to get labels to nodes mapping
* for specified labels in existing cluster
*
*
* @param labels labels for which labels to nodes mapping has to be retrieved
* @return labels to nodes mappings for specific labels
* @throws YarnException
* @throws IOException
*/
@Public
@Unstable
public abstract Map> getLabelsToNodes(Set labels)
throws YarnException, IOException;
/**
*
* The interface used by client to get node labels in the cluster
*
*
* @return cluster node labels collection
* @throws YarnException
* @throws IOException
*/
@Public
@Unstable
public abstract Set getClusterNodeLabels()
throws YarnException, IOException;
}