org.apache.flink.yarn

Class YarnResourceManager

java.lang.Object

org.apache.flink.runtime.rpc.RpcEndpoint

org.apache.flink.runtime.rpc.FencedRpcEndpoint<org.apache.flink.runtime.resourcemanager.ResourceManagerId>

org.apache.flink.runtime.resourcemanager.ResourceManager<WorkerType>

org.apache.flink.runtime.resourcemanager.ActiveResourceManager<YarnWorkerNode>

org.apache.flink.yarn.YarnResourceManager

All Implemented Interfaces:

AutoCloseable, org.apache.flink.runtime.leaderelection.LeaderContender, org.apache.flink.runtime.resourcemanager.ResourceManagerGateway, org.apache.flink.runtime.rpc.FencedRpcGateway<org.apache.flink.runtime.resourcemanager.ResourceManagerId>, org.apache.flink.runtime.rpc.RpcGateway, org.apache.flink.util.AutoCloseableAsync, org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler, org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler

public class YarnResourceManager
extends org.apache.flink.runtime.resourcemanager.ActiveResourceManager<YarnWorkerNode>
implements org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler, org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler

The yarn implementation of the resource manager. Used when the system is started via the resource framework YARN.

Nested Class Summary

Nested classes/interfaces inherited from class org.apache.flink.runtime.rpc.RpcEndpoint

org.apache.flink.runtime.rpc.RpcEndpoint.MainThreadExecutor

Field Summary

Fields inherited from class org.apache.flink.runtime.resourcemanager.ActiveResourceManager

defaultMemoryMB, env, flinkClientConfig, flinkConfig, numSlotsPerTaskManager, resourceProfilesPerWorker, taskExecutorProcessSpec

Fields inherited from class org.apache.flink.runtime.resourcemanager.ResourceManager

RESOURCE_MANAGER_NAME

Fields inherited from class org.apache.flink.runtime.rpc.RpcEndpoint

log, rpcServer

Constructor Summary

Constructors

Constructor and Description

YarnResourceManager(org.apache.flink.runtime.rpc.RpcService rpcService, String resourceManagerEndpointId, org.apache.flink.runtime.clusterframework.types.ResourceID resourceId, org.apache.flink.configuration.Configuration flinkConfig, Map<String,String> env, org.apache.flink.runtime.highavailability.HighAvailabilityServices highAvailabilityServices, org.apache.flink.runtime.heartbeat.HeartbeatServices heartbeatServices, org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager slotManager, org.apache.flink.runtime.resourcemanager.JobLeaderIdService jobLeaderIdService, org.apache.flink.runtime.entrypoint.ClusterInformation clusterInformation, org.apache.flink.runtime.rpc.FatalErrorHandler fatalErrorHandler, String webInterfaceUrl, org.apache.flink.runtime.metrics.groups.ResourceManagerMetricGroup resourceManagerMetricGroup)

Method Summary

Modifier and Type	Method and Description
protected org.apache.hadoop.yarn.client.api.async.NMClientAsync	createAndStartNodeManagerClient(org.apache.hadoop.yarn.conf.YarnConfiguration yarnConfiguration)
protected org.apache.hadoop.yarn.client.api.async.AMRMClientAsync<org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest>	createAndStartResourceManagerClient(org.apache.hadoop.yarn.conf.YarnConfiguration yarnConfiguration, int yarnHeartbeatIntervalMillis, String webInterfaceUrl)
protected double	getCpuCores(org.apache.flink.configuration.Configuration configuration)
float	getProgress()
protected void	initialize()
protected void	internalDeregisterApplication(org.apache.flink.runtime.clusterframework.ApplicationStatus finalStatus, String diagnostics)
protected org.apache.flink.configuration.Configuration	loadClientConfiguration()
void	onContainersAllocated(List<org.apache.hadoop.yarn.api.records.Container> containers)
void	onContainersCompleted(List<org.apache.hadoop.yarn.api.records.ContainerStatus> statuses)
void	onContainerStarted(org.apache.hadoop.yarn.api.records.ContainerId containerId, Map<String,ByteBuffer> map)
void	onContainerStatusReceived(org.apache.hadoop.yarn.api.records.ContainerId containerId, org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus)
void	onContainerStopped(org.apache.hadoop.yarn.api.records.ContainerId containerId)
void	onError(Throwable error)
void	onGetContainerStatusError(org.apache.hadoop.yarn.api.records.ContainerId containerId, Throwable throwable)
void	onNodesUpdated(List<org.apache.hadoop.yarn.api.records.NodeReport> list)
void	onShutdownRequest()
void	onStartContainerError(org.apache.hadoop.yarn.api.records.ContainerId containerId, Throwable t)
CompletableFuture<Void>	onStop()
void	onStopContainerError(org.apache.hadoop.yarn.api.records.ContainerId containerId, Throwable throwable)
Collection<org.apache.flink.runtime.clusterframework.types.ResourceProfile>	startNewWorker(org.apache.flink.runtime.clusterframework.types.ResourceProfile resourceProfile)
boolean	stopWorker(YarnWorkerNode workerNode)
protected YarnWorkerNode	workerStarted(org.apache.flink.runtime.clusterframework.types.ResourceID resourceID)

Methods inherited from class org.apache.flink.runtime.resourcemanager.ActiveResourceManager

getStopTerminationFutureOrCompletedExceptionally

Methods inherited from class org.apache.flink.runtime.resourcemanager.ResourceManager

cancelSlotRequest, clearStateAsync, closeJobManagerConnection, closeTaskManagerConnection, deregisterApplication, disconnectJobManager, disconnectTaskManager, getNumberOfRegisteredTaskManagers, getNumberRequiredTaskManagerSlots, grantLeadership, handleError, heartbeatFromJobManager, heartbeatFromTaskManager, jobLeaderLostLeadership, notifySlotAvailable, onFatalError, onStart, prepareLeadershipAsync, registerJobManager, registerTaskExecutor, releaseResource, removeJob, requestResourceOverview, requestSlot, requestTaskManagerFileUpload, requestTaskManagerInfo, requestTaskManagerInfo, requestTaskManagerMetricQueryServiceAddresses, revokeLeadership, sendSlotReport, setFailUnfulfillableRequest, startServicesOnLeadership

Methods inherited from class org.apache.flink.runtime.rpc.FencedRpcEndpoint

callAsyncWithoutFencing, getFencingToken, getMainThreadExecutor, getUnfencedMainThreadExecutor, runAsyncWithoutFencing, setFencingToken

Methods inherited from class org.apache.flink.runtime.rpc.RpcEndpoint

callAsync, closeAsync, getAddress, getEndpointId, getHostname, getRpcService, getSelfGateway, getTerminationFuture, internalCallOnStart, internalCallOnStop, isRunning, runAsync, scheduleRunAsync, scheduleRunAsync, start, stop, validateRunsInMainThread

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.flink.runtime.rpc.FencedRpcGateway

getFencingToken

Methods inherited from interface org.apache.flink.runtime.rpc.RpcGateway

getAddress, getHostname

Methods inherited from interface org.apache.flink.runtime.leaderelection.LeaderContender

getDescription

Methods inherited from interface org.apache.flink.util.AutoCloseableAsync

close

Constructor Detail

YarnResourceManager

public YarnResourceManager(org.apache.flink.runtime.rpc.RpcService rpcService,
                           String resourceManagerEndpointId,
                           org.apache.flink.runtime.clusterframework.types.ResourceID resourceId,
                           org.apache.flink.configuration.Configuration flinkConfig,
                           Map<String,String> env,
                           org.apache.flink.runtime.highavailability.HighAvailabilityServices highAvailabilityServices,
                           org.apache.flink.runtime.heartbeat.HeartbeatServices heartbeatServices,
                           org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager slotManager,
                           org.apache.flink.runtime.resourcemanager.JobLeaderIdService jobLeaderIdService,
                           org.apache.flink.runtime.entrypoint.ClusterInformation clusterInformation,
                           org.apache.flink.runtime.rpc.FatalErrorHandler fatalErrorHandler,
                           @Nullable
                           String webInterfaceUrl,
                           org.apache.flink.runtime.metrics.groups.ResourceManagerMetricGroup resourceManagerMetricGroup)

Method Detail

createAndStartResourceManagerClient

protected org.apache.hadoop.yarn.client.api.async.AMRMClientAsync<org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest> createAndStartResourceManagerClient(org.apache.hadoop.yarn.conf.YarnConfiguration yarnConfiguration,
                                                                                                                                                                     int yarnHeartbeatIntervalMillis,
                                                                                                                                                                     @Nullable
                                                                                                                                                                     String webInterfaceUrl)
                                                                                                                                                              throws Exception

Throws:

Exception

createAndStartNodeManagerClient

protected org.apache.hadoop.yarn.client.api.async.NMClientAsync createAndStartNodeManagerClient(org.apache.hadoop.yarn.conf.YarnConfiguration yarnConfiguration)

loadClientConfiguration

protected org.apache.flink.configuration.Configuration loadClientConfiguration()

Specified by:

loadClientConfiguration in class org.apache.flink.runtime.resourcemanager.ActiveResourceManager<YarnWorkerNode>

initialize

protected void initialize()
                   throws org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException

Specified by:

initialize in class org.apache.flink.runtime.resourcemanager.ResourceManager<YarnWorkerNode>

Throws:

org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException

onStop

public CompletableFuture<Void> onStop()

Overrides:

onStop in class org.apache.flink.runtime.resourcemanager.ResourceManager<YarnWorkerNode>

internalDeregisterApplication

protected void internalDeregisterApplication(org.apache.flink.runtime.clusterframework.ApplicationStatus finalStatus,
                                             @Nullable
                                             String diagnostics)

Specified by:

internalDeregisterApplication in class org.apache.flink.runtime.resourcemanager.ResourceManager<YarnWorkerNode>

startNewWorker

public Collection<org.apache.flink.runtime.clusterframework.types.ResourceProfile> startNewWorker(org.apache.flink.runtime.clusterframework.types.ResourceProfile resourceProfile)

Specified by:

startNewWorker in class org.apache.flink.runtime.resourcemanager.ResourceManager<YarnWorkerNode>

stopWorker

public boolean stopWorker(YarnWorkerNode workerNode)

Specified by:

stopWorker in class org.apache.flink.runtime.resourcemanager.ResourceManager<YarnWorkerNode>

workerStarted

protected YarnWorkerNode workerStarted(org.apache.flink.runtime.clusterframework.types.ResourceID resourceID)

Specified by:

workerStarted in class org.apache.flink.runtime.resourcemanager.ResourceManager<YarnWorkerNode>

getProgress

public float getProgress()

Specified by:

getProgress in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onContainersCompleted

public void onContainersCompleted(List<org.apache.hadoop.yarn.api.records.ContainerStatus> statuses)

Specified by:

onContainersCompleted in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onContainersAllocated

public void onContainersAllocated(List<org.apache.hadoop.yarn.api.records.Container> containers)

Specified by:

onContainersAllocated in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onShutdownRequest

public void onShutdownRequest()

Specified by:

onShutdownRequest in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onNodesUpdated

public void onNodesUpdated(List<org.apache.hadoop.yarn.api.records.NodeReport> list)

Specified by:

onNodesUpdated in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onError

public void onError(Throwable error)

Specified by:

onError in interface org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler

onContainerStarted

public void onContainerStarted(org.apache.hadoop.yarn.api.records.ContainerId containerId,
                               Map<String,ByteBuffer> map)

Specified by:

onContainerStarted in interface org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler

onContainerStatusReceived

public void onContainerStatusReceived(org.apache.hadoop.yarn.api.records.ContainerId containerId,
                                      org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus)

Specified by:

onContainerStatusReceived in interface org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler

onContainerStopped

public void onContainerStopped(org.apache.hadoop.yarn.api.records.ContainerId containerId)

Specified by:

onContainerStopped in interface org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler

onStartContainerError

public void onStartContainerError(org.apache.hadoop.yarn.api.records.ContainerId containerId,
                                  Throwable t)

Specified by:

onStartContainerError in interface org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler

onGetContainerStatusError

public void onGetContainerStatusError(org.apache.hadoop.yarn.api.records.ContainerId containerId,
                                      Throwable throwable)

Specified by:

onGetContainerStatusError in interface org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler

onStopContainerError

public void onStopContainerError(org.apache.hadoop.yarn.api.records.ContainerId containerId,
                                 Throwable throwable)

Specified by:

onStopContainerError in interface org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler

getCpuCores

protected double getCpuCores(org.apache.flink.configuration.Configuration configuration)

Specified by:

getCpuCores in class org.apache.flink.runtime.resourcemanager.ActiveResourceManager<YarnWorkerNode>