Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.io.network;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.io.disk.iomanager.IOManager.IOMode;
import org.apache.flink.runtime.io.network.buffer.BufferPool;
import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
import org.apache.flink.runtime.io.network.partition.InternalResultPartition;
import org.apache.flink.runtime.io.network.partition.ResultPartitionManager;
import org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.query.KvStateClientProxy;
import org.apache.flink.runtime.query.KvStateRegistry;
import org.apache.flink.runtime.query.KvStateServer;
import org.apache.flink.runtime.query.TaskKvStateRegistry;
import org.apache.flink.runtime.state.internal.InternalKvState;
import org.apache.flink.runtime.taskmanager.Task;
import org.apache.flink.runtime.taskmanager.TaskManager;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* Network I/O components of each {@link TaskManager} instance. The network environment contains
* the data structures that keep track of all intermediate results and all data exchanges.
*/
public class NetworkEnvironment {
private static final Logger LOG = LoggerFactory.getLogger(NetworkEnvironment.class);
private final Object lock = new Object();
private final NetworkBufferPool networkBufferPool;
private final ConnectionManager connectionManager;
private final ResultPartitionManager resultPartitionManager;
private final TaskEventDispatcher taskEventDispatcher;
/** Server for {@link InternalKvState} requests. */
private KvStateServer kvStateServer;
/** Proxy for the queryable state client. */
private KvStateClientProxy kvStateProxy;
/** Registry for {@link InternalKvState} instances. */
private final KvStateRegistry kvStateRegistry;
private final IOManager.IOMode defaultIOMode;
private final int partitionRequestInitialBackoff;
private final int partitionRequestMaxBackoff;
/** Number of network buffers to use for each input channel. */
private final int networkBuffersPerChannel;
/** Number of extra network buffers to use for each input gate. */
private final int extraNetworkBuffersPerGate;
/** Number of network buffers to use for each external shuffle blocking input channel. */
private final int networkBuffersPerExternalBlockingChannel;
/** Number of extra network buffers to use for each external shuffle blocking input gate. */
private final int extraNetworkBuffersPerExternalBlockingGate;
/** Number of network buffers to use for each internal result subpartition. */
private final int networkBuffersPerSubpartition;
private final boolean enableCreditBased;
private boolean isShutdown;
public NetworkEnvironment(
NetworkBufferPool networkBufferPool,
ConnectionManager connectionManager,
ResultPartitionManager resultPartitionManager,
TaskEventDispatcher taskEventDispatcher,
KvStateRegistry kvStateRegistry,
KvStateServer kvStateServer,
KvStateClientProxy kvStateClientProxy,
IOMode defaultIOMode,
int partitionRequestInitialBackoff,
int partitionRequestMaxBackoff,
int networkBuffersPerChannel,
int extraNetworkBuffersPerGate,
int networkBuffersPerExternalBlockingChannel,
int extraNetworkBuffersPerExternalBlockingGate,
int networkBuffersPerSubpartition,
boolean enableCreditBased) {
this.networkBufferPool = checkNotNull(networkBufferPool);
this.connectionManager = checkNotNull(connectionManager);
this.resultPartitionManager = checkNotNull(resultPartitionManager);
this.taskEventDispatcher = checkNotNull(taskEventDispatcher);
this.kvStateRegistry = checkNotNull(kvStateRegistry);
this.kvStateServer = kvStateServer;
this.kvStateProxy = kvStateClientProxy;
this.defaultIOMode = defaultIOMode;
this.partitionRequestInitialBackoff = partitionRequestInitialBackoff;
this.partitionRequestMaxBackoff = partitionRequestMaxBackoff;
isShutdown = false;
this.networkBuffersPerChannel = networkBuffersPerChannel;
this.extraNetworkBuffersPerGate = extraNetworkBuffersPerGate;
this.networkBuffersPerExternalBlockingChannel = networkBuffersPerExternalBlockingChannel;
this.extraNetworkBuffersPerExternalBlockingGate = extraNetworkBuffersPerExternalBlockingGate;
this.networkBuffersPerSubpartition = networkBuffersPerSubpartition;
this.enableCreditBased = enableCreditBased;
}
// --------------------------------------------------------------------------------------------
// Properties
// --------------------------------------------------------------------------------------------
public ResultPartitionManager getResultPartitionManager() {
return resultPartitionManager;
}
public TaskEventDispatcher getTaskEventDispatcher() {
return taskEventDispatcher;
}
public ConnectionManager getConnectionManager() {
return connectionManager;
}
public NetworkBufferPool getNetworkBufferPool() {
return networkBufferPool;
}
public IOMode getDefaultIOMode() {
return defaultIOMode;
}
public int getPartitionRequestInitialBackoff() {
return partitionRequestInitialBackoff;
}
public int getPartitionRequestMaxBackoff() {
return partitionRequestMaxBackoff;
}
public boolean isCreditBased() {
return enableCreditBased;
}
public KvStateRegistry getKvStateRegistry() {
return kvStateRegistry;
}
public KvStateServer getKvStateServer() {
return kvStateServer;
}
public KvStateClientProxy getKvStateProxy() {
return kvStateProxy;
}
public TaskKvStateRegistry createKvStateTaskRegistry(JobID jobId, JobVertexID jobVertexId) {
return kvStateRegistry.createTaskRegistry(jobId, jobVertexId);
}
// --------------------------------------------------------------------------------------------
// Task operations
// --------------------------------------------------------------------------------------------
public void registerTask(Task task) throws IOException {
final List resultPartitions = task.getInternalPartitions();
synchronized (lock) {
if (isShutdown) {
throw new IllegalStateException("NetworkEnvironment is shut down");
}
for (final InternalResultPartition partition : resultPartitions) {
setupPartition(partition);
}
// Setup the buffer pool for each buffer reader
final SingleInputGate[] inputGates = task.getAllInputGates();
for (SingleInputGate gate : inputGates) {
setupInputGate(gate);
}
}
}
@VisibleForTesting
public void setupPartition(InternalResultPartition partition) throws IOException {
BufferPool bufferPool = null;
try {
int extraNetworkBufferThisPartition = extraNetworkBuffersPerGate >= 0 ?
extraNetworkBuffersPerGate : partition.getNumberOfSubpartitions() * networkBuffersPerSubpartition;
int numberOfMemorySegments = partition.getNumberOfSubpartitions() * networkBuffersPerSubpartition
+ extraNetworkBufferThisPartition;
bufferPool = networkBufferPool.createBufferPool(numberOfMemorySegments, numberOfMemorySegments);
partition.registerBufferPool(bufferPool);
resultPartitionManager.registerResultPartition(partition);
} catch (Throwable t) {
if (bufferPool != null) {
bufferPool.lazyDestroy();
}
if (t instanceof IOException) {
throw (IOException) t;
} else {
throw new IOException(t.getMessage(), t);
}
}
taskEventDispatcher.registerPartition(partition.getPartitionId());
}
@VisibleForTesting
public void setupInputGate(SingleInputGate gate) throws IOException {
BufferPool bufferPool = null;
int maxNumberOfMemorySegments;
int extraNetworkBufferThisGate;
try {
if (enableCreditBased) {
if (gate.isPartitionRequestRestricted()) {
extraNetworkBufferThisGate = Math.max(extraNetworkBuffersPerExternalBlockingGate, 0);
gate.setNetworkProperties(networkBufferPool, networkBuffersPerExternalBlockingChannel);
// for external shuffle, extra buffer is not used currently
bufferPool = networkBufferPool.createBufferPool(0, extraNetworkBufferThisGate);
} else {
extraNetworkBufferThisGate = extraNetworkBuffersPerGate >= 0 ?
extraNetworkBuffersPerGate : gate.getNumberOfInputChannels() * networkBuffersPerChannel;
gate.setNetworkProperties(networkBufferPool, networkBuffersPerChannel);
bufferPool = networkBufferPool.createBufferPool(0, extraNetworkBufferThisGate);
}
} else {
extraNetworkBufferThisGate = extraNetworkBuffersPerGate >= 0 ?
extraNetworkBuffersPerGate : gate.getNumberOfInputChannels() * networkBuffersPerChannel;
maxNumberOfMemorySegments = gate.getNumberOfInputChannels() * networkBuffersPerChannel
+ extraNetworkBufferThisGate;
bufferPool = networkBufferPool.createBufferPool(gate.getNumberOfInputChannels(),
maxNumberOfMemorySegments);
}
gate.setBufferPool(bufferPool);
} catch (Throwable t) {
if (bufferPool != null) {
bufferPool.lazyDestroy();
}
ExceptionUtils.rethrowIOException(t);
}
}
public void unregisterTask(Task task) {
LOG.debug("Unregister task {} from network environment (state: {}).",
task.getTaskInfo().getTaskNameWithSubtasks(), task.getExecutionState());
final ExecutionAttemptID executionId = task.getExecutionId();
synchronized (lock) {
if (isShutdown) {
// no need to do anything when we are not operational
return;
}
if (task.isCanceledOrFailed()) {
resultPartitionManager.releasePartitionsProducedBy(executionId, task.getFailureCause());
}
for (InternalResultPartition partition : task.getInternalPartitions()) {
taskEventDispatcher.unregisterPartition(partition.getPartitionId());
partition.destroyBufferPool();
}
final SingleInputGate[] inputGates = task.getAllInputGates();
if (inputGates != null) {
for (SingleInputGate gate : inputGates) {
try {
if (gate != null) {
gate.releaseAllResources();
}
}
catch (IOException e) {
LOG.error("Error during release of reader resources: " + e.getMessage(), e);
}
}
}
}
}
public void start() throws IOException {
synchronized (lock) {
Preconditions.checkState(!isShutdown, "The NetworkEnvironment has already been shut down.");
LOG.info("Starting the network environment and its components.");
try {
LOG.debug("Starting network connection manager");
connectionManager.start(resultPartitionManager, taskEventDispatcher);
} catch (IOException t) {
throw new IOException("Failed to instantiate network connection manager.", t);
}
if (kvStateServer != null) {
try {
kvStateServer.start();
} catch (Throwable ie) {
kvStateServer.shutdown();
kvStateServer = null;
throw new IOException("Failed to start the Queryable State Data Server.", ie);
}
}
if (kvStateProxy != null) {
try {
kvStateProxy.start();
} catch (Throwable ie) {
kvStateProxy.shutdown();
kvStateProxy = null;
throw new IOException("Failed to start the Queryable State Client Proxy.", ie);
}
}
}
}
/**
* Tries to shut down all network I/O components.
*/
public void shutdown() {
synchronized (lock) {
if (isShutdown) {
return;
}
LOG.info("Shutting down the network environment and its components.");
if (kvStateProxy != null) {
try {
LOG.debug("Shutting down Queryable State Client Proxy.");
kvStateProxy.shutdown();
} catch (Throwable t) {
LOG.warn("Cannot shut down Queryable State Client Proxy.", t);
}
}
if (kvStateServer != null) {
try {
LOG.debug("Shutting down Queryable State Data Server.");
kvStateServer.shutdown();
} catch (Throwable t) {
LOG.warn("Cannot shut down Queryable State Data Server.", t);
}
}
// terminate all network connections
try {
LOG.debug("Shutting down network connection manager");
connectionManager.shutdown();
}
catch (Throwable t) {
LOG.warn("Cannot shut down the network connection manager.", t);
}
// shutdown all intermediate results
try {
LOG.debug("Shutting down intermediate result partition manager");
resultPartitionManager.shutdown();
}
catch (Throwable t) {
LOG.warn("Cannot shut down the result partition manager.", t);
}
taskEventDispatcher.clearAll();
// make sure that the global buffer pool re-acquires all buffers
networkBufferPool.destroyAllBufferPools();
// destroy the buffer pool
try {
networkBufferPool.destroy();
}
catch (Throwable t) {
LOG.warn("Network buffer pool did not shut down properly.", t);
}
isShutdown = true;
}
}
public boolean isShutdown() {
synchronized (lock) {
return isShutdown;
}
}
}
