eu.stratosphere.nephele.taskmanager.bytebuffered.ByteBufferedChannelManager Maven / Gradle / Ivy
/***********************************************************************************************************************
* Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu)
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
**********************************************************************************************************************/
package eu.stratosphere.nephele.taskmanager.bytebuffered;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import eu.stratosphere.configuration.GlobalConfiguration;
import eu.stratosphere.nephele.execution.Environment;
import eu.stratosphere.nephele.executiongraph.ExecutionVertexID;
import eu.stratosphere.nephele.instance.InstanceConnectionInfo;
import eu.stratosphere.nephele.io.AbstractID;
import eu.stratosphere.nephele.io.GateID;
import eu.stratosphere.nephele.io.channels.Buffer;
import eu.stratosphere.nephele.io.channels.ChannelID;
import eu.stratosphere.nephele.io.channels.ChannelType;
import eu.stratosphere.nephele.jobgraph.JobID;
import eu.stratosphere.nephele.protocols.ChannelLookupProtocol;
import eu.stratosphere.nephele.taskmanager.Task;
import eu.stratosphere.nephele.taskmanager.bufferprovider.BufferProvider;
import eu.stratosphere.nephele.taskmanager.bufferprovider.BufferProviderBroker;
import eu.stratosphere.nephele.taskmanager.bufferprovider.GlobalBufferPool;
import eu.stratosphere.nephele.taskmanager.bufferprovider.LocalBufferPool;
import eu.stratosphere.nephele.taskmanager.bufferprovider.LocalBufferPoolOwner;
import eu.stratosphere.nephele.taskmanager.transferenvelope.TransferEnvelope;
import eu.stratosphere.nephele.taskmanager.transferenvelope.TransferEnvelopeDispatcher;
import eu.stratosphere.nephele.taskmanager.transferenvelope.TransferEnvelopeReceiverList;
public final class ByteBufferedChannelManager implements TransferEnvelopeDispatcher, BufferProviderBroker {
/**
* The log object used to report problems and errors.
*/
private static final Log LOG = LogFactory.getLog(ByteBufferedChannelManager.class);
private static final boolean DEFAULT_ALLOW_SENDER_SIDE_SPILLING = false;
private static final boolean DEFAULT_MERGE_SPILLED_BUFFERS = true;
// TODO: Make this configurable
private static final int NUMBER_OF_CHANNELS_FOR_MULTICAST = 10;
private final Map registeredChannels = new ConcurrentHashMap();
private final Map localBufferPoolOwner = new ConcurrentHashMap();
private final NetworkConnectionManager networkConnectionManager;
private final ChannelLookupProtocol channelLookupService;
private final InstanceConnectionInfo localConnectionInfo;
private final LocalBufferPool transitBufferPool;
private final boolean allowSenderSideSpilling;
private final boolean mergeSpilledBuffers;
private final boolean multicastEnabled = true;
/**
* This map caches transfer envelope receiver lists.
*/
private final Map receiverCache = new ConcurrentHashMap();
public ByteBufferedChannelManager(final ChannelLookupProtocol channelLookupService,
final InstanceConnectionInfo localInstanceConnectionInfo) throws IOException {
this.channelLookupService = channelLookupService;
this.localConnectionInfo = localInstanceConnectionInfo;
// Initialize the global buffer pool
GlobalBufferPool.getInstance();
// Initialize the transit buffer pool
this.transitBufferPool = new LocalBufferPool(128, true);
this.networkConnectionManager = new NetworkConnectionManager(this,
localInstanceConnectionInfo.getAddress(), localInstanceConnectionInfo.getDataPort());
this.allowSenderSideSpilling = GlobalConfiguration.getBoolean("channel.network.allowSenderSideSpilling",
DEFAULT_ALLOW_SENDER_SIDE_SPILLING);
this.mergeSpilledBuffers = GlobalConfiguration.getBoolean("channel.network.mergeSpilledBuffers",
DEFAULT_MERGE_SPILLED_BUFFERS);
LOG.info("Initialized byte buffered channel manager with sender-side spilling "
+ (this.allowSenderSideSpilling ? "enabled" : "disabled")
+ (this.mergeSpilledBuffers ? " and spilled buffer merging enabled" : ""));
}
/**
* Registers the given task with the byte buffered channel manager.
*
* @param task
* the task to be registered
* @param the
* set of output channels which are initially active
* @throws InsufficientResourcesException
* thrown if the channel manager does not have enough memory buffers to safely run this task
*/
public void register(final Task task, final Set activeOutputChannels)
throws InsufficientResourcesException {
// Check if we can safely run this task with the given resources
checkBufferAvailability(task);
final Environment environment = task.getEnvironment();
final TaskContext taskContext = task.createTaskContext(this,
this.localBufferPoolOwner.remove(task.getVertexID()));
final Set outputGateIDs = environment.getOutputGateIDs();
for (final Iterator gateIt = outputGateIDs.iterator(); gateIt.hasNext();) {
final GateID gateID = gateIt.next();
final OutputGateContext outputGateContext = taskContext.createOutputGateContext(gateID);
final Set outputChannelIDs = environment.getOutputChannelIDsOfGate(gateID);
for (final Iterator channelIt = outputChannelIDs.iterator(); channelIt.hasNext();) {
final ChannelID channelID = channelIt.next();
final OutputChannelContext previousContext = (OutputChannelContext) this.registeredChannels
.get(channelID);
final boolean isActive = true;/* activeOutputChannels.contains(channelID); */
final OutputChannelContext outputChannelContext = outputGateContext.createOutputChannelContext(
channelID, previousContext, isActive, this.mergeSpilledBuffers);
// Add routing entry to receiver cache to reduce latency
if (outputChannelContext.getType() == ChannelType.INMEMORY) {
addReceiverListHint(outputChannelContext.getChannelID(),
outputChannelContext.getConnectedChannelID());
}
// Add routing entry to receiver cache to save lookup for data arriving at the output channel
if (outputChannelContext.getType() == ChannelType.NETWORK) {
addReceiverListHint(outputChannelContext.getConnectedChannelID(),
outputChannelContext.getChannelID());
}
if (LOG.isDebugEnabled()) {
LOG.debug("Registering byte buffered output channel " + outputChannelContext.getChannelID() + " ("
+ (isActive ? "active" : "inactive") + ")");
}
this.registeredChannels.put(outputChannelContext.getChannelID(), outputChannelContext);
}
}
final Set inputGateIDs = environment.getInputGateIDs();
for (final Iterator gateIt = inputGateIDs.iterator(); gateIt.hasNext();) {
final GateID gateID = gateIt.next();
final InputGateContext inputGateContext = taskContext.createInputGateContext(gateID);
final Set inputChannelIDs = environment.getInputChannelIDsOfGate(gateID);
for (final Iterator channelIt = inputChannelIDs.iterator(); channelIt.hasNext();) {
final ChannelID channelID = channelIt.next();
final InputChannelContext previousContext = (InputChannelContext) this.registeredChannels
.get(channelID);
final InputChannelContext inputChannelContext = inputGateContext.createInputChannelContext(
channelID, previousContext);
// Add routing entry to receiver cache to reduce latency
if (inputChannelContext.getType() == ChannelType.INMEMORY) {
addReceiverListHint(inputChannelContext.getChannelID(), inputChannelContext.getConnectedChannelID());
}
this.registeredChannels.put(inputChannelContext.getChannelID(), inputChannelContext);
}
// Add input gate context to set of local buffer pool owner
final LocalBufferPoolOwner bufferPoolOwner = inputGateContext.getLocalBufferPoolOwner();
if (bufferPoolOwner != null) {
this.localBufferPoolOwner.put(inputGateContext.getGateID(), bufferPoolOwner);
}
}
this.localBufferPoolOwner.put(task.getVertexID(), taskContext);
redistributeGlobalBuffers();
}
/**
* Unregisters the given task from the byte buffered channel manager.
*
* @param vertexID
* the ID of the task to be unregistered
* @param task
* the task to be unregistered
*/
public void unregister(final ExecutionVertexID vertexID, final Task task) {
final Environment environment = task.getEnvironment();
Iterator channelIterator = environment.getOutputChannelIDs().iterator();
while (channelIterator.hasNext()) {
final ChannelID outputChannelID = channelIterator.next();
final ChannelContext context = this.registeredChannels.remove(outputChannelID);
if (context != null) {
context.destroy();
}
this.receiverCache.remove(outputChannelID);
}
channelIterator = environment.getInputChannelIDs().iterator();
while (channelIterator.hasNext()) {
final ChannelID outputChannelID = channelIterator.next();
final ChannelContext context = this.registeredChannels.remove(outputChannelID);
if (context != null) {
context.destroy();
}
this.receiverCache.remove(outputChannelID);
}
final Iterator inputGateIterator = environment.getInputGateIDs().iterator();
while (inputGateIterator.hasNext()) {
final GateID inputGateID = inputGateIterator.next();
final LocalBufferPoolOwner owner = this.localBufferPoolOwner.remove(inputGateID);
if (owner != null) {
owner.clearLocalBufferPool();
}
}
final LocalBufferPoolOwner owner = this.localBufferPoolOwner.remove(vertexID);
if (owner != null) {
owner.clearLocalBufferPool();
}
redistributeGlobalBuffers();
}
/**
* Shuts down the byte buffered channel manager and stops all its internal processes.
*/
public void shutdown() {
this.networkConnectionManager.shutDown();
}
public NetworkConnectionManager getNetworkConnectionManager() {
return this.networkConnectionManager;
}
private void recycleBuffer(final TransferEnvelope envelope) {
final Buffer buffer = envelope.getBuffer();
if (buffer != null) {
buffer.recycleBuffer();
}
}
private void sendReceiverNotFoundEvent(final TransferEnvelope envelope, final ChannelID receiver)
throws IOException, InterruptedException {
if (ReceiverNotFoundEvent.isReceiverNotFoundEvent(envelope)) {
LOG.info("Dropping request to send ReceiverNotFoundEvent as response to ReceiverNotFoundEvent");
return;
}
final JobID jobID = envelope.getJobID();
final TransferEnvelope transferEnvelope = ReceiverNotFoundEvent.createEnvelopeWithEvent(jobID, receiver,
envelope.getSequenceNumber());
final TransferEnvelopeReceiverList receiverList = getReceiverList(jobID, receiver);
if (receiverList == null) {
return;
}
processEnvelopeEnvelopeWithoutBuffer(transferEnvelope, receiverList);
}
private void processEnvelope(final TransferEnvelope transferEnvelope, final boolean freeSourceBuffer)
throws IOException, InterruptedException {
TransferEnvelopeReceiverList receiverList = null;
try {
receiverList = getReceiverList(transferEnvelope.getJobID(),
transferEnvelope.getSource());
} catch (InterruptedException e) {
recycleBuffer(transferEnvelope);
throw e;
} catch (IOException e) {
recycleBuffer(transferEnvelope);
throw e;
}
if (receiverList == null) {
recycleBuffer(transferEnvelope);
return;
}
// This envelope is known to have either no buffer or an memory-based input buffer
if (transferEnvelope.getBuffer() == null) {
processEnvelopeEnvelopeWithoutBuffer(transferEnvelope, receiverList);
} else {
processEnvelopeWithBuffer(transferEnvelope, receiverList, freeSourceBuffer);
}
}
private void processEnvelopeWithBuffer(final TransferEnvelope transferEnvelope,
final TransferEnvelopeReceiverList receiverList, final boolean freeSourceBuffer)
throws IOException, InterruptedException {
// Handle the most common (unicast) case first
if (!freeSourceBuffer) {
final List localReceivers = receiverList.getLocalReceivers();
if (localReceivers.size() != 1) {
LOG.error("Expected receiver list to have exactly one element");
}
final ChannelID localReceiver = localReceivers.get(0);
final ChannelContext cc = this.registeredChannels.get(localReceiver);
if (cc == null) {
try {
sendReceiverNotFoundEvent(transferEnvelope, localReceiver);
} finally {
recycleBuffer(transferEnvelope);
}
return;
}
if (!cc.isInputChannel()) {
LOG.error("Local receiver " + localReceiver
+ " is not an input channel, but is supposed to accept a buffer");
}
cc.queueTransferEnvelope(transferEnvelope);
return;
}
// This is the in-memory or multicast case
final Buffer srcBuffer = transferEnvelope.getBuffer();
try {
if (receiverList.hasLocalReceivers()) {
final List localReceivers = receiverList.getLocalReceivers();
for (final ChannelID localReceiver : localReceivers) {
final ChannelContext cc = this.registeredChannels.get(localReceiver);
if (cc == null) {
sendReceiverNotFoundEvent(transferEnvelope, localReceiver);
continue;
}
if (!cc.isInputChannel()) {
LOG.error("Local receiver " + localReceiver
+ " is not an input channel, but is supposed to accept a buffer");
continue;
}
final InputChannelContext inputChannelContext = (InputChannelContext) cc;
Buffer destBuffer = null;
try {
destBuffer = inputChannelContext.requestEmptyBufferBlocking(srcBuffer.size());
srcBuffer.copyToBuffer(destBuffer);
} catch (IOException e) {
if (destBuffer != null) {
destBuffer.recycleBuffer();
}
throw e;
}
// TODO: See if we can save one duplicate step here
final TransferEnvelope dup = transferEnvelope.duplicateWithoutBuffer();
dup.setBuffer(destBuffer);
inputChannelContext.queueTransferEnvelope(dup);
}
}
if (receiverList.hasRemoteReceivers()) {
final List remoteReceivers = receiverList.getRemoteReceivers();
// Generate sender hint before sending the first envelope over the network
if (transferEnvelope.getSequenceNumber() == 0) {
generateSenderHint(transferEnvelope, remoteReceivers);
}
for (final RemoteReceiver remoteReceiver : remoteReceivers) {
TransferEnvelope dup = transferEnvelope.duplicate();
this.networkConnectionManager.queueEnvelopeForTransfer(remoteReceiver, dup);
}
}
} finally {
// Recycle the source buffer
srcBuffer.recycleBuffer();
}
}
private void processEnvelopeEnvelopeWithoutBuffer(final TransferEnvelope transferEnvelope,
final TransferEnvelopeReceiverList receiverList) throws IOException, InterruptedException {
// No need to copy anything
final Iterator localIt = receiverList.getLocalReceivers().iterator();
while (localIt.hasNext()) {
final ChannelID localReceiver = localIt.next();
final ChannelContext channelContext = this.registeredChannels.get(localReceiver);
if (channelContext == null) {
sendReceiverNotFoundEvent(transferEnvelope, localReceiver);
continue;
}
channelContext.queueTransferEnvelope(transferEnvelope);
}
if (!receiverList.hasRemoteReceivers()) {
return;
}
// Generate sender hint before sending the first envelope over the network
final List remoteReceivers = receiverList.getRemoteReceivers();
if (transferEnvelope.getSequenceNumber() == 0) {
generateSenderHint(transferEnvelope, remoteReceivers);
}
final Iterator remoteIt = remoteReceivers.iterator();
while (remoteIt.hasNext()) {
final RemoteReceiver remoteReceiver = remoteIt.next();
this.networkConnectionManager.queueEnvelopeForTransfer(remoteReceiver, transferEnvelope);
}
}
private void addReceiverListHint(final ChannelID source, final ChannelID localReceiver) {
final TransferEnvelopeReceiverList receiverList = new TransferEnvelopeReceiverList(localReceiver);
if (this.receiverCache.put(source, receiverList) != null) {
LOG.warn("Receiver cache already contained entry for " + source);
}
}
private void addReceiverListHint(final ChannelID source, final RemoteReceiver remoteReceiver) {
final TransferEnvelopeReceiverList receiverList = new TransferEnvelopeReceiverList(remoteReceiver);
if (this.receiverCache.put(source, receiverList) != null) {
LOG.warn("Receiver cache already contained entry for " + source);
}
}
private void generateSenderHint(final TransferEnvelope transferEnvelope, final List remoteReceivers) {
final ChannelContext channelContext = this.registeredChannels.get(transferEnvelope.getSource());
if (channelContext == null) {
LOG.error("Cannot find channel context for channel ID " + transferEnvelope.getSource());
return;
}
// Only generate sender hints for output channels
if (channelContext.isInputChannel()) {
return;
}
final ChannelID remoteSourceID = channelContext.getConnectedChannelID();
final int connectionIndex = remoteReceivers.get(0).getConnectionIndex();
final InetSocketAddress isa = new InetSocketAddress(this.localConnectionInfo.getAddress(),
this.localConnectionInfo.getDataPort());
final RemoteReceiver remoteReceiver = new RemoteReceiver(isa, connectionIndex);
final TransferEnvelope senderHint = SenderHintEvent.createEnvelopeWithEvent(transferEnvelope, remoteSourceID,
remoteReceiver);
final Iterator remoteIt = remoteReceivers.iterator();
while (remoteIt.hasNext()) {
final RemoteReceiver rr = remoteIt.next();
this.networkConnectionManager.queueEnvelopeForTransfer(rr, senderHint);
}
}
/**
* Returns the list of receivers for transfer envelopes produced by the channel with the given source channel ID.
*
* @param jobID
* the ID of the job the given channel ID belongs to
* @param sourceChannelID
* the source channel ID for which the receiver list shall be retrieved
* @return the list of receivers or null if the receiver could not be determined
* @throws IOException
* @throws InterruptedExcption
*/
private TransferEnvelopeReceiverList getReceiverList(final JobID jobID, final ChannelID sourceChannelID)
throws IOException, InterruptedException {
TransferEnvelopeReceiverList receiverList = this.receiverCache.get(sourceChannelID);
if (receiverList != null) {
return receiverList;
}
while (true) {
if (Thread.currentThread().isInterrupted()) {
break;
}
ConnectionInfoLookupResponse lookupResponse;
synchronized (this.channelLookupService) {
lookupResponse = this.channelLookupService.lookupConnectionInfo(
this.localConnectionInfo, jobID, sourceChannelID);
}
if (lookupResponse.isJobAborting()) {
break;
}
if (lookupResponse.receiverNotFound()) {
LOG.error("Cannot find task(s) waiting for data from source channel with ID " + sourceChannelID);
break;
}
if (lookupResponse.receiverNotReady()) {
Thread.sleep(500);
continue;
}
if (lookupResponse.receiverReady()) {
receiverList = new TransferEnvelopeReceiverList(lookupResponse);
break;
}
}
if (receiverList != null) {
this.receiverCache.put(sourceChannelID, receiverList);
if (LOG.isDebugEnabled()) {
final StringBuilder sb = new StringBuilder();
sb.append("Receiver list for source channel ID " + sourceChannelID + " at task manager "
+ this.localConnectionInfo + "\n");
if (receiverList.hasLocalReceivers()) {
sb.append("\tLocal receivers:\n");
final Iterator it = receiverList.getLocalReceivers().iterator();
while (it.hasNext()) {
sb.append("\t\t" + it.next() + "\n");
}
}
if (receiverList.hasRemoteReceivers()) {
sb.append("Remote receivers:\n");
final Iterator it = receiverList.getRemoteReceivers().iterator();
while (it.hasNext()) {
sb.append("\t\t" + it.next() + "\n");
}
}
LOG.debug(sb.toString());
}
}
return receiverList;
}
@Override
public void processEnvelopeFromOutputChannel(final TransferEnvelope transferEnvelope) throws IOException,
InterruptedException {
processEnvelope(transferEnvelope, true);
}
@Override
public void processEnvelopeFromInputChannel(final TransferEnvelope transferEnvelope) throws IOException,
InterruptedException {
processEnvelope(transferEnvelope, false);
}
@Override
public void processEnvelopeFromNetwork(final TransferEnvelope transferEnvelope, boolean freeSourceBuffer)
throws IOException, InterruptedException {
// Check if the envelope is the special envelope with the sender hint event
if (SenderHintEvent.isSenderHintEvent(transferEnvelope)) {
// Check if this is the final destination of the sender hint event before adding it
final SenderHintEvent seh = (SenderHintEvent) transferEnvelope.getEventList().get(0);
if (this.registeredChannels.get(seh.getSource()) != null) {
addReceiverListHint(seh.getSource(), seh.getRemoteReceiver());
return;
}
}
processEnvelope(transferEnvelope, freeSourceBuffer);
}
/**
* Triggers the byte buffer channel manager write the current utilization of its read and write buffers to the logs.
* This method is primarily for debugging purposes.
*/
public void logBufferUtilization() {
System.out.println("Buffer utilization at " + System.currentTimeMillis());
System.out.println("\tUnused global buffers: " + GlobalBufferPool.getInstance().getCurrentNumberOfBuffers());
System.out.println("\tLocal buffer pool status:");
final Iterator it = this.localBufferPoolOwner.values().iterator();
while (it.hasNext()) {
it.next().logBufferUtilization();
}
this.networkConnectionManager.logBufferUtilization();
System.out.println("\tIncoming connections:");
final Iterator> it2 = this.registeredChannels.entrySet()
.iterator();
while (it2.hasNext()) {
final Map.Entry entry = it2.next();
final ChannelContext context = entry.getValue();
if (context.isInputChannel()) {
final InputChannelContext inputChannelContext = (InputChannelContext) context;
inputChannelContext.logQueuedEnvelopes();
}
}
}
@Override
public BufferProvider getBufferProvider(final JobID jobID, final ChannelID sourceChannelID) throws IOException,
InterruptedException {
final TransferEnvelopeReceiverList receiverList = getReceiverList(jobID, sourceChannelID);
// Receiver could not be determined, use transit buffer pool to read data from channel
if (receiverList == null) {
return this.transitBufferPool;
}
if (receiverList.hasLocalReceivers() && !receiverList.hasRemoteReceivers()) {
final List localReceivers = receiverList.getLocalReceivers();
if (localReceivers.size() == 1) {
// Unicast case, get final buffer provider
final ChannelID localReceiver = localReceivers.get(0);
final ChannelContext cc = this.registeredChannels.get(localReceiver);
if (cc == null) {
// Use the transit buffer for this purpose, data will be discarded in most cases anyway.
return this.transitBufferPool;
}
if (!cc.isInputChannel()) {
throw new IOException("Channel context for local receiver " + localReceiver
+ " is not an input channel context");
}
final InputChannelContext icc = (InputChannelContext) cc;
return icc;
}
}
return this.transitBufferPool;
}
/**
* Checks if the byte buffered channel manager has enough resources available to safely execute the given task.
*
* @param task
* the task to be executed
* @throws InsufficientResourcesException
* thrown if the byte buffered manager currently does not have enough resources available to execute the
* task
*/
private void checkBufferAvailability(final Task task) throws InsufficientResourcesException {
final int totalNumberOfBuffers = GlobalBufferPool.getInstance().getTotalNumberOfBuffers();
int numberOfAlreadyRegisteredChannels = this.registeredChannels.size();
if (this.multicastEnabled) {
numberOfAlreadyRegisteredChannels += NUMBER_OF_CHANNELS_FOR_MULTICAST;
}
final Environment env = task.getEnvironment();
final int numberOfNewChannels = env.getNumberOfOutputChannels() + env.getNumberOfInputChannels();
final int totalNumberOfChannels = numberOfAlreadyRegisteredChannels + numberOfNewChannels;
final double buffersPerChannel = (double) totalNumberOfBuffers
/ (double) totalNumberOfChannels;
if (buffersPerChannel < 1.0) {
// Construct error message
final StringBuilder sb = new StringBuilder(this.localConnectionInfo.getHostName());
sb.append(" has not enough buffers available to safely execute ");
sb.append(env.getTaskName());
sb.append(" (");
sb.append(totalNumberOfChannels - totalNumberOfBuffers);
sb.append(" buffers are currently missing)");
throw new InsufficientResourcesException(sb.toString());
}
}
/**
* Redistributes the global buffers among the registered tasks.
*/
private void redistributeGlobalBuffers() {
final int totalNumberOfBuffers = GlobalBufferPool.getInstance().getTotalNumberOfBuffers();
int totalNumberOfChannels = this.registeredChannels.size();
if (this.multicastEnabled) {
totalNumberOfChannels += NUMBER_OF_CHANNELS_FOR_MULTICAST;
}
final double buffersPerChannel = (double) totalNumberOfBuffers / (double) totalNumberOfChannels;
if (buffersPerChannel < 1.0) {
LOG.warn("System is low on memory buffers. This may result in reduced performance.");
}
if (LOG.isDebugEnabled()) {
LOG.debug("Total number of buffers is " + totalNumberOfBuffers);
LOG.debug("Total number of channels is " + totalNumberOfChannels);
}
if (this.localBufferPoolOwner.isEmpty()) {
return;
}
final Iterator it = this.localBufferPoolOwner.values().iterator();
while (it.hasNext()) {
final LocalBufferPoolOwner lbpo = it.next();
lbpo.setDesignatedNumberOfBuffers((int) Math.ceil(buffersPerChannel * lbpo.getNumberOfChannels()));
}
if (this.multicastEnabled) {
this.transitBufferPool.setDesignatedNumberOfBuffers((int) Math.ceil(buffersPerChannel
* NUMBER_OF_CHANNELS_FOR_MULTICAST));
}
}
/**
* Invalidates the entries identified by the given channel IDs from the receiver lookup cache.
*
* @param channelIDs
* the channel IDs identifying the cache entries to invalidate
*/
public void invalidateLookupCacheEntries(final Set channelIDs) {
final Iterator it = channelIDs.iterator();
while (it.hasNext()) {
this.receiverCache.remove(it.next());
}
}
public void reportAsynchronousEvent(final ExecutionVertexID vertexID) {
final LocalBufferPoolOwner lbpo = this.localBufferPoolOwner.get(vertexID);
if (lbpo == null) {
System.out.println("Cannot find local buffer pool owner for " + vertexID);
return;
}
lbpo.reportAsynchronousEvent();
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy