eu.stratosphere.nephele.taskmanager.runtime.RuntimeInputChannelContext Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of stratosphere-runtime Show documentation
There is a newer version: 0.5.2-hadoop2
/***********************************************************************************************************************
 * Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
 * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations under the License.
 **********************************************************************************************************************/

package eu.stratosphere.nephele.taskmanager.runtime;

import java.io.IOException;
import java.util.ArrayDeque;
import java.util.Iterator;
import java.util.Queue;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import eu.stratosphere.nephele.event.task.AbstractEvent;
import eu.stratosphere.nephele.io.channels.Buffer;
import eu.stratosphere.nephele.io.channels.ChannelID;
import eu.stratosphere.nephele.io.channels.ChannelType;
import eu.stratosphere.nephele.io.channels.bytebuffered.AbstractByteBufferedInputChannel;
import eu.stratosphere.nephele.io.channels.bytebuffered.BufferOrEvent;
import eu.stratosphere.nephele.io.channels.bytebuffered.ByteBufferedInputChannelBroker;
import eu.stratosphere.nephele.jobgraph.JobID;
import eu.stratosphere.nephele.taskmanager.bufferprovider.BufferAvailabilityListener;
import eu.stratosphere.nephele.taskmanager.bytebuffered.InputChannelContext;
import eu.stratosphere.nephele.taskmanager.bytebuffered.ReceiverNotFoundEvent;
import eu.stratosphere.nephele.taskmanager.transferenvelope.TransferEnvelope;
import eu.stratosphere.nephele.taskmanager.transferenvelope.TransferEnvelopeDispatcher;


final class RuntimeInputChannelContext implements InputChannelContext, ByteBufferedInputChannelBroker {

	private static final Log LOG = LogFactory.getLog(RuntimeInputChannelContext.class);

	private final RuntimeInputGateContext inputGateContext;

	private final AbstractByteBufferedInputChannel byteBufferedInputChannel;

	private final TransferEnvelopeDispatcher transferEnvelopeDispatcher;

	private final Queue queuedEnvelopes = new ArrayDeque();
	
	private Iterator pendingEvents;

	private int lastReceivedEnvelope = -1;

	private boolean destroyCalled = false;

	RuntimeInputChannelContext(final RuntimeInputGateContext inputGateContext,
			final TransferEnvelopeDispatcher transferEnvelopeDispatcher,
			final AbstractByteBufferedInputChannel byteBufferedInputChannel) {

		this.inputGateContext = inputGateContext;
		this.transferEnvelopeDispatcher = transferEnvelopeDispatcher;
		this.byteBufferedInputChannel = byteBufferedInputChannel;
		this.byteBufferedInputChannel.setInputChannelBroker(this);
	}


	@Override
	public BufferOrEvent getNextBufferOrEvent() throws IOException {
		// return pending events first
		if (this.pendingEvents != null) {
			// if the field is not null, it must always have a next value!
			BufferOrEvent next = new BufferOrEvent(this.pendingEvents.next());
			if (!this.pendingEvents.hasNext()) {
				this.pendingEvents = null;
			}
			return next;
		}

		// if no events are pending, get the next buffer
		TransferEnvelope nextEnvelope;
		synchronized (this.queuedEnvelopes) {
			if (this.queuedEnvelopes.isEmpty()) {
				return null;
			}
			nextEnvelope = this.queuedEnvelopes.poll();
		}

		// schedule events as pending, because events come always after the buffer!
		if (nextEnvelope.getEventList() != null) {
			Iterator events = nextEnvelope.getEventList().iterator();
			if (events.hasNext()) {
				this.pendingEvents = events;
			}
		}
		
		// get the buffer, if there is one
		if (nextEnvelope.getBuffer() != null) {
			return new BufferOrEvent(nextEnvelope.getBuffer());
		}
		else if (this.pendingEvents != null) {
			// if the field is not null, it must always have a next value!
			BufferOrEvent next = new BufferOrEvent(this.pendingEvents.next());
			if (!this.pendingEvents.hasNext()) {
				this.pendingEvents = null;
			}
			
			return next;
		}
		else {
			// no buffer and no events, this should be an error
			throw new IOException("Received an envelope with neither data nor events.");
		}
	}

	@Override
	public void transferEventToOutputChannel(AbstractEvent event) throws IOException, InterruptedException {
		TransferEnvelope ephemeralTransferEnvelope = new TransferEnvelope(0, getJobID(), getChannelID());
		ephemeralTransferEnvelope.addEvent(event);
		
		this.transferEnvelopeDispatcher.processEnvelopeFromInputChannel(ephemeralTransferEnvelope);
	}

	@Override
	public void queueTransferEnvelope(TransferEnvelope transferEnvelope) {

		if (ReceiverNotFoundEvent.isReceiverNotFoundEvent(transferEnvelope)) {
			return;
		}
		
		// The sequence number of the envelope to be queued
		final int sequenceNumber = transferEnvelope.getSequenceNumber();

		synchronized (this.queuedEnvelopes) {

			if (this.destroyCalled) {
				final Buffer buffer = transferEnvelope.getBuffer();
				if (buffer != null) {
					buffer.recycleBuffer();
				}
				return;
			}

			final int expectedSequenceNumber = this.lastReceivedEnvelope + 1;
			if (sequenceNumber != expectedSequenceNumber) {
				// This is a problem, now we are actually missing some data
				this.byteBufferedInputChannel.reportIOException(new IOException("Expected data packet "
						+ expectedSequenceNumber + " but received " + sequenceNumber));
				
				// notify that something (an exception) is available
				this.byteBufferedInputChannel.notifyGateThatInputIsAvailable();

				if (LOG.isDebugEnabled()) {
					LOG.debug("Input channel " + getChannelName() + " expected envelope " + expectedSequenceNumber
						+ " but received " + sequenceNumber);
				}

				// rescue the buffer
				final Buffer buffer = transferEnvelope.getBuffer();
				if (buffer != null) {
					buffer.recycleBuffer();
				}
			} else {

				this.queuedEnvelopes.add(transferEnvelope);
				this.lastReceivedEnvelope = sequenceNumber;

				// Notify the channel about the new data. notify as much as there is (buffer plus once per event)
				if (transferEnvelope.getBuffer() != null) {
					this.byteBufferedInputChannel.notifyGateThatInputIsAvailable();
				}
				if (transferEnvelope.getEventList() != null) {
					for (int i = 0; i < transferEnvelope.getEventList().size(); i++) {
						this.byteBufferedInputChannel.notifyGateThatInputIsAvailable();
					}
				}
			}
		}
	}

	@Override
	public ChannelID getChannelID() {
		return this.byteBufferedInputChannel.getID();
	}

	@Override
	public ChannelID getConnectedChannelID() {
		return this.byteBufferedInputChannel.getConnectedChannelID();
	}

	@Override
	public JobID getJobID() {
		return this.byteBufferedInputChannel.getJobID();
	}

	@Override
	public boolean isInputChannel() {
		return this.byteBufferedInputChannel.isInputChannel();
	}

	@Override
	public void destroy() {
		final Queue buffersToRecycle = new ArrayDeque();

		synchronized (this.queuedEnvelopes) {
			this.destroyCalled = true;

			while (!this.queuedEnvelopes.isEmpty()) {
				final TransferEnvelope envelope = this.queuedEnvelopes.poll();
				if (envelope.getBuffer() != null) {
					buffersToRecycle.add(envelope.getBuffer());
				}
			}
		}

		while (!buffersToRecycle.isEmpty()) {
			buffersToRecycle.poll().recycleBuffer();
		}
	}

	@Override
	public void logQueuedEnvelopes() {
		int numberOfQueuedEnvelopes = 0;
		int numberOfQueuedEnvelopesWithMemoryBuffers = 0;
		int numberOfQueuedEnvelopesWithFileBuffers = 0;

		synchronized (this.queuedEnvelopes) {

			final Iterator it = this.queuedEnvelopes.iterator();
			while (it.hasNext()) {

				final TransferEnvelope envelope = it.next();
				++numberOfQueuedEnvelopes;
				final Buffer buffer = envelope.getBuffer();
				if (buffer == null) {
					continue;
				}

				if (buffer.isBackedByMemory()) {
					++numberOfQueuedEnvelopesWithMemoryBuffers;
				} else {
					++numberOfQueuedEnvelopesWithFileBuffers;
				}
			}
		}

		System.out.println("\t\t" + getChannelName() + ": " + numberOfQueuedEnvelopes + " ("
			+ numberOfQueuedEnvelopesWithMemoryBuffers + ", " + numberOfQueuedEnvelopesWithFileBuffers + ")");

	}

	@Override
	public Buffer requestEmptyBuffer(int minimumSizeOfBuffer) throws IOException {
		return this.inputGateContext.requestEmptyBuffer(minimumSizeOfBuffer);
	}

	@Override
	public Buffer requestEmptyBufferBlocking(int minimumSizeOfBuffer) throws IOException, InterruptedException {
		return this.inputGateContext.requestEmptyBufferBlocking(minimumSizeOfBuffer);
	}

	@Override
	public int getMaximumBufferSize() {
		return this.inputGateContext.getMaximumBufferSize();
	}

	@Override
	public boolean isShared() {
		return this.inputGateContext.isShared();
	}

	@Override
	public void reportAsynchronousEvent() {
		this.inputGateContext.reportAsynchronousEvent();
	}

	@Override
	public ChannelType getType() {
		return this.byteBufferedInputChannel.getType();
	}

	/**
	 * Constructs and returns a human-readable name of this channel used for debugging.
	 * 
	 * @return a human-readable name of this channel used for debugging
	 */
	private String getChannelName() {
		StringBuilder sb = new StringBuilder(this.inputGateContext.getTaskName());
		sb.append(' ');
		sb.append('(');
		sb.append(this.byteBufferedInputChannel.getChannelIndex());
		sb.append(',');
		sb.append(' ');
		sb.append(this.byteBufferedInputChannel.getID());
		sb.append(')');
		return sb.toString();
	}

	@Override
	public boolean registerBufferAvailabilityListener(final BufferAvailabilityListener bufferAvailabilityListener) {
		return this.inputGateContext.registerBufferAvailabilityListener(bufferAvailabilityListener);
	}
}