eu.stratosphere.nephele.io.RuntimeOutputGate Maven / Gradle / Ivy
/***********************************************************************************************************************
* Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu)
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
**********************************************************************************************************************/
package eu.stratosphere.nephele.io;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import eu.stratosphere.core.io.IOReadableWritable;
import eu.stratosphere.nephele.event.task.AbstractEvent;
import eu.stratosphere.nephele.io.channels.AbstractOutputChannel;
import eu.stratosphere.nephele.io.channels.ChannelID;
import eu.stratosphere.nephele.io.channels.ChannelType;
import eu.stratosphere.nephele.io.channels.bytebuffered.InMemoryOutputChannel;
import eu.stratosphere.nephele.io.channels.bytebuffered.NetworkOutputChannel;
import eu.stratosphere.nephele.jobgraph.JobID;
/**
* In Nephele output gates are a specialization of general gates and connect
* record writers and output channels. As channels, output gates are always
* parameterized to a specific type of record which they can transport.
*
* This class is in general not thread-safe.
*
* @param
* the type of record that can be transported through this gate
*/
public class RuntimeOutputGate extends AbstractGate implements OutputGate {
/**
* The log object used for debugging.
*/
private static final Log LOG = LogFactory.getLog(OutputGate.class);
/**
* The list of output channels attached to this gate.
*/
private final ArrayList> outputChannels = new ArrayList>();
/**
* Channel selector to determine which channel is supposed receive the next record.
*/
private final ChannelSelector channelSelector;
/**
* The class of the record transported through this output gate.
*/
private final Class type;
/**
* Stores whether all records passed to this output gate shall be transmitted through all connected output channels.
*/
private final boolean isBroadcast;
/**
* Constructs a new runtime output gate.
*
* @param jobID
* the ID of the job this input gate belongs to
* @param gateID
* the ID of the gate
* @param inputClass
* the class of the record that can be transported through this
* gate
* @param index
* the index assigned to this output gate at the {@link Environment} object
* @param channelSelector
* the channel selector to be used for this output gate
* @param isBroadcast
* true if every records passed to this output gate shall be transmitted through all connected
* output channels, false otherwise
*/
public RuntimeOutputGate(final JobID jobID, final GateID gateID, final Class inputClass, final int index,
final ChannelSelector channelSelector, final boolean isBroadcast) {
super(jobID, gateID, index);
this.isBroadcast = isBroadcast;
this.type = inputClass;
if (this.isBroadcast) {
this.channelSelector = null;
} else {
if (channelSelector == null) {
this.channelSelector = new DefaultChannelSelector();
} else {
this.channelSelector = channelSelector;
}
}
}
@Override
public final Class getType() {
return this.type;
}
/**
* Adds a new output channel to the output gate.
*
* @param outputChannel
* the output channel to be added.
*/
private void addOutputChannel(AbstractOutputChannel outputChannel) {
if (!this.outputChannels.contains(outputChannel)) {
this.outputChannels.add(outputChannel);
}
}
/**
* Removes the output channel with the given ID from the output gate if it
* exists.
*
* @param outputChannelID
* the ID of the channel to be removed
*/
public void removeOutputChannel(ChannelID outputChannelID) {
for (int i = 0; i < this.outputChannels.size(); i++) {
final AbstractOutputChannel outputChannel = this.outputChannels.get(i);
if (outputChannel.getID().equals(outputChannelID)) {
this.outputChannels.remove(i);
return;
}
}
LOG.debug("Cannot find output channel with ID " + outputChannelID + " to remove");
}
/**
* Removes all output channels from the output gate.
*/
public void removeAllOutputChannels() {
this.outputChannels.clear();
}
@Override
public boolean isInputGate() {
return false;
}
@Override
public int getNumberOfOutputChannels() {
return this.outputChannels.size();
}
/**
* Returns the output channel from position pos of the gate's
* internal channel list.
*
* @param pos
* the position to retrieve the channel from
* @return the channel from the given position or null if such
* position does not exist.
*/
public AbstractOutputChannel getOutputChannel(int pos) {
if (pos < this.outputChannels.size()) {
return this.outputChannels.get(pos);
} else {
return null;
}
}
@Override
public NetworkOutputChannel createNetworkOutputChannel(final OutputGate outputGate,
final ChannelID channelID, final ChannelID connectedChannelID) {
final NetworkOutputChannel enoc = new NetworkOutputChannel(outputGate, this.outputChannels.size(),
channelID, connectedChannelID);
addOutputChannel(enoc);
return enoc;
}
@Override
public InMemoryOutputChannel createInMemoryOutputChannel(final OutputGate outputGate,
final ChannelID channelID, final ChannelID connectedChannelID) {
final InMemoryOutputChannel einoc = new InMemoryOutputChannel(outputGate, this.outputChannels.size(),
channelID, connectedChannelID);
addOutputChannel(einoc);
return einoc;
}
@Override
public void requestClose() throws IOException, InterruptedException {
// Close all output channels
for (int i = 0; i < this.getNumberOfOutputChannels(); i++) {
final AbstractOutputChannel outputChannel = this.getOutputChannel(i);
outputChannel.requestClose();
}
}
@Override
public boolean isClosed() throws IOException, InterruptedException {
boolean allClosed = true;
for (int i = 0; i < this.getNumberOfOutputChannels(); i++) {
final AbstractOutputChannel outputChannel = this.getOutputChannel(i);
if (!outputChannel.isClosed()) {
allClosed = false;
}
}
return allClosed;
}
@Override
public void writeRecord(final T record) throws IOException, InterruptedException {
if (this.isBroadcast) {
if (getChannelType() == ChannelType.INMEMORY) {
final int numberOfOutputChannels = this.outputChannels.size();
for (int i = 0; i < numberOfOutputChannels; ++i) {
this.outputChannels.get(i).writeRecord(record);
}
} else {
// Use optimization for byte buffered channels
this.outputChannels.get(0).writeRecord(record);
}
} else {
// Non-broadcast gate, use channel selector to select output channels
final int numberOfOutputChannels = this.outputChannels.size();
final int[] selectedOutputChannels = this.channelSelector.selectChannels(record, numberOfOutputChannels);
if (selectedOutputChannels == null) {
return;
}
for (int i = 0; i < selectedOutputChannels.length; ++i) {
if (selectedOutputChannels[i] < numberOfOutputChannels) {
final AbstractOutputChannel outputChannel = this.outputChannels.get(selectedOutputChannels[i]);
outputChannel.writeRecord(record);
}
}
}
}
@Override
public List> getOutputChannels() {
return this.outputChannels;
}
@Override
public String toString() {
return "Output " + super.toString();
}
@Override
public void publishEvent(AbstractEvent event) throws IOException, InterruptedException {
// Copy event to all connected channels
final Iterator> it = this.outputChannels.iterator();
while (it.hasNext()) {
it.next().transferEvent(event);
}
}
@Override
public void flush() throws IOException, InterruptedException {
// Flush all connected channels
final Iterator> it = this.outputChannels.iterator();
while (it.hasNext()) {
it.next().flush();
}
}
@Override
public boolean isBroadcast() {
return this.isBroadcast;
}
@Override
public ChannelSelector getChannelSelector() {
return this.channelSelector;
}
@Override
public void releaseAllChannelResources() {
final Iterator> it = this.outputChannels.iterator();
while (it.hasNext()) {
it.next().releaseAllResources();
}
}
}