org.apache.flink.graph.Graph Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of flink-gelly Show documentation
There is a newer version: 1.16.3
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.graph;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.List;
import java.util.Arrays;

import org.apache.flink.api.common.functions.CoGroupFunction;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.api.common.functions.JoinFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.functions.FunctionAnnotation.ForwardedFields;
import org.apache.flink.api.java.functions.FunctionAnnotation.ForwardedFieldsFirst;
import org.apache.flink.api.java.functions.FunctionAnnotation.ForwardedFieldsSecond;
import org.apache.flink.api.java.tuple.Tuple1;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple4;
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.graph.gsa.ApplyFunction;
import org.apache.flink.graph.gsa.GSAConfiguration;
import org.apache.flink.graph.gsa.GatherFunction;
import org.apache.flink.graph.gsa.GatherSumApplyIteration;
import org.apache.flink.graph.gsa.SumFunction;
import org.apache.flink.graph.spargel.MessagingFunction;
import org.apache.flink.graph.spargel.VertexCentricConfiguration;
import org.apache.flink.graph.spargel.VertexCentricIteration;
import org.apache.flink.graph.spargel.VertexUpdateFunction;
import org.apache.flink.graph.utils.EdgeToTuple3Map;
import org.apache.flink.graph.utils.Tuple2ToVertexMap;
import org.apache.flink.graph.utils.Tuple3ToEdgeMap;
import org.apache.flink.graph.utils.VertexToTuple2Map;
import org.apache.flink.graph.validation.GraphValidator;
import org.apache.flink.util.Collector;
import org.apache.flink.types.NullValue;

/**
 * Represents a Graph consisting of {@link Edge edges} and {@link Vertex
 * vertices}.
 * 
 * 
 * @see org.apache.flink.graph.Edge
 * @see org.apache.flink.graph.Vertex
 * 
 * @param  the key type for edge and vertex identifiers
 * @param  the value type for vertices
 * @param  the value type for edges
 */
@SuppressWarnings("serial")
public class Graph {

	private final ExecutionEnvironment context;
	private final DataSet> vertices;
	private final DataSet> edges;

	/**
	 * Creates a graph from two DataSets: vertices and edges
	 * 
	 * @param vertices a DataSet of vertices.
	 * @param edges a DataSet of edges.
	 * @param context the flink execution environment.
	 */
	private Graph(DataSet> vertices, DataSet> edges, ExecutionEnvironment context) {
		this.vertices = vertices;
		this.edges = edges;
		this.context = context;
	}

	/**
	 * Creates a graph from a Collection of vertices and a Collection of edges.
	 * 
	 * @param vertices a Collection of vertices.
	 * @param edges a Collection of edges.
	 * @param context the flink execution environment.
	 * @return the newly created graph.
	 */
	public static  Graph fromCollection(Collection> vertices,
			Collection> edges, ExecutionEnvironment context) {

		return fromDataSet(context.fromCollection(vertices),
				context.fromCollection(edges), context);
	}

	/**
	 * Creates a graph from a Collection of edges, vertices are induced from the
	 * edges. Vertices are created automatically and their values are set to
	 * NullValue.
	 * 
	 * @param edges a Collection of vertices.
	 * @param context the flink execution environment.
	 * @return the newly created graph.
	 */
	public static  Graph fromCollection(Collection> edges,
			ExecutionEnvironment context) {

		return fromDataSet(context.fromCollection(edges), context);
	}

	/**
	 * Creates a graph from a Collection of edges, vertices are induced from the
	 * edges and vertex values are calculated by a mapper function. Vertices are
	 * created automatically and their values are set by applying the provided
	 * map function to the vertex ids.
	 * 
	 * @param edges a Collection of edges.
	 * @param mapper the mapper function.
	 * @param context the flink execution environment.
	 * @return the newly created graph.
	 */
	public static  Graph fromCollection(Collection> edges,
			final MapFunction mapper,ExecutionEnvironment context) {

		return fromDataSet(context.fromCollection(edges), mapper, context);
	}

	/**
	 * Creates a graph from a DataSet of vertices and a DataSet of edges.
	 * 
	 * @param vertices a DataSet of vertices.
	 * @param edges a DataSet of edges.
	 * @param context the flink execution environment.
	 * @return the newly created graph.
	 */
	public static  Graph fromDataSet(DataSet> vertices,
			DataSet> edges, ExecutionEnvironment context) {

		return new Graph(vertices, edges, context);
	}

	/**
	 * Creates a graph from a DataSet of edges, vertices are induced from the
	 * edges. Vertices are created automatically and their values are set to
	 * NullValue.
	 * 
	 * @param edges a DataSet of edges.
	 * @param context the flink execution environment.
	 * @return the newly created graph.
	 */
	public static  Graph fromDataSet(
			DataSet> edges, ExecutionEnvironment context) {

		DataSet> vertices = edges.flatMap(new EmitSrcAndTarget()).distinct();

		return new Graph(vertices, edges, context);
	}

	private static final class EmitSrcAndTarget implements FlatMapFunction<
		Edge, Vertex> {

		public void flatMap(Edge edge, Collector> out) {
			out.collect(new Vertex(edge.f0, NullValue.getInstance()));
			out.collect(new Vertex(edge.f1, NullValue.getInstance()));
		}
	}

	/**
	 * Creates a graph from a DataSet of edges, vertices are induced from the
	 * edges and vertex values are calculated by a mapper function. Vertices are
	 * created automatically and their values are set by applying the provided
	 * map function to the vertex ids.
	 * 
	 * @param edges a DataSet of edges.
	 * @param mapper the mapper function.
	 * @param context the flink execution environment.
	 * @return the newly created graph.
	 */
	public static  Graph fromDataSet(DataSet> edges,
			final MapFunction mapper, ExecutionEnvironment context) {

		TypeInformation keyType = ((TupleTypeInfo) edges.getType()).getTypeAt(0);

		TypeInformation valueType = TypeExtractor.createTypeInfo(
				MapFunction.class, mapper.getClass(), 1, null, null);

		@SuppressWarnings({ "unchecked", "rawtypes" })
		TypeInformation> returnType = (TypeInformation>) new TupleTypeInfo(
				Vertex.class, keyType, valueType);

		DataSet> vertices = edges
				.flatMap(new EmitSrcAndTargetAsTuple1()).distinct()
				.map(new MapFunction, Vertex>() {
					public Vertex map(Tuple1 value) throws Exception {
						return new Vertex(value.f0, mapper.map(value.f0));
					}
				}).returns(returnType).withForwardedFields("f0");

		return new Graph(vertices, edges, context);
	}

	private static final class EmitSrcAndTargetAsTuple1 implements FlatMapFunction<
		Edge, Tuple1> {

		public void flatMap(Edge edge, Collector> out) {
			out.collect(new Tuple1(edge.f0));
			out.collect(new Tuple1(edge.f1));
		}
	}

	/**
	 * Creates a graph from a DataSet of Tuple objects for vertices and edges.
	 * 
	 * Vertices with value are created from Tuple2, Edges with value are created
	 * from Tuple3.
	 * 
	 * @param vertices a DataSet of Tuple2.
	 * @param edges a DataSet of Tuple3.
	 * @param context the flink execution environment.
	 * @return the newly created graph.
	 */
	public static  Graph fromTupleDataSet(DataSet> vertices,
			DataSet> edges, ExecutionEnvironment context) {

		DataSet> vertexDataSet = vertices.map(new Tuple2ToVertexMap());
		DataSet> edgeDataSet = edges.map(new Tuple3ToEdgeMap());
		return fromDataSet(vertexDataSet, edgeDataSet, context);
	}

	/**
	 * Creates a graph from a DataSet of Tuple objects for edges, vertices are
	 * induced from the edges.
	 * 
	 * Edges with value are created from Tuple3. Vertices are created
	 * automatically and their values are set to NullValue.
	 * 
	 * @param edges a DataSet of Tuple3.
	 * @param context the flink execution environment.
	 * @return the newly created graph.
	 */
	public static  Graph fromTupleDataSet(DataSet> edges,
			ExecutionEnvironment context) {

		DataSet> edgeDataSet = edges.map(new Tuple3ToEdgeMap());
		return fromDataSet(edgeDataSet, context);
	}

	/**
	 * Creates a graph from a DataSet of Tuple objects for edges, vertices are
	 * induced from the edges and vertex values are calculated by a mapper
	 * function. Edges with value are created from Tuple3. Vertices are created
	 * automatically and their values are set by applying the provided map
	 * function to the vertex ids.
	 * 
	 * @param edges a DataSet of Tuple3.
	 * @param mapper the mapper function.
	 * @param context the flink execution environment.
	 * @return the newly created graph.
	 */
	public static  Graph fromTupleDataSet(DataSet> edges,
			final MapFunction mapper, ExecutionEnvironment context) {

		DataSet> edgeDataSet = edges.map(new Tuple3ToEdgeMap());
		return fromDataSet(edgeDataSet, mapper, context);
	}

	/**
	 * @return the flink execution environment.
	 */
	public ExecutionEnvironment getContext() {
		return this.context;
	}

	/**
	 * Function that checks whether a Graph is a valid Graph,
	 * as defined by the given {@link GraphValidator}.
	 * 
	 * @return true if the Graph is valid.
	 */
	public Boolean validate(GraphValidator validator) throws Exception {
		return validator.validate(this);
	}

	/**
	 * @return the vertex DataSet.
	 */
	public DataSet> getVertices() {
		return vertices;
	}

	/**
	 * @return the edge DataSet.
	 */
	public DataSet> getEdges() {
		return edges;
	}

	/**
	 * @return the vertex DataSet as Tuple2.
	 */
	public DataSet> getVerticesAsTuple2() {
		return vertices.map(new VertexToTuple2Map());
	}

	/**
	 * @return the edge DataSet as Tuple3.
	 */
	public DataSet> getEdgesAsTuple3() {
		return edges.map(new EdgeToTuple3Map());
	}

	/**
	 * This method allows access to the graph's edge values along with its source and target vertex values.
	 *
	 * @return a triplet DataSet consisting of (srcVertexId, trgVertexId, srcVertexValue, trgVertexValue, edgeValue)
	 */
	public DataSet> getTriplets() {
		return this.getVertices().join(this.getEdges()).where(0).equalTo(0)
				.with(new ProjectEdgeWithSrcValue())
				.join(this.getVertices()).where(1).equalTo(0)
				.with(new ProjectEdgeWithVertexValues());
	}

	@ForwardedFieldsFirst("f1->f2")
	@ForwardedFieldsSecond("f0; f1; f2->f3")
	private static final class ProjectEdgeWithSrcValue implements
			FlatJoinFunction, Edge, Tuple4> {

		@Override
		public void join(Vertex vertex, Edge edge, Collector> collector)
				throws Exception {

			collector.collect(new Tuple4(edge.getSource(), edge.getTarget(), vertex.getValue(),
					edge.getValue()));
		}
	}

	@ForwardedFieldsFirst("f0; f1; f2; f3->f4")
	@ForwardedFieldsSecond("f1->f3")
	private static final class ProjectEdgeWithVertexValues implements
			FlatJoinFunction, Vertex, Triplet> {

		@Override
		public void join(Tuple4 tripletWithSrcValSet,
						Vertex vertex, Collector> collector) throws Exception {

			collector.collect(new Triplet(tripletWithSrcValSet.f0, tripletWithSrcValSet.f1,
					tripletWithSrcValSet.f2, vertex.getValue(), tripletWithSrcValSet.f3));
		}
	}


	/**
	 * Apply a function to the attribute of each vertex in the graph.
	 * 
	 * @param mapper the map function to apply.
	 * @return a new graph
	 */
	@SuppressWarnings({ "unchecked", "rawtypes" })
	public  Graph mapVertices(final MapFunction, NV> mapper) {

		TypeInformation keyType = ((TupleTypeInfo) vertices.getType()).getTypeAt(0);

		TypeInformation valueType = TypeExtractor.createTypeInfo(MapFunction.class, mapper.getClass(), 1, null, null);

		TypeInformation> returnType = (TypeInformation>) new TupleTypeInfo(
				Vertex.class, keyType, valueType);

		DataSet> mappedVertices = vertices.map(
				new MapFunction, Vertex>() {
					public Vertex map(Vertex value) throws Exception {
						return new Vertex(value.f0, mapper.map(value));
					}
				})
				.returns(returnType)
				.withForwardedFields("f0");

		return new Graph(mappedVertices, this.edges, this.context);
	}

	/**
	 * Apply a function to the attribute of each edge in the graph.
	 * 
	 * @param mapper the map function to apply.
	 * @return a new graph
	 */
	@SuppressWarnings({ "unchecked", "rawtypes" })
	public  Graph mapEdges(final MapFunction, NV> mapper) {

		TypeInformation keyType = ((TupleTypeInfo) edges.getType()).getTypeAt(0);

		TypeInformation valueType = TypeExtractor.createTypeInfo(MapFunction.class, mapper.getClass(), 1, null, null);

		TypeInformation> returnType = (TypeInformation>) new TupleTypeInfo(
				Edge.class, keyType, keyType, valueType);

		DataSet> mappedEdges = edges.map(
				new MapFunction, Edge>() {
					public Edge map(Edge value) throws Exception {
						return new Edge(value.f0, value.f1, mapper
								.map(value));
					}
				})
				.returns(returnType)
				.withForwardedFields("f0; f1");

		return new Graph(this.vertices, mappedEdges, this.context);
	}

	/**
	 * Joins the vertex DataSet of this graph with an input DataSet and applies
	 * a UDF on the resulted values.
	 * 
	 * @param inputDataSet the DataSet to join with.
	 * @param mapper the UDF map function to apply.
	 * @return a new graph where the vertex values have been updated.
	 */
	public  Graph joinWithVertices(DataSet> inputDataSet, 
			final MapFunction, VV> mapper) {

		DataSet> resultedVertices = this.getVertices()
				.coGroup(inputDataSet).where(0).equalTo(0)
				.with(new ApplyCoGroupToVertexValues(mapper));
		return new Graph(resultedVertices, this.edges, this.context);
	}

	private static final class ApplyCoGroupToVertexValues
			implements CoGroupFunction, Tuple2, Vertex> {

		private MapFunction, VV> mapper;

		public ApplyCoGroupToVertexValues(MapFunction, VV> mapper) {
			this.mapper = mapper;
		}

		@Override
		public void coGroup(Iterable> vertices,
				Iterable> input, Collector> collector) throws Exception {

			final Iterator> vertexIterator = vertices.iterator();
			final Iterator> inputIterator = input.iterator();

			if (vertexIterator.hasNext()) {
				if (inputIterator.hasNext()) {
					final Tuple2 inputNext = inputIterator.next();

					collector.collect(new Vertex(inputNext.f0, mapper
							.map(new Tuple2(vertexIterator.next().f1,
									inputNext.f1))));
				} else {
					collector.collect(vertexIterator.next());
				}

			}
		}
	}

	/**
	 * Joins the edge DataSet with an input DataSet on a composite key of both
	 * source and target and applies a UDF on the resulted values.
	 * 
	 * @param inputDataSet the DataSet to join with.
	 * @param mapper the UDF map function to apply.
	 * @param  the return type
	 * @return a new graph where the edge values have been updated.
	 */
	public  Graph joinWithEdges(DataSet> inputDataSet,
			final MapFunction, EV> mapper) {

		DataSet> resultedEdges = this.getEdges()
				.coGroup(inputDataSet).where(0, 1).equalTo(0, 1)
				.with(new ApplyCoGroupToEdgeValues(mapper));
		return new Graph(this.vertices, resultedEdges, this.context);
	}

	private static final class ApplyCoGroupToEdgeValues
			implements CoGroupFunction, Tuple3, Edge> {

		private MapFunction, EV> mapper;

		public ApplyCoGroupToEdgeValues(MapFunction, EV> mapper) {
			this.mapper = mapper;
		}

		@Override
		public void coGroup(Iterable> edges, Iterable> input,
				Collector> collector) throws Exception {

			final Iterator> edgesIterator = edges.iterator();
			final Iterator> inputIterator = input.iterator();

			if (edgesIterator.hasNext()) {
				if (inputIterator.hasNext()) {
					final Tuple3 inputNext = inputIterator.next();

					collector.collect(new Edge(inputNext.f0,
							inputNext.f1, mapper.map(new Tuple2(
									edgesIterator.next().f2, inputNext.f2))));
				} else {
					collector.collect(edgesIterator.next());
				}
			}
		}
	}

	/**
	 * Joins the edge DataSet with an input DataSet on the source key of the
	 * edges and the first attribute of the input DataSet and applies a UDF on
	 * the resulted values. In case the inputDataSet contains the same key more
	 * than once, only the first value will be considered.
	 * 
	 * @param inputDataSet the DataSet to join with.
	 * @param mapper the UDF map function to apply.
	 * @param  the return type
	 * @return a new graph where the edge values have been updated.
	 */
	public  Graph joinWithEdgesOnSource(DataSet> inputDataSet,
			final MapFunction, EV> mapper) {

		DataSet> resultedEdges = this.getEdges()
				.coGroup(inputDataSet).where(0).equalTo(0)
				.with(new ApplyCoGroupToEdgeValuesOnEitherSourceOrTarget(mapper));

		return new Graph(this.vertices, resultedEdges, this.context);
	}

	private static final class ApplyCoGroupToEdgeValuesOnEitherSourceOrTarget
			implements CoGroupFunction, Tuple2, Edge> {

		private MapFunction, EV> mapper;

		public ApplyCoGroupToEdgeValuesOnEitherSourceOrTarget(
				MapFunction, EV> mapper) {
			this.mapper = mapper;
		}

		@Override
		public void coGroup(Iterable> edges,
				Iterable> input, Collector> collector) throws Exception {

			final Iterator> edgesIterator = edges.iterator();
			final Iterator> inputIterator = input.iterator();

			if (inputIterator.hasNext()) {
				final Tuple2 inputNext = inputIterator.next();

				while (edgesIterator.hasNext()) {
					Edge edgesNext = edgesIterator.next();

					collector.collect(new Edge(edgesNext.f0,
							edgesNext.f1, mapper.map(new Tuple2(
									edgesNext.f2, inputNext.f1))));
				}

			} else {
				while (edgesIterator.hasNext()) {
					collector.collect(edgesIterator.next());
				}
			}
		}
	}

	/**
	 * Joins the edge DataSet with an input DataSet on the target key of the
	 * edges and the first attribute of the input DataSet and applies a UDF on
	 * the resulted values. Should the inputDataSet contain the same key more
	 * than once, only the first value will be considered.
	 * 
	 * @param inputDataSet the DataSet to join with.
	 * @param mapper the UDF map function to apply.
	 * @param  the return type
	 * @return a new graph where the edge values have been updated.
	 */
	public  Graph joinWithEdgesOnTarget(DataSet> inputDataSet,
			final MapFunction, EV> mapper) {

		DataSet> resultedEdges = this.getEdges()
				.coGroup(inputDataSet).where(1).equalTo(0)
				.with(new ApplyCoGroupToEdgeValuesOnEitherSourceOrTarget(mapper));

		return new Graph(this.vertices, resultedEdges, this.context);
	}

	/**
	 * Apply filtering functions to the graph and return a sub-graph that
	 * satisfies the predicates for both vertices and edges.
	 * 
	 * @param vertexFilter the filter function for vertices.
	 * @param edgeFilter the filter function for edges.
	 * @return the resulting sub-graph.
	 */
	public Graph subgraph(FilterFunction> vertexFilter, FilterFunction> edgeFilter) {

		DataSet> filteredVertices = this.vertices.filter(vertexFilter);

		DataSet> remainingEdges = this.edges.join(filteredVertices)
				.where(0).equalTo(0).with(new ProjectEdge())
				.join(filteredVertices).where(1).equalTo(0)
				.with(new ProjectEdge());

		DataSet> filteredEdges = remainingEdges.filter(edgeFilter);

		return new Graph(filteredVertices, filteredEdges,
				this.context);
	}

	/**
	 * Apply a filtering function to the graph and return a sub-graph that
	 * satisfies the predicates only for the vertices.
	 * 
	 * @param vertexFilter the filter function for vertices.
	 * @return the resulting sub-graph.
	 */
	public Graph filterOnVertices(FilterFunction> vertexFilter) {

		DataSet> filteredVertices = this.vertices.filter(vertexFilter);

		DataSet> remainingEdges = this.edges.join(filteredVertices)
				.where(0).equalTo(0).with(new ProjectEdge())
				.join(filteredVertices).where(1).equalTo(0)
				.with(new ProjectEdge());

		return new Graph(filteredVertices, remainingEdges, this.context);
	}

	/**
	 * Apply a filtering function to the graph and return a sub-graph that
	 * satisfies the predicates only for the edges.
	 * 
	 * @param edgeFilter the filter function for edges.
	 * @return the resulting sub-graph.
	 */
	public Graph filterOnEdges(FilterFunction> edgeFilter) {
		DataSet> filteredEdges = this.edges.filter(edgeFilter);

		return new Graph(this.vertices, filteredEdges, this.context);
	}

	@ForwardedFieldsFirst("f0; f1; f2")
	private static final class ProjectEdge implements FlatJoinFunction<
		Edge, Vertex, Edge> {
		public void join(Edge first, Vertex second, Collector> out) {
			out.collect(first);
		}
	}

	/**
	 * Return the out-degree of all vertices in the graph
	 * 
	 * @return A DataSet of Tuple2
	 */
	public DataSet> outDegrees() {

		return vertices.coGroup(edges).where(0).equalTo(0).with(new CountNeighborsCoGroup());
	}

	private static final class CountNeighborsCoGroup
			implements CoGroupFunction, Edge, Tuple2> {
		@SuppressWarnings("unused")
		public void coGroup(Iterable> vertex,	Iterable> outEdges,
				Collector> out) {
			long count = 0;
			for (Edge edge : outEdges) {
				count++;
			}

			Iterator> vertexIterator = vertex.iterator();

			if(vertexIterator.hasNext()) {
				out.collect(new Tuple2(vertexIterator.next().f0, count));
			} else {
				throw new NoSuchElementException("The edge src/trg id could not be found within the vertexIds");
			}
		}
	}

	/**
	 * Return the in-degree of all vertices in the graph
	 * 
	 * @return A DataSet of Tuple2
	 */
	public DataSet> inDegrees() {

		return vertices.coGroup(edges).where(0).equalTo(1).with(new CountNeighborsCoGroup());
	}

	/**
	 * Return the degree of all vertices in the graph
	 * 
	 * @return A DataSet of Tuple2
	 */
	public DataSet> getDegrees() {
		return outDegrees().union(inDegrees()).groupBy(0).sum(1);
	}

	/**
	 * This operation adds all inverse-direction edges to the graph.
	 * 
	 * @return the undirected graph.
	 */
	public Graph getUndirected() {

		DataSet> undirectedEdges = edges.flatMap(new RegularAndReversedEdgesMap());
		return new Graph(vertices, undirectedEdges, this.context);
	}

	/**
	 * Compute an aggregate over the edges of each vertex. The function applied
	 * on the edges has access to the vertex value.
	 * 
	 * @param edgesFunction
	 *            the function to apply to the neighborhood
	 * @param direction
	 *            the edge direction (in-, out-, all-)
	 * @param 
	 *            the output type
	 * @return a dataset of a T
	 * @throws IllegalArgumentException
	 */
	public  DataSet groupReduceOnEdges(EdgesFunctionWithVertexValue edgesFunction,
											EdgeDirection direction) throws IllegalArgumentException {

		switch (direction) {
		case IN:
			return vertices.coGroup(edges).where(0).equalTo(1)
					.with(new ApplyCoGroupFunction(edgesFunction));
		case OUT:
			return vertices.coGroup(edges).where(0).equalTo(0)
					.with(new ApplyCoGroupFunction(edgesFunction));
		case ALL:
			return vertices.coGroup(edges.flatMap(new EmitOneEdgePerNode()))
					.where(0).equalTo(0).with(new ApplyCoGroupFunctionOnAllEdges(edgesFunction));
		default:
			throw new IllegalArgumentException("Illegal edge direction");
		}
	}

	/**
	 * Compute an aggregate over the edges of each vertex. The function applied
	 * on the edges only has access to the vertex id (not the vertex value).
	 * 
	 * @param edgesFunction
	 *            the function to apply to the neighborhood
	 * @param direction
	 *            the edge direction (in-, out-, all-)
	 * @param 
	 *            the output type
	 * @return a dataset of T
	 * @throws IllegalArgumentException
	 */
	public  DataSet groupReduceOnEdges(EdgesFunction edgesFunction,
											EdgeDirection direction) throws IllegalArgumentException {

		switch (direction) {
		case IN:
			return edges.map(new ProjectVertexIdMap(1))
					.withForwardedFields("f1->f0")
					.groupBy(0).reduceGroup(new ApplyGroupReduceFunction(edgesFunction));
		case OUT:
			return edges.map(new ProjectVertexIdMap(0))
					.withForwardedFields("f0")
					.groupBy(0).reduceGroup(new ApplyGroupReduceFunction(edgesFunction));
		case ALL:
			return edges.flatMap(new EmitOneEdgePerNode())
					.groupBy(0).reduceGroup(new ApplyGroupReduceFunction(edgesFunction));
		default:
			throw new IllegalArgumentException("Illegal edge direction");
		}
	}

	private static final class ProjectVertexIdMap implements MapFunction<
		Edge, Tuple2>> {

		private int fieldPosition;

		public ProjectVertexIdMap(int position) {
			this.fieldPosition = position;
		}

		@SuppressWarnings("unchecked")
		public Tuple2> map(Edge edge) {
			return new Tuple2>((K) edge.getField(fieldPosition),	edge);
		}
	}

	private static final class ProjectVertexWithEdgeValueMap	implements MapFunction<
		Edge, Tuple2> {

		private int fieldPosition;

		public ProjectVertexWithEdgeValueMap(int position) {
			this.fieldPosition = position;
		}

		@SuppressWarnings("unchecked")
		public Tuple2 map(Edge edge) {
			return new Tuple2((K) edge.getField(fieldPosition),	edge.getValue());
		}
	}

	private static final class ApplyGroupReduceFunction implements GroupReduceFunction<
		Tuple2>, T>,	ResultTypeQueryable {

		private EdgesFunction function;

		public ApplyGroupReduceFunction(EdgesFunction fun) {
			this.function = fun;
		}

		public void reduce(Iterable>> edges, Collector out) throws Exception {
			function.iterateEdges(edges, out);
		}

		@Override
		public TypeInformation getProducedType() {
			return TypeExtractor.createTypeInfo(EdgesFunction.class, function.getClass(), 2, null, null);
		}
	}

	private static final class EmitOneEdgePerNode implements FlatMapFunction<
		Edge, Tuple2>> {

		public void flatMap(Edge edge, Collector>> out) {
			out.collect(new Tuple2>(edge.getSource(), edge));
			out.collect(new Tuple2>(edge.getTarget(), edge));
		}
	}

	private static final class EmitOneVertexWithEdgeValuePerNode	implements FlatMapFunction<
		Edge, Tuple2> {

		public void flatMap(Edge edge, Collector> out) {
			out.collect(new Tuple2(edge.getSource(), edge.getValue()));
			out.collect(new Tuple2(edge.getTarget(), edge.getValue()));
		}
	}

	private static final class EmitOneEdgeWithNeighborPerNode implements FlatMapFunction<
		Edge, Tuple3>> {

		public void flatMap(Edge edge, Collector>> out) {
			out.collect(new Tuple3>(edge.getSource(), edge.getTarget(), edge));
			out.collect(new Tuple3>(edge.getTarget(), edge.getSource(), edge));
		}
	}

	private static final class ApplyCoGroupFunction implements CoGroupFunction<
		Vertex, Edge, T>, ResultTypeQueryable {

		private EdgesFunctionWithVertexValue function;

		public ApplyCoGroupFunction(EdgesFunctionWithVertexValue fun) {
			this.function = fun;
		}

		public void coGroup(Iterable> vertex,
				Iterable> edges, Collector out) throws Exception {
			function.iterateEdges(vertex.iterator().next(), edges, out);
		}

		@Override
		public TypeInformation getProducedType() {
			return TypeExtractor.createTypeInfo(EdgesFunctionWithVertexValue.class, function.getClass(), 3,
					null, null);
		}
	}

	private static final class ApplyCoGroupFunctionOnAllEdges
			implements	CoGroupFunction, Tuple2>, T>, ResultTypeQueryable {

		private EdgesFunctionWithVertexValue function;

		public ApplyCoGroupFunctionOnAllEdges(EdgesFunctionWithVertexValue fun) {
			this.function = fun;
		}

		public void coGroup(Iterable> vertex,	final Iterable>> keysWithEdges,
				Collector out) throws Exception {

			final Iterator> edgesIterator = new Iterator>() {

				final Iterator>> keysWithEdgesIterator = keysWithEdges.iterator();

				@Override
				public boolean hasNext() {
					return keysWithEdgesIterator.hasNext();
				}

				@Override
				public Edge next() {
					return keysWithEdgesIterator.next().f1;
				}

				@Override
				public void remove() {
					keysWithEdgesIterator.remove();
				}
			};

			Iterable> edgesIterable = new Iterable>() {
				public Iterator> iterator() {
					return edgesIterator;
				}
			};

			function.iterateEdges(vertex.iterator().next(),	edgesIterable, out);
		}

		@Override
		public TypeInformation getProducedType() {
			return TypeExtractor.createTypeInfo(EdgesFunctionWithVertexValue.class, function.getClass(), 3,
					null, null);
		}
	}

	@ForwardedFields("f0->f1; f1->f0; f2")
	private static final class ReverseEdgesMap
			implements MapFunction, Edge> {

		public Edge map(Edge value) {
			return new Edge(value.f1, value.f0, value.f2);
		}
	}

	private static final class RegularAndReversedEdgesMap
			implements FlatMapFunction, Edge> {

		@Override
		public void flatMap(Edge edge, Collector> out) throws Exception {
			out.collect(new Edge(edge.f0, edge.f1, edge.f2));
			out.collect(new Edge(edge.f1, edge.f0, edge.f2));
		}
	}

	/**
	 * Reverse the direction of the edges in the graph
	 * 
	 * @return a new graph with all edges reversed
	 * @throws UnsupportedOperationException
	 */
	public Graph reverse() throws UnsupportedOperationException {
		DataSet> reversedEdges = edges.map(new ReverseEdgesMap());
		return new Graph(vertices, reversedEdges, this.context);
	}

	/**
	 * @return a long integer representing the number of vertices
	 */
	public long numberOfVertices() throws Exception {
		return vertices.count();
	}

	/**
	 * @return a long integer representing the number of edges
	 */
	public long numberOfEdges() throws Exception {
		return edges.count();
	}

	/**
	 * @return The IDs of the vertices as DataSet
	 */
	public DataSet getVertexIds() {
		return vertices.map(new ExtractVertexIDMapper());
	}

	private static final class ExtractVertexIDMapper
			implements MapFunction, K> {
		@Override
		public K map(Vertex vertex) {
			return vertex.f0;
		}
	}

	/**
	 * @return The IDs of the edges as DataSet
	 */
	public DataSet> getEdgeIds() {
		return edges.map(new ExtractEdgeIDsMapper());
	}

	@ForwardedFields("f0; f1")
	private static final class ExtractEdgeIDsMapper
			implements MapFunction, Tuple2> {
		@Override
		public Tuple2 map(Edge edge) throws Exception {
			return new Tuple2(edge.f0, edge.f1);
		}
	}

	/**
	 * Adds the input vertex to the graph. If the vertex already
	 * exists in the graph, it will not be added again.
	 * 
	 * @param vertex the vertex to be added
	 * @return the new graph containing the existing vertices as well as the one just added
	 */
	@SuppressWarnings("unchecked")
	public Graph addVertex(final Vertex vertex) {
		List> newVertex = new ArrayList>();
		newVertex.add(vertex);

		return addVertices(newVertex);
	}

	/**
	 * Adds the list of vertices, passed as input, to the graph.
	 * If the vertices already exist in the graph, they will not be added once more.
	 *
	 * @param verticesToAdd the list of vertices to add
	 * @return the new graph containing the existing and newly added vertices
	 */
	@SuppressWarnings("unchecked")
	public Graph addVertices(List> verticesToAdd) {
		// Add the vertices
		DataSet> newVertices = this.vertices.union(this.context.fromCollection(verticesToAdd)).distinct();

		return new Graph(newVertices, this.edges, this.context);
	}

	/**
	 * Adds the given edge to the graph. If the source and target vertices do
	 * not exist in the graph, they will also be added.
	 * 
	 * @param source the source vertex of the edge
	 * @param target the target vertex of the edge
	 * @param edgeValue the edge value
	 * @return the new graph containing the existing vertices and edges plus the
	 *         newly added edge
	 */
	@SuppressWarnings("unchecked")
	public Graph addEdge(Vertex source, Vertex target, EV edgeValue) {
		Graph partialGraph = fromCollection(Arrays.asList(source, target),
				Arrays.asList(new Edge(source.f0, target.f0, edgeValue)),
				this.context);
		return this.union(partialGraph);
	}

	/**
	 * Adds the given list edges to the graph.
	 *
	 * When adding an edge for a non-existing set of vertices, the edge is considered invalid and ignored.
	 *
	 * @param newEdges the data set of edges to be added
	 * @return a new graph containing the existing edges plus the newly added edges.
	 */
	@SuppressWarnings("unchecked")
	public Graph addEdges(List> newEdges) {

		DataSet> newEdgesDataSet = this.context.fromCollection(newEdges);

		DataSet> validNewEdges = this.getVertices().join(newEdgesDataSet)
				.where(0).equalTo(0)
				.with(new JoinVerticesWithEdgesOnSrc())
				.join(this.getVertices()).where(1).equalTo(0)
				.with(new JoinWithVerticesOnTrg());

		return Graph.fromDataSet(this.vertices, this.edges.union(validNewEdges), this.context);
	}

	@ForwardedFieldsSecond("f0; f1; f2")
	private static final class JoinVerticesWithEdgesOnSrc implements
			JoinFunction, Edge, Edge> {

		@Override
		public Edge join(Vertex vertex, Edge edge) throws Exception {
			return edge;
		}
	}

	@ForwardedFieldsFirst("f0; f1; f2")
	private static final class JoinWithVerticesOnTrg implements
			JoinFunction, Vertex, Edge> {

		@Override
		public Edge join(Edge edge, Vertex vertex) throws Exception {
			return edge;
		}
	}

	/**
	 * Removes the given vertex and its edges from the graph.
	 * 
	 * @param vertex the vertex to remove
	 * @return the new graph containing the existing vertices and edges without
	 *         the removed vertex and its edges
	 */
	public Graph removeVertex(Vertex vertex) {

		List> vertexToBeRemoved = new ArrayList>();
		vertexToBeRemoved.add(vertex);

		return removeVertices(vertexToBeRemoved);
	}

	/**
	 * Removes the given list of vertices and its edges from the graph.
	 *
	 * @param verticesToBeRemoved the list of vertices to be removed
	 * @return the resulted graph containing the initial vertices and edges minus the vertices
	 * 		   and edges removed.
	 */
	public Graph removeVertices(List> verticesToBeRemoved) {

		DataSet> newVertices = getVertices().coGroup(this.context.fromCollection(verticesToBeRemoved)).where(0).equalTo(0)
				.with(new VerticesRemovalCoGroup());

		DataSet < Edge < K, EV >> newEdges = newVertices.join(getEdges()).where(0).equalTo(0)
				// if the edge source was removed, the edge will also be removed
				.with(new ProjectEdgeToBeRemoved())
				// if the edge target was removed, the edge will also be removed
				.join(newVertices).where(1).equalTo(0)
				.with(new ProjectEdge());

		return new Graph(newVertices, newEdges, context);
	}

	private static final class VerticesRemovalCoGroup implements CoGroupFunction, Vertex, Vertex> {

		@Override
		public void coGroup(Iterable> vertex, Iterable> vertexToBeRemoved,
							Collector> out) throws Exception {

			final Iterator> vertexIterator = vertex.iterator();
			final Iterator> vertexToBeRemovedIterator = vertexToBeRemoved.iterator();
			Vertex next;

			if (vertexIterator.hasNext()) {
				if (!vertexToBeRemovedIterator.hasNext()) {
					next = vertexIterator.next();
					out.collect(next);
				}
			}
		}
	}

	@ForwardedFieldsSecond("f0; f1; f2")
	private static final class ProjectEdgeToBeRemoved implements JoinFunction, Edge, Edge> {
		@Override
		public Edge join(Vertex vertex, Edge edge) throws Exception {
			return edge;
		}
	}

	 /**
	 * Removes all edges that match the given edge from the graph.
	 * 
	 * @param edge the edge to remove
	 * @return the new graph containing the existing vertices and edges without
	 *         the removed edges
	 */
	public Graph removeEdge(Edge edge) {
		DataSet> newEdges = getEdges().filter(new EdgeRemovalEdgeFilter(edge));
		return new Graph(this.vertices, newEdges, this.context);
	}

	private static final class EdgeRemovalEdgeFilter
			implements FilterFunction> {
		private Edge edgeToRemove;

		public EdgeRemovalEdgeFilter(Edge edge) {
			edgeToRemove = edge;
		}

		@Override
		public boolean filter(Edge edge) {
			return (!(edge.f0.equals(edgeToRemove.f0) && edge.f1
					.equals(edgeToRemove.f1)));
		}
	}

	/**
	 * Removes all the edges that match the edges in the given data set from the graph.
	 *
	 * @param edgesToBeRemoved the list of edges to be removed
	 * @return a new graph where the edges have been removed and in which the vertices remained intact
	 */
	public Graph removeEdges(List> edgesToBeRemoved) {

		DataSet> newEdges = getEdges().coGroup(this.context.fromCollection(edgesToBeRemoved))
				.where(0,1).equalTo(0,1).with(new EdgeRemovalCoGroup());

		return new Graph(this.vertices, newEdges, context);
	}

	private static final class EdgeRemovalCoGroup implements CoGroupFunction, Edge, Edge> {

		@Override
		public void coGroup(Iterable> edge, Iterable> edgeToBeRemoved,
							Collector> out) throws Exception {

			final Iterator> edgeIterator = edge.iterator();
			final Iterator> edgeToBeRemovedIterator = edgeToBeRemoved.iterator();
			Edge next;

			if (edgeIterator.hasNext()) {
				if (!edgeToBeRemovedIterator.hasNext()) {
					next = edgeIterator.next();
					out.collect(next);
				}
			}
		}
	}

	/**
	 * Performs union on the vertices and edges sets of the input graphs
	 * removing duplicate vertices but maintaining duplicate edges.
	 * 
	 * @param graph the graph to perform union with
	 * @return a new graph
	 */
	public Graph union(Graph graph) {

		DataSet> unionedVertices = graph.getVertices().union(this.getVertices()).distinct();
		DataSet> unionedEdges = graph.getEdges().union(this.getEdges());
		return new Graph(unionedVertices, unionedEdges, this.context);
	}

	/**
	 * Runs a Vertex-Centric iteration on the graph.
	 * No configuration options are provided.
	 *
	 * @param vertexUpdateFunction the vertex update function
	 * @param messagingFunction the messaging function
	 * @param maximumNumberOfIterations maximum number of iterations to perform
	 * 
	 * @return the updated Graph after the vertex-centric iteration has converged or
	 * after maximumNumberOfIterations.
	 */
	public  Graph runVertexCentricIteration(
			VertexUpdateFunction vertexUpdateFunction,
			MessagingFunction messagingFunction,
			int maximumNumberOfIterations) {

		return this.runVertexCentricIteration(vertexUpdateFunction, messagingFunction,
				maximumNumberOfIterations, null);
	}

	/**
	 * Runs a Vertex-Centric iteration on the graph with configuration options.
	 * 
	 * @param vertexUpdateFunction the vertex update function
	 * @param messagingFunction the messaging function
	 * @param maximumNumberOfIterations maximum number of iterations to perform
	 * @param parameters the iteration configuration parameters
	 * 
	 * @return the updated Graph after the vertex-centric iteration has converged or
	 * after maximumNumberOfIterations.
	 */
	public  Graph runVertexCentricIteration(
			VertexUpdateFunction vertexUpdateFunction,
			MessagingFunction messagingFunction,
			int maximumNumberOfIterations, VertexCentricConfiguration parameters) {

		VertexCentricIteration iteration = VertexCentricIteration.withEdges(
				edges, vertexUpdateFunction, messagingFunction, maximumNumberOfIterations);

		iteration.configure(parameters);

		DataSet> newVertices = this.getVertices().runOperation(iteration);

		return new Graph(newVertices, this.edges, this.context);
	}

	/**
	 * Runs a Gather-Sum-Apply iteration on the graph.
	 * No configuration options are provided.
	 *
	 * @param gatherFunction the gather function collects information about adjacent vertices and edges
	 * @param sumFunction the sum function aggregates the gathered information
	 * @param applyFunction the apply function updates the vertex values with the aggregates
	 * @param maximumNumberOfIterations maximum number of iterations to perform
	 * @param  the intermediate type used between gather, sum and apply
	 *
	 * @return the updated Graph after the gather-sum-apply iteration has converged or
	 * after maximumNumberOfIterations.
	 */
	public  Graph runGatherSumApplyIteration(
			GatherFunction gatherFunction, SumFunction sumFunction,
			ApplyFunction applyFunction, int maximumNumberOfIterations) {

		return this.runGatherSumApplyIteration(gatherFunction, sumFunction, applyFunction,
				maximumNumberOfIterations, null);
	}

	/**
	 * Runs a Gather-Sum-Apply iteration on the graph with configuration options.
	 *
	 * @param gatherFunction the gather function collects information about adjacent vertices and edges
	 * @param sumFunction the sum function aggregates the gathered information
	 * @param applyFunction the apply function updates the vertex values with the aggregates
	 * @param maximumNumberOfIterations maximum number of iterations to perform
	 * @param parameters the iteration configuration parameters
	 * @param  the intermediate type used between gather, sum and apply
	 *
	 * @return the updated Graph after the gather-sum-apply iteration has converged or
	 * after maximumNumberOfIterations.
	 */
	public  Graph runGatherSumApplyIteration(
			GatherFunction gatherFunction, SumFunction sumFunction,
			ApplyFunction applyFunction, int maximumNumberOfIterations,
			GSAConfiguration parameters) {

		GatherSumApplyIteration iteration = GatherSumApplyIteration.withEdges(
				edges, gatherFunction, sumFunction, applyFunction, maximumNumberOfIterations);

		iteration.configure(parameters);

		DataSet> newVertices = vertices.runOperation(iteration);

		return new Graph(newVertices, this.edges, this.context);
	}

	public Graph run(GraphAlgorithm algorithm) throws Exception {
		return algorithm.run(this);
	}

	/**
	 * Compute an aggregate over the neighbors (edges and vertices) of each
	 * vertex. The function applied on the neighbors has access to the vertex
	 * value.
	 * 
	 * @param neighborsFunction the function to apply to the neighborhood
	 * @param direction the edge direction (in-, out-, all-)
	 * @param  the output type
	 * @return a dataset of a T
	 * @throws IllegalArgumentException
	 */
	public  DataSet groupReduceOnNeighbors(NeighborsFunctionWithVertexValue neighborsFunction,
												EdgeDirection direction) throws IllegalArgumentException {
		switch (direction) {
		case IN:
			// create  pairs
			DataSet, Vertex>> edgesWithSources = edges
					.join(this.vertices).where(0).equalTo(0);
			return vertices.coGroup(edgesWithSources)
					.where(0).equalTo("f0.f1")
					.with(new ApplyNeighborCoGroupFunction(neighborsFunction));
		case OUT:
			// create  pairs
			DataSet, Vertex>> edgesWithTargets = edges
					.join(this.vertices).where(1).equalTo(0);
			return vertices.coGroup(edgesWithTargets)
					.where(0).equalTo("f0.f0")
					.with(new ApplyNeighborCoGroupFunction(neighborsFunction));
		case ALL:
			// create  pairs
			DataSet, Vertex>> edgesWithNeighbors = edges
					.flatMap(new EmitOneEdgeWithNeighborPerNode())
					.join(this.vertices).where(1).equalTo(0)
					.with(new ProjectEdgeWithNeighbor());

			return vertices.coGroup(edgesWithNeighbors)
					.where(0).equalTo(0)
					.with(new ApplyCoGroupFunctionOnAllNeighbors(neighborsFunction));
		default:
			throw new IllegalArgumentException("Illegal edge direction");
		}
	}

	/**
	 * Compute an aggregate over the neighbors (edges and vertices) of each
	 * vertex. The function applied on the neighbors only has access to the
	 * vertex id (not the vertex value).
	 * 
	 * @param neighborsFunction the function to apply to the neighborhood
	 * @param direction the edge direction (in-, out-, all-)
	 * @param  the output type
	 * @return a dataset of a T
	 * @throws IllegalArgumentException
	 */
	public  DataSet groupReduceOnNeighbors(NeighborsFunction neighborsFunction,
												EdgeDirection direction) throws IllegalArgumentException {
		switch (direction) {
		case IN:
			// create  pairs
			DataSet, Vertex>> edgesWithSources = edges
					.join(this.vertices).where(0).equalTo(0)
					.with(new ProjectVertexIdJoin(1))
					.withForwardedFieldsFirst("f1->f0");
			return edgesWithSources.groupBy(0).reduceGroup(
					new ApplyNeighborGroupReduceFunction(neighborsFunction));
		case OUT:
			// create  pairs
			DataSet, Vertex>> edgesWithTargets = edges
					.join(this.vertices).where(1).equalTo(0)
					.with(new ProjectVertexIdJoin(0))
					.withForwardedFieldsFirst("f0");
			return edgesWithTargets.groupBy(0).reduceGroup(
					new ApplyNeighborGroupReduceFunction(neighborsFunction));
		case ALL:
			// create  pairs
			DataSet, Vertex>> edgesWithNeighbors = edges
					.flatMap(new EmitOneEdgeWithNeighborPerNode())
					.join(this.vertices).where(1).equalTo(0)
					.with(new ProjectEdgeWithNeighbor());

			return edgesWithNeighbors.groupBy(0).reduceGroup(
					new ApplyNeighborGroupReduceFunction(neighborsFunction));
		default:
			throw new IllegalArgumentException("Illegal edge direction");
		}
	}

	private static final class ApplyNeighborGroupReduceFunction
			implements GroupReduceFunction, Vertex>, T>, ResultTypeQueryable {

		private NeighborsFunction function;

		public ApplyNeighborGroupReduceFunction(NeighborsFunction fun) {
			this.function = fun;
		}

		public void reduce(Iterable, Vertex>> edges, Collector out) throws Exception {
			function.iterateNeighbors(edges, out);
		}

		@Override
		public TypeInformation getProducedType() {
			return TypeExtractor.createTypeInfo(NeighborsFunction.class, function.getClass(), 3, null, null);
		}
	}

	@ForwardedFieldsSecond("f1")
	private static final class ProjectVertexWithNeighborValueJoin
			implements FlatJoinFunction, Vertex, Tuple2> {

		private int fieldPosition;

		public ProjectVertexWithNeighborValueJoin(int position) {
			this.fieldPosition = position;
		}

		@SuppressWarnings("unchecked")
		public void join(Edge edge, Vertex otherVertex, 
				Collector> out) {
			out.collect(new Tuple2((K) edge.getField(fieldPosition), otherVertex.getValue()));
		}
	}

	private static final class ProjectVertexIdJoin implements FlatJoinFunction<
		Edge, Vertex, Tuple3, Vertex>> {

		private int fieldPosition;

		public ProjectVertexIdJoin(int position) {
			this.fieldPosition = position;
		}

		@SuppressWarnings("unchecked")
		public void join(Edge edge, Vertex otherVertex,
						Collector, Vertex>> out) {
			out.collect(new Tuple3, Vertex>((K) edge.getField(fieldPosition), edge, otherVertex));
		}
	}

	@ForwardedFieldsFirst("f0")
	@ForwardedFieldsSecond("f1")
	private static final class ProjectNeighborValue implements FlatJoinFunction<
		Tuple3>, Vertex, Tuple2> {

		public void join(Tuple3> keysWithEdge, Vertex neighbor,
				Collector> out) {

			out.collect(new Tuple2(keysWithEdge.f0, neighbor.getValue()));
		}
	}

	@ForwardedFieldsFirst("f0; f2->f1")
	@ForwardedFieldsSecond("*->f2")
	private static final class ProjectEdgeWithNeighbor implements FlatJoinFunction<
		Tuple3>, Vertex, Tuple3, Vertex>> {

		public void join(Tuple3> keysWithEdge, Vertex neighbor,
						Collector, Vertex>> out) {
			out.collect(new Tuple3, Vertex>(keysWithEdge.f0, keysWithEdge.f2, neighbor));
		}
	}

	private static final class ApplyNeighborCoGroupFunction implements CoGroupFunction<
		Vertex, Tuple2, Vertex>, T>, ResultTypeQueryable {

		private NeighborsFunctionWithVertexValue function;

		public ApplyNeighborCoGroupFunction(NeighborsFunctionWithVertexValue fun) {
			this.function = fun;
		}

		public void coGroup(Iterable> vertex, Iterable, Vertex>> neighbors,
				Collector out) throws Exception {
			function.iterateNeighbors(vertex.iterator().next(),	neighbors, out);
		}

		@Override
		public TypeInformation getProducedType() {
			return TypeExtractor.createTypeInfo(NeighborsFunctionWithVertexValue.class,	function.getClass(), 3, null, null);
		}
	}

	private static final class ApplyCoGroupFunctionOnAllNeighbors
			implements CoGroupFunction, Tuple3, Vertex>, T>, ResultTypeQueryable {

		private NeighborsFunctionWithVertexValue function;

		public ApplyCoGroupFunctionOnAllNeighbors(NeighborsFunctionWithVertexValue fun) {
			this.function = fun;
		}

		public void coGroup(Iterable> vertex,
				final Iterable, Vertex>> keysWithNeighbors, 
				Collector out) throws Exception {

			final Iterator, Vertex>> neighborsIterator = new Iterator, Vertex>>() {

				final Iterator, Vertex>> keysWithEdgesIterator = keysWithNeighbors.iterator();

				@Override
				public boolean hasNext() {
					return keysWithEdgesIterator.hasNext();
				}

				@Override
				public Tuple2, Vertex> next() {
					Tuple3, Vertex> next = keysWithEdgesIterator.next();
					return new Tuple2, Vertex>(next.f1, next.f2);
				}

				@Override
				public void remove() {
					keysWithEdgesIterator.remove();
				}
			};

			Iterable, Vertex>> neighborsIterable = new Iterable, Vertex>>() {
				public Iterator, Vertex>> iterator() {
					return neighborsIterator;
				}
			};

			function.iterateNeighbors(vertex.iterator().next(), neighborsIterable, out);
		}

		@Override
		public TypeInformation getProducedType() {
			return TypeExtractor.createTypeInfo(NeighborsFunctionWithVertexValue.class,	function.getClass(), 3, null, null);
		}
	}

	/**
	 * Compute an aggregate over the neighbor values of each
	 * vertex.
	 *
	 * @param reduceNeighborsFunction the function to apply to the neighborhood
	 * @param direction the edge direction (in-, out-, all-)
	 * @return a Dataset containing one value per vertex (vertex id, aggregate vertex value)
	 * @throws IllegalArgumentException
	 */
	public DataSet> reduceOnNeighbors(ReduceNeighborsFunction reduceNeighborsFunction,
									EdgeDirection direction) throws IllegalArgumentException {
		switch (direction) {
			case IN:
				// create  pairs
				final DataSet> verticesWithSourceNeighborValues = edges
						.join(this.vertices).where(0).equalTo(0)
						.with(new ProjectVertexWithNeighborValueJoin(1))
						.withForwardedFieldsFirst("f1->f0");
				return verticesWithSourceNeighborValues.groupBy(0).reduce(new ApplyNeighborReduceFunction(
						reduceNeighborsFunction));
			case OUT:
				// create  pairs
				DataSet> verticesWithTargetNeighborValues = edges
						.join(this.vertices).where(1).equalTo(0)
						.with(new ProjectVertexWithNeighborValueJoin(0))
						.withForwardedFieldsFirst("f0");
				return verticesWithTargetNeighborValues.groupBy(0).reduce(new ApplyNeighborReduceFunction(
						reduceNeighborsFunction));
			case ALL:
				// create  pairs
				DataSet> verticesWithNeighborValues = edges
						.flatMap(new EmitOneEdgeWithNeighborPerNode())
						.join(this.vertices).where(1).equalTo(0)
						.with(new ProjectNeighborValue());

				return verticesWithNeighborValues.groupBy(0).reduce(new ApplyNeighborReduceFunction(
						reduceNeighborsFunction));
			default:
				throw new IllegalArgumentException("Illegal edge direction");
		}
	}

	@ForwardedFields("f0")
	private static final class ApplyNeighborReduceFunction implements ReduceFunction> {

		private ReduceNeighborsFunction function;

		public ApplyNeighborReduceFunction(ReduceNeighborsFunction fun) {
			this.function = fun;
		}

		@Override
		public Tuple2 reduce(Tuple2 first, Tuple2 second) throws Exception {
			first.setField(function.reduceNeighbors(first.f1, second.f1), 1);
			return first;
		}
	}

	/**
	 * Compute an aggregate over the edge values of each vertex.
	 *
	 * @param reduceEdgesFunction
	 *            the function to apply to the neighborhood
	 * @param direction
	 *            the edge direction (in-, out-, all-)
	 * @return a Dataset containing one value per vertex(vertex key, aggegate edge value)
	 * @throws IllegalArgumentException
	 */
	public DataSet> reduceOnEdges(ReduceEdgesFunction reduceEdgesFunction,
								EdgeDirection direction) throws IllegalArgumentException {

		switch (direction) {
			case IN:
				return edges.map(new ProjectVertexWithEdgeValueMap(1))
						.withForwardedFields("f1->f0")
						.groupBy(0).reduce(new ApplyReduceFunction(reduceEdgesFunction));
			case OUT:
				return edges.map(new ProjectVertexWithEdgeValueMap(0))
						.withForwardedFields("f0->f0")
						.groupBy(0).reduce(new ApplyReduceFunction(reduceEdgesFunction));
			case ALL:
				return edges.flatMap(new EmitOneVertexWithEdgeValuePerNode())
						.withForwardedFields("f2->f1")
						.groupBy(0).reduce(new ApplyReduceFunction(reduceEdgesFunction));
			default:
				throw new IllegalArgumentException("Illegal edge direction");
		}
	}

	@ForwardedFields("f0")
	private static final class ApplyReduceFunction implements ReduceFunction> {

		private ReduceEdgesFunction function;

		public ApplyReduceFunction(ReduceEdgesFunction fun) {
			this.function = fun;
		}

		@Override
		public Tuple2 reduce(Tuple2 first, Tuple2 second) throws Exception {
			first.setField(function.reduceEdges(first.f1, second.f1), 1);
			return first;
		}
	}
}