org.apache.flink.examples.java.graph.TransitiveClosureNaive Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of flink-java-examples Show documentation
There is a newer version: 0.10.2-hadoop1
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


package org.apache.flink.examples.java.graph;

import org.apache.flink.api.common.functions.CoGroupFunction;
import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.functions.JoinFunction;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.operators.IterativeDataSet;
import org.apache.flink.api.common.ProgramDescription;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.examples.java.graph.util.ConnectedComponentsData;
import org.apache.flink.util.Collector;

import java.util.HashSet;
import java.util.Set;

@SuppressWarnings("serial")
public class TransitiveClosureNaive implements ProgramDescription {


	public static void main (String... args) throws Exception{

		if (!parseParameters(args)) {
			return;
		}

		// set up execution environment
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

		DataSet> edges = getEdgeDataSet(env);

		IterativeDataSet> paths = edges.iterate(maxIterations);

		DataSet> nextPaths = paths
				.join(edges)
				.where(1)
				.equalTo(0)
				.with(new JoinFunction, Tuple2, Tuple2>() {
					@Override
					/**
						left: Path (z,x) - x is reachable by z
						right: Edge (x,y) - edge x-->y exists
						out: Path (z,y) - y is reachable by z
					 */
					public Tuple2 join(Tuple2 left, Tuple2 right) throws Exception {
						return new Tuple2(left.f0, right.f1);
					}
				}).withForwardedFieldsFirst("0").withForwardedFieldsSecond("1")
				.union(paths)
				.groupBy(0, 1)
				.reduceGroup(new GroupReduceFunction, Tuple2>() {
					@Override
					public void reduce(Iterable> values, Collector> out) throws Exception {
						out.collect(values.iterator().next());
					}
				}).withForwardedFields("0;1");

		DataSet> newPaths = paths
				.coGroup(nextPaths)
				.where(0).equalTo(0)
				.with(new CoGroupFunction, Tuple2, Tuple2>() {
					Set> prevSet = new HashSet>();
					@Override
					public void coGroup(Iterable> prevPaths, Iterable> nextPaths, Collector> out) throws Exception {
						for (Tuple2 prev : prevPaths) {
							prevSet.add(prev);
						}
						for (Tuple2 next: nextPaths) {
							if (!prevSet.contains(next)) {
								out.collect(next);
							}
						}
					}
				}).withForwardedFieldsFirst("0").withForwardedFieldsSecond("0");

		DataSet> transitiveClosure = paths.closeWith(nextPaths, newPaths);


		// emit result
		if (fileOutput) {
			transitiveClosure.writeAsCsv(outputPath, "\n", " ");

			// execute program explicitly, because file sinks are lazy
			env.execute("Transitive Closure Example");
		} else {
			transitiveClosure.print();
		}
	}

	@Override
	public String getDescription() {
		return "Parameters:   ";
	}

	// *************************************************************************
	//     UTIL METHODS
	// *************************************************************************

	private static boolean fileOutput = false;
	private static String edgesPath = null;
	private static String outputPath = null;
	private static int maxIterations = 10;

	private static boolean parseParameters(String[] programArguments) {

		if (programArguments.length > 0) {
			// parse input arguments
			fileOutput = true;
			if (programArguments.length == 3) {
				edgesPath = programArguments[0];
				outputPath = programArguments[1];
				maxIterations = Integer.parseInt(programArguments[2]);
			} else {
				System.err.println("Usage: TransitiveClosure   ");
				return false;
			}
		} else {
			System.out.println("Executing TransitiveClosure example with default parameters and built-in default data.");
			System.out.println("  Provide parameters to read input data from files.");
			System.out.println("  See the documentation for the correct format of input files.");
			System.out.println("  Usage: TransitiveClosure   ");
		}
		return true;
	}


	private static DataSet> getEdgeDataSet(ExecutionEnvironment env) {

		if(fileOutput) {
			return env.readCsvFile(edgesPath).fieldDelimiter(" ").types(Long.class, Long.class);
		} else {
			return ConnectedComponentsData.getDefaultEdgeDataSet(env);
		}
	}

}