archetype-resources.src.main.java.WordCountJob Maven / Gradle / Ivy

package ${package};

import eu.stratosphere.api.java.DataSet;
import eu.stratosphere.api.java.ExecutionEnvironment;
import eu.stratosphere.api.java.aggregation.Aggregations;
import eu.stratosphere.api.java.functions.FlatMapFunction;
import eu.stratosphere.api.java.tuple.Tuple2;
import eu.stratosphere.util.Collector;

/**
 * Implements the "WordCount" program that computes a simple word occurrence histogram
 * over some sample data
 * 
 * 
 * This example shows how to:
 * 

 * write a simple Stratosphere program.
 * 
use Tuple data types.
 * 
write and use user-defined functions. 
 * 
 * 
 */
@SuppressWarnings("serial")
public class WordCountJob {
	
	//
	//	Program. 
	//
	
	public static void main(String[] args) throws Exception {
		
		// set up the execution environment
		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		
		// get input data
		DataSet text = env.fromElements(
				"To be, or not to be,--that is the question:--",
				"Whether 'tis nobler in the mind to suffer",
				"The slings and arrows of outrageous fortune",
				"Or to take arms against a sea of troubles,"
				);
		
		DataSet> counts = 
				// split up the lines in pairs (2-tuples) containing: (word,1)
				text.flatMap(new LineSplitter())
				// group by the tuple field "0" and sum up tuple field "1"
				.groupBy(0)
				.aggregate(Aggregations.SUM, 1);

		// emit result
		counts.print();
		
		// execute program
		env.execute("WordCount Example");
	}
	
	//
	// 	User Functions
	//
	
	/**
	 * Implements the string tokenizer that splits sentences into words as a user-defined
	 * FlatMapFunction. The function takes a line (String) and splits it into 
	 * multiple pairs in the form of "(word,1)" (Tuple2).
	 */
	public static final class LineSplitter extends FlatMapFunction> {

		@Override
		public void flatMap(String value, Collector> out) {
			// normalize and split the line
			String[] tokens = value.toLowerCase().split("\\W+");
			
			// emit the pairs
			for (String token : tokens) {
				if (token.length() > 0) {
					out.collect(new Tuple2(token, 1));
				}
			}
		}
	}
}