spark.streaming.examples.JavaNetworkWordCount Maven / Gradle / Ivy

Go to download
package spark.streaming.examples;

import com.google.common.collect.Lists;
import scala.Tuple2;
import spark.api.java.function.FlatMapFunction;
import spark.api.java.function.Function2;
import spark.api.java.function.PairFunction;
import spark.streaming.Duration;
import spark.streaming.api.java.JavaDStream;
import spark.streaming.api.java.JavaPairDStream;
import spark.streaming.api.java.JavaStreamingContext;

/**
 * Counts words in UTF8 encoded, '\n' delimited text received from the network every second.
 * Usage: NetworkWordCount   
 *    is the Spark master URL. In local mode,  should be 'local[n]' with n > 1.
 *    and  describe the TCP server that Spark Streaming would connect to receive data.
 *
 * To run this on your local machine, you need to first run a Netcat server
 *    `$ nc -lk 9999`
 * and then run the example
 *    `$ ./run spark.streaming.examples.JavaNetworkWordCount local[2] localhost 9999`
 */
public class JavaNetworkWordCount {
  public static void main(String[] args) {
    if (args.length < 3) {
      System.err.println("Usage: NetworkWordCount   \n" +
          "In local mode,  should be 'local[n]' with n > 1");
      System.exit(1);
    }

    // Create the context with a 1 second batch size
    JavaStreamingContext ssc = new JavaStreamingContext(args[0], "NetworkWordCount",
            new Duration(1000), System.getenv("SPARK_HOME"), System.getenv("SPARK_EXAMPLES_JAR"));

    // Create a NetworkInputDStream on target ip:port and count the
    // words in input stream of \n delimited test (eg. generated by 'nc')
    JavaDStream lines = ssc.socketTextStream(args[1], Integer.parseInt(args[2]));
    JavaDStream words = lines.flatMap(new FlatMapFunction() {
      @Override
      public Iterable call(String x) {
        return Lists.newArrayList(x.split(" "));
      }
    });
    JavaPairDStream wordCounts = words.map(
      new PairFunction() {
        @Override
        public Tuple2 call(String s) throws Exception {
          return new Tuple2(s, 1);
        }
      }).reduceByKey(new Function2() {
        @Override
        public Integer call(Integer i1, Integer i2) throws Exception {
          return i1 + i2;
        }
      });

    wordCounts.print();
    ssc.start();

  }
}