com.nextbreakpoint.flink.dummies.TemperatureMeasurementMain Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of com.nextbreakpoint.flink.dummies Show documentation
Collection of dummy jobs for Apache Flink
There is a newer version: 0.0.6
Show newest version
/*
 * This file is part of Flink Dummies
 * https://github.com/nextbreakpoint/flink-dummies
 */
package com.nextbreakpoint.flink.dummies;

import lombok.extern.slf4j.Slf4j;
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.configuration.MemorySize;
import org.apache.flink.connector.file.sink.FileSink;
import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.Path;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.DefaultRollingPolicy;
import org.apache.flink.streaming.api.windowing.assigners.SlidingEventTimeWindows;

import java.net.URI;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import static com.nextbreakpoint.flink.dummies.Parameters.getOptionalParam;
import static com.nextbreakpoint.flink.dummies.Utilities.getLocations;
import static com.nextbreakpoint.flink.dummies.Utilities.getProperties;
import static com.nextbreakpoint.flink.dummies.Utilities.getSensors;

@Slf4j
public class TemperatureMeasurementMain {

    public static void main(String[] args) throws Exception {
        final ParameterTool parameters = ParameterTool.fromArgs(args);

        final int parallelism = Integer.parseInt(getOptionalParam(parameters, Parameters.PARALLELISM, "1"));
        final String configFile = getOptionalParam(parameters, Parameters.CONFIG_FILE, "file:///var/config/measurement.properties");
        final boolean debugMode = Boolean.parseBoolean(getOptionalParam(parameters, Parameters.DEBUG_MODE, "false"));

        log.info("{}={}", Parameters.PARALLELISM, parallelism);
        log.info("{}={}", Parameters.CONFIG_FILE, configFile);
        log.info("{}={}", Parameters.DEBUG_MODE, debugMode);

        final FileSystem fileSystem = FileSystem.getUnguardedFileSystem(new URI(configFile));

        final Map properties = getProperties(fileSystem, new Path(configFile));
        properties.forEach((key, value) -> log.info("{}={}", key, value));

        final Map flinkProperties = properties.entrySet()
                .stream()
                .filter(entry -> entry.getKey().startsWith("flink."))
                .collect(Collectors.toMap(entry -> entry.getKey().substring(6), Map.Entry::getValue));

        final Map customProperties = properties.entrySet()
                .stream()
                .filter(entry -> !entry.getKey().startsWith("flink."))
                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));

        final List sensors = getSensors(10, getLocations(3));

        final StreamExecutionEnvironment environment = Environment.getStreamExecutionEnvironment(debugMode, flinkProperties);

        environment.setParallelism(parallelism > 0 ? parallelism : environment.getParallelism());

        final TemperatureMeasurementConfiguration configuration = TemperatureMeasurementConfiguration.fromProperties(customProperties);

        final SampledTemperatureSource source = new SampledTemperatureSource(configuration.getSourceMaxCount(), configuration.getSourceMaxDelay(), sensors);

        final WatermarkStrategy watermarkStrategy = WatermarkStrategy.forBoundedOutOfOrderness(Duration.ofMillis(configuration.getMaxOutOfOrderness()))
                .withIdleness(Duration.ofMillis(configuration.getIdleTimeout()))
                .withTimestampAssigner(new SampledTemperatureTimestampAssigner());

        final DataStream samples = environment
                .fromSource(source, watermarkStrategy, "samples")
                .uid("samples");

        final Path samplesPath = new Path(configuration.getBucketOutputPath() + "/samples");

        final FileSink samplesSink = FileSink.forRowFormat(samplesPath, new SampledTemperatureEncoder())
                .withBucketCheckInterval(configuration.getBucketCheckInterval())
                .withBucketAssigner(new SampledTemperatureBucketAssigner())
                .withRollingPolicy(
                        DefaultRollingPolicy.builder()
                                .withRolloverInterval(Duration.ofMillis(configuration.getBucketRolloverInterval()))
                                .withInactivityInterval(Duration.ofMillis(configuration.getBucketInactivityInterval()))
                                .withMaxPartSize(MemorySize.ofMebiBytes(configuration.getBucketMaxPartSize()))
                                .build()
                )
                .build();

        samples.sinkTo(samplesSink)
                .uid("samples-write")
                .name("samples-write");

        if (configuration.getLogVerbosity() > 1) {
            samples.sinkTo(new LoggingSink<>())
                    .setParallelism(1)
                    .name("samples-print")
                    .uid("samples-print");
        }

        final DataStream measures = samples.keyBy(SampledTemperature::getLocationId)
                .window(SlidingEventTimeWindows.of(Duration.ofMillis(configuration.getWindowSize()), Duration.ofMillis(configuration.getWindowSlide())))
                .aggregate(new TemperatureAggregator(), new MeasuredTemperatureProcess())
                .uid("measures")
                .name("measures");

        final Path measuresPath = new Path(configuration.getBucketOutputPath() + "/measures");

        final FileSink measuresSink = FileSink.forRowFormat(measuresPath, new MeasuredTemperatureEncoder())
                .withBucketCheckInterval(configuration.getBucketCheckInterval())
                .withBucketAssigner(new MeasuredTemperatureBucketAssigner())
                .withRollingPolicy(
                        DefaultRollingPolicy.builder()
                                .withRolloverInterval(Duration.ofMillis(configuration.getBucketRolloverInterval()))
                                .withInactivityInterval(Duration.ofMillis(configuration.getBucketInactivityInterval()))
                                .withMaxPartSize(MemorySize.ofMebiBytes(configuration.getBucketMaxPartSize()))
                                .build()
                )
                .build();

        measures.sinkTo(measuresSink)
                .uid("measures-write")
                .name("measures-write");

        if (configuration.getLogVerbosity() > 0) {
            measures.sinkTo(new LoggingSink<>())
                    .setParallelism(1)
                    .name("measures-print")
                    .uid("measures-print");
        }

        final JobExecutionResult result = environment.execute(configuration.getJobName());

        log.info(result.toString());
    }
}