io.activej.etl.LogOTProcessor Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of activej-etl Show documentation
Various tools for extracting, transforming and loading data.
There is a newer version: 6.0-rc1
/*
 * Copyright (C) 2020 ActiveJ LLC.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package io.activej.etl;

import io.activej.async.AsyncAccumulator;
import io.activej.async.function.AsyncSupplier;
import io.activej.async.service.EventloopService;
import io.activej.common.initializer.WithInitializer;
import io.activej.datastream.StreamConsumerWithResult;
import io.activej.datastream.StreamSupplierWithResult;
import io.activej.datastream.processor.StreamUnion;
import io.activej.datastream.stats.StreamStats;
import io.activej.datastream.stats.StreamStatsBasic;
import io.activej.datastream.stats.StreamStatsDetailed;
import io.activej.eventloop.Eventloop;
import io.activej.eventloop.jmx.EventloopJmxBeanWithStats;
import io.activej.jmx.api.attribute.JmxAttribute;
import io.activej.jmx.api.attribute.JmxOperation;
import io.activej.multilog.LogPosition;
import io.activej.multilog.Multilog;
import io.activej.promise.Promise;
import io.activej.promise.jmx.PromiseStats;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.time.Duration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static io.activej.async.function.AsyncSuppliers.reuse;
import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;

/**
 * Processes logs. Creates new aggregation logs and persists to {@link LogDataConsumer} .
 */
@SuppressWarnings("rawtypes") // JMX doesn't work with generic types
public final class LogOTProcessor implements EventloopService, EventloopJmxBeanWithStats, WithInitializer> {
	private static final Logger logger = LoggerFactory.getLogger(LogOTProcessor.class);

	private final Eventloop eventloop;
	private final Multilog multilog;
	private final LogDataConsumer logStreamConsumer;

	private final String log;
	private final List partitions;

	private final LogOTState state;

	// JMX
	private boolean enabled = true;
	private boolean detailed;
	private final StreamStatsBasic streamStatsBasic = StreamStats.basic();
	private final StreamStatsDetailed streamStatsDetailed = StreamStats.detailed();
	private final PromiseStats promiseProcessLog = PromiseStats.create(Duration.ofMinutes(5));

	private LogOTProcessor(Eventloop eventloop, Multilog multilog, LogDataConsumer logStreamConsumer,
			String log, List partitions, LogOTState state) {
		this.eventloop = eventloop;
		this.multilog = multilog;
		this.logStreamConsumer = logStreamConsumer;
		this.log = log;
		this.partitions = partitions;
		this.state = state;
	}

	public static  LogOTProcessor create(Eventloop eventloop, Multilog multilog,
			LogDataConsumer logStreamConsumer,
			String log, List partitions, LogOTState state) {
		return new LogOTProcessor<>(eventloop, multilog, logStreamConsumer, log, partitions, state);
	}

	@Override
	public @NotNull Eventloop getEventloop() {
		return eventloop;
	}

	@Override
	public @NotNull Promise start() {
		return Promise.complete();
	}

	@Override
	public @NotNull Promise stop() {
		return Promise.complete();
	}

	private final AsyncSupplier> processLog = reuse(this::doProcessLog);

	public Promise> processLog() {
		return processLog.get();
	}

	private @NotNull Promise> doProcessLog() {
		if (!enabled) return Promise.of(LogDiff.of(emptyMap(), emptyList()));
		logger.trace("processLog_gotPositions called. Positions: {}", state.getPositions());

		StreamSupplierWithResult> supplier = getSupplier();
		StreamConsumerWithResult> consumer = logStreamConsumer.consume();
		return supplier.streamTo(consumer)
				.whenComplete(promiseProcessLog.recordStats())
				.map(result -> LogDiff.of(result.getValue1(), result.getValue2()))
				.whenResult(logDiff ->
						logger.info("Log '{}' processing complete. Positions: {}", log, logDiff.getPositions()));
	}

	private StreamSupplierWithResult> getSupplier() {
		AsyncAccumulator> logPositionsAccumulator = AsyncAccumulator.create(new HashMap<>());
		StreamUnion streamUnion = StreamUnion.create();
		for (String partition : partitions) {
			String logName = logName(partition);
			LogPosition logPosition = state.getPositions().get(logName);
			if (logPosition == null) {
				logPosition = LogPosition.initial();
			}
			logger.info("Starting reading '{}' from position {}", logName, logPosition);

			LogPosition logPositionFrom = logPosition;
			logPositionsAccumulator.addPromise(
					StreamSupplierWithResult.ofPromise(
									multilog.read(partition, logPosition.getLogFile(), logPosition.getPosition(), null))
							.streamTo(streamUnion.newInput()),
					(logPositions, logPositionTo) -> {
						if (!logPositionTo.equals(logPositionFrom)) {
							logPositions.put(logName, new LogPositionDiff(logPositionFrom, logPositionTo));
						}
					});
		}
		return StreamSupplierWithResult.of(
				streamUnion.getOutput()
						.transformWith(detailed ? streamStatsDetailed : streamStatsBasic),
				logPositionsAccumulator.run().promise());
	}

	private String logName(String partition) {
		return log != null && !log.isEmpty() ? log + "." + partition : partition;
	}

	@JmxAttribute
	public boolean isEnabled() {
		return enabled;
	}

	@JmxAttribute
	public void setEnabled(boolean enabled) {
		this.enabled = enabled;
	}

	@JmxAttribute
	public PromiseStats getPromiseProcessLog() {
		return promiseProcessLog;
	}

	@JmxAttribute
	public StreamStatsBasic getStreamStatsBasic() {
		return streamStatsBasic;
	}

	@JmxAttribute
	public StreamStatsDetailed getStreamStatsDetailed() {
		return streamStatsDetailed;
	}

	@JmxOperation
	public void startDetailedMonitoring() {
		detailed = true;
	}

	@JmxOperation
	public void stopDetailedMonitoring() {
		detailed = false;
	}
}