All Downloads are FREE. Search and download functionalities are using the official Maven repository.

io.datakernel.multilog.MultilogImpl Maven / Gradle / Ivy

There is a newer version: 3.1.0
Show newest version
/*
 * Copyright (C) 2015 SoftIndex LLC.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package io.datakernel.multilog;

import io.datakernel.bytebuf.ByteBuf;
import io.datakernel.common.MemSize;
import io.datakernel.common.Preconditions;
import io.datakernel.common.Stopwatch;
import io.datakernel.common.parse.TruncatedDataException;
import io.datakernel.csp.process.ChannelLZ4Compressor;
import io.datakernel.csp.process.ChannelLZ4Decompressor;
import io.datakernel.datastream.StreamConsumer;
import io.datakernel.datastream.StreamSupplier;
import io.datakernel.datastream.StreamSupplierWithResult;
import io.datakernel.datastream.csp.ChannelDeserializer;
import io.datakernel.datastream.csp.ChannelSerializer;
import io.datakernel.datastream.stats.StreamRegistry;
import io.datakernel.datastream.stats.StreamStats;
import io.datakernel.datastream.stats.StreamStatsDetailed;
import io.datakernel.eventloop.Eventloop;
import io.datakernel.eventloop.jmx.EventloopJmxMBeanEx;
import io.datakernel.promise.Promise;
import io.datakernel.promise.SettablePromise;
import io.datakernel.remotefs.FileMetadata;
import io.datakernel.remotefs.FsClient;
import io.datakernel.serializer.BinarySerializer;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.time.Duration;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;

import static io.datakernel.datastream.stats.StreamStatsSizeCounter.forByteBufs;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toList;

public final class MultilogImpl implements Multilog, EventloopJmxMBeanEx {
	private static final Logger logger = LoggerFactory.getLogger(MultilogImpl.class);

	public static final MemSize DEFAULT_BUFFER_SIZE = MemSize.kilobytes(256);

	private final Eventloop eventloop;
	private final FsClient client;
	private final LogNamingScheme namingScheme;
	private final BinarySerializer serializer;

	private MemSize bufferSize = DEFAULT_BUFFER_SIZE;
	private Duration autoFlushInterval = null;

	private final StreamRegistry streamReads = StreamRegistry.create();
	private final StreamRegistry streamWrites = StreamRegistry.create();

	private final StreamStatsDetailed streamReadStats = StreamStats.detailed(forByteBufs());
	private final StreamStatsDetailed streamWriteStats = StreamStats.detailed(forByteBufs());

	private MultilogImpl(Eventloop eventloop, FsClient client, BinarySerializer serializer,
			LogNamingScheme namingScheme) {
		this.eventloop = eventloop;
		this.client = client;
		this.serializer = serializer;
		this.namingScheme = namingScheme;
	}

	public static  MultilogImpl create(Eventloop eventloop, FsClient client,
			BinarySerializer serializer, LogNamingScheme namingScheme) {
		return new MultilogImpl<>(eventloop, client, serializer, namingScheme);
	}

	public MultilogImpl withBufferSize(int bufferSize) {
		this.bufferSize = MemSize.of(bufferSize);
		return this;
	}

	public MultilogImpl withBufferSize(MemSize bufferSize) {
		this.bufferSize = bufferSize;
		return this;
	}

	public MultilogImpl withAutoFlushInterval(Duration autoFlushInterval) {
		this.autoFlushInterval = autoFlushInterval;
		return this;
	}

	@Override
	public Promise> write(@NotNull String logPartition) {
		validateLogPartition(logPartition);

		return Promise.of(StreamConsumer.ofSupplier(
				supplier -> supplier
						.transformWith(ChannelSerializer.create(serializer)
								.withAutoFlushInterval(autoFlushInterval)
								.withInitialBufferSize(bufferSize)
								.withSkipSerializationErrors())
						.transformWith(ChannelLZ4Compressor.createFastCompressor())
						.transformWith(streamWrites.register(logPartition))
						.transformWith(streamWriteStats)
						.bindTo(new LogStreamChunker(eventloop, client, namingScheme, logPartition)))
				.withLateBinding());
	}

	@Override
	public Promise> read(@NotNull String logPartition,
			@NotNull LogFile startLogFile, long startOffset,
			@Nullable LogFile endLogFile) {
		validateLogPartition(logPartition);
		LogPosition startPosition = LogPosition.create(startLogFile, startOffset);
		return client.list(namingScheme.getListGlob(logPartition))
				.map(files ->
						files.stream()
								.map(FileMetadata::getName)
								.map(namingScheme::parse)
								.filter(Objects::nonNull)
								.filter(partitionAndFile -> partitionAndFile.getLogPartition().equals(logPartition))
								.map(PartitionAndFile::getLogFile)
								.collect(toList()))
				.map(logFiles ->
						logFiles.stream()
								.filter(logFile -> isFileInRange(logFile, startPosition, endLogFile))
								.map(logFile -> logFile.equals(startPosition.getLogFile()) ?
										startPosition : LogPosition.create(logFile, 0)
								)
								.sorted()
								.collect(toList()))
				.map(logFilesToRead -> readLogFiles(logPartition, startPosition, logFilesToRead));
	}

	private StreamSupplierWithResult readLogFiles(@NotNull String logPartition, @NotNull LogPosition startPosition, @NotNull List logFiles) {
		SettablePromise positionPromise = new SettablePromise<>();

		Iterator> logFileStreams = new Iterator>() {
			final Stopwatch sw = Stopwatch.createUnstarted();

			final Iterator it = logFiles.iterator();
			LogPosition currentPosition;
			long inputStreamPosition;

			@Override
			public boolean hasNext() {
				if (it.hasNext()) return true;
				positionPromise.set(getLogPosition());
				return false;
			}

			LogPosition getLogPosition() {
				if (currentPosition == null)
					return startPosition;

				return LogPosition.create(currentPosition.getLogFile(), currentPosition.getPosition() + inputStreamPosition);
			}

			@Override
			public StreamSupplier next() {
				currentPosition = it.next();
				long position = currentPosition.getPosition();
				LogFile currentLogFile = currentPosition.getLogFile();
				if (logger.isTraceEnabled())
					logger.trace("Read log file `{}` from: {}", currentLogFile, position);

				return StreamSupplier.ofPromise(
						client.download(namingScheme.path(logPartition, currentLogFile), position)
								.map(fileStream -> {
									inputStreamPosition = 0L;
									sw.reset().start();
									return fileStream
											.transformWith(streamReads.register(logPartition + ":" + currentLogFile + "@" + position))
											.transformWith(streamReadStats)
											.transformWith(ChannelLZ4Decompressor.create()
													.withInspector(new ChannelLZ4Decompressor.Inspector() {
														@Override
														public  Q lookup(Class type) {
															throw new UnsupportedOperationException();
														}

														@Override
														public void onBlock(ChannelLZ4Decompressor self, ChannelLZ4Decompressor.Header header, ByteBuf inputBuf, ByteBuf outputBuf) {
															inputStreamPosition += ChannelLZ4Decompressor.HEADER_LENGTH + header.compressedLen;
														}
													}))
											.transformWith(supplier ->
													supplier.withEndOfStream(eos ->
															eos.thenEx(($, e) -> (e == null || e instanceof TruncatedDataException) ?
																	Promise.complete() :
																	Promise.ofException(e))))
											.transformWith(ChannelDeserializer.create(serializer))
											.withEndOfStream(eos ->
													eos.whenComplete(($, e) -> log(e)))
											.withLateBinding();
								}));
			}

			private void log(Throwable e) {
				if (e == null && logger.isTraceEnabled()) {
					logger.trace("Finish log file {}:`{}` in {}, compressed bytes: {} ({} bytes/s)",
							client, namingScheme.path(logPartition, currentPosition.getLogFile()),
							sw, inputStreamPosition, inputStreamPosition / Math.max(sw.elapsed(SECONDS), 1));
				} else if (e != null && logger.isErrorEnabled()) {
					logger.error("Error on log file {}:`{}` in {}, compressed bytes: {} ({} bytes/s)",
							client, namingScheme.path(logPartition, currentPosition.getLogFile()),
							sw, inputStreamPosition, inputStreamPosition / Math.max(sw.elapsed(SECONDS), 1), e);
				}
			}
		};

		return StreamSupplierWithResult.of(
				StreamSupplier.concat(logFileStreams).withLateBinding(),
				positionPromise);
	}

	private static void validateLogPartition(@NotNull String logPartition) {
		Preconditions.checkArgument(!logPartition.contains("-"), "Using dash (-) in log partition name is not allowed");
	}

	private static boolean isFileInRange(@NotNull LogFile logFile, @NotNull LogPosition startPosition, @Nullable LogFile endFile) {
		if (logFile.compareTo(startPosition.getLogFile()) < 0)
			return false;

		return endFile == null || logFile.compareTo(endFile) <= 0;
	}

	@NotNull
	@Override
	public Eventloop getEventloop() {
		return eventloop;
	}

}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy