io.datakernel.multilog.MultilogImpl Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of datakernel-multilog Show documentation
Show all versions of datakernel-multilog Show documentation
Package provides tools for working with logs stored on different partitions.
/*
* Copyright (C) 2015 SoftIndex LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.datakernel.multilog;
import io.datakernel.bytebuf.ByteBuf;
import io.datakernel.common.MemSize;
import io.datakernel.common.Preconditions;
import io.datakernel.common.Stopwatch;
import io.datakernel.common.parse.TruncatedDataException;
import io.datakernel.csp.process.ChannelLZ4Compressor;
import io.datakernel.csp.process.ChannelLZ4Decompressor;
import io.datakernel.datastream.StreamConsumer;
import io.datakernel.datastream.StreamSupplier;
import io.datakernel.datastream.StreamSupplierWithResult;
import io.datakernel.datastream.csp.ChannelDeserializer;
import io.datakernel.datastream.csp.ChannelSerializer;
import io.datakernel.datastream.stats.StreamRegistry;
import io.datakernel.datastream.stats.StreamStats;
import io.datakernel.datastream.stats.StreamStatsDetailed;
import io.datakernel.eventloop.Eventloop;
import io.datakernel.eventloop.jmx.EventloopJmxMBeanEx;
import io.datakernel.promise.Promise;
import io.datakernel.promise.SettablePromise;
import io.datakernel.remotefs.FileMetadata;
import io.datakernel.remotefs.FsClient;
import io.datakernel.serializer.BinarySerializer;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.time.Duration;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import static io.datakernel.datastream.stats.StreamStatsSizeCounter.forByteBufs;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toList;
public final class MultilogImpl implements Multilog, EventloopJmxMBeanEx {
private static final Logger logger = LoggerFactory.getLogger(MultilogImpl.class);
public static final MemSize DEFAULT_BUFFER_SIZE = MemSize.kilobytes(256);
private final Eventloop eventloop;
private final FsClient client;
private final LogNamingScheme namingScheme;
private final BinarySerializer serializer;
private MemSize bufferSize = DEFAULT_BUFFER_SIZE;
private Duration autoFlushInterval = null;
private final StreamRegistry streamReads = StreamRegistry.create();
private final StreamRegistry streamWrites = StreamRegistry.create();
private final StreamStatsDetailed streamReadStats = StreamStats.detailed(forByteBufs());
private final StreamStatsDetailed streamWriteStats = StreamStats.detailed(forByteBufs());
private MultilogImpl(Eventloop eventloop, FsClient client, BinarySerializer serializer,
LogNamingScheme namingScheme) {
this.eventloop = eventloop;
this.client = client;
this.serializer = serializer;
this.namingScheme = namingScheme;
}
public static MultilogImpl create(Eventloop eventloop, FsClient client,
BinarySerializer serializer, LogNamingScheme namingScheme) {
return new MultilogImpl<>(eventloop, client, serializer, namingScheme);
}
public MultilogImpl withBufferSize(int bufferSize) {
this.bufferSize = MemSize.of(bufferSize);
return this;
}
public MultilogImpl withBufferSize(MemSize bufferSize) {
this.bufferSize = bufferSize;
return this;
}
public MultilogImpl withAutoFlushInterval(Duration autoFlushInterval) {
this.autoFlushInterval = autoFlushInterval;
return this;
}
@Override
public Promise> write(@NotNull String logPartition) {
validateLogPartition(logPartition);
return Promise.of(StreamConsumer.ofSupplier(
supplier -> supplier
.transformWith(ChannelSerializer.create(serializer)
.withAutoFlushInterval(autoFlushInterval)
.withInitialBufferSize(bufferSize)
.withSkipSerializationErrors())
.transformWith(ChannelLZ4Compressor.createFastCompressor())
.transformWith(streamWrites.register(logPartition))
.transformWith(streamWriteStats)
.bindTo(new LogStreamChunker(eventloop, client, namingScheme, logPartition)))
.withLateBinding());
}
@Override
public Promise> read(@NotNull String logPartition,
@NotNull LogFile startLogFile, long startOffset,
@Nullable LogFile endLogFile) {
validateLogPartition(logPartition);
LogPosition startPosition = LogPosition.create(startLogFile, startOffset);
return client.list(namingScheme.getListGlob(logPartition))
.map(files ->
files.stream()
.map(FileMetadata::getName)
.map(namingScheme::parse)
.filter(Objects::nonNull)
.filter(partitionAndFile -> partitionAndFile.getLogPartition().equals(logPartition))
.map(PartitionAndFile::getLogFile)
.collect(toList()))
.map(logFiles ->
logFiles.stream()
.filter(logFile -> isFileInRange(logFile, startPosition, endLogFile))
.map(logFile -> logFile.equals(startPosition.getLogFile()) ?
startPosition : LogPosition.create(logFile, 0)
)
.sorted()
.collect(toList()))
.map(logFilesToRead -> readLogFiles(logPartition, startPosition, logFilesToRead));
}
private StreamSupplierWithResult readLogFiles(@NotNull String logPartition, @NotNull LogPosition startPosition, @NotNull List logFiles) {
SettablePromise positionPromise = new SettablePromise<>();
Iterator> logFileStreams = new Iterator>() {
final Stopwatch sw = Stopwatch.createUnstarted();
final Iterator it = logFiles.iterator();
LogPosition currentPosition;
long inputStreamPosition;
@Override
public boolean hasNext() {
if (it.hasNext()) return true;
positionPromise.set(getLogPosition());
return false;
}
LogPosition getLogPosition() {
if (currentPosition == null)
return startPosition;
return LogPosition.create(currentPosition.getLogFile(), currentPosition.getPosition() + inputStreamPosition);
}
@Override
public StreamSupplier next() {
currentPosition = it.next();
long position = currentPosition.getPosition();
LogFile currentLogFile = currentPosition.getLogFile();
if (logger.isTraceEnabled())
logger.trace("Read log file `{}` from: {}", currentLogFile, position);
return StreamSupplier.ofPromise(
client.download(namingScheme.path(logPartition, currentLogFile), position)
.map(fileStream -> {
inputStreamPosition = 0L;
sw.reset().start();
return fileStream
.transformWith(streamReads.register(logPartition + ":" + currentLogFile + "@" + position))
.transformWith(streamReadStats)
.transformWith(ChannelLZ4Decompressor.create()
.withInspector(new ChannelLZ4Decompressor.Inspector() {
@Override
public Q lookup(Class type) {
throw new UnsupportedOperationException();
}
@Override
public void onBlock(ChannelLZ4Decompressor self, ChannelLZ4Decompressor.Header header, ByteBuf inputBuf, ByteBuf outputBuf) {
inputStreamPosition += ChannelLZ4Decompressor.HEADER_LENGTH + header.compressedLen;
}
}))
.transformWith(supplier ->
supplier.withEndOfStream(eos ->
eos.thenEx(($, e) -> (e == null || e instanceof TruncatedDataException) ?
Promise.complete() :
Promise.ofException(e))))
.transformWith(ChannelDeserializer.create(serializer))
.withEndOfStream(eos ->
eos.whenComplete(($, e) -> log(e)))
.withLateBinding();
}));
}
private void log(Throwable e) {
if (e == null && logger.isTraceEnabled()) {
logger.trace("Finish log file {}:`{}` in {}, compressed bytes: {} ({} bytes/s)",
client, namingScheme.path(logPartition, currentPosition.getLogFile()),
sw, inputStreamPosition, inputStreamPosition / Math.max(sw.elapsed(SECONDS), 1));
} else if (e != null && logger.isErrorEnabled()) {
logger.error("Error on log file {}:`{}` in {}, compressed bytes: {} ({} bytes/s)",
client, namingScheme.path(logPartition, currentPosition.getLogFile()),
sw, inputStreamPosition, inputStreamPosition / Math.max(sw.elapsed(SECONDS), 1), e);
}
}
};
return StreamSupplierWithResult.of(
StreamSupplier.concat(logFileStreams).withLateBinding(),
positionPromise);
}
private static void validateLogPartition(@NotNull String logPartition) {
Preconditions.checkArgument(!logPartition.contains("-"), "Using dash (-) in log partition name is not allowed");
}
private static boolean isFileInRange(@NotNull LogFile logFile, @NotNull LogPosition startPosition, @Nullable LogFile endFile) {
if (logFile.compareTo(startPosition.getLogFile()) < 0)
return false;
return endFile == null || logFile.compareTo(endFile) <= 0;
}
@NotNull
@Override
public Eventloop getEventloop() {
return eventloop;
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy