/*
* Copyright (C) 2020 ActiveJ LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.activej.fs;
import io.activej.async.service.EventloopService;
import io.activej.bytebuf.ByteBuf;
import io.activej.common.ApplicationSettings;
import io.activej.common.CollectorsEx;
import io.activej.common.MemSize;
import io.activej.common.collection.CollectionUtils;
import io.activej.common.exception.MalformedDataException;
import io.activej.common.exception.UncheckedException;
import io.activej.common.time.CurrentTimeProvider;
import io.activej.common.tuple.Tuple2;
import io.activej.csp.ChannelConsumer;
import io.activej.csp.ChannelSupplier;
import io.activej.csp.dsl.ChannelConsumerTransformer;
import io.activej.csp.file.ChannelFileReader;
import io.activej.csp.file.ChannelFileWriter;
import io.activej.eventloop.Eventloop;
import io.activej.eventloop.jmx.EventloopJmxBeanEx;
import io.activej.fs.LocalFileUtils.*;
import io.activej.fs.exception.*;
import io.activej.jmx.api.attribute.JmxAttribute;
import io.activej.promise.Promise;
import io.activej.promise.Promise.BlockingCallable;
import io.activej.promise.Promise.BlockingRunnable;
import io.activej.promise.jmx.PromiseStats;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.time.Duration;
import java.util.*;
import java.util.concurrent.Executor;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collector;
import static io.activej.async.util.LogUtils.Level.TRACE;
import static io.activej.async.util.LogUtils.toLogger;
import static io.activej.common.Checks.checkArgument;
import static io.activej.common.collection.CollectionUtils.*;
import static io.activej.csp.dsl.ChannelConsumerTransformer.identity;
import static io.activej.fs.LocalFileUtils.*;
import static io.activej.fs.util.RemoteFsUtils.batchEx;
import static io.activej.fs.util.RemoteFsUtils.ofFixedSize;
import static java.nio.file.StandardOpenOption.*;
import static java.util.Collections.*;
/**
* An implementation of {@link ActiveFs} which operates on a real underlying filesystem, no networking involved.
*
* Only permits file operations to be made within a specified storage path.
*
* This implementation does not define new limitations, other than those defined in {@link ActiveFs} interface.
*/
public final class LocalActiveFs implements ActiveFs, EventloopService, EventloopJmxBeanEx {
private static final Logger logger = LoggerFactory.getLogger(LocalActiveFs.class);
public static final String DEFAULT_TEMP_DIR = ".upload";
public static final boolean DEFAULT_FSYNC_UPLOADS = ApplicationSettings.getBoolean(LocalActiveFs.class, "fsyncUploads", false);
public static final boolean DEFAULT_FSYNC_DIRECTORIES = ApplicationSettings.getBoolean(LocalActiveFs.class, "fsyncDirectories", false);
public static final boolean DEFAULT_FSYNC_APPENDS = ApplicationSettings.getBoolean(LocalActiveFs.class, "fsyncAppends", false);
private static final char SEPARATOR_CHAR = SEPARATOR.charAt(0);
private static final Function toLocalName = File.separatorChar == SEPARATOR_CHAR ?
Function.identity() :
s -> s.replace(SEPARATOR_CHAR, File.separatorChar);
private static final Function toRemoteName = File.separatorChar == SEPARATOR_CHAR ?
Function.identity() :
s -> s.replace(File.separatorChar, SEPARATOR_CHAR);
private final Eventloop eventloop;
private final Path storage;
private final Executor executor;
private final Set appendOptions = CollectionUtils.set(WRITE);
private final Set appendNewOptions = CollectionUtils.set(WRITE, CREATE);
private MemSize readerBufferSize = MemSize.kilobytes(256);
private boolean hardlinkOnCopy = false;
private Path tempDir;
private boolean fsyncUploads = DEFAULT_FSYNC_UPLOADS;
private boolean fsyncDirectories = DEFAULT_FSYNC_DIRECTORIES;
CurrentTimeProvider now;
//region JMX
private final PromiseStats uploadBeginPromise = PromiseStats.create(Duration.ofMinutes(5));
private final PromiseStats uploadFinishPromise = PromiseStats.create(Duration.ofMinutes(5));
private final PromiseStats appendBeginPromise = PromiseStats.create(Duration.ofMinutes(5));
private final PromiseStats appendFinishPromise = PromiseStats.create(Duration.ofMinutes(5));
private final PromiseStats downloadBeginPromise = PromiseStats.create(Duration.ofMinutes(5));
private final PromiseStats downloadFinishPromise = PromiseStats.create(Duration.ofMinutes(5));
private final PromiseStats listPromise = PromiseStats.create(Duration.ofMinutes(5));
private final PromiseStats infoPromise = PromiseStats.create(Duration.ofMinutes(5));
private final PromiseStats infoAllPromise = PromiseStats.create(Duration.ofMinutes(5));
private final PromiseStats copyPromise = PromiseStats.create(Duration.ofMinutes(5));
private final PromiseStats copyAllPromise = PromiseStats.create(Duration.ofMinutes(5));
private final PromiseStats movePromise = PromiseStats.create(Duration.ofMinutes(5));
private final PromiseStats moveAllPromise = PromiseStats.create(Duration.ofMinutes(5));
private final PromiseStats deletePromise = PromiseStats.create(Duration.ofMinutes(5));
private final PromiseStats deleteAllPromise = PromiseStats.create(Duration.ofMinutes(5));
//endregion
// region creators
private LocalActiveFs(Eventloop eventloop, Path storage, Executor executor) {
this.eventloop = eventloop;
this.executor = executor;
this.storage = storage;
this.tempDir = storage.resolve(DEFAULT_TEMP_DIR);
now = eventloop;
if (DEFAULT_FSYNC_APPENDS) {
appendOptions.add(SYNC);
appendNewOptions.add(SYNC);
}
}
public static LocalActiveFs create(Eventloop eventloop, Executor executor, Path storageDir) {
return new LocalActiveFs(eventloop, storageDir, executor);
}
/**
* Sets the buffer size for reading files from the filesystem.
*/
public LocalActiveFs withReaderBufferSize(MemSize size) {
readerBufferSize = size;
return this;
}
/**
* If set to {@code true}, an attempt to create a hard link will be made when copying files
*/
@SuppressWarnings("UnusedReturnValue")
public LocalActiveFs withHardLinkOnCopy(boolean hardLinkOnCopy) {
this.hardlinkOnCopy = hardLinkOnCopy;
return this;
}
/**
* Sets a temporary directory for files to be stored while uploading.
*/
public LocalActiveFs withTempDir(Path tempDir) {
this.tempDir = tempDir;
return this;
}
/**
* If set to {@code true}, all uploaded files will be synchronously persisted to the storage device.
*
* Note: may be slow when there are a lot of new files uploaded
*/
public LocalActiveFs withFSyncUploads(boolean fsync) {
this.fsyncUploads = fsync;
return this;
}
/**
* If set to {@code true}, all newly created directories as well all changes to the directories
* (e.g. adding new files, updating existing, etc.)
* will be synchronously persisted to the storage device.
*
* Note: may be slow when there are a lot of new directories created or or changed
*/
public LocalActiveFs withFSyncDirectories(boolean fsync) {
this.fsyncDirectories = fsync;
return this;
}
/**
* If set to {@code true}, each write to {@link #append)} consumer will be synchronously written to the storage device.
*
* Note: significantly slows down appends
*/
public LocalActiveFs withFSyncAppends(boolean fsync) {
if (fsync) {
appendOptions.add(SYNC);
appendNewOptions.add(SYNC);
} else {
appendOptions.remove(SYNC);
appendNewOptions.remove(SYNC);
}
return this;
}
/**
* Sets file persistence options
*
* @see #withFSyncUploads(boolean)
* @see #withFSyncDirectories(boolean)
* @see #withFSyncAppends(boolean)
*/
public LocalActiveFs withFSync(boolean fsyncUploads, boolean fsyncDirectories, boolean fsyncAppends) {
this.fsyncUploads = fsyncUploads;
this.fsyncDirectories = fsyncDirectories;
return withFSyncAppends(fsyncAppends);
}
// endregion
@Override
public Promise> upload(@NotNull String name) {
return uploadImpl(name, identity())
.whenComplete(toLogger(logger, TRACE, "upload", name, this));
}
@Override
public Promise> upload(@NotNull String name, long size) {
return uploadImpl(name, ofFixedSize(size))
.whenComplete(toLogger(logger, TRACE, "upload", name, size, this));
}
@Override
public Promise> append(@NotNull String name, long offset) {
checkArgument(offset >= 0, "Offset cannot be less than 0");
return execute(
() -> {
Path path = resolve(name);
FileChannel channel;
if (offset == 0) {
channel = ensureTarget(null, path, () -> FileChannel.open(path, appendNewOptions));
if (fsyncDirectories) {
tryFsync(path.getParent());
}
} else {
channel = FileChannel.open(path, appendOptions);
}
long size = channel.size();
if (size < offset) {
throw new IllegalOffsetException("Offset " + offset + " exceeds file size " + size);
}
return channel;
})
.thenEx(translateScalarErrors(name))
.whenComplete(appendBeginPromise.recordStats())
.map(channel -> {
ChannelFileWriter writer = ChannelFileWriter.create(executor, channel)
.withOffset(offset);
if (fsyncUploads && !appendOptions.contains(SYNC)) {
writer.withForceOnClose(true);
}
return writer
.withAcknowledgement(ack -> ack
.thenEx(translateScalarErrors(name))
.whenComplete(appendFinishPromise.recordStats())
.whenComplete(toLogger(logger, TRACE, "onAppendComplete", name, offset, this)));
})
.whenComplete(toLogger(logger, TRACE, "append", name, offset, this));
}
@Override
public Promise> download(@NotNull String name, long offset, long limit) {
checkArgument(offset >= 0, "offset < 0");
checkArgument(limit >= 0, "limit < 0");
return resolveAsync(name)
.then(path -> execute(() -> {
FileChannel channel = FileChannel.open(path, READ);
long size = channel.size();
if (size < offset) {
throw new IllegalOffsetException("Offset " + offset + " exceeds file size " + size);
}
return channel;
}))
.map(channel -> ChannelFileReader.create(executor, channel)
.withBufferSize(readerBufferSize)
.withOffset(offset)
.withLimit(limit)
.withEndOfStream(eos -> eos
.thenEx(translateScalarErrors(name))
.whenComplete(downloadFinishPromise.recordStats())
.whenComplete(toLogger(logger, TRACE, "onDownloadComplete", name, offset, limit))))
.thenEx(translateScalarErrors(name))
.whenComplete(toLogger(logger, TRACE, "download", name, offset, limit, this))
.whenComplete(downloadBeginPromise.recordStats());
}
@Override
public Promise