com.goeuro.sync4j.sync.Synchronizer Maven / Gradle / Ivy

Go to download
package com.goeuro.sync4j.sync;

import com.goeuro.sync4j.fs.*;
import com.goeuro.sync4j.fs.Detail.Kind;
import com.goeuro.sync4j.fs.FileSystem.WriteListener;
import com.goeuro.sync4j.sync.ElementFilter.Action;
import com.goeuro.sync4j.sync.Lock.LockResult;

import javax.annotation.Nonnegative;
import javax.annotation.Nonnull;
import javax.annotation.concurrent.Immutable;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.math.BigInteger;
import java.time.Duration;
import java.util.Collection;
import java.util.Map;
import java.util.Optional;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;
import java.util.logging.Logger;
import java.util.stream.Stream;

import static com.goeuro.sync4j.fs.Detail.Kind.directory;
import static com.goeuro.sync4j.fs.Detail.Kind.file;
import static com.goeuro.sync4j.fs.Paths.hasSameContent;
import static com.goeuro.sync4j.sync.ElementFilter.Action.*;
import static com.goeuro.sync4j.sync.Lock.lock;
import static com.goeuro.sync4j.sync.SynchronizeStep.step;
import static com.goeuro.sync4j.sync.Synchronizer.Result.*;
import static com.goeuro.sync4j.utils.IOStreams.closeQuietly;
import static java.util.Optional.empty;

@Immutable
public class Synchronizer {

    @Nonnull
    private static final Logger LOG = Logger.getLogger(Synchronizer.class.getName());
    @Nonnull
    protected static final ProgressListener DEFAULT_LISTENER = new ProgressListener() {
    };

    @Nonnull
    public static Builder synchronizer() {
        return new Builder();
    }

    @Nonnull
    private final ProgressListener progressListener;

    protected Synchronizer(
        @Nonnull ProgressListener progressListener
    ) {
        this.progressListener = progressListener;
    }

    public  void execute(@Nonnull SynchronizeTask task) {
        final AtomicReference result = new AtomicReference<>(failed);
        final SynchronizeStep step = step(task).build();
        final Optional> writeLock = createWriteLockInstanceIfRequired(task);
        if (!writeLock.map(instance -> tryLock(step, instance)).orElse(true)) {
            result.set(rejected);
            return;
        }
        writeLock.ifPresent(candidate -> progressListener().onAfterWriteLocked(step, candidate));
        try {
            final Optional> readLock = createReadLockInstanceIfRequired(task);
            if (!readLock.map(instance -> tryLock(step, instance)).orElse(true)) {
                result.set(rejected);
                return;
            }
            readLock.ifPresent(candidate -> progressListener().onAfterReadLocked(step, candidate));
            try {
                execute(step, true);
                result.set(success);
            } finally {
                try {
                    readLock.ifPresent(candidate -> progressListener().onBeforeReadUnlock(step, candidate, result.get()));
                } finally {
                    closeQuietly(readLock);
                }
            }
        } finally {
            try {
                writeLock.ifPresent(candidate -> progressListener().onBeforeWriteUnlock(step, candidate, result.get()));
            } finally {
                closeQuietly(writeLock);
            }
        }
    }

    @Nonnull
    protected  Optional> createReadLockInstanceIfRequired(@Nonnull SynchronizeTask task) {
        return task.readLockFilename().map(lockFilename -> {
            final Path lockFile = task.retriablePathOperations().resolve(task.from(), lockFilename);
            return lock(lockFile)
                .whichRefreshesEvery(task.readLockRefreshEvery())
                .withTimeoutAfter(task.readLockTimeout())
                .withRetryEvery(task.readLockRetryEvery())
                .withOwner(task.lockOwner().orElse(null))
                .build();
        });
    }

    @Nonnull
    protected  Optional> createWriteLockInstanceIfRequired(@Nonnull SynchronizeTask task) {
        return task.writeLockFilename().map(lockFilename -> {
            final Path lockFile = task.retriablePathOperations().resolve(task.to(), lockFilename);
            return lock(lockFile)
                .whichRefreshesEvery(task.writeLockRefreshEvery())
                .withTimeoutAfter(task.writeLockTimeout())
                .withRetryEvery(task.writeLockRetryEvery())
                .withOwner(task.lockOwner().orElse(null))
                .build();
        });
    }

    protected  boolean tryLock(@Nonnull SynchronizeStep step, @Nonnull Lock lock) {
        final LockResult result = lock.tryLock(
            step.task().writeLockAcquireTimeout().orElse(null),
            (l, lr, elapsed, timeout) -> progressListener().nextAcquireLockRetryAllowed(step, l, lr, elapsed, timeout)
        );
        progressListener().onAfterWriteLockAcquireAttempt(step, lock, result);
        return result.success();
    }

    @Nonnull
    protected   Optional> detailsOf(@Nonnull SynchronizeStep step, @Nonnull Path path) {
        return step.task().retriablePathOperations().detailsOf(path);
    }

    protected  void execute(@Nonnull SynchronizeStep step, boolean isRootElement) {
        detailsOf(step, step.from()).ifPresent(candidate -> {
            if (shouldFromBeRespected(step) != include) {
                return;
            }
            if (candidate.kind() == directory) {
                executeForDirectory(step);
            } else {
                if (isRootElement) {
                    if (detailsOf(step, step.to())
                        .flatMap(to -> hasSameContent(candidate, to))
                        .orElse(false)) {
                        return;
                    }
                }
                executeForFile(step, (File) candidate);
            }
        });
    }

    protected  void executeForDirectory(@Nonnull SynchronizeStep step) {
        final Map, Detail> notHandledToFiles = listOf(step, step.to());
        createToDirectory(step);

        try (final Stream> list = step.task().retriablePathOperations().list(step.from())) {
            list
                .forEach(subFromDetails -> executeForSpecificPathWhichShouldResolveTo(step.forActualFrom(subFromDetails), notHandledToFiles));
        }

        cleanOrphanedPathsOnRemote(step, notHandledToFiles);
    }

    protected  void executeForFile(@Nonnull SynchronizeStep step, @Nonnull File file) {
        final boolean wantTransferProgress = progressListener().wantTransferProgress(step.to().loose());
        try {
            step.task().retriablePathOperations().copy(step.from(), step.to(), new WriteListener() {
                @Override
                public void onTransferProgress(@Nonnull LoosePath loosePath, @Nonnegative @Nonnull BigInteger transferred) {
                    if (wantTransferProgress) {
                        progressListener().onTransferProgress(step, file.size(), transferred);
                    }
                }

                @Override
                public void onTransferDone(@Nonnull LoosePath loosePath, @Nonnegative @Nonnull BigInteger total) {
                    progressListener().onTransferDone(step, total);
                }
            }).orElseThrow(() -> new IllegalStateException("Source file '" + step.from() + "' does not longer exists but just a second before?"));
        } catch (final IOException e) {
            throw new UncheckedIOException("Could not copy '" + step.from() + "' -> '" + step.to() + "'.", e);
        }
    }

    protected  void executeForSpecificPathWhichShouldResolveTo(@Nonnull SynchronizeStep step, @Nonnull Map, Detail> notHandledToFiles) {
        kindOfFrom(step);
        final String relativePath = step.task().retriablePathOperations().relativize(step.task().from(), step.from());
        final Path relativeToFromTo = step.task().retriablePathOperations().resolve(step.task().to(), relativePath);
        final SynchronizeStep newTask = step.forActualTo(relativeToFromTo);
        executeSpecific(newTask, notHandledToFiles);
    }

    protected  void executeSpecific(@Nonnull SynchronizeStep step, @Nonnull Map, Detail> notHandledToFiles) {
        final Kind kind = kindOfFrom(step);
        final Action action = shouldFromBeRespected(step);
        if (action == ignore) {
            notHandledToFiles.remove(step.to());
            return;
        }
        if (action == exclude) {
            return;
        }
        final Path existingToDetail = notHandledToFiles.remove(step.to());
        if (existingToDetail == null) {
            executeForAbsent(step);
        } else {
            if (hasSameContent(existingToDetail, step.from()).orElse(false)) {
                executeForAlreadyUpToDate(step);
            } else if (kind == directory) {
                execute(step, false);
            } else {
                executeForExistingThatNeedsUpdate(step);
            }
        }
    }

    protected  void executeForAbsent(@Nonnull SynchronizeStep step) {
        if (kindOfFrom(step) == file) {
            progressListener().onCreatingNewFile(step);
        }
        execute(step, false);
    }

    protected  void executeForAlreadyUpToDate(@Nonnull SynchronizeStep step) {
        progressListener().onSkippedBecauseAlreadyUpToDate(step);
    }

    protected  void executeForExistingThatNeedsUpdate(@Nonnull SynchronizeStep step) {
        progressListener().onExistingWillBeUpdated(step);
        execute(step, false);
    }

    @Nonnull
    protected  Kind kindOfFrom(@Nonnull SynchronizeStep step) {
        final Path from = step.from();
        if (from instanceof Detail) {
            return ((Detail) from).kind();
        }
        throw new IllegalArgumentException("The to part of a step provided to this method should be detailed but got: " + step.getClass());
    }

    protected  void createToDirectory(@Nonnull SynchronizeStep step) {
        final Path target = step.to();
        if (!detailsOf(step, target).isPresent()) {
            progressListener().onCreatingNewDirectory(step);
            try {
                Paths.createDirectory(target);
            } catch (final IOException e) {
                throw new UncheckedIOException("Could not create directory '" + target + "'.", e);
            }
        }
    }

    @Nonnull
    protected  Action shouldFromBeRespected(@Nonnull SynchronizeStep step) {
        return shouldBeRespected(step, SynchronizeTask::from, SynchronizeStep::from);
    }

    @Nonnull
    protected  Action shouldToBeRespected(@Nonnull SynchronizeStep step) {
        return shouldBeRespected(step, SynchronizeTask::to, SynchronizeStep::to);
    }

    @Nonnull
    protected  Action shouldBeRespected(
        @Nonnull SynchronizeStep step,
        @Nonnull Function, Path> resolveBase,
        @Nonnull Function, Path> resolveActual
    ) {
        final SynchronizeTask task = step.task();
        final Path base = resolveBase.apply(task);
        final Path actual = resolveActual.apply(step);
        final String relativized = step.task().retriablePathOperations().relativize(base, actual);
        if (relativized.isEmpty()) {
            return include;
        }
        final Path resolved = step.task().retriablePathOperations().resolve(base, relativized);
        final Action action = task.filter().handle(resolved, relativized);
        if (action == exclude) {
            progressListener().onIgnoredBecauseNotIncluded(step);
        }
        return action;
    }

    protected  void cleanOrphanedPathsOnRemote(@Nonnull SynchronizeStep step, @Nonnull Map, Detail> notHandledToFiles) {
        cleanOrphanedPathsOnRemote(step, notHandledToFiles.values());
    }

    protected  void cleanOrphanedPathsOnRemote(@Nonnull SynchronizeStep step, @Nonnull Collection> notHandledToFiles) {
        notHandledToFiles.forEach(path -> step.task().retriablePathOperations().delete(path, candidate -> {
            final SynchronizeStep newStep = step.forActualTo(candidate);
            if (shouldToBeRespected(newStep) == ignore) {
                return false;
            }
            progressListener().onDeletingOldOne(newStep);
            return true;
        }));
    }

    @Nonnull
    protected  Map, Detail> listOf(@Nonnull SynchronizeStep step, @Nonnull Path path) {
        final Map, Detail> results = new TreeMap<>();
        try (final Stream> paths = step.task().retriablePathOperations().list(path)) {
            paths
                .forEach(detail -> results.put(detail, detail));
        }
        return results;
    }

    @Nonnull
    protected ProgressListener progressListener() {
        return progressListener;
    }

    public static class Builder {

        @Nonnull
        private Optional progressListener = empty();

        protected Builder() {}

        @Nonnull
        public Builder withProgressListener(@Nonnull ProgressListener progressListener) {
            this.progressListener = Optional.of(progressListener);
            return this;
        }

        @Nonnull
        public Synchronizer build() {
            return new Synchronizer(
                progressListener.orElse(DEFAULT_LISTENER)
            );
        }
    }

    public interface ProgressListener {

        default  boolean nextAcquireLockRetryAllowed(
            @Nonnull SynchronizeStep step,
            @Nonnull Lock lock,
            @Nonnull LockResult lockResult,
            @Nonnull Duration elapsed,
            @Nonnull Duration timeout
        ) {
            LOG.info(() -> {
                final String specific = lockResult.otherContent()
                    .map(content -> content.owner()
                        .map(owner -> " owned by '" + owner + "'")
                        .orElseGet(() -> "") +
                        " #" + content.id() + " which last pinged at " + content.lastPing()
                    ).orElseGet(() -> "");
                return "Could not acquire lock since " + elapsed + " because blocked by other instance" + specific + "." +
                    " Retry until " + timeout + ".";
            });
            return true;
        }

        default  void onAfterWriteLockAcquireAttempt(
            @Nonnull SynchronizeStep step,
            @Nonnull Lock lock,
            @Nonnull LockResult lockResult
        ) {
            if (!lockResult.success()) {
                throw new IllegalStateException("Could not create write lock. Got: " + lockResult);
            }
        }

        default  void onAfterReadLocked(@Nonnull SynchronizeStep step, @Nonnull Lock lock) {
        }

        default  void onBeforeReadUnlock(@Nonnull SynchronizeStep step, @Nonnull Lock lock, @Nonnull Result result) {
        }

        default  void onAfterWriteLocked(@Nonnull SynchronizeStep step, @Nonnull Lock lock) {
        }

        default  void onBeforeWriteUnlock(@Nonnull SynchronizeStep step, @Nonnull Lock lock, @Nonnull Result result) {
        }

        default  void onIgnoredBecauseNotIncluded(@Nonnull SynchronizeStep step) {
        }

        default  void onSkippedBecauseAlreadyUpToDate(@Nonnull SynchronizeStep step) {
        }

        default  void onExistingWillBeUpdated(@Nonnull SynchronizeStep step) {
        }

        default  void onCreatingNewDirectory(@Nonnull SynchronizeStep step) {
        }

        default  void onCreatingNewFile(@Nonnull SynchronizeStep step) {
        }

        default  void onDeletingOldOne(@Nonnull SynchronizeStep step) {
        }

        default  void onTransferProgress(@Nonnull SynchronizeStep step, @Nonnegative @Nonnull BigInteger total, @Nonnegative @Nonnull BigInteger transferred) {
        }

        default  void onTransferDone(@Nonnull SynchronizeStep step, @Nonnegative @Nonnull BigInteger total) {
        }

        default boolean wantTransferProgress(@Nonnull LoosePath target) {
            return false;
        }
    }

    public enum Result {
        success,
        failed,
        rejected
    }
}