tv.hd3g.jobkit.watchfolder.Watchfolders Maven / Gradle / Ivy

Go to download
/*
 * This file is part of jobkit-engine.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 *
 * Copyright (C) hdsdi3g for hd3g.tv 2020
 *
 */
package tv.hd3g.jobkit.watchfolder;

import static java.time.Duration.ZERO;
import static java.time.temporal.ChronoUnit.MILLIS;
import static java.util.function.Predicate.not;
import static java.util.stream.Collectors.toUnmodifiableMap;
import static tv.hd3g.jobkit.watchfolder.RetryScanPolicyOnUserError.RETRY_FOUNDED_FILE;
import static tv.hd3g.jobkit.watchfolder.WatchFolderPickupType.FILES_ONLY;

import java.time.Duration;
import java.util.Collection;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Supplier;

import lombok.extern.slf4j.Slf4j;
import tv.hd3g.jobkit.engine.BackgroundService;
import tv.hd3g.jobkit.engine.JobKitEngine;

@Slf4j
public class Watchfolders {
	static final int DEFAULT_RETRY_AFTER_TIME = 10;

	private final Map observedFoldersServices;
	private final FolderActivity folderActivity;
	private final Duration defaultTimeBetweenScans;
	private final JobKitEngine jobKitEngine;
	private final String defaultSpoolScans;
	private final String defaultSpoolEvents;

	public Watchfolders(final Collection allObservedFolders,
						final FolderActivity folderActivity,
						final Duration defaultTimeBetweenScans,
						final JobKitEngine jobKitEngine,
						final String defaultSpoolScans,
						final String defaultSpoolEvents,
						final Supplier watchedFilesDbBuilder) {
		this.folderActivity = Objects.requireNonNull(folderActivity);
		this.defaultTimeBetweenScans = Objects.requireNonNull(defaultTimeBetweenScans);
		this.jobKitEngine = Objects.requireNonNull(jobKitEngine);
		this.defaultSpoolScans = Objects.requireNonNull(defaultSpoolScans);
		this.defaultSpoolEvents = Objects.requireNonNull(defaultSpoolEvents);
		Objects.requireNonNull(watchedFilesDbBuilder);

		final var allLabelsCount = (int) allObservedFolders.stream()
				.map(ObservedFolder::getLabel)
				.distinct()
				.count();
		if (allObservedFolders.size() != allLabelsCount) {
			throw new IllegalArgumentException(
					"ObservedFolders setup fail: you must have separate labels name for each entry");
		}

		observedFoldersServices = Objects.requireNonNull(allObservedFolders).stream()
				.filter(not(ObservedFolder::isDisabled))
				.collect(toUnmodifiableMap(
						of -> of,
						oF -> createService(oF, watchedFilesDbBuilder.get())));
		if (observedFoldersServices.isEmpty()) {
			log.warn("No configured watchfolders");
		}
	}

	Map getObservedFoldersServices() {
		return observedFoldersServices;
	}

	private void justLogAfterBadUserRun(final Exception e) {
		if (e != null) {
			log.error("Can't send event", e);
		}
	}

	BackgroundService createService(final ObservedFolder observedFolder, final WatchedFilesDb db) {
		final var name = observedFolder.getLabel();

		if (observedFolder.getSpoolEvents() == null
			|| observedFolder.getSpoolEvents().equals("")) {
			observedFolder.setSpoolEvents(defaultSpoolEvents);
		}
		if (observedFolder.getSpoolScans() == null
			|| observedFolder.getSpoolScans().equals("")) {
			observedFolder.setSpoolScans(defaultSpoolScans);
		}
		if (observedFolder.getTimeBetweenScans() == null
			|| observedFolder.getTimeBetweenScans().equals(ZERO)) {
			observedFolder.setTimeBetweenScans(defaultTimeBetweenScans);
		}
		if (observedFolder.getRetryAfterTimeFactor() < 1) {
			observedFolder.setRetryAfterTimeFactor(DEFAULT_RETRY_AFTER_TIME);
		}

		final var pickUp = folderActivity.getPickUpType(observedFolder);
		db.setup(observedFolder, Optional.ofNullable(pickUp).orElse(FILES_ONLY));

		return jobKitEngine.createService(
				"Watchfolder for " + name,
				observedFolder.getSpoolScans(),
				() -> {
					try (var fs = observedFolder.createFileSystem()) {
						log.trace("Start Watchfolder scan for {} :: {}", name, fs);
						jobKitEngine.runOneShot(
								"Watchfolder start dir scan for " + name,
								observedFolder.getSpoolEvents(),
								observedFolder.getJobsPriority(),
								() -> folderActivity.onBeforeScan(observedFolder), this::justLogAfterBadUserRun);
						final var startTime = System.currentTimeMillis();
						final var scanResult = db.update(observedFolder, fs);
						final var scanTime = Duration.of(System.currentTimeMillis() - startTime, MILLIS);

						jobKitEngine.runOneShot(
								"On event on watchfolder scan for " + name,
								observedFolder.getSpoolEvents(),
								observedFolder.getJobsPriority(),
								() -> folderActivity.onAfterScan(observedFolder, scanTime, scanResult),
								e -> {
									if (e == null) {
										return;
									}
									final var policy = folderActivity.retryScanPolicyOnUserError(
											observedFolder, scanResult, e);
									final var founded = scanResult.founded();
									if (founded.isEmpty() == false) {
										log.error("Can't process user event of onAfterScan ({} founded), policy is {}",
												founded.size(), policy, e);
										if (policy == RETRY_FOUNDED_FILE) {
											db.reset(observedFolder, founded);
										}
									} else {
										log.error("Can't process user event of onAfterScan", e);
									}
								});
						log.trace("Ends Watchfolder scan for {} :: {}", name, fs);
					} catch (final Exception e) {
						folderActivity.onScanErrorFolder(observedFolder, e);
						throw e;
					}
				},
				() -> folderActivity.onStopScan(observedFolder))
				.setTimedInterval(observedFolder.getTimeBetweenScans())
				.setRetryAfterTimeFactor(observedFolder.getRetryAfterTimeFactor())
				.setPriority(observedFolder.getJobsPriority());
	}

	public synchronized void startScans() {
		observedFoldersServices.forEach((oF, service) -> {
			if (service.isEnabled()) {
				return;
			}
			jobKitEngine.runOneShot(
					"Start (enable) watchfolder scans for " + oF.getLabel(),
					oF.getSpoolEvents(),
					oF.getJobsPriority(),
					() -> {
						folderActivity.onStartScan(oF);
						service.enable();
					}, this::justLogAfterBadUserRun);
		});
	}

	public synchronized void stopScans() {
		observedFoldersServices.values().stream()
				.forEach(BackgroundService::disable);
	}

}