com.metaeffekt.artifact.analysis.flow.ObserveFolderFlow Maven / Gradle / Ivy
/*
* Copyright 2021-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.metaeffekt.artifact.analysis.flow;
import com.metaeffekt.artifact.analysis.utils.FileUtils;
import com.metaeffekt.artifact.analysis.utils.PropertyUtils;
import com.metaeffekt.flow.common.AbstractFlow;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
public class ObserveFolderFlow extends AbstractFlow {
private static final Logger LOG = LoggerFactory.getLogger(ObserveFolderFlow.class);
private static final String KEY_STATUS = "status";
private static final String KEY_CHECKSUM = "checksum";
private static final String STATUS_NONE = "none";
private static final String STATUS_COMPLETED = "completed";
private static final String STATUS_INITIALIZED = "initialized";
private static final String STATUS_PROCESSING = "processing";
public void process(ObserveFolderFlowParam observeFolderFlowParam) {
final File observationDir = observeFolderFlowParam.getObservationDir();
final File statusBaseDir = observeFolderFlowParam.getStatusBaseDir();
final File tmpBaseDir = observeFolderFlowParam.getTmpBaseDir();
final int delayInSeconds = observeFolderFlowParam.getDelayInSeconds();
// check pre-requisites
validateObservationDir(observationDir);
final Map fileToChecksumMap = new HashMap<>();
final ExecutorService executor = Executors.newFixedThreadPool(observeFolderFlowParam.getNumThreads());
// run scan / trigger loop
while (true) {
LOG.debug("Running loop in [{}]", observationDir);
// inspect observation directory for files and folders
final File[] files = observationDir.listFiles(pathname -> {
// skip fs-level temp files
if (pathname.getName().startsWith(".DS_Store")) return false;
// skip empty dirs
if (pathname.isDirectory() && Objects.requireNonNull(pathname.listFiles(
path -> !path.getName().startsWith(".DS_Store"))).length == 0) return false;
return true;
});
LOG.debug("Running observe loop part.");
if (files != null) {
for (File file : files) {
observeFolderAndManageTriggerFile(file, fileToChecksumMap, statusBaseDir, tmpBaseDir);
}
LOG.debug("Running trigger loop part.");
for (File file : files) {
observeTriggerFile(statusBaseDir, file, executor, observeFolderFlowParam.getConsumer());
}
}
try {
Thread.sleep(delayInSeconds * 1000);
} catch (InterruptedException e) {
break;
}
}
awaitTermination(executor);
}
protected void awaitTermination(ExecutorService executor) {
// shutdown and wait until all commands have finished or an error occurred
executor.shutdown();
while (!executor.isTerminated()) {
try {
executor.awaitTermination(200, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
// nothing to do
}
}
}
private void observeTriggerFile(File statusBaseDir, File file, ExecutorService executor, Consumer consumer) {
final File triggerPropertiesFile = new File(statusBaseDir, file.getName() + ".trigger.properties");
if (triggerPropertiesFile.exists()) {
final Properties triggerProperties = PropertyUtils.loadProperties(triggerPropertiesFile);
final String status = triggerProperties.getProperty(KEY_STATUS, STATUS_NONE);
if (status.equals(STATUS_INITIALIZED)) {
LOG.warn("Triggering process execution for [{}].", file);
triggerProperties.setProperty(KEY_STATUS, STATUS_PROCESSING);
PropertyUtils.saveProperties(triggerPropertiesFile, triggerProperties);
executor.submit(() -> {
LOG.info("Processing [{}]...", file);
consumer.accept(file);
final Properties p = PropertyUtils.loadProperties(triggerPropertiesFile);
p.setProperty(KEY_STATUS, STATUS_COMPLETED);
PropertyUtils.saveProperties(triggerPropertiesFile, p);
LOG.info("Processing [{}] completed.", file);
});
}
}
}
private void observeFolderAndManageTriggerFile(File file, Map fileToChecksumMap, File statusBaseDir, File tmpBaseDir) {
final String md5Checksum;
try {
if (file.isDirectory()) {
// compute recursive checksum using tmp directory
final File contentChecksumFile = new File(tmpBaseDir, file.getName() + ".content.md5");
try {
FileUtils.createDirectoryContentChecksumFile(file, contentChecksumFile);
md5Checksum = FileUtils.computeChecksum(contentChecksumFile);
} finally {
FileUtils.forceDelete(contentChecksumFile);
}
} else {
// compute single checksum
md5Checksum = FileUtils.computeChecksum(file);
}
final String previousChecksum = fileToChecksumMap.get(file);
if (previousChecksum == null) {
// if no checksum exists is the first time, we encounter the file; put into map; do nothing more
LOG.debug("Observing [{}] the first time.", file.getAbsolutePath());
fileToChecksumMap.put(file, md5Checksum);
} else {
// if there is already a checksum, we compare
if (previousChecksum.equals(md5Checksum)) {
// they are equal; meaning that since last attempt no change to the file/directory is observed
// trigger scan if not already done / running / performed
LOG.debug("Observing process status for [{}].", file.getAbsolutePath());
final File triggerPropertiesFile = new File(statusBaseDir, file.getName() + ".trigger.properties");
final Properties triggerProperties;
if (triggerPropertiesFile.exists()) {
triggerProperties = PropertyUtils.loadProperties(triggerPropertiesFile);
} else {
triggerProperties = new Properties();
}
final String triggerChecksum = triggerProperties.getProperty(KEY_CHECKSUM, "");
final String triggerStatus = triggerProperties.getProperty(KEY_STATUS, STATUS_NONE);
switch (triggerStatus) {
case STATUS_NONE:
initializeTrigger(file, md5Checksum, triggerPropertiesFile);
break;
case STATUS_COMPLETED:
case STATUS_INITIALIZED:
if (triggerChecksum.equals(md5Checksum)) {
// process has already been initiated/completed with the same content; done
} else {
initializeTrigger(file, md5Checksum, triggerPropertiesFile);
}
break;
default:
// for any other anticipated status (failed, in progress, ...) we skip
LOG.info("Skipping process trigger for [{}]. Status is [{}].", file.getAbsolutePath(), triggerStatus);
}
} else {
// checksums are different; there are changes since last attempt; update checksum and continue loop
fileToChecksumMap.put(file, md5Checksum);
LOG.info("Observed changes in [{}].", file.getAbsolutePath());
}
}
} catch (IOException e) {
LOG.warn("Exception [{}] while observing folder [{}]", e.getMessage(), file.getAbsolutePath());
}
}
private void initializeTrigger(File file, String md5Checksum, File triggerPropertiesFile) {
LOG.info("Initializing process trigger for [{}].", file.getAbsolutePath());
final Properties triggerProperties = new Properties();
triggerProperties.setProperty(KEY_CHECKSUM, md5Checksum);
triggerProperties.setProperty(KEY_STATUS, STATUS_INITIALIZED);
PropertyUtils.saveProperties(triggerPropertiesFile, triggerProperties);
}
private void validateObservationDir(File observationDir) {
if (!observationDir.exists()) {
throw new IllegalStateException(String.format("Observation directory [%s] must exist.", observationDir.getAbsolutePath()));
}
if (!observationDir.isDirectory()) {
throw new IllegalStateException(String.format("Observation directory [%s] not a directory.", observationDir.getAbsolutePath()));
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy