net.aequologica.neo.dagr.Dags Maven / Gradle / Ivy
The newest version!
package net.aequologica.neo.dagr;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.StringWriter;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.nio.file.Paths;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Observable;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.type.TypeReference;
import net.aequologica.neo.dagr.DagOnSteroids.DagCleaner;
import net.aequologica.neo.dagr.bus.Bus;
import net.aequologica.neo.dagr.model.Dag;
import net.aequologica.neo.dagr.model.Dag.Node;
import net.aequologica.neo.geppaequo.document.DocumentHelper;
import net.aequologica.neo.geppaequo.document.impl.DocumentHelperFactory;
import net.aequologica.neo.serioulizer.Serioulizer;
import net.aequologica.neo.serioulizer.jackson.SeriousReaderWriter;
@Singleton
public class Dags extends Observable {
@Inject
private Serioulizer serioulizer;
private SeriousReaderWriter serializer;
private final Logger log = LoggerFactory.getLogger(Dags.class);
private final FileSystem fileSystem = FileSystems.getDefault();
private final String DAGS_DIRECTORY = "/dags";
private final DocumentHelper documentHelper = DocumentHelperFactory.DOCUMENTHELPERFACTORY.getDocumentHelper();
private final Lock lockOnIO;
private final Map map;
Dags() {
this.lockOnIO = new ReentrantLock();
this.map = new HashMap<>();
}
@PostConstruct
void postConstruct() {
serializer= new SeriousReaderWriter<>(new TypeReference() {}, serioulizer);
loadDags();
DagBerk.PROFOND_DESESPOIR.cdiEstDeLaMerdeEnPot(this);
}
/////////////////////////////
// collections getters
public Set getDagNames() {
return this.map.keySet();
}
public Collection getDagOnSteroidss() {
return this.map.values();
}
public List getDAGs() {
return this.map.values().stream().map(b -> b.getDag()).collect(Collectors.toList());
}
/////////////////////////////
// elements getters
public DagOnSteroids getDagOnSteroids(final String dagkey) {
return this.map.get(dagkey);
}
public Dag getDAG(final String dagkey) {
DagOnSteroids dagOnSteroids = this.map.get(dagkey);
return dagOnSteroids == null ? null : dagOnSteroids.getDag();
}
public Bus getBus(final String dagkey, final Scope scope) {
final DagOnSteroids dagOnSteroids = this.map.get(dagkey);
if (dagOnSteroids == null) {
return null;
}
final DagCleaner dagCleaner = dagOnSteroids.getDagCleaner(scope);
if (dagCleaner == null) {
return null;
}
return dagCleaner.getBus();
}
/////////////////////////////
// loaders
public Collection>>> loadDags() {
this.lockOnIO.lock(); // block until condition holds
try {
final Collection>>> allExceptions = new ArrayList<>();
// notify before clear
setChanged();
notifyObservers(Boolean.TRUE);
this.map.clear();
List sources;
try {
sources = privateGetDagSources();
} catch (Exception e) {
Collection> bigError = new ArrayList<>();
bigError.add(tuple(e.getClass().getSimpleName(), e.getMessage()));
allExceptions.add(tuple("", bigError));
return allExceptions;
}
for (Path source : sources) {
PathMatcher matcher = fileSystem.getPathMatcher("regex:^.*\\.(properties|yml|hbs|gitmodules|xml|pom)$");
if (matcher.matches(source.getFileName())) {
continue;
}
allExceptions.add(privateLoadDag(null, source));
}
// notify after puts
setChanged();
notifyObservers(allExceptions);
return allExceptions;
} finally {
this.lockOnIO.unlock();
}
}
public Map.Entry>> loadDag(String dagName) {
this.lockOnIO.lock(); // block until condition holds
try {
// notify before clear
setChanged();
notifyObservers(dagName);
final Dag dag = getDAG(dagName);
final Path source;
if (dag != null) {
source = Paths.get(dag.getSource());
} else {
try {
source = privateLocateSourceWithDagName(dagName);
} catch (IOException e) {
Collection> exceptions = new ArrayList<>();
exceptions.add(tuple(e.getClass().getSimpleName(), e.getMessage()));
return tuple(dagName, exceptions);
}
}
final Map.Entry>> thisDagExceptions = privateLoadDag(dagName, source);
// notify after puts
setChanged();
notifyObservers(thisDagExceptions);
return thisDagExceptions;
} finally {
this.lockOnIO.unlock();
}
}
private class PropertiesAndExceptions {
Map properties;
final Collection> exceptions = new ArrayList<>();
void load(Path propertiesPath) {
// properties is optional, not an issue if it does not exists, simply return null
try (final InputStream inputStream = documentHelper.getInputStream(propertiesPath);) {
if (inputStream == null) {
throw new IOException("no ["+ propertiesPath + "] in document service");
}
try {
final Properties properties = new Properties();
properties.load(new InputStreamReader(inputStream, "UTF-8"));
this.properties = properties.entrySet().stream().collect(Collectors.toMap(e -> e.getKey().toString(), e -> e.getValue().toString()));
} catch (Exception accumulated) {
// if properties exists, and there is an error loading / java8streamingCollecting it, please report the error
exceptions.add(tuple(propertiesPath.getFileName().toString(), accumulated.getClass().getSimpleName() + " - " + accumulated.getMessage()));
this.properties = null;
}
} catch (Exception ignored) {
log.info("ignored exception {} {}", ignored.getClass().getSimpleName(), ignored.getMessage());
this.properties = null;
}
}
}
private Map.Entry>> privateLoadDag(String dagName, final Path source) {
if (source == null) {
throw new IllegalArgumentException("source must be not null");
}
final Collection> exceptions = new ArrayList<>();
try {
final Dag dag = serializer.read(source);
dag.setSource(source.toString());
if (dagName == null) {
dagName = dag.getName();
}
this.map.remove(dagName);
final Path sourceWithoutJSONExtensionIFAny;
int indexJsonExtension = source.toString().lastIndexOf(".json");
if (0 < indexJsonExtension) {
sourceWithoutJSONExtensionIFAny = Paths.get(source.toString().substring(0, indexJsonExtension));
} else {
sourceWithoutJSONExtensionIFAny = source;
}
// aliases
PropertiesAndExceptions aliasesPAE = new PropertiesAndExceptions();
aliasesPAE.load(Paths.get(sourceWithoutJSONExtensionIFAny+".aliases.properties"));
if (aliasesPAE.exceptions != null && aliasesPAE.exceptions.size() > 0) {
exceptions.addAll(aliasesPAE.exceptions);
}
// cleaner
PropertiesAndExceptions cleanerPAE = new PropertiesAndExceptions();
cleanerPAE.load(Paths.get(sourceWithoutJSONExtensionIFAny+".cleaner.properties"));
if (cleanerPAE.exceptions != null && cleanerPAE.exceptions.size() > 0) {
exceptions.addAll(cleanerPAE.exceptions);
}
loadSubs(dag);
map.put(dag.getName(), new DagOnSteroids(dag, aliasesPAE.properties, cleanerPAE.properties));
} catch (Exception e) {
exceptions.add(tuple(e.getClass().getSimpleName(), e.getMessage()));
}
return tuple(dagName!=null ? dagName : source.getFileName().toString(), exceptions);
}
public Path saveSub(Dag dag, String subDagId, String description) throws IOException {
final Path path = Paths.get(DAGS_DIRECTORY, dag.getName(), "subs", description+"__"+subDagId);
final List nodes = dag.getNodes(subDagId).stream().map(n -> n.getName()).collect(Collectors.toList());
Collections.sort(nodes);
StringWriter sw = new StringWriter();
for (String node : nodes) {
sw.write(node);
sw.write('\n');
}
documentHelper.write(path, Serioulizer.createStream(sw.toString()));
return path;
}
private void loadSubs(Dag dag) throws IOException {
List subs = Collections.emptyList();;
try {
subs = documentHelper.list(Paths.get(DAGS_DIRECTORY, dag.getName(), "subs"));
} catch (Exception ignored) {
return;
}
for (final Path sub : subs) {
String nodeNamesAsString = Serioulizer.read(documentHelper.getInputStream(sub));
Set nodeNames = new HashSet(Arrays.asList(nodeNamesAsString.split("\n")));
dag.registerSubDag(nodeNames);
}
}
void packageDumpDag(Path path, Dag dag) throws IOException {
serializer.write(path, dag);
}
private Map.Entry tuple(final String a, final T b) {
return new AbstractMap.SimpleImmutableEntry<>(a, b);
}
private Path privateLocateSourceWithDagName(final String dagName) throws IOException {
final List sources = privateGetDagSources();
for (final Path source : sources) {
final Dag dag = serializer.read(source);
if (dag.getName().equals(dagName)) {
return source;
}
}
List collect = sources
.stream()
.map(s -> s.getFileName().toString())
.collect(Collectors.toList());
throw new IOException("no dag with name /" + dagName + "/ found in " + collect);
}
private List privateGetDagSources() throws IOException {
final List potentials = documentHelper.list(Paths.get(DAGS_DIRECTORY));
final List sources = new ArrayList<>();
final PathMatcher extExclude = fileSystem.getPathMatcher("regex:^.*\\.(properties|yml|hbs|gitmodules|xml|pom|txt)$");
final PathMatcher dirExclude = fileSystem.getPathMatcher("regex:^.*[\\/\\\\]+(subs)[\\/\\\\]+.*$");
for (final Path source : potentials) {
if (extExclude.matches(source.getFileName())) {
continue;
}
if (dirExclude.matches(source)) {
continue;
}
sources.add(source);
}
return sources;
}
} © 2015 - 2025 Weber Informatics LLC | Privacy Policy