Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
com.regnosys.rosetta.translate.Translator Maven / Gradle / Ivy
package com.regnosys.rosetta.translate;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeoutException;
import java.util.jar.JarOutputStream;
import java.util.stream.Collectors;
import java.util.zip.Deflater;
import java.util.zip.ZipEntry;
import com.regnosys.rosetta.common.compile.JavaCSourceCancellableCompiler;
import com.regnosys.rosetta.common.compile.JavaCancellableCompiler;
import com.regnosys.rosetta.common.compile.JavaCompilationResult;
import com.regnosys.rosetta.common.compile.JavaCompileReleaseFlag;
import org.apache.log4j.Logger;
import org.eclipse.emf.common.util.URI;
import org.eclipse.emf.ecore.resource.Resource;
import org.eclipse.emf.ecore.resource.ResourceSet;
import org.eclipse.xtext.resource.XtextResourceSet;
import com.google.common.base.Stopwatch;
import com.regnosys.rosetta.RosettaStandaloneSetup;
import com.regnosys.rosetta.common.util.ClassPathUtils;
import com.regnosys.rosetta.common.util.StreamUtils;
import com.regnosys.rosetta.common.util.UrlUtils;
import com.regnosys.rosetta.rosetta.RosettaEnumeration;
import com.regnosys.rosetta.rosetta.RosettaModel;
import com.regnosys.rosetta.rosetta.RosettaRootElement;
import com.regnosys.rosetta.rosetta.RosettaSynonymSource;
import com.regnosys.rosetta.rosetta.RosettaType;
import com.regnosys.rosetta.rosetta.simple.Data;
import com.regnosys.rosetta.translate.IngesterGenerator.GeneratedIngesters;
import com.regnosys.rosetta.translate.MappingError.MappingErrorLevel;
public class Translator {
private static final Logger LOGGER = Logger.getLogger(Translator.class);
private final TranslatorOptions options;
private final ClassLoader classLoader;
private final IngesterGenerator generator; // TODO: inject instead
private final SynonymToEnumMapGenerator synonymToEnumMapGenerator; // TODO: inject instead
public Translator(TranslatorOptions options, ClassLoader classLoader, IngesterGenerator generator, SynonymToEnumMapGenerator synonymToEnumMapGenerator) {
this.options = options;
this.classLoader = classLoader;
this.generator = generator;
this.synonymToEnumMapGenerator = synonymToEnumMapGenerator;
}
public GeneratedClasses generateClassesFromXmlSchema(Path baseDir) throws IOException, ClassNotFoundException {
return generateClassesFromXmlSchema(baseDir, true);
}
public GeneratedClasses generateClassesFromJsonSchema(Path baseDir) throws IOException, ClassNotFoundException {
return generateClassesFromJsonSchema(baseDir, true);
}
@SuppressWarnings("unchecked")
public GeneratedClasses generateClassesFromXmlSchema(Path baseDir, boolean generate) throws IOException, ClassNotFoundException {
Path classesPath = generateAndCompile(baseDir, generate);
// Load XmlHandlerFactory class
URLClassLoader mappingsClassLoader = URLClassLoader.newInstance(new URL[] { classesPath.toUri().toURL() }, classLoader);
String factoryName = String.format("%s.%s", options.getGeneratedPackage(), options.getGeneratedFactoryName());
Class factory = mappingsClassLoader.loadClass(factoryName);
// Basic sanity check
if (!XmlHandlerFactory.class.isAssignableFrom(factory)) {
throw new IllegalStateException(factory + " is not assignable from " + XmlHandlerFactory.class);
}
Class xmlFactory = (Class) factory;
Class synonymToEnumMap = createSynonymToEnumMap(mappingsClassLoader);
return new GeneratedClasses<>(xmlFactory, synonymToEnumMap);
}
@SuppressWarnings("unchecked")
public GeneratedClasses generateClassesFromJsonSchema(Path baseDir, boolean generate) throws IOException, ClassNotFoundException {
Path classesPath = generateAndCompile(baseDir, generate);
// Load XmlHandlerFactory class
URLClassLoader mappingsClassLoader = URLClassLoader.newInstance(new URL[] { classesPath.toUri().toURL() }, classLoader);
String factoryName = String.format("%s.%s", options.getGeneratedPackage(), options.getGeneratedFactoryName());
Class factory = mappingsClassLoader.loadClass(factoryName);
// Basic sanity check
if (!JsonHandlerFactory.class.isAssignableFrom(factory)) {
throw new IllegalStateException(factory + " is not assignable from " + JsonHandlerFactory.class);
}
Class jsonFactory = (Class) factory;
Class synonymToEnumMap = createSynonymToEnumMap(mappingsClassLoader);
return new GeneratedClasses<>(jsonFactory, synonymToEnumMap);
}
@SuppressWarnings("unchecked")
private Class createSynonymToEnumMap(URLClassLoader mappingsClassLoader) throws IOException, ClassNotFoundException {
// Load SynonymToEnumMap class
String className = String.format("%s.%s", options.getGeneratedPackage(), SynonymToEnumMapGenerator.GENERATED_CLASS_NAME);
Class mapBuilder = mappingsClassLoader.loadClass(className);
// Basic sanity check
if (!SynonymToEnumMapBuilder.class.isAssignableFrom(mapBuilder)) {
throw new IllegalStateException(mapBuilder + " is not assignable from " + SynonymToEnumMapBuilder.class);
}
return (Class) mapBuilder;
}
// Separate this work from above XMLhandlerFactory creation, since json does not need it
public Path generateAndCompile(Path baseDir, boolean generate) throws IOException {
Path srcPath = Files.createDirectories(baseDir.resolve("translate-src"));
Path classesDir = baseDir.resolve("translate-classes");
boolean hasClasses = Files.exists(classesDir) && Files.walk(classesDir).filter(Files::isRegularFile).findAny().isPresent();
Path classesPath = Files.createDirectories(classesDir);
if (generate) {
// Remove old src and classes
if (options.clean() && Files.exists(baseDir)) {
Files.walk(baseDir)
.sorted(Comparator.reverseOrder())
.map(Path::toFile)
.forEach(File::delete);
}
// Generate new src and classes
List generateJavaPaths = generateJava(srcPath, hasClasses);
// if the generation finishes and the compilation fails, no classes are written,
// re-running the code results in skipping compilation all together, resulting in missing classes
// TODO: consider check generated file path for corresponding class file and compile if different
if (!generateJavaPaths.isEmpty()) {
compileToClasses(srcPath, generateJavaPaths, classesPath, true);
}
}
if (!Files.exists(classesPath)) {
throw new IllegalStateException("Output classes path does not exist [" + classesPath + "]");
}
return classesPath;
}
public List generateJava(Path outputPath, boolean hasClasses) throws IOException {
LOGGER.info("Starting Java Code Generation to " + outputPath);
Stopwatch stopwatch = Stopwatch.createStarted();
List expandedModelPaths = ClassPathUtils.findPathsFromClassPath(options.getModelClasspath(), options.getModelFileDirIncludeRegex(),
options.getModelFileDirExcludeRegex(), classLoader);
List rosettaRootElements = loadRosettaRootElements(expandedModelPaths);
List rosettaClasses = loadRosettaClasses(options, rosettaRootElements);
List generatorParams = new ArrayList<>();
for (RosettaType rosettaClass : rosettaClasses) {
URL expandedXsdURL = getXsdURL(options, rosettaClass.getName());
LOGGER.info("Using xsd path: " + expandedXsdURL);
Collection synonymSourcesAsString = options.getSynonymSources(rosettaClass.getName());
List synonymSources = synonymSourcesAsString.stream()
.map(s -> loadRosettaSource(rosettaRootElements, s))
.collect(Collectors.toList());
Collection topLevelTags = options.getTopLevelTags(rosettaClass.getName());
GeneratorParams params = new GeneratorParams(
expandedXsdURL, Collections.singleton(rosettaClass), synonymSources, topLevelTags,
options.getChildPackageName(rosettaClass.getName()));
generatorParams.add(params);
}
String generatedFactoryName = options.getGeneratedFactoryName();
LOGGER.info("Generating handlers in " + options.getGeneratedPackage() + " with factory class " + generatedFactoryName);
GeneratedIngesters ingesters = options.isJson() ? generator.generateJson(options.getGeneratedPackage(), generatedFactoryName, generatorParams) :
generator.generateXml(options.getGeneratedPackage(), generatedFactoryName, generatorParams);
LOGGER.info("Generated " + ingesters.getGeneratedHandlers().size() + " Xml Handlers");
Set errors = new HashSet<>(ingesters.getErrors());
for (MappingError error : errors) {
if (error.getLevel() == MappingErrorLevel.ERROR) {
LOGGER.debug(error.getMessage());
}
if (error.getLevel() == MappingErrorLevel.WARNING) {
// LOGGER.debug(error.getMessage());
}
}
long warnCount = ingesters.getErrors().stream()
.filter(e -> e.getLevel() == MappingErrorLevel.WARNING)
.count();
LOGGER.warn("Model comparison generated " + warnCount + " warnings");
List generatedJavaPaths = writeOutJava(ingesters, outputPath, hasClasses);
LOGGER.info("Generating SynonymToEnumMap in " + options.getGeneratedPackage());
List rosettaEnumerations = loadRosettaEnumerations(rosettaRootElements);
GeneratedNameAndSource generatedSynonymToEnumMap = synonymToEnumMapGenerator.generate(options.getGeneratedPackage(), generatorParams, rosettaEnumerations);
writeClass(outputPath, generatedSynonymToEnumMap.getClassName(), generatedSynonymToEnumMap.getSource(), hasClasses).ifPresent(generatedJavaPaths::add);
LOGGER.info("Finished Java Code Generation. Took " + stopwatch.toString());
return generatedJavaPaths;
}
@SuppressWarnings("unused")
private Path compileToJar(Path javaSourcePath, Path outputJarPath, boolean useSystemClassPath, Path... additionalClassPaths) throws IOException {
List javaPaths = ClassPathUtils.expandPaths(Collections.singletonList(javaSourcePath), ".*\\.java", Optional.empty());
Path classesDir = compileToClasses(javaSourcePath, javaPaths, Files.createTempDirectory("translate-classes"), useSystemClassPath, additionalClassPaths);
Files.createDirectories(outputJarPath.getParent());
cleanupOldJars(outputJarPath);
return jar(outputJarPath, classesDir);
}
private Path jar(Path outputJarPath, Path outputClassesDir) throws IOException {
LOGGER.info("Creating JAR file " + outputJarPath.toAbsolutePath() + " from " + outputClassesDir.toAbsolutePath());
Path zipFilePath = Files.createFile(outputJarPath);
try (JarOutputStream zipOutputStream = new JarOutputStream(Files.newOutputStream(zipFilePath))) {
zipOutputStream.setLevel(Deflater.BEST_COMPRESSION);
List paths = Files.walk(outputClassesDir)
.filter(path -> !Files.isDirectory(path))
.collect(Collectors.toList());
for (Path path : paths) {
ZipEntry zipEntry = new ZipEntry(outputClassesDir.relativize(path).toString());
LOGGER.debug("Writing JAR file entry " + zipEntry);
zipOutputStream.putNextEntry(zipEntry);
zipOutputStream.write(Files.readAllBytes(path));
zipOutputStream.closeEntry();
zipOutputStream.flush();
}
zipOutputStream.flush();
}
LOGGER.info("Finished Creating JAR file " + outputJarPath.toAbsolutePath() + " from " + outputClassesDir.toAbsolutePath());
return zipFilePath;
}
private void cleanupOldJars(Path outputJarPath) throws IOException {
if (Files.exists(outputJarPath)) {
String date = DateTimeFormatter.ofPattern("yyyyMMddHHmmss").format(LocalDateTime.now());
Files.move(outputJarPath, outputJarPath.resolveSibling(date + "-" + outputJarPath.getFileName().toString()));
}
List sortedJarFiles = Files.list(outputJarPath.getParent())
.filter(p -> p.getFileName().toString().endsWith(outputJarPath.getFileName().toString()))
.sorted()
.collect(Collectors.toList());
// Delete everything but the last 5
for (int i = 0; i < sortedJarFiles.size() - 5; i++) {
LOGGER.info("Deleting old jar file " + sortedJarFiles.get(i) + ".");
Files.delete(sortedJarFiles.get(i));
}
}
public Path compileToClasses(Path javaSourcePath, List generateJavaPaths, Path outputClassesDir, boolean useSystemClassPath,
Path... additionalClassPaths) throws IOException {
Files.createDirectories(outputClassesDir);
LOGGER.info("Starting Compiling " + javaSourcePath.toString() + " to " + outputClassesDir.toString());
LOGGER.debug(String.format("useSystemClassPath %s - %s", useSystemClassPath, Arrays.toString(additionalClassPaths)) );
Stopwatch stopwatch = Stopwatch.createStarted();
ExecutorService executorService = Executors.newSingleThreadExecutor();
JavaCancellableCompiler javaCompiler = new JavaCSourceCancellableCompiler(executorService, useSystemClassPath, true, options.isVerbose(), JavaCompileReleaseFlag.JAVA_11, additionalClassPaths);
try {
JavaCompilationResult compile = javaCompiler.compile(generateJavaPaths, outputClassesDir, () -> false);
if (!compile.isCompilationSuccessful()) {
LOGGER.error("Compilation Failed: " + compile.getDiagnostics());
}
} catch (ExecutionException e) {
LOGGER.error("Error thrown during compilation", e);
throw new RuntimeException(e);
} catch (InterruptedException e) {
LOGGER.error("Unexpected interrupt during compilation in Translator", e);
throw new IllegalStateException(e);
} catch (TimeoutException e) {
LOGGER.error("Timed out during compilation", e);
throw new RuntimeException(e);
} finally {
executorService.shutdown();
}
LOGGER.info("Finished Compiling. Took " + stopwatch.toString());
return outputClassesDir;
}
// TODO A lot of this work is repeated in ModelLoader
private List loadRosettaRootElements(List expandedModelPaths) {
LOGGER.info("Loading rosetta root elements");
RosettaStandaloneSetup.doSetup();
ResourceSet resourceSet = createResourceSet(expandedModelPaths);
List rootElements = resourceSet.getResources()
.stream()
.map(Resource::getContents)
.flatMap(Collection::stream)
.map(r -> (RosettaModel) r)
.filter(Objects::nonNull)
.map(RosettaModel::getElements)
.flatMap(Collection::stream)
.collect(Collectors.toList());
LOGGER.info("Found " + rootElements.size() + " root elements");
return rootElements;
}
private List loadRosettaClasses(TranslatorOptions options, List rosettaRootElements) {
LOGGER.info("Trying to load model classes : " + getFullClassName(options));
List rosettaClasses = rosettaRootElements.stream()
.filter(c -> c instanceof Data)
.map(c -> (Data) c)
.filter(c -> getFullClassName(options).contains(getFullClassName(c)))
.filter(StreamUtils.distinctByKey(c->c.getName()))
.collect(Collectors.toList());
LOGGER.info("Found model classes : " + rosettaClasses.stream()
.map(this::getFullClassName)
.collect(Collectors.toList()));
return rosettaClasses;
}
private List loadRosettaEnumerations(List rosettaRootElements) {
LOGGER.info("Load model enumerations");
List rosettaEnums = rosettaRootElements.stream()
.filter(e -> e instanceof RosettaEnumeration)
.map(e -> (RosettaEnumeration) e)
.filter(StreamUtils.distinctByKey(c->c.getName()))
.collect(Collectors.toList());
LOGGER.info("Found " + rosettaEnums.size() + " model enumerations");
return rosettaEnums;
}
private RosettaSynonymSource loadRosettaSource(List rosettaRootElements, String sourceName) {
return rosettaRootElements.stream()
.filter(c -> c instanceof RosettaSynonymSource)
.map(c -> (RosettaSynonymSource) c)
.filter(c -> c.getName().equals(sourceName))
.findAny()
.orElseThrow(() -> new IllegalArgumentException("Could not find source with name " + sourceName));
}
private List getFullClassName(TranslatorOptions options) {
return options.getRosettaClasses().stream()
.map(x -> options.getFullClassname(x))
.collect(Collectors.toList());
}
private String getFullClassName(Data data) {
String namespace = Optional.ofNullable(data.getModel()).map(RosettaModel::getName).map(ns -> ns + ".").orElse("");
return namespace + data.getName();
}
private URL getXsdURL(TranslatorOptions options, String rosettaClassName) {
String xsdFilePath = options.getXsdFilePath(rosettaClassName);
Path path = Paths.get(xsdFilePath);
if (Files.exists(path)) {
return UrlUtils.toUrl(path);
}
URL resource = classLoader.getResource(xsdFilePath);
if (resource == null) {
throw new IngestException("Error reading xsd file - " + xsdFilePath + " could not be found");
}
return resource;
}
private static List writeOutJava(GeneratedIngesters ingesters, Path outputPath, boolean hasClasses) throws IOException {
LOGGER.info("Writing Java mapping handlers to " + outputPath);
List javaPaths = new ArrayList<>();
GeneratedNameAndSource generatedFactory = ingesters.getGeneratedFactory();
writeClass(outputPath, generatedFactory.getClassName(), generatedFactory.getSource(), hasClasses).ifPresent(javaPaths::add);
List generatedXmlHandlers = ingesters.getGeneratedHandlers();
for (GeneratedNameAndSource gen : generatedXmlHandlers) {
writeClass(outputPath, gen.getClassName(), gen.getSource(), hasClasses).ifPresent(javaPaths::add);
}
LOGGER.info("Wrote " + javaPaths.size() + " java classes");
return javaPaths;
}
private static Optional writeClass(Path outputPath, String qualifiedClassName, GenerationResult classContents, boolean hasClasses) throws IOException {
String pathName = qualifiedClassName.replace('.', File.separatorChar);
Path path = outputPath.resolve(pathName + ".java");
Files.createDirectories(path.getParent());
LOGGER.debug("Writing Java mapping file " + path.toAbsolutePath()
.toString());
if (classContents.isUnchanged()) {
return Optional.empty();
}
if (!Files.exists(path) || !hasClasses) {
Files.writeString(path, classContents.getChangedValue(), StandardCharsets.UTF_8);
return Optional.of(path);
}
if (!Files.readString(path, StandardCharsets.UTF_8).equals(classContents.getChangedValue())) {
Files.write(outputPath.resolve(pathName + ".old"), Files.readAllBytes(path));
Files.writeString(outputPath.resolve(pathName + ".new"), classContents.getChangedValue(), StandardCharsets.UTF_8);
Files.writeString(path, classContents.getChangedValue(), StandardCharsets.UTF_8);
return Optional.of(path);
}
return Optional.empty();
}
private XtextResourceSet createResourceSet(List expandedModelPaths) {
XtextResourceSet resourceSet = new XtextResourceSet();
expandedModelPaths.forEach(f -> resourceSet.getResource(URI.createURI(f.toUri().toString(), true), true));
return resourceSet;
}
public static class GeneratedClasses {
private final Class handlerFactory;
private final Class synonymToEnumMap;
public GeneratedClasses(Class handlerFactory, Class synonymToEnumMap) {
this.handlerFactory = handlerFactory;
this.synonymToEnumMap = synonymToEnumMap;
}
public Class getHandlerFactory() {
return handlerFactory;
}
public Class getSynonymToEnumMap() {
return synonymToEnumMap;
}
}
}
