cdc.applic.tools.ExtractS1000DDmApplic Maven / Gradle / Ivy
package cdc.applic.tools;
import java.io.File;
import java.io.IOException;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import cdc.applic.dictionaries.Dictionary;
import cdc.applic.dictionaries.checks.SemanticChecker;
import cdc.applic.dictionaries.core.visitors.CollectPropertyNames;
import cdc.applic.dictionaries.handles.DictionaryHandle;
import cdc.applic.dictionaries.impl.RepositoryImpl;
import cdc.applic.dictionaries.impl.io.RepositoryIo;
import cdc.applic.expressions.Expression;
import cdc.applic.expressions.checks.ApplicIssue;
import cdc.applic.expressions.literals.Name;
import cdc.applic.proofs.ProverFeatures;
import cdc.applic.s1000d.ExpressionToS1000DConverter;
import cdc.applic.s1000d.S1000DToExpressionConverter;
import cdc.applic.s1000d.core.DataXmlHandler;
import cdc.applic.s1000d.core.DataXmlSource;
import cdc.applic.s1000d.core.ExpressionToS1000DConverterImpl;
import cdc.applic.s1000d.core.S1000DApplicToRepositoryImpl;
import cdc.applic.s1000d.core.S1000DSupport;
import cdc.applic.s1000d.core.S1000DToExpressionConverterImpl;
import cdc.applic.simplification.Simplifier;
import cdc.applic.simplification.SimplifierFeatures;
import cdc.applic.simplification.core.SimplifierImpl;
import cdc.io.data.Element;
import cdc.io.data.xml.XmlDataWriter;
import cdc.io.xml.XmlWriter;
import cdc.issues.Diagnosis;
import cdc.issues.Issue;
import cdc.office.ss.SheetLoader;
import cdc.office.ss.WorkbookWriter;
import cdc.office.ss.WorkbookWriterFactory;
import cdc.office.ss.WorkbookWriterFeatures;
import cdc.office.tables.Row;
import cdc.office.tables.TableSection;
import cdc.util.cli.AbstractMainSupport;
import cdc.util.cli.FeatureMask;
import cdc.util.cli.MainResult;
import cdc.util.cli.OptionEnum;
import cdc.util.lang.FailureReaction;
import cdc.util.time.Chronometer;
/**
* Utility used to extract applicabilities from S1000D Data Modules.
*
* It converts them to Expressions and generates =an Office file.
*/
public final class ExtractS1000DDmApplic {
private static final Logger LOGGER = LogManager.getLogger(ExtractS1000DDmApplic.class);
private static final DataXmlSource DATA_XML_SOURCE = new DataXmlSource();
private static final SimplifierFeatures SIMPLIFIER_FEATURES =
SimplifierFeatures.builder()
.hint(SimplifierFeatures.Hint.REMOVE_INEQUALITIES)
.hint(SimplifierFeatures.Hint.NORMALIZE_BOOLEAN_PROPERTIES)
.hint(SimplifierFeatures.Hint.REMOVE_NEGATION)
.proverFeatures(ProverFeatures.EXCLUDE_ASSERTIONS_ALL_POSSIBLE_RESERVES)
.build();
private final MainArgs margs;
private final Map buckets = new HashMap<>();
private int total;
private S1000DToExpressionConverter toExpression;
private SemanticChecker semanticChecker;
private ExpressionToS1000DConverter toS1000D;
private Simplifier simplifier;
private final Chronometer chronoLoading = new Chronometer();
private final Chronometer chronoComputing = new Chronometer();
private final Chronometer chronoSaving = new Chronometer();
private int bucketId = 1;
private final Set whiteList = new HashSet<>();
private static class Bucket {
final int id;
/** The S1000D applic element. */
final Element applic;
/** Conversion of applic to an Expression. */
final Expression expression;
/** Exception thrown by conversion of applic to expression. */
final RuntimeException conversionToExpressionException;
final Diagnosis diagnosis;
/** The simplification of expression. */
final Expression simplifiedExpression;
/** Exception thrown by simplification of expression. */
final RuntimeException simplificationException;
/** Conversion of simplifiedExpression to an S1000D applic element. */
final Element simplifiedApplic;
/** Exception thrown by conversion of simplifiedExpression to simplifiedApplic. */
final Exception conversionToS1000DException;
/** Number of occurrences of the S1000D applic in the input data modules. */
int count = 0;
/**
* Creates a Bucket from an S1000 applic element
*
* @param element The {@code } element.
* @param owner The owning class, used for its utilities.
*/
public Bucket(Element element,
ExtractS1000DDmApplic owner) {
this.id = owner.bucketId++;
// Clone element and remove its attributes
this.applic = element.clone(true);
if (this.applic.hasAttributes()) {
this.applic.removeAttributes(); // NPE in removeAttributes
}
// Convert applic to an expression
Expression x = null;
RuntimeException rtex = null;
try {
x = owner.toExpression.convertToExpression(element, DATA_XML_SOURCE);
rtex = null;
} catch (final RuntimeException e) {
x = null;
rtex = e;
}
this.expression = x;
this.conversionToExpressionException = rtex;
if (this.expression == null) {
this.diagnosis = null;
} else {
this.diagnosis = owner.semanticChecker.check(expression);
}
// Simplify expression and convert back to S1000D
if (this.expression == null) {
this.simplifiedExpression = null;
this.simplificationException = null;
this.simplifiedApplic = this.applic;
this.conversionToS1000DException = null;
} else {
// simplify expression
RuntimeException sex = null;
Expression s = null;
try {
s = owner.simplifier.simplify(this.expression,
SIMPLIFIER_FEATURES)
.getValue();
sex = null;
} catch (final RuntimeException e) {
s = null;
sex = e;
}
this.simplifiedExpression = s;
this.simplificationException = sex;
// Convert simplified expression to S1000D
final DataXmlHandler handler = new DataXmlHandler();
Exception ex = null;
Element r = null;
if (simplifiedExpression == null) {
ex = null;
r = null;
} else {
try {
owner.toS1000D.convertToS1000D(simplifiedExpression, handler);
r = handler.getRoot();
ex = null;
} catch (IOException | RuntimeException e) {
r = null;
ex = e;
}
}
if (r == null) {
this.simplifiedApplic = this.applic;
} else {
this.simplifiedApplic = r;
}
this.conversionToS1000DException = ex;
}
}
}
public static class MainArgs {
/** The directory that contains Data Modules to analyze. */
public File inputDir;
/** The File of the generated Office synthesis. */
public File output;
public File act;
public Locale locale = Locale.ENGLISH;
public String dictionaryName = "Dictionary";
/** The File containing the repository. */
public File repository;
/** The path of the dictionary to use in the repository. */
public String dictionaryPath;
/** The optional file containing the DM to analyze. */
public File whiteList;
/** Set of enabled features. */
public final FeatureMask features = new FeatureMask<>();
public enum Feature implements OptionEnum {
VERBOSE("verbose", "Prints messages."),
TRANSFORM_TYPES(S1000DApplicToRepository.MainArgs.Feature.TRANSFORM_TYPES);
private final String name;
private final String description;
private Feature(String name,
String description) {
this.name = name;
this.description = description;
}
private Feature(OptionEnum model) {
this.name = model.getName();
this.description = model.getDescription();
}
@Override
public final String getName() {
return name;
}
@Override
public final String getDescription() {
return description;
}
}
}
private ExtractS1000DDmApplic(MainArgs margs) {
this.margs = margs;
}
private void log(String message) {
if (margs.features.isEnabled(MainArgs.Feature.VERBOSE)) {
LOGGER.info(message);
}
}
private static String toString(Element element) {
return XmlDataWriter.toString(element,
false,
" ",
XmlWriter.Feature.PRETTY_PRINT);
}
private static String toKey(Element applic) {
final Element e = applic.clone(true);
if (e.hasAttributes()) {
e.removeAttributes(); // NPE in removeAttributes
}
return toString(e);
}
private static String toString(Diagnosis diag) {
if (diag == null) {
return "";
} else {
final StringBuilder builder = new StringBuilder();
if (diag.isOk()) {
builder.append("OK");
} else {
builder.append("KO");
for (final Issue issue : diag.getIssues()) {
builder.append("\n")
.append(issue);
}
}
return builder.toString();
}
}
private void execute() throws IOException {
log("Start");
final Chronometer chrono = new Chronometer();
// Create or load repository
final RepositoryImpl repository;
if (margs.repository != null) {
log("Load repository " + margs.repository);
chrono.start();
repository = RepositoryIo.load(margs.repository, FailureReaction.FAIL);
chrono.suspend();
log("Loaded repository " + margs.repository + "(" + chrono + ")");
} else {
log("Generate repository from " + margs.act);
chrono.start();
final S1000DApplicToRepositoryImpl generator =
new S1000DApplicToRepositoryImpl(margs.act,
margs.locale,
margs.dictionaryName,
margs.features.contains(MainArgs.Feature.TRANSFORM_TYPES));
generator.execute();
chrono.suspend();
log("Generated repository (" + chrono + ")");
repository = generator.getRepository();
}
final Dictionary dictionary = repository.getDictionary(margs.dictionaryPath);
this.toExpression = new S1000DToExpressionConverterImpl<>(dictionary);
this.toS1000D = new ExpressionToS1000DConverterImpl(dictionary);
final DictionaryHandle handle = new DictionaryHandle(dictionary);
this.simplifier = new SimplifierImpl(handle);
this.semanticChecker = new SemanticChecker(dictionary);
if (margs.whiteList != null) {
log("Load " + margs.whiteList);
chrono.start();
final SheetLoader loader = new SheetLoader();
final List rows = loader.load(margs.whiteList, null, 0);
for (final Row row : rows) {
if (row.size() > 0 && row.getValue(0).toLowerCase().endsWith(".xml")) {
whiteList.add(row.getValue(0));
}
}
chrono.suspend();
log("Loaded " + margs.whiteList + "(" + chrono + ")");
}
final WorkbookWriterFactory factory = new WorkbookWriterFactory();
factory.setEnabled(WorkbookWriterFactory.Hint.POI_STREAMING, true);
log("Generate " + margs.output);
chrono.start();
try (final WorkbookWriter writer = factory.create(margs.output, WorkbookWriterFeatures.STANDARD_FAST)) {
generateDataSheet(writer);
generateApplicsSheet(writer, dictionary);
generateDuplicateApplicsSheet(writer);
}
chrono.suspend();
log("Generated " + margs.output + " (" + chrono + ")");
log("Loading: " + chronoLoading);
log("Computing: " + chronoComputing);
log("Saving: " + chronoSaving);
}
private void generateDataSheet(WorkbookWriter writer) throws IOException {
chronoSaving.resume();
writer.beginSheet("Data");
writer.beginRow(TableSection.HEADER);
writer.addCellAndComment("File",
"Name of the DM file containing .");
writer.addCellAndComment("Path",
"Path of the in File.");
writer.addCellAndComment("DM ",
"S1000D XML extracted from File.");
writer.addCellAndComment(" ID",
" content identifier.");
writer.addCellAndComment("CDC Expression",
"Conversion of S1000D to CDC Expression.");
writer.addCellAndComment("Exception ( to CDC)",
"Exception thrown during conversion of S1000D to CDC Expression.");
writer.addCellAndComment("CDC Expression check",
"Semantic analysis of CDC Expression.");
writer.addCellAndComment("Simplified CDC Expression)",
"Simplification of CDC Expression."
+ "\nSimplification may fail. In that case the exception is logged.");
writer.addCellAndComment("Exception (CDC Simplification)",
"Exception thrown during simplification of CDC Expression.");
writer.addCellAndComment("Simplified DM ",
"Conversion of CDC simplified Expression to S1000D ."
+ "\nConversion may fail. In that case the exception is logged.");
writer.addCellAndComment("Exception (CDC to )",
"Exception thrown by conversion of CDC simplified Expression to S1000D .");
chronoSaving.suspend();
log("List files of " + margs.inputDir);
chronoLoading.resume();
final File[] files = margs.inputDir.listFiles((dir,
name) -> name.toLowerCase().endsWith(".xml"));
chronoLoading.suspend();
total = files.length;
int index = 0;
int skipped = 0;
for (final File file : files) {
index++;
if (whiteList.isEmpty() || whiteList.contains(file.getName())) {
processFile(writer, file, index);
} else {
skipped++;
}
}
log("Processed " + (total - skipped) + " file(s)");
if (skipped > 0) {
log("Skipped " + skipped + " file(s)");
}
}
private void generateApplicsSheet(WorkbookWriter writer,
Dictionary dictionary) throws IOException {
chronoSaving.resume();
writer.beginSheet("Applics");
writer.beginRow(TableSection.HEADER);
writer.addCellAndComment("DM ",
"S1000D DM with @id removed.");
writer.addCellAndComment(" ID",
" content identifier.");
writer.addCellAndComment("CDC Expression",
"Conversion of S1000D to CDC Expression."
+ "\nIt may fail. In that case FAILURE is written and the exception is logged.");
writer.addCellAndComment("Exception ( to CDC)",
"Exception thrown during conversion of S1000D to CDC Expression.");
writer.addCellAndComment("CDC Expression check",
"Semantic analysis of CDC Expression.");
writer.addCellAndComment("Simplified CDC Expression)",
"Simplification of CDC Expression."
+ "\nSimplification may fail. In that case the exception is logged.");
writer.addCellAndComment("Exception (CDC Simplification)",
"Exception thrown during simplification of CDC Expression.");
writer.addCellAndComment("Simplified DM ",
"Conversion of CDC simplified Expression to S1000D ."
+ "\nConversion may fail. In that case the exception is logged.");
writer.addCellAndComment("Exception (CDC to )",
"Exception thrown by conversion of CDC simplified Expression to S1000D .");
writer.addCellAndComment("# Occurrences",
"Number of occurrences of in the input Data Modules.");
writer.addCellAndComment("# Properties",
"Number of properties (product attributes and condtions) used in the .");
writer.addCellAndComment("Properties",
"Properties (product attributes and condtions) used in the .");
for (final String key : buckets.keySet().stream().sorted().toList()) {
final Bucket bucket = buckets.get(key);
writer.addRow(TableSection.DATA,
key,
bucket.id,
bucket.expression == null ? "FAILURE" : bucket.expression,
bucket.conversionToExpressionException,
toString(bucket.diagnosis),
bucket.simplifiedExpression == null ? "" : bucket.simplifiedExpression,
bucket.simplificationException,
toString(bucket.simplifiedApplic),
bucket.conversionToS1000DException,
bucket.count,
bucket.expression == null
? "?"
: CollectPropertyNames.collect(bucket.expression.getRootNode(), dictionary).size(),
bucket.expression == null
? "?"
: CollectPropertyNames.collect(bucket.expression.getRootNode(), dictionary)
.stream()
.sorted()
.map(Name::getNonEscapedLiteral)
.collect(Collectors.joining("\n")));
}
chronoSaving.suspend();
}
private void generateDuplicateApplicsSheet(WorkbookWriter writer) throws IOException {
// Duplicates
final Map> map = new HashMap<>();
for (final Bucket bucket : buckets.values()) {
if (bucket.simplifiedExpression != null) {
final String key = bucket.simplifiedExpression.getContent();
final Set set = map.computeIfAbsent(key, k -> new HashSet<>());
set.add(bucket);
}
}
chronoSaving.resume();
writer.beginSheet("Duplicate Applics");
writer.beginRow(TableSection.HEADER);
writer.addCellAndComment("Simplified CDC Expression",
"Simplified CDC Expression.");
writer.addCellAndComment("# ",
"Number of corresponding .");
writer.addCellAndComment("",
"Corresponding .");
for (final String key : map.keySet()
.stream()
.sorted()
.toList()) {
final Set set = map.get(key);
writer.beginRow(TableSection.DATA);
writer.addCell(key);
writer.addCell(set.size());
writer.addCell(set.stream()
.map(b -> toString(b.applic))
.collect(Collectors.joining("\n\n")));
}
chronoSaving.suspend();
}
private void processFile(WorkbookWriter writer,
File file,
int index) throws IOException {
log(index + "/" + total + " " + file.getPath());
chronoLoading.resume();
final Element root = S1000DSupport.loadDmRoot(file);
chronoLoading.suspend();
processRec(writer, file.getName(), root);
}
private Bucket add(Element applic) {
chronoComputing.resume();
try {
final String key = toKey(applic);
final Bucket bucket = buckets.computeIfAbsent(key, k -> new Bucket(applic, this));
bucket.count++;
return bucket;
} finally {
chronoComputing.suspend();
}
}
private void processRec(WorkbookWriter writer,
String filename,
Element element) throws IOException {
if (element.getName().equals("applic")) {
final Bucket bucket = add(element);
chronoSaving.resume();
writer.addRow(TableSection.DATA,
filename,
element.getQName(),
toString(element),
bucket.id,
bucket.expression == null ? "FAILURE" : bucket.expression,
bucket.conversionToExpressionException,
toString(bucket.diagnosis),
bucket.simplifiedExpression == null ? "" : bucket.simplifiedExpression,
bucket.simplificationException,
toString(bucket.simplifiedApplic),
bucket.conversionToS1000DException);
chronoSaving.suspend();
}
for (final Element child : element.getChildren(Element.class)) {
processRec(writer, filename, child);
}
}
public static void execute(MainArgs margs) throws IOException {
final ExtractS1000DDmApplic instance = new ExtractS1000DDmApplic(margs);
instance.execute();
}
public static MainResult exec(String... args) {
final MainSupport support = new MainSupport();
support.main(args);
return support.getResult();
}
public static void main(String... args) {
final int code = exec(args).getCode();
System.exit(code);
}
private static class MainSupport extends AbstractMainSupport {
private static final String ACT = "act";
private static final String LOCALE = "locale";
private static final String DICTIONARY_NAME = "dictionary-name";
private static final String DICTIONARY_PATH = "dictionary-path";
private static final String REPOSITORY = "repository";
private static final String WHITE_LIST = "white-list";
public MainSupport() {
super(ExtractS1000DDmApplic.class,
LOGGER);
}
@Override
protected String getVersion() {
return Config.VERSION;
}
@Override
protected boolean addArgsFileOption(Options options) {
return true;
}
@Override
protected String getHelpHeader() {
return """
Extract applicability from S1000D Data Modules (Issues 3(?), 4 and 5) and convert them to Expressions.
An Office (XLS, XLSX, ...) file is generated.
A repository must be passed or can be generated from an ACT/CCT/PCT.""";
}
@Override
protected void addSpecificOptions(Options options) {
options.addOption(Option.builder()
.longOpt(INPUT_DIR)
.desc("Mandatory name of the intput directory that contains S1000D Data Modules.")
.hasArg()
.required()
.build());
options.addOption(Option.builder()
.longOpt(OUTPUT)
.desc("""
Mandatory name of the output file.
Its extension must be a recognized one: xls, xlsx, ods, ...
Generating a csv is possible, but not advised because several sheets are produced.""")
.hasArg()
.required()
.build());
options.addOption(Option.builder()
.longOpt(REPOSITORY)
.desc("Optional name of the applic repository to load."
+ "\nMust be defined if the dictionary is not generated from ACT.")
.hasArg()
.build());
options.addOption(Option.builder()
.longOpt(DICTIONARY_PATH)
.desc("Mandatory name of the applic dictionary path in the repository.")
.hasArg()
.required()
.build());
options.addOption(Option.builder()
.longOpt(WHITE_LIST)
.desc("Optional name of a white-list file containing the names of DM to analyze.")
.hasArg()
.build());
options.addOption(Option.builder()
.longOpt(ACT)
.desc("Optional name of the input ACT file used to generate the repository."
+ "\nMust be used when a preexisting repository is not passed.")
.hasArg()
.build());
options.addOption(Option.builder()
.longOpt(LOCALE)
.desc("Optional locale of descriptions in ACT/CCT/PCT (default: en)."
+ "\nUsed when ACT is loaded to generate the repository.")
.hasArg()
.build());
options.addOption(Option.builder()
.longOpt(DICTIONARY_NAME)
.desc("Optional name of the generated dictionary (default: Dictionary)."
+ "\nUsed when ACT is loaded to generate the repository.")
.hasArg()
.build());
addNoArgOptions(options, MainArgs.Feature.class);
createGroup(options, true, ACT, REPOSITORY);
}
@Override
protected MainArgs analyze(CommandLine cl) throws ParseException {
final MainArgs margs = new MainArgs();
margs.inputDir = getValueAsResolvedFile(cl, INPUT_DIR, IS_DIRECTORY);
margs.output = getValueAsResolvedFile(cl, OUTPUT, IS_TRUE);
margs.repository = getValueAsResolvedFile(cl, REPOSITORY, IS_NULL_OR_FILE);
margs.dictionaryPath = getValueAsString(cl, DICTIONARY_PATH, null);
margs.whiteList = getValueAsResolvedFile(cl, WHITE_LIST, IS_NULL_OR_FILE);
margs.act = getValueAsResolvedFile(cl, ACT, IS_NULL_OR_FILE);
margs.dictionaryName = getValueAsString(cl, DICTIONARY_NAME, "Dictionary");
margs.locale = getValue(cl, LOCALE, Locale.ENGLISH, Locale::forLanguageTag);
setMask(cl, MainArgs.Feature.class, margs.features::setEnabled);
return margs;
}
@Override
protected Void execute(MainArgs margs) throws IOException, SQLException {
ExtractS1000DDmApplic.execute(margs);
return null;
}
}
}