Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
gov.nasa.pds.citool.CIToolValidator Maven / Gradle / Ivy
The Legacy Catalog Tool provides functionality for ingesting PDS3 catalog files into the PDS4 infrastructure including the Registry Service.
package gov.nasa.pds.citool;
import gov.nasa.pds.citool.report.NewStandardValuesReport;
import gov.nasa.pds.citool.report.ReferentialIntegrityReport;
import gov.nasa.pds.citool.report.Report;
import gov.nasa.pds.citool.ri.NewValidValueFinder;
import gov.nasa.pds.citool.ri.RIType;
import gov.nasa.pds.citool.ri.ReferentialIntegrityValidator;
import gov.nasa.pds.citool.target.Target;
import gov.nasa.pds.citool.util.Utility;
import gov.nasa.pds.citool.validate.ReferenceFinderValidator;
import gov.nasa.pds.citool.validate.ReferenceValidator;
import gov.nasa.pds.tools.LabelParserException;
import gov.nasa.pds.tools.constants.Constants.ProblemType;
import gov.nasa.pds.tools.dict.Dictionary;
import gov.nasa.pds.tools.dict.parser.DictionaryParser;
import gov.nasa.pds.tools.label.AttributeStatement;
import gov.nasa.pds.tools.label.IncludePointer;
import gov.nasa.pds.tools.label.Label;
import gov.nasa.pds.tools.label.ManualPathResolver;
import gov.nasa.pds.tools.label.ObjectStatement;
import gov.nasa.pds.tools.label.PointerStatement;
import gov.nasa.pds.tools.label.Statement;
import gov.nasa.pds.tools.label.parser.DefaultLabelParser;
import gov.nasa.pds.tools.label.validate.Validator;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
public class CIToolValidator {
private ManualPathResolver resolver;
private DefaultLabelParser parser;
private Report report;
private URL allrefs;
public CIToolValidator(Report report) {
this(report, new ArrayList());
}
public CIToolValidator(Report report, List includePaths) {
this.report = report;
this.allrefs = null;
resolver = new ManualPathResolver();
if (!includePaths.isEmpty()) {
resolver.setIncludePaths(includePaths);
}
parser = new DefaultLabelParser(true, true, true, resolver);
}
public Dictionary parseDictionary(List dictionaries,
boolean aliasing)
throws URISyntaxException, CIToolValidatorException {
Dictionary dictionary = null;
boolean dictionaryPassed = true;
URL url = null;
Iterator i = dictionaries.iterator();
try {
url = Utility.toURL(i.next());
dictionary = DictionaryParser.parse(url, aliasing, true);
report.addConfiguration(" Dictionary version "
+ dictionary.getVersion());
if (dictionary.getProblems().size() != 0) {
dictionaryPassed = false;
report.record(dictionary.getDictionaryURI(),
dictionary.getProblems());
}
while (i.hasNext()) {
url = Utility.toURL(i.next());
Dictionary mergedDictionary = DictionaryParser.parse(url,
aliasing, true);
dictionary.merge(mergedDictionary);
report.addConfiguration(" Dictionary version "
+ mergedDictionary.getVersion());
if (mergedDictionary.getProblems().size() != 0) {
dictionaryPassed = false;
report.record(mergedDictionary.getDictionaryURI(),
mergedDictionary.getProblems());
}
}
} catch (LabelParserException le) {
List problems =
new ArrayList();
problems.add(le);
report.record(url.toURI(), problems);
dictionaryPassed = false;
} catch (IOException io) {
report.recordSkip(url.toURI(), io);
dictionaryPassed = false;
}
if (!dictionaryPassed) {
throw new CIToolValidatorException("Dictionary did not pass");
}
return dictionary;
}
public void setAllrefs(URL url) {
this.allrefs = url;
}
public Report doReferentialIntegrity(List catalogs) {
Set typesFound = new HashSet();
for(Label catalog : catalogs) {
if(!catalog.getObjects(
RIType.REFERENCE.getName().toUpperCase()).isEmpty()) {
typesFound.add(RIType.REFERENCE);
} else if(!catalog.getObjects(
RIType.PERSONNEL.getName().toUpperCase()).isEmpty()) {
typesFound.add(RIType.PERSONNEL);
} else if(!catalog.getObjects(
RIType.TARGET.getName().toUpperCase()).isEmpty()) {
typesFound.add(RIType.TARGET);
} else if(!catalog.getObjects(
RIType.INSTRUMENT_HOST.getName().toUpperCase())
.isEmpty()) {
typesFound.add(RIType.INSTRUMENT_HOST);
} else if(!catalog.getObjects(
RIType.INSTRUMENT.getName().toUpperCase()).isEmpty()) {
typesFound.add(RIType.INSTRUMENT);
} else if(!catalog.getObjects(
RIType.MISSION.getName().toUpperCase()).isEmpty()) {
typesFound.add(RIType.MISSION);
} else if(!catalog.getObjects(
RIType.DATA_SET.getName().toUpperCase()).isEmpty()) {
typesFound.add(RIType.DATA_SET);
}
}
ReferentialIntegrityReport riReport = new ReferentialIntegrityReport();
riReport.setOutput(report.getOutput());
ReferentialIntegrityValidator riv = new ReferentialIntegrityValidator(
riReport);
for (RIType typeFound : typesFound) {
riv.addRIChecker(typeFound);
}
riv.isValid(catalogs, allrefs);
return riReport;
}
/**
* Finds new standard values in the given set of label files.
*
* @param catalogs A list of catalog files.
* @param dictionary A PDS data dictionary
*/
public NewStandardValuesReport findNewStandardValues(List catalogs,
Dictionary dictionary) {
List newValues =
new ArrayList();
NewStandardValuesReport svReport = new NewStandardValuesReport();
svReport.setOutput(report.getOutput());
for (Label catalog : catalogs) {
List statements = (List)
catalog.getStatements();
for (Statement s : statements) {
for (AttributeStatement newValue :
NewValidValueFinder.getNewValues(s, dictionary)) {
if (!newValues.contains(newValue)) {
newValues.add(newValue);
}
}
}
}
svReport.printHeader();
if (!newValues.isEmpty()) {
for (AttributeStatement newValue : newValues) {
String strippedValue = Utility.stripOnlyWhitespaceAndNewLine(
newValue.getValue().toString());
svReport.printStandardValue(newValue.getIdentifier().getId(),
strippedValue);
}
}
return svReport;
}
public void validate(Target target, List dictionaries,
boolean recurse, boolean aliasing)
throws CIToolValidatorException {
try {
List catalogs = new ArrayList();
Dictionary dictionary = parseDictionary(dictionaries, aliasing);
if (target.isDirectory()) {
List urls = target.traverse(recurse);
for (Iterator i = urls.iterator(); i.hasNext();) {
Label catalog = validate(i.next(), dictionary);
if (catalog != null) {
catalogs.add(catalog);
}
}
} else {
Label catalog = validate(target.toURL(), dictionary);
if (catalog != null) {
catalogs.add(catalog);
}
}
Report riReport = doReferentialIntegrity(catalogs);
NewStandardValuesReport svReport =
findNewStandardValues(catalogs, dictionary);
riReport.printFooter();
svReport.printFooter();
} catch (CIToolValidatorException ve) {
//Don't do anything. This means that dictionary validation
//failed.
} catch (Exception e) {
e.printStackTrace();
throw new CIToolValidatorException(e.getMessage());
}
}
public Label validate(URL url, Dictionary dictionary)
throws URISyntaxException {
Label catalog = null;
try {
resolver.setBaseURI(ManualPathResolver.getBaseURI(url.toURI()));
catalog = parser.parseLabel(url);
} catch (LabelParserException lpe) {
//Product tools library records files that have a missing
//PDS_VERSION_ID as an error. However, we want VTool to record
//this as a warning, so we need to instantiate a new
//LabelParserException.
if ("parser.error.missingVersion".equals(lpe.getKey())) {
report.recordSkip(url.toURI(), new LabelParserException(
lpe.getSourceFile(),
null,
null,
lpe.getKey(), ProblemType.INVALID_LABEL_WARNING,
lpe.getArguments()));
} else {
report.recordSkip(url.toURI(), lpe);
}
} catch (IOException ie) {
report.recordSkip(url.toURI(), ie);
} catch (Exception e) {
report.recordSkip(url.toURI(), e);
}
if (catalog != null) {
Validator validator = new Validator();
validator.setDuplicateIdCheck(true);
validator.setDictionaryCheck(true);
validator.addValidator(new ReferenceFinderValidator());
if (!catalog.getObjects(
RIType.REFERENCE.getName().toUpperCase()).isEmpty()) {
validator.addValidator(new ReferenceValidator());
}
validator.validate(catalog, dictionary);
checkFileRefs(catalog);
report.record(url.toURI(), catalog.getProblems());
}
return catalog;
}
/**
* Check for the existence of referenced files specified in a label
*
* @param label
*/
private void checkFileRefs(Label label) {
List externalPointers = new ArrayList();
List localPointers = new ArrayList();
for (PointerStatement pointer : getPointersRecursively(label)) {
//Do not check include pointers. Its already being
//handled at the parser level.
if (!(pointer instanceof IncludePointer)) {
if (!label.equals(pointer.getLabel())) {
externalPointers.add(pointer);
} else {
localPointers.add(pointer);
}
}
for (PointerStatement lp : localPointers) {
try {
resolver.resolveURIs(lp);
} catch (IOException i) {
}
}
//Pointers found in a fragment file are handled differently.
Set externalProblems =
new LinkedHashSet();
for(PointerStatement ep : externalPointers) {
//Missing referenced files are captured in the label
//associated with the external pointer
try {
resolver.resolveURIs(ep);
externalProblems.addAll(ep.getLabel().getProblems());
} catch (IOException i) {
}
}
//Add these external problems back into the parent label
for(LabelParserException externalProblem : externalProblems) {
//Check for duplication of messages
if(!label.getProblems().contains(externalProblem)) {
label.addProblem(externalProblem.getSourceURI(), externalProblem);
}
}
}
}
private List getPointersRecursively(Label label) {
List result = new ArrayList();
//TODO: See if we can put this functionality into the Label class
result.addAll(label.getPointers());
for(ObjectStatement object : label.getObjects()) {
result.addAll(getPointersRecursively(object));
}
Collections.sort(result);
return result;
}
private List getPointersRecursively(ObjectStatement object) {
List result = new ArrayList();
result.addAll(object.getPointers());
for(ObjectStatement nestedObject : object.getObjects()) {
result.addAll(getPointersRecursively(nestedObject));
}
return result;
}
}
