gov.nasa.pds.citool.ri.ReferenceRIChecker Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of catalog-legacy Show documentation
The Legacy Catalog Tool provides functionality for ingesting PDS3 catalog files into the PDS4 infrastructure including the Registry Service.
There is a newer version: 3.1.0
Show newest version
package gov.nasa.pds.citool.ri;

import gov.nasa.pds.citool.diff.DiffException;
import gov.nasa.pds.citool.diff.DiffRecord;
import gov.nasa.pds.citool.diff.ValueDiff;
import gov.nasa.pds.citool.util.Utility;
import gov.nasa.pds.tools.LabelParserException;
import gov.nasa.pds.tools.constants.Constants;
import gov.nasa.pds.tools.label.AttributeStatement;
import gov.nasa.pds.tools.label.Label;
import gov.nasa.pds.tools.label.ObjectStatement;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.io.IOUtils;

/**
 * Class to do referential integrity checking on references.
 *
 * @author mcayanan
 *
 */
public class ReferenceRIChecker extends RIChecker {
    private Map idToDescriptions;
    private List refFiles;
    private List parentIDs;
    private final String ID = "REFERENCE_KEY_ID";
    private final String DESCRIPTION = "REFERENCE_DESC";

    /**
     * Constructor
     *
     */
    public ReferenceRIChecker() {
        idToDescriptions = new HashMap();
        parentIDs = new ArrayList();
        refFiles = new ArrayList();
    }

    /**
     * Determines whether the supplied list of catalog files has referential
     * integrity when compared to the REF.CAT file.
     *
     * @param parentLabels A set of catalog files containing the REFERENCE
     * object.
     * @param childLabels A set of catalog files that do not contain the
     * REFERENCE object.
     */
    public void performCheck(List parentLabels, List childLabels) {
        List children = new ArrayList();
        //Get reference attributes
        setParentAttributes(parentLabels);

        //Check references against the allrefs dictionary support file
        if (supportFile != null) {
            addProblems(validate(supportFile));
        }
        //Check that reference key IDs in the parent are all found in the
        //child labels
        Map> parentMap = new HashMap>();
        parentMap.put(ID, parentIDs);
        for (Label child : childLabels) {
            List childStmts = StatementFinder.getStatementsRecursively(child, ID);
            children.addAll(childStmts);
            Map results = getUnmatchedValues(parentIDs, childStmts);
            for (Map.Entry entry : results.entrySet()) {
                AttributeStatement a = entry.getValue();
                String statement = a.getIdentifier().getId() + " = " + entry.getKey();
                Object[] arguments = {statement, "ref.cat"};
                LabelParserException le = new LabelParserException(
                        a.getSourceURI(), null, null,
                        "referentialIntegrity.error.missingIdInParent",
                        Constants.ProblemType.MISSING_ID, arguments);
                addProblem(le);
            }
        }
        //Check that each ref key ID in the parent references is specified
        //in at least one other catalog file
        ValueMatcher matcher = new ValueMatcher(children);
        Map missingFromChildren = matcher.getUnmatched(parentIDs);
        if (!missingFromChildren.isEmpty()) {
            for (Map.Entry entry : missingFromChildren.entrySet()) {
                AttributeStatement a = entry.getValue();
                //TODO: Do we really need this?
/*
                log.log(new ToolsLogRecord(CIToolLevel.INFO,
                        a.getIdentifier() + " = " + entry.getKey() + " is not specified in a non REF.CAT file.",
                        a.getFilename(), refFiles.toString(), a.getLineNumber()));
*/
            }
        }
    }

    private void setParentAttributes(List parents) {
        for (Label parent : parents) {
            List references = (List)
            parent.getObjects(RIType.REFERENCE.getName().toUpperCase());
            for (ObjectStatement ref : references) {
                AttributeStatement id = null;
                if (ref.getAttribute(ID) != null) {
                    id = ref.getAttribute(ID);
                    if (!Utility.isNull(id.getValue().toString())) {
                        parentIDs.add(id);
                        idToDescriptions.put(id, null);
                    }
                }
                idToDescriptions.put(id, ref.getAttribute(DESCRIPTION));
            }
        }
    }

    /**
     * Validates that all reference key IDs and descriptions in the parent
     * exist in the allrefs dictionary support file.
     *
     * @param allrefs dictionary support file for references
     * @return true if all reference key IDs and descriptions in the parent
     * exist in the allrefs file.
     */
    private List validate(URL allrefs) {
        List problems =
            new ArrayList();
        try {
            Map allrefsReferences = parse(allrefs);
            for (AttributeStatement idStmt : idToDescriptions.keySet()) {
                String id = idStmt.getValue().toString();
                if (allrefsReferences.containsKey(id)) {
                    AttributeStatement descStmt = idToDescriptions.get(idStmt);
                    String description = descStmt.getValue().toString()
                    .replaceAll("\\s+", " ").trim();
                    String allrefsDesc = allrefsReferences.get(id)
                    .replaceAll("\\s+", " ").trim();
                    if (!description.equals(allrefsDesc)) {
                        URI uri = null;
                        try {
                            uri = allrefs.toURI();
                        } catch (URISyntaxException u) {
                            //Don't do anything
                        }
                        List diffRecords = ValueDiff.diff(
                                allrefsDesc, description);
                        DiffException de = new DiffException(uri, null,
                                descStmt.getLineNumber(),
                                "referenceDescription.error.allRefsMisMatch",
                                diffRecords, new Object[0]);
                        problems.add(de);
/*
                        log.log(new ToolsLogRecord(CIToolLevel.SEVERE, "Description for " + id + " does not match description in allrefs dictionary.", refFiles.toString(), descStmt.getLineNumber()));
                        log.log(new ToolsLogRecord(CIToolLevel.SEVERE_NOTIFY, "Source", allrefs.toString(), refFiles.toString()));
                        DiffLogger.logRecord(CIToolLevel.DIFF, diffRecords, refFiles.toString());
*/
                    }
                } else {
                    LabelParserException le = new LabelParserException(
                            new File(refFiles.toString()),
                            new Integer(idStmt.getLineNumber()),
                            null, "allrefs.error.missingId",
                            Constants.ProblemType.MISSING_ID, id);
                    problems.add(le);
                }
            }
        } catch (IOException io) {
            problems.add(new LabelParserException(io, null, null, Constants.ProblemType.INVALID_LABEL_WARNING));
//            log.log(new ToolsLogRecord(CIToolLevel.SEVERE, io.getMessage(), refFiles.toString()));
//            isValid = false;
        }
        return problems;
    }

    /**
     * Parses the allrefs file.
     *
     * @param allrefs The allrefs file.
     *
     * @return A mapping of reference IDs to its description.
     * @throws IOException
     */
    private Map parse(URL allrefs) throws IOException {
        Map results = new HashMap();
        List lines = null;
        InputStream in = null;
        String id = null;
        String desc = null;
        try {
            in = allrefs.openStream();
            lines = (List) IOUtils.readLines(new InputStreamReader(in));
        } finally {
            IOUtils.closeQuietly(in);
        }
        for (String line : lines) {
            String[] values = line.split("[|]", 3);
            if (values[1].equals("1")) {
                if ((id != null) && (id != null))
                    results.put(id, desc);
                id = values[0];
                desc = values[2];
            }
            else {
                desc = desc + " " + values[2];
            }
        }
        if ((id != null) && (desc != null)) {
            results.put(id, desc);
        }
        return results;
    }

    @Override
    public RIType getType() {
        return RIType.REFERENCE;
    }
}