gov.nasa.pds.citool.ri.ReferenceRIChecker Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of catalog Show documentation
The Catalog Tool provides functionality for ingesting PDS3 catalog files into the the PDS4 infrastructure including the Registry and Storage Services.
There is a newer version: 2.2.0
Show newest version
// Copyright 2009, by the California Institute of Technology.
// ALL RIGHTS RESERVED. United States Government sponsorship acknowledged.
// Any commercial use must be negotiated with the Office of Technology Transfer
// at the California Institute of Technology.
//
// This software is subject to U. S. export control laws and regulations
// (22 C.F.R. 120-130 and 15 C.F.R. 730-774). To the extent that the software
// is subject to U.S. export control laws and regulations, the recipient has
// the responsibility to obtain export licenses or other export authority as
// may be required before exporting such information to foreign countries or
// providing access to foreign nationals.
//
// $Id$

package gov.nasa.pds.citool.ri;

import gov.nasa.pds.citool.diff.DiffException;
import gov.nasa.pds.citool.diff.DiffRecord;
import gov.nasa.pds.citool.diff.ValueDiff;
import gov.nasa.pds.citool.util.Utility;
import gov.nasa.pds.tools.LabelParserException;
import gov.nasa.pds.tools.constants.Constants;
import gov.nasa.pds.tools.label.AttributeStatement;
import gov.nasa.pds.tools.label.Label;
import gov.nasa.pds.tools.label.ObjectStatement;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.io.IOUtils;

/**
 * Class to do referential integrity checking on references.
 *
 * @author mcayanan
 *
 */
public class ReferenceRIChecker extends RIChecker {
    private Map idToDescriptions;
    private List refFiles;
    private List parentIDs;
    private final String ID = "REFERENCE_KEY_ID";
    private final String DESCRIPTION = "REFERENCE_DESC";

    /**
     * Constructor
     *
     */
    public ReferenceRIChecker() {
        idToDescriptions = new HashMap();
        parentIDs = new ArrayList();
        refFiles = new ArrayList();
    }

    /**
     * Determines whether the supplied list of catalog files has referential
     * integrity when compared to the REF.CAT file.
     *
     * @param parentLabels A set of catalog files containing the REFERENCE
     * object.
     * @param childLabels A set of catalog files that do not contain the
     * REFERENCE object.
     */
    public void performCheck(List parentLabels, List childLabels) {
        List children = new ArrayList();
        //Get reference attributes
        setParentAttributes(parentLabels);

        //Check references against the allrefs dictionary support file
        if (supportFile != null) {
            addProblems(validate(supportFile));
        }
        //Check that reference key IDs in the parent are all found in the
        //child labels
        Map> parentMap = new HashMap>();
        parentMap.put(ID, parentIDs);
        for (Label child : childLabels) {
            List childStmts = StatementFinder.getStatementsRecursively(child, ID);
            children.addAll(childStmts);
            Map results = getUnmatchedValues(parentIDs, childStmts);
            for (Map.Entry entry : results.entrySet()) {
                AttributeStatement a = entry.getValue();
                String statement = a.getIdentifier().getId() + " = " + entry.getKey();
                Object[] arguments = {statement, "ref.cat"};
                LabelParserException le = new LabelParserException(
                        a.getSourceURI(), null, null,
                        "referentialIntegrity.error.missingIdInParent",
                        Constants.ProblemType.MISSING_ID, arguments);
                addProblem(le);
            }
        }
        //Check that each ref key ID in the parent references is specified
        //in at least one other catalog file
        ValueMatcher matcher = new ValueMatcher(children);
        Map missingFromChildren = matcher.getUnmatched(parentIDs);
        if (!missingFromChildren.isEmpty()) {
            for (Map.Entry entry : missingFromChildren.entrySet()) {
                AttributeStatement a = entry.getValue();
                //TODO: Do we really need this?
/*
                log.log(new ToolsLogRecord(CIToolLevel.INFO,
                        a.getIdentifier() + " = " + entry.getKey() + " is not specified in a non REF.CAT file.",
                        a.getFilename(), refFiles.toString(), a.getLineNumber()));
*/
            }
        }
    }

    private void setParentAttributes(List parents) {
        for (Label parent : parents) {
            List references = (List)
            parent.getObjects(RIType.REFERENCE.getName().toUpperCase());
            for (ObjectStatement ref : references) {
                AttributeStatement id = null;
                if (ref.getAttribute(ID) != null) {
                    id = ref.getAttribute(ID);
                    if (!Utility.isNull(id.getValue().toString())) {
                        parentIDs.add(id);
                        idToDescriptions.put(id, null);
                    }
                }
                idToDescriptions.put(id, ref.getAttribute(DESCRIPTION));
            }
        }
    }

    /**
     * Validates that all reference key IDs and descriptions in the parent
     * exist in the allrefs dictionary support file.
     *
     * @param allrefs dictionary support file for references
     * @return true if all reference key IDs and descriptions in the parent
     * exist in the allrefs file.
     */
    private List validate(URL allrefs) {
        List problems =
            new ArrayList();
        try {
            Map allrefsReferences = parse(allrefs);
            for (AttributeStatement idStmt : idToDescriptions.keySet()) {
                String id = idStmt.getValue().toString();
                if (allrefsReferences.containsKey(id)) {
                    AttributeStatement descStmt = idToDescriptions.get(idStmt);
                    String description = descStmt.getValue().toString()
                    .replaceAll("\\s+", " ").trim();
                    String allrefsDesc = allrefsReferences.get(id)
                    .replaceAll("\\s+", " ").trim();
                    if (!description.equals(allrefsDesc)) {
                        URI uri = null;
                        try {
                            uri = allrefs.toURI();
                        } catch (URISyntaxException u) {
                            //Don't do anything
                        }
                        List diffRecords = ValueDiff.diff(
                                allrefsDesc, description);
                        DiffException de = new DiffException(uri, null,
                                descStmt.getLineNumber(),
                                "referenceDescription.error.allRefsMisMatch",
                                diffRecords, new Object[0]);
                        problems.add(de);
/*
                        log.log(new ToolsLogRecord(CIToolLevel.SEVERE, "Description for " + id + " does not match description in allrefs dictionary.", refFiles.toString(), descStmt.getLineNumber()));
                        log.log(new ToolsLogRecord(CIToolLevel.SEVERE_NOTIFY, "Source", allrefs.toString(), refFiles.toString()));
                        DiffLogger.logRecord(CIToolLevel.DIFF, diffRecords, refFiles.toString());
*/
                    }
                } else {
                    LabelParserException le = new LabelParserException(
                            new File(refFiles.toString()),
                            new Integer(idStmt.getLineNumber()),
                            null, "allrefs.error.missingId",
                            Constants.ProblemType.MISSING_ID, id);
                    problems.add(le);
                }
            }
        } catch (IOException io) {
            problems.add(new LabelParserException(io, null, null, Constants.ProblemType.INVALID_LABEL_WARNING));
//            log.log(new ToolsLogRecord(CIToolLevel.SEVERE, io.getMessage(), refFiles.toString()));
//            isValid = false;
        }
        return problems;
    }

    /**
     * Parses the allrefs file.
     *
     * @param allrefs The allrefs file.
     *
     * @return A mapping of reference IDs to its description.
     * @throws IOException
     */
    private Map parse(URL allrefs) throws IOException {
        Map results = new HashMap();
        List lines = null;
        InputStream in = null;
        String id = null;
        String desc = null;
        try {
            in = allrefs.openStream();
            lines = (List) IOUtils.readLines(new InputStreamReader(in));
        } finally {
            IOUtils.closeQuietly(in);
        }
        for (String line : lines) {
            String[] values = line.split("[|]", 3);
            if (values[1].equals("1")) {
                if ((id != null) && (id != null))
                    results.put(id, desc);
                id = values[0];
                desc = values[2];
            }
            else {
                desc = desc + " " + values[2];
            }
        }
        if ((id != null) && (desc != null)) {
            results.put(id, desc);
        }
        return results;
    }

    @Override
    public RIType getType() {
        return RIType.REFERENCE;
    }
}