com.okworx.ilcd.validation.LinkValidator Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of ilcd-validation Show documentation
A Java library for performing technical validation of data in ILCD data format.
There is a newer version: 2.7.2
package com.okworx.ilcd.validation;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import javax.xml.transform.Templates;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.dom.DOMResult;
import javax.xml.transform.stream.StreamSource;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import net.java.truevfs.access.TFileInputStream;

import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import com.okworx.ilcd.validation.common.DatasetType;
import com.okworx.ilcd.validation.events.IValidationEvent;
import com.okworx.ilcd.validation.events.Severity;
import com.okworx.ilcd.validation.events.ValidationEvent;
import com.okworx.ilcd.validation.reference.DatasetReference;
import com.okworx.ilcd.validation.reference.IDatasetReference;
import com.okworx.ilcd.validation.reference.ReferenceCache;
import com.okworx.ilcd.validation.util.AbstractDatasetsTask;
import com.okworx.ilcd.validation.util.ILCDNameSpaceContext;
import com.okworx.ilcd.validation.util.PartitionedList;
import com.okworx.ilcd.validation.util.TaskResult;
import com.okworx.ilcd.validation.util.TransletCache;

/**
 * Checks whether local references (links) within datasets are valid.
 *
 * @author oliver.kusche
 * @version $Id: $Id
 */
public class LinkValidator extends AbstractReferenceObjectsAwareValidator implements IValidator {

	/** Constant PARAM_IGNORE_REFS_TO_LCIAMETHODS="ignoreReferencesToLCIAMethods" */
	public static final String PARAM_IGNORE_REFS_TO_LCIAMETHODS = "ignoreReferencesToLCIAMethods";

	/** Constant PARAM_IGNORE_COMPLEMENTINGPROCESS="ignoreComplementingProcess" */
	public static final String PARAM_IGNORE_COMPLEMENTINGPROCESS = "ignoreComplementingProcess";

	/** Constant PARAM_IGNORE_INCLUDEDPROCESSES="ignoreIncludedProcesses" */
	public static final String PARAM_IGNORE_INCLUDEDPROCESSES = "ignoreIncludedProcesses";

	/** Constant PARAM_IGNORE_REFERENCE_OBJECTS="ignoreReferenceObjects" */
	public static final String PARAM_IGNORE_REFERENCE_OBJECTS = "ignoreReferenceObjects";

	/** {@inheritDoc} */
	@Override
	public String getAspectName() {
		return "Links";
	}

	/**
	 * getAspectDescription.
	 *
	 * @return a {@link java.lang.String} object.
	 */
	public String getAspectDescription() {
		return "Checks whether local references (links) within a set datasets are valid.";
	}

	protected ReferenceCache referenceObjectsCache;

	// TODO refactor out link extraction routine
	/**
	 * validate.
	 *
	 * @return a boolean.
	 * @throws java.lang.InterruptedException if any.
	 */
	public boolean validate() throws InterruptedException {
		super.validate();

		updateStatusValidating();

		this.unitsTotal = this.objectsToValidate.size();

		log.debug("initializing reference objects cache...");
		this.referenceObjectsCache = new ReferenceCache();
		this.referenceObjectsCache.put(this.objectsToValidate);
		log.debug("done - " + this.referenceObjectsCache.getLinks().size() + " objects in cache.");

		boolean result = true;

		try {

			XPathExpression expr = setupXpathRef();

			PartitionedList partList = new PartitionedList(
					this.objectsToValidate.values());

			log.debug(this.unitsTotal + " total objects split into " + partList.getPartitions().size() + " chunks");

			Collection> tasks = new ArrayList>();

			for (List refList : partList.getPartitions()) {
				tasks.add(new LinkExtractorTask(refList, TransletCache.getInstance().getTranslet(), expr, this));
			}

			ExecutorService executor = Executors.newFixedThreadPool(partList.getNumThreads());

			try {
				List> taskResults = executor.invokeAll(tasks);
				for (Future taskResult : taskResults) {
					if (taskResult.get() != null) {
						TaskResult res = taskResult.get();
						this.eventsList.addAll(res.getValidationEvents());
						this.statistics.add(res.getStatistics());
					}
				}
				executor.shutdown();
			} catch (InterruptedException e) {
				executor.shutdown();
				interrupted(e);
			} catch (Exception e) {
				log.error(e);
				e.printStackTrace();
			}

			log.info(this.eventsList.getEvents().size() + " events");

			updateProgress(1);
			updateStatusDone();

			return this.getEventsList().isPositive();
		}

		catch (Exception e) {
			log.error(e);
			e.printStackTrace();
		}

		return result;
	}

	private XPathExpression setupXpathRef() throws XPathExpressionException {
		XPathFactory xpathFactory = XPathFactory.newInstance();
		XPath xpath = xpathFactory.newXPath();
		xpath.setNamespaceContext(new ILCDNameSpaceContext());
		XPathExpression expr = xpath.compile("/*/*");
		return expr;
	}

	final class LinkExtractorTask extends AbstractDatasetsTask implements Callable {

		private static final String MESSAGE_INVALID_REFERENCE = "Invalid (because empty) reference, neither UUID nor URI specified. Reference to ";

		private static final String MESSAGE_TO = " to ";

		private static final String MESSAGE_COULD_NOT_RESOLVE_REFERENCE = "could not resolve reference ";

		private XPathExpression expr;

		private Templates templates;

		LinkExtractorTask(List files, Templates templates, XPathExpression expr,
				LinkValidator validator) {
			this.files = files;
			this.expr = expr;
			this.templates = templates;
			this.validator = validator;
		}

		public TaskResult call() throws Exception {
			return new TaskResult(extract(files), this.statistics);
		}

		private Collection extract(Collection files) throws Exception {

			Collection events = new ArrayList();

			Transformer transformer = templates.newTransformer();

			int count = 0;

			for (IDatasetReference reference : files) {
				if (Thread.currentThread().isInterrupted()) {
					log.info("operation was interrupted, aborting");
					updateStatusCancelled();
					break;
				}

				int eventCount = events.size();
				
				if (!reference.getType().equals(DatasetType.EXTERNAL_FILE))
					checkReference(events, transformer, reference);

				boolean success = (eventCount == events.size());
				this.statistics.update(reference, success);
				
				if (success && LinkValidator.this.reportSuccesses)
					events.add(new ValidationEvent(LinkValidator.this.getAspectName(), Severity.SUCCESS, reference, ValidationEvent.SUCCESS_MESSAGE));

				count = updateChunkCount(count);
			}

			return events;
		}

		private void checkReference(Collection events, Transformer transformer,
				IDatasetReference reference) throws TransformerException, XPathExpressionException, IOException {

			NodeList refs = extractNodeList(transformer, reference);

			List links = new ArrayList();

			for (int i = 0; i < refs.getLength(); i++) {
				Node ref = refs.item(i);
				String uuid = ref.getAttributes().getNamedItem(ATTRIBUTE_REF_OBJECT_ID).getNodeValue().trim();
				String uri = ref.getAttributes().getNamedItem(ATTRIBUTE_URI).getNodeValue().trim();
				String origin = ref.getAttributes().getNamedItem(ATTRIBUTE_ORIGIN).getNodeValue().trim();
				String name = null;
				
				try {
					name = ref.getAttributes().getNamedItem(ATTRIBUTE_NAME).getNodeValue().trim();
				} catch (NullPointerException e1) {
				}
				
				// skip remote links
				if (StringUtils.startsWithIgnoreCase(uri.trim(), URL_PREFIX_HTTP) || StringUtils.startsWithIgnoreCase(uri.trim(), URL_PREFIX_HTTPS))
					continue;

				String type = ref.getAttributes().getNamedItem(ATTRIBUTE_TYPE).getNodeValue();
				DatasetType dsType = null;
				try {
					dsType = DatasetType.fromValue(type);
				} catch (Exception e) {
					log.error("invalid dataset type " + type);
				}
				IDatasetReference dsRef = new DatasetReference(uuid, uri, dsType, null,	dsType.equals(DatasetType.EXTERNAL_FILE) ? FilenameUtils.getName(uri.trim()) : null, origin);
				dsRef.setName(name);
				links.add(dsRef);
			}

			log.debug("checking " + links.size() + " links");

			boolean skipReferenceObjects = BooleanUtils.isTrue((Boolean) this.validator.parameters.get(LinkValidator.PARAM_IGNORE_REFERENCE_OBJECTS));
			
			for (IDatasetReference ref : links) {
				if (!LinkValidator.this.referenceObjectsCache.contains(ref) && !ref.equals(reference)) {
					
					// if skipReferenceObjects param is set, check whether it's listed there
					if (skipReferenceObjects) {
						if (((LinkValidator) this.validator).referenceElementaryFlows.containsKey(ref.getUuid()) || ((LinkValidator) this.validator).referenceObjectsOther.containsKey(ref.getUuid())) {
							if (log.isDebugEnabled())
								log.debug("ignoring object found in reference list: " + ref.getUuid());
							continue;
						}
					}
					
					// check whether we might have an external document
					if (ref.getType().equals(DatasetType.EXTERNAL_FILE)) {
						if (log.isDebugEnabled())
							log.debug(ref.getShortFileName()
								+ " "
								+ LinkValidator.this.referenceObjectsCache.getLinks().keySet()
										.contains(ref.getShortFileName()));
						if (LinkValidator.this.referenceObjectsCache.getLinks().keySet()
								.contains(ref.getShortFileName()))
							continue;
					}

					StringBuilder messagePrefix = new StringBuilder(MESSAGE_COULD_NOT_RESOLVE_REFERENCE);
					messagePrefix.append(ref.getOrigin());
					messagePrefix.append(MESSAGE_TO);
					StringBuilder messageSuffix = new StringBuilder(" (");
					messageSuffix.append(ref.getName()).append(")");
					
					if (StringUtils.isBlank(ref.getUuid()) && StringUtils.isBlank(ref.getUri())) {
						messagePrefix = new StringBuilder(MESSAGE_INVALID_REFERENCE).append(ref.getType().getValue());
						messageSuffix = new StringBuilder();
					}

					StringBuilder message = new StringBuilder();
					message.append(" ");
					message.append(ref.getType().getValue());
					message.append(" ");
					message.append(StringUtils.isNotBlank(ref.getUuid()) ? ref.getUuid() : ref.getUri());
					
					message.insert(0, messagePrefix);
					message.append(messageSuffix);
					
					IValidationEvent event = new ValidationEvent(LinkValidator.this.getAspectName(), Severity.ERROR, reference, message.toString());
					event.setMessageReference(ref);
					if (ref.getType().equals(DatasetType.EXTERNAL_FILE))
						ref.setName(ref.getShortFileName());
					event.setAltMessage(messagePrefix.toString());
					events.add(event);
				}
			}
		}

		private NodeList extractNodeList(Transformer transformer, IDatasetReference reference)
				throws TransformerException, XPathExpressionException, IOException {
			DOMResult transformResult = new DOMResult();
			
			boolean paramSkipSiam = BooleanUtils.isTrue((Boolean) this.validator.getParameter(PARAM_IGNORE_REFS_TO_LCIAMETHODS));
			if (paramSkipSiam) {
				if (LinkValidator.this.log.isDebugEnabled())
					LinkValidator.this.log.debug("setting skipSupportedImpactAssessmentMethods=true");
				transformer.setParameter(PARAM_IGNORE_REFS_TO_LCIAMETHODS, new Boolean(true));
			}

			boolean paramSkipCP = BooleanUtils.isTrue((Boolean) this.validator.getParameter(PARAM_IGNORE_COMPLEMENTINGPROCESS));
			if (paramSkipCP) {
				if (LinkValidator.this.log.isDebugEnabled())
					LinkValidator.this.log.debug("setting skipComplementingProcess=true");
				transformer.setParameter(PARAM_IGNORE_COMPLEMENTINGPROCESS, new Boolean(true));
			}

			boolean paramSkipIP = BooleanUtils.isTrue((Boolean) this.validator.getParameter(PARAM_IGNORE_INCLUDEDPROCESSES));
			if (paramSkipIP) {
				if (LinkValidator.this.log.isDebugEnabled())
					LinkValidator.this.log.debug("setting skipIncludedProcesses=true");
				transformer.setParameter(PARAM_IGNORE_INCLUDEDPROCESSES, new Boolean(true));
			}

			transformer.transform(new StreamSource(wrapInputStream(new TFileInputStream(reference.getAbsoluteFileName()))),
					transformResult);

			Document resultDoc = (Document) transformResult.getNode();

			NodeList refs = (NodeList) expr.evaluate(resultDoc, XPathConstants.NODESET);
			return refs;
		}
	}
}