azkaban.project.validator.XmlValidatorManager Maven / Gradle / Ivy

Go to download
package azkaban.project.validator;

import azkaban.project.Project;
import azkaban.utils.Props;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.log4j.Logger;
import org.w3c.dom.Document;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

/**
 * Xml implementation of the ValidatorManager.
 *
 * Looks for the property project.validators.xml.file in the azkaban properties.
 *
 * 
The xml to be in the following form:
 * 
{@code
 * 
 *   
 *     
 *      
 *       ...
 *   
 * 
 * }
 */
public class XmlValidatorManager implements ValidatorManager {

  public static final String VALIDATOR_TAG = "validator";
  public static final String CLASSNAME_ATTR = "classname";
  public static final String ITEM_TAG = "property";
  private static final Logger logger = Logger.getLogger(XmlValidatorManager.class);
  private static final Map resourceTimestamps = new HashMap<>();
  private static ValidatorClassLoader validatorLoader;
  private final String validatorDirPath;
  private Map validators;

  /**
   * Load the validator plugins from the validator directory (default being validators/) into the
   * validator ClassLoader. This enables creating instances of these validators in the
   * loadValidators() method.
   */
  // Todo jamiesjc: guicify XmlValidatorManager class
  public XmlValidatorManager(final Props props) {
    this.validatorDirPath = props
        .getString(ValidatorConfigs.VALIDATOR_PLUGIN_DIR, ValidatorConfigs.DEFAULT_VALIDATOR_DIR);
    final File validatorDir = new File(this.validatorDirPath);
    if (!validatorDir.canRead() || !validatorDir.isDirectory()) {
      logger.warn("Validator directory " + this.validatorDirPath
          + " does not exist or is not a directory.");
    }

    // Check for updated validator JAR files
    checkResources();

    // Load the validators specified in the xml file.
    try {
      loadValidators(props, logger);
    } catch (final Exception e) {
      logger.error("Cannot load all the validators.");
      throw new ValidatorManagerException(e);
    }
  }

  private void checkResources() {
    final File validatorDir = new File(this.validatorDirPath);
    final List resources = new ArrayList<>();
    boolean reloadResources = false;
    try {
      if (validatorDir.canRead() && validatorDir.isDirectory()) {
        for (final File f : validatorDir.listFiles()) {
          if (f.getName().endsWith(".jar")) {
            resources.add(f.toURI().toURL());
            if (resourceTimestamps.get(f.getName()) == null
                || resourceTimestamps.get(f.getName()) != f.lastModified()) {
              reloadResources = true;
              logger.info("Resource " + f.getName() + " is updated. Reload the classloader.");
              resourceTimestamps.put(f.getName(), f.lastModified());
            }
          }
        }
      }
    } catch (final MalformedURLException e) {
      throw new ValidatorManagerException(e);
    }

    if (reloadResources) {
      if (validatorLoader != null) {
        try {
          // Since we cannot use Java 7 feature inside Azkaban (....), we need a customized class loader
          // that does the close for us.
          validatorLoader.close();
        } catch (final ValidatorManagerException e) {
          logger.error("Cannot reload validator classloader because failure "
              + "to close the validator classloader.", e);
          // We do not throw the ValidatorManagerException because we do not want to crash Azkaban at runtime.
        }
      }
      validatorLoader = new ValidatorClassLoader(resources.toArray(new URL[resources.size()]));
    }
  }

  /**
   * Instances of the validators are created here rather than in the constructors. This is because
   * some validators might need to maintain project-specific states. By instantiating the validators
   * here, it ensures that the validator objects are project-specific, rather than global.
   *
   * {@inheritDoc}
   *
   * @see azkaban.project.validator.ValidatorManager#loadValidators(azkaban.utils.Props,
   * org.apache.log4j.Logger)
   */
  @Override
  public void loadValidators(final Props props, final Logger log) {
    this.validators = new LinkedHashMap<>();
    if (!props.containsKey(ValidatorConfigs.XML_FILE_PARAM)) {
      logger.warn(
          "Azkaban properties file does not contain the key " + ValidatorConfigs.XML_FILE_PARAM);
      return;
    }
    final String xmlPath = props.get(ValidatorConfigs.XML_FILE_PARAM);
    final File file = new File(xmlPath);
    if (!file.exists()) {
      logger.error("Azkaban validator configuration file " + xmlPath + " does not exist.");
      return;
    }

    // Creating the document builder to parse xml.
    final DocumentBuilderFactory docBuilderFactory =
        DocumentBuilderFactory.newInstance();
    DocumentBuilder builder = null;
    try {
      builder = docBuilderFactory.newDocumentBuilder();
    } catch (final ParserConfigurationException e) {
      throw new ValidatorManagerException(
          "Exception while parsing validator xml. Document builder not created.", e);
    }

    Document doc = null;
    try {
      doc = builder.parse(file);
    } catch (final SAXException e) {
      throw new ValidatorManagerException("Exception while parsing " + xmlPath
          + ". Invalid XML.", e);
    } catch (final IOException e) {
      throw new ValidatorManagerException("Exception while parsing " + xmlPath
          + ". Error reading file.", e);
    }

    final NodeList tagList = doc.getChildNodes();
    final Node azkabanValidators = tagList.item(0);

    final NodeList azkabanValidatorsList = azkabanValidators.getChildNodes();
    for (int i = 0; i < azkabanValidatorsList.getLength(); ++i) {
      final Node node = azkabanValidatorsList.item(i);
      if (node.getNodeType() == Node.ELEMENT_NODE) {
        if (node.getNodeName().equals(VALIDATOR_TAG)) {
          parseValidatorTag(node, props, log);
        }
      }
    }
  }

  private void parseValidatorTag(final Node node, final Props props, final Logger log) {
    final NamedNodeMap validatorAttrMap = node.getAttributes();
    final Node classNameAttr = validatorAttrMap.getNamedItem(CLASSNAME_ATTR);
    if (classNameAttr == null) {
      throw new ValidatorManagerException(
          "Error loading validator. The validator 'classname' attribute doesn't exist");
    }

    final NodeList keyValueItemsList = node.getChildNodes();
    for (int i = 0; i < keyValueItemsList.getLength(); i++) {
      final Node keyValuePair = keyValueItemsList.item(i);
      if (keyValuePair.getNodeName().equals(ITEM_TAG)) {
        parseItemTag(keyValuePair, props);
      }
    }
    final String className = classNameAttr.getNodeValue();
    try {
      final Class validatorClass =
          (Class) validatorLoader.loadClass(className);
      final Constructor validatorConstructor =
          validatorClass.getConstructor(Logger.class);
      final ProjectValidator validator = (ProjectValidator) validatorConstructor.newInstance(log);
      validator.initialize(props);
      this.validators.put(validator.getValidatorName(), validator);
      logger.info("Added validator " + className + " to list of validators.");
    } catch (final Exception e) {
      logger.error("Could not instantiate ProjectValidator " + className);
      throw new ValidatorManagerException(e);
    }
  }

  private void parseItemTag(final Node node, final Props props) {
    final NamedNodeMap keyValueMap = node.getAttributes();
    final Node keyAttr = keyValueMap.getNamedItem("key");
    final Node valueAttr = keyValueMap.getNamedItem("value");
    if (keyAttr == null || valueAttr == null) {
      throw new ValidatorManagerException("Error loading validator key/value "
          + "pair. The 'key' or 'value' attribute doesn't exist");
    }
    props.put(keyAttr.getNodeValue(), valueAttr.getNodeValue());
  }

  @Override
  public Map validate(final Project project, final File projectDir) {
    final Map reports = new LinkedHashMap<>();
    for (final Entry validator : this.validators.entrySet()) {
      reports.put(validator.getKey(), validator.getValue().validateProject(project, projectDir));
      logger.info("Validation status of validator " + validator.getKey() + " is "
          + reports.get(validator.getKey()).getStatus());
    }
    return reports;
  }

  @Override
  public List getValidatorsInfo() {
    final List info = new ArrayList<>();
    for (final String key : this.validators.keySet()) {
      info.add(key);
    }
    return info;
  }

}