org.xbib.gradle.task.elasticsearch.qa.ThirdPartyAuditTask.groovy Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of gradle-plugin-elasticsearch-build Show documentation
Gradle plugins for the developer kit for building and testing Elasticsearch and Elasticsearch plugins
The newest version!
package org.xbib.gradle.task.elasticsearch.qa

import org.apache.tools.ant.BuildEvent
import org.apache.tools.ant.BuildException
import org.apache.tools.ant.BuildListener
import org.apache.tools.ant.BuildLogger
import org.apache.tools.ant.DefaultLogger
import org.apache.tools.ant.Project
import org.gradle.api.artifacts.Configuration
import org.gradle.api.file.FileCollection
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.OutputFile
import org.xbib.gradle.task.elasticsearch.qa.forbiddenapis.ForbiddenApisAntTask

import java.nio.file.FileVisitResult
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.SimpleFileVisitor
import java.nio.file.attribute.BasicFileAttributes
import java.util.regex.Matcher
import java.util.regex.Pattern

/**
 * Basic static checking to keep tabs on third party JARs
 */
class ThirdPartyAuditTask extends AntTask {

    // patterns for classes to exclude, because we understand their issues
    @Input
    List excludes = []

    /**
     * Input for the task. Set javadoc for {#link getJars} for more. Protected
     * so the afterEvaluate closure in the constructor can write it.
     */
    protected FileCollection jars

    /**
     * Classpath against which to run the third patty audit. Protected so the
     * afterEvaluate closure in the constructor can write it.
     */
    protected FileCollection classpath

    /**
     * We use a simple "marker" file that we touch when the task succeeds
     * as the task output. This is compared against the modified time of the
     * inputs (ie the jars/class files).
     */
    @OutputFile
    File successMarker = new File(project.buildDir, 'markers/thirdPartyAudit')

    ThirdPartyAuditTask() {
        // we depend on this because its the only reliable configuration
        // this probably makes the build slower: gradle you suck here when it comes to configurations, you pay the price.
        dependsOn(project.configurations.testCompile)
        description = "Checks third party JAR bytecode for missing classes, use of internal APIs, and other horrors'"
        project.afterEvaluate {
            Configuration configuration = project.configurations.findByName('runtime')
            if (configuration == null) {
                // some projects apparently do not have 'runtime'? what a nice inconsistency,
                // basically only serves to waste time in build logic!
                configuration = project.configurations.findByName('testCompile')
            }
            classpath = configuration
            // we only want third party dependencies
            jars = configuration.fileCollection({ dependency ->
                !dependency.group.startsWith("org.elasticsearch")
            })
            // we don't want provided dependencies, which we have already scanned. e.g. don't
            // scan ES core's dependencies for every single plugin
            Configuration provided = project.configurations.findByName('provided')
            if (provided != null) {
                jars -= provided
            }
            inputs.files(jars)
            onlyIf {
                !jars.isEmpty()
            }
        }
    }

    static class EvilLogger extends DefaultLogger {

        // yes, we parse forbiddenApis errors to find missing classes, and to keep a continuous audit
        static final Pattern MISSING_CLASS_PATTERN = Pattern.compile(/class '(.*)' cannot be loaded/)

        static final Pattern VIOLATION_PATTERN = Pattern.compile(/\s\sin ([a-zA-Z0-9\$\.]+) \(.*\)/)

        final Set missingClasses = new TreeSet<>()

        final Map> violations = new TreeMap<>()

        String previousLine = null

        @Override
        void messageLogged(BuildEvent event) {
            if (event.getTask().getClass() == ForbiddenApisAntTask) {
                if (event.getPriority() == Project.MSG_WARN) {
                    Matcher m = MISSING_CLASS_PATTERN.matcher(event.getMessage())
                    if (m.matches()) {
                        missingClasses.add(m.group(1).replace('.', '/') + ".class")
                    }
                    // Reset the priority of the event to DEBUG, so it doesn't
                    // pollute the build output
                    event.setMessage(event.getMessage(), Project.MSG_DEBUG)
                } else if (event.getPriority() == Project.MSG_ERR) {
                    Matcher m = VIOLATION_PATTERN.matcher(event.getMessage())
                    if (m.matches()) {
                        String violation = previousLine + '\n' + event.getMessage()
                        String clazz = m.group(1).replace('.', '/') + ".class"
                        List current = violations.get(clazz)
                        if (current == null) {
                            current = new ArrayList<>()
                            violations.put(clazz, current)
                        }
                        current.add(violation)
                    }
                    previousLine = event.getMessage()
                }
            }
            super.messageLogged(event)
        }
    }


    @Override
    protected BuildLogger makeLogger(PrintStream stream, int outputLevel) {
        EvilLogger log = new EvilLogger()
        log.errorPrintStream = stream
        log.outputPrintStream = stream
        log.messageOutputLevel = outputLevel
        log
    }

    @Override
    protected void runAnt(AntBuilder ant) {
        ant.project.addTaskDefinition('thirdPartyAudit', ForbiddenApisAntTask)
        // print which jars we are going to scan, always
        // this is not the time to try to be succinct! Forbidden will print plenty on its own!
        Set names = new TreeSet<>()
        for (File jar : jars) {
            names.add(jar.getName())
        }
        // TODO: forbidden-apis + zipfileset gives O(n^2) behavior unless we dump to a tmpdir first,
        // and then remove our temp dir afterwards. don't complain: try it yourself.
        // we don't use gradle temp dir handling, just google it, or try it yourself.
        File tmpDir = new File(project.buildDir, 'tmp/thirdPartyAudit')
        // clean up any previous mess (if we failed), then unzip everything to one directory
        ant.delete(dir: tmpDir.getAbsolutePath())
        tmpDir.mkdirs()
        for (File jar : jars) {
            ant.unzip(src: jar.getAbsolutePath(), dest: tmpDir.getAbsolutePath())
        }
        for (String s : excludes) {
            if (s.indexOf('*') != -1) {
                throw new IllegalArgumentException("illegal third party audit exclusion: '" + s + "', wildcards are not permitted")
            }
        }
        // convert exclusion class names to binary file names
        List excludedFiles = excludes.sort().collect {
            it.replace('.', '/') + ".class"
        }
        Set excludedSet = new TreeSet<>(excludedFiles)
        Set sheistySet = getSheistyClasses(tmpDir.toPath())
        try {
            ant.thirdPartyAudit(failOnUnsupportedJava: false,
                            failOnMissingClasses: false,
                            classpath: classpath.asPath) {
                fileset(dir: tmpDir)
                signatures {
                    string(value: getClass().getResourceAsStream('/forbidden/third-party-audit.txt').getText('UTF-8'))
                }
            }
        } catch (BuildException e) {
            logger.error(e.getMessage() as String, e)
        }

        EvilLogger evilLogger = null
        for (BuildListener listener : ant.project.getBuildListeners()) {
            if (listener instanceof EvilLogger) {
                evilLogger = (EvilLogger) listener
                break
            }
        }
        // keep our whitelist up to date
        Set bogusExclusions = new TreeSet<>(excludedSet)
        bogusExclusions.removeAll(sheistySet)
        bogusExclusions.removeAll(evilLogger.missingClasses)
        bogusExclusions.removeAll(evilLogger.violations.keySet())

        if (!bogusExclusions.isEmpty()) {
            throw new IllegalStateException("Invalid exclusions, nothing is wrong with these classes: " + bogusExclusions)
        }
        // don't duplicate classes with the JDK
        sheistySet.removeAll(excludedSet)
        if (!sheistySet.isEmpty()) {
            throw new IllegalStateException("JAR HELL WITH JDK! " + sheistySet)
        }
        // don't allow a broken classpath
        evilLogger.missingClasses.removeAll(excludedSet)
        if (!evilLogger.missingClasses.isEmpty()) {
            throw new IllegalStateException("CLASSES ARE MISSING! " + evilLogger.missingClasses)
        }
        // don't use internal classes
        evilLogger.violations.keySet().removeAll(excludedSet)
        if (!evilLogger.violations.isEmpty()) {
            throw new IllegalStateException("VIOLATIONS WERE FOUND! " + evilLogger.violations)
        }
        // clean up our mess (if we succeed)
        ant.delete(dir: tmpDir.getAbsolutePath())
        successMarker.setText("", 'UTF-8')
    }

    /**
     * check for sheisty classes: if they also exist in the extensions classloader, its jar hell with the jdk
     */
    private static Set getSheistyClasses(Path root) {
        // system.parent = extensions loader.
        // note: for jigsaw, this evilness will need modifications (e.g. use jrt filesystem).
        // but groovy/gradle needs to work at all first
        ClassLoader ext = ClassLoader.getSystemClassLoader().getParent()
        Set sheistySet = new TreeSet<>()
        Files.walkFileTree(root, new SimpleFileVisitor() {
            @Override
            FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                String entry = root.relativize(file).toString().replace('\\', '/')
                if (entry.endsWith(".class")) {
                    if (ext.getResource(entry) != null && !entry.equals('module-info.class')) {
                        sheistySet.add(entry)
                    }
                }
                return FileVisitResult.CONTINUE
            }
        })
        sheistySet
    }
}