edu.berkeley.cs.jqf.plugin.FuzzGoal Maven / Gradle / Ivy
/*
* Copyright (c) 2017-2018 The Regents of the University of California
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package edu.berkeley.cs.jqf.plugin;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.net.MalformedURLException;
import java.net.URLClassLoader;
import java.time.Duration;
import java.time.format.DateTimeParseException;
import java.util.List;
import edu.berkeley.cs.jqf.fuzz.ei.ExecutionIndexingGuidance;
import edu.berkeley.cs.jqf.fuzz.ei.ZestGuidance;
import edu.berkeley.cs.jqf.fuzz.junit.GuidedFuzzing;
import edu.berkeley.cs.jqf.instrument.InstrumentingClassLoader;
import org.apache.maven.artifact.DependencyResolutionRequiredException;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.apache.maven.plugin.logging.Log;
import org.apache.maven.plugins.annotations.LifecyclePhase;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.apache.maven.plugins.annotations.ResolutionScope;
import org.apache.maven.project.MavenProject;
import org.junit.runner.Result;
import static edu.berkeley.cs.jqf.instrument.InstrumentingClassLoader.stringsToUrls;
/**
* Maven plugin for feedback-directed fuzzing using JQF.
*
* Performs code-coverage-guided generator-based fuzz testing
* using a provided entry point.
*
* @author Rohan Padhye
*/
@Mojo(name="fuzz",
requiresDependencyResolution= ResolutionScope.TEST,
defaultPhase=LifecyclePhase.VERIFY)
public class FuzzGoal extends AbstractMojo {
@Parameter(defaultValue="${project}", required=true, readonly=true)
MavenProject project;
@Parameter(property="target", defaultValue="${project.build.directory}", readonly=true)
private File target;
/**
* The fully-qualified name of the test class containing methods
* to fuzz.
*
* This class will be loaded using the Maven project's test
* classpath. It must be annotated with {@code @RunWith(JQF.class)}
*/
@Parameter(property="class", required=true)
private String testClassName;
/**
* The name of the method to fuzz.
*
* This method must be annotated with {@code @Fuzz}, and take
* one or more arguments (with optional junit-quickcheck
* annotations) whose values will be fuzzed by JQF.
*
* If more than one method of this name exists in the
* test class or if the method is not declared
* {@code public void}, then the fuzzer will not launch.
*/
@Parameter(property="method", required=true)
private String testMethod;
/**
* Comma-separated list of FQN prefixes to exclude from
* coverage instrumentation.
*
* Example: org/mozilla/javascript/gen,org/slf4j/logger,
* will exclude classes auto-generated by Mozilla Rhino's CodeGen and
* logging classes.
*/
@Parameter(property="excludes")
private String excludes;
/**
* Comma-separated list of FQN prefixes to forcibly include,
* even if they match an exclude.
*
* Typically, these will be a longer prefix than a prefix
* in the excludes clauses.
*/
@Parameter(property="includes")
private String includes;
/**
* The duration of time for which to run fuzzing.
*
* If this property is not provided, the fuzzing session is
* run for an unlimited time until the process is terminated
* by the user (e.g. via kill or CTRL+C).
*
* Valid time durations are non-empty strings in the format
* [Nh][Nm][Ns], such as "60s" or "2h30m".
*/
@Parameter(property="time")
private String time;
/**
* Whether to generate inputs blindly without taking into
* account coverage feedback. Blind input generation is equivalent
* to running QuickCheck.
*
* If this property is set to true, then the fuzzing
* algorithm does not maintain a queue. Every input is randomly
* generated from scratch. The program under test is still instrumented
* in order to provide coverage statistics. This mode is mainly useful
* for comparing coverage-guided fuzzing with plain-old QuickCheck.
*/
@Parameter(property="blind")
private boolean blind;
/**
* The fuzzing engine.
*
* One of 'zest' and 'zeal'. Default is 'zest'.
*/
@Parameter(property="engine", defaultValue="zest")
private String engine;
/**
* Whether to disable code-coverage instrumentation.
*
* Disabling instrumentation speeds up test case execution, but
* provides no feedback about code coverage in the status screen and
* to the fuzzing guidance.
*
* This setting only makes sense when used with {@code -Dblind}.
*
*/
@Parameter(property="noCov")
private boolean disableCoverage;
/**
* The name of the input directory containing seed files.
*
* If not provided, then fuzzing starts with randomly generated
* initial inputs.
*/
@Parameter(property="in")
private String inputDirectory;
/**
* The name of the output directory where fuzzing results will
* be stored.
*
* The directory will be created inside the standard Maven
* project build directory.
*
* If not provided, defaults to
* jqf-fuzz/${testClassName}/${$testMethod}.
*/
@Parameter(property="out")
private String outputDirectory;
/**
* Whether to save ALL inputs generated during fuzzing, even
* the ones that do not have any unique code coverage.
*
* This setting leads to a very large number of files being
* created in the output directory, and could potentially
* reduce the overall performance of fuzzing.
*/
@Parameter(property="saveAll")
private boolean saveAll;
/**
* Weather to use libFuzzer like output instead of AFL like stats
* screen
*
* If this property is set to true, then output will look like libFuzzer output
* https://llvm.org/docs/LibFuzzer.html#output
* .
*/
@Parameter(property="libFuzzerCompatOutput")
private String libFuzzerCompatOutput;
/**
* Whether to avoid printing fuzzing statistics progress in the console.
*
* If not provided, defaults to {@code false}.
*/
@Parameter(property="quiet")
private boolean quiet;
/**
* Whether to stop fuzzing once a crash is found.
*
* If this property is set to true, then the fuzzing
* will exit on first crash. Useful for continuous fuzzing when you dont wont to consume resource
* once a crash is found. Also fuzzing will be more effective once the crash is fixed.
*/
@Parameter(property="exitOnCrash")
private String exitOnCrash;
/**
* The timeout for each individual trial, in milliseconds.
*
* If not provided, defaults to 0 (unlimited).
*/
@Parameter(property="runTimeout")
private int runTimeout;
@Override
public void execute() throws MojoExecutionException, MojoFailureException {
ClassLoader loader;
ZestGuidance guidance;
Log log = getLog();
PrintStream out = System.out; // TODO: Re-route to logger from super.getLog()
Result result;
// Configure classes to instrument
if (excludes != null) {
System.setProperty("janala.excludes", excludes);
}
if (includes != null) {
System.setProperty("janala.includes", includes);
}
// Configure Zest Guidance
if (saveAll) {
System.setProperty("jqf.ei.SAVE_ALL_INPUTS", "true");
}
if (libFuzzerCompatOutput != null) {
System.setProperty("jqf.ei.LIBFUZZER_COMPAT_OUTPUT", libFuzzerCompatOutput);
}
if (quiet) {
System.setProperty("jqf.ei.QUIET_MODE", "true");
}
if (exitOnCrash != null) {
System.setProperty("jqf.ei.EXIT_ON_CRASH", exitOnCrash);
}
if (runTimeout > 0) {
System.setProperty("jqf.ei.TIMEOUT", String.valueOf(runTimeout));
}
Duration duration = null;
if (time != null && !time.isEmpty()) {
try {
duration = Duration.parse("PT"+time);
} catch (DateTimeParseException e) {
throw new MojoExecutionException("Invalid time duration: " + time);
}
}
if (outputDirectory == null || outputDirectory.isEmpty()) {
outputDirectory = "fuzz-results" + File.separator + testClassName + File.separator + testMethod;
}
try {
List classpathElements = project.getTestClasspathElements();
if (disableCoverage) {
loader = new URLClassLoader(
stringsToUrls(classpathElements.toArray(new String[0])),
getClass().getClassLoader());
} else {
loader = new InstrumentingClassLoader(
classpathElements.toArray(new String[0]),
getClass().getClassLoader());
}
} catch (DependencyResolutionRequiredException|MalformedURLException e) {
throw new MojoExecutionException("Could not get project classpath", e);
}
try {
File resultsDir = new File(target, outputDirectory);
String targetName = testClassName + "#" + testMethod;
File seedsDir = inputDirectory == null ? null : new File(inputDirectory);
switch (engine) {
case "zest":
guidance = new ZestGuidance(targetName, duration, resultsDir, seedsDir);
break;
case "zeal":
System.setProperty("jqf.traceGenerators", "true");
guidance = new ExecutionIndexingGuidance(targetName, duration, resultsDir, seedsDir);
break;
default:
throw new MojoExecutionException("Unknown fuzzing engine: " + engine);
}
guidance.setBlind(blind);
} catch (FileNotFoundException e) {
throw new MojoExecutionException("File not found", e);
} catch (IOException e) {
throw new MojoExecutionException("I/O error", e);
}
try {
result = GuidedFuzzing.run(testClassName, testMethod, loader, guidance, out);
} catch (ClassNotFoundException e) {
throw new MojoExecutionException("Could not load test class", e);
} catch (IllegalArgumentException e) {
throw new MojoExecutionException("Bad request", e);
} catch (RuntimeException e) {
throw new MojoExecutionException("Internal error", e);
}
if (!result.wasSuccessful()) {
throw new MojoFailureException("Fuzzing revealed errors. " +
"Use mvn jqf:repro to reproduce failing test case.");
}
}
}