edu.berkeley.cs.jqf.plugin.FuzzGoal Maven / Gradle / Ivy
Show all versions of jqf-maven-plugin Show documentation
/*
* Copyright (c) 2017-2018 The Regents of the University of California
* Copyright (c) 2020-2021 Rohan Padhye
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package edu.berkeley.cs.jqf.plugin;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.net.MalformedURLException;
import java.net.URLClassLoader;
import java.time.Duration;
import java.time.format.DateTimeParseException;
import java.util.List;
import java.util.Random;
import edu.berkeley.cs.jqf.fuzz.ei.ExecutionIndexingGuidance;
import edu.berkeley.cs.jqf.fuzz.ei.ZestGuidance;
import edu.berkeley.cs.jqf.fuzz.guidance.Guidance;
import edu.berkeley.cs.jqf.fuzz.guidance.GuidanceException;
import edu.berkeley.cs.jqf.fuzz.junit.GuidedFuzzing;
import edu.berkeley.cs.jqf.instrument.InstrumentingClassLoader;
import org.apache.maven.artifact.DependencyResolutionRequiredException;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.apache.maven.plugin.logging.Log;
import org.apache.maven.plugins.annotations.LifecyclePhase;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.apache.maven.plugins.annotations.ResolutionScope;
import org.apache.maven.project.MavenProject;
import org.junit.runner.Result;
import static edu.berkeley.cs.jqf.instrument.InstrumentingClassLoader.stringsToUrls;
/**
* Maven plugin for feedback-directed fuzzing using JQF.
*
* Performs code-coverage-guided generator-based fuzz testing
* using a provided entry point.
*
* @author Rohan Padhye
*/
@Mojo(name="fuzz",
requiresDependencyResolution= ResolutionScope.TEST,
defaultPhase=LifecyclePhase.VERIFY)
public class FuzzGoal extends AbstractMojo {
@Parameter(defaultValue="${project}", required=true, readonly=true)
MavenProject project;
@Parameter(property="target", defaultValue="${project.build.directory}", readonly=true)
private File target;
/**
* The fully-qualified name of the test class containing methods
* to fuzz.
*
* This class will be loaded using the Maven project's test
* classpath. It must be annotated with {@code @RunWith(JQF.class)}
*/
@Parameter(property="class", required=true)
private String testClassName;
/**
* The name of the method to fuzz.
*
* This method must be annotated with {@code @Fuzz}, and take
* one or more arguments (with optional junit-quickcheck
* annotations) whose values will be fuzzed by JQF.
*
* If more than one method of this name exists in the
* test class or if the method is not declared
* {@code public void}, then the fuzzer will not launch.
*/
@Parameter(property="method", required=true)
private String testMethod;
/**
* Comma-separated list of FQN prefixes to exclude from
* coverage instrumentation.
*
* Example: org/mozilla/javascript/gen,org/slf4j/logger,
* will exclude classes auto-generated by Mozilla Rhino's CodeGen and
* logging classes.
*/
@Parameter(property="excludes")
private String excludes;
/**
* Comma-separated list of FQN prefixes to forcibly include,
* even if they match an exclude.
*
* Typically, these will be a longer prefix than a prefix
* in the excludes clauses.
*/
@Parameter(property="includes")
private String includes;
/**
* The duration of time for which to run fuzzing.
*
*
* If neither this property nor {@code trials} are provided, the fuzzing
* session is run for an unlimited time until the process is terminated by the
* user (e.g. via kill or CTRL+C).
*
*
*
* Valid time durations are non-empty strings in the format [Nh][Nm][Ns], such
* as "60s" or "2h30m".
*
*/
@Parameter(property="time")
private String time;
/**
* The number of trials for which to run fuzzing.
*
*
* If neither this property nor {@code time} are provided, the fuzzing
* session is run for an unlimited time until the process is terminated by the
* user (e.g. via kill or CTRL+C).
*
*/
@Parameter(property="trials")
private Long trials;
/**
* A number to seed the source of randomness in the fuzzing algorithm.
*
*
* Setting this to any value will make the result of running the same fuzzer
* with on the same input the same. This is useful for testing the fuzzer, but
* shouldn't be used on code attempting to find real bugs. By default, the
* seed is chosen randomly based on system state.
*
*/
@Parameter(property="randomSeed")
private Long randomSeed;
/**
* Whether to generate inputs blindly without taking into
* account coverage feedback. Blind input generation is equivalent
* to running QuickCheck.
*
* If this property is set to true, then the fuzzing
* algorithm does not maintain a queue. Every input is randomly
* generated from scratch. The program under test is still instrumented
* in order to provide coverage statistics. This mode is mainly useful
* for comparing coverage-guided fuzzing with plain-old QuickCheck.
*/
@Parameter(property="blind")
private boolean blind;
/**
* The fuzzing engine.
*
* One of 'zest' and 'zeal'. Default is 'zest'.
*/
@Parameter(property="engine", defaultValue="zest")
private String engine;
/**
* Whether to disable code-coverage instrumentation.
*
* Disabling instrumentation speeds up test case execution, but
* provides no feedback about code coverage in the status screen and
* to the fuzzing guidance.
*
* This setting only makes sense when used with {@code -Dblind}.
*
*/
@Parameter(property="noCov")
private boolean disableCoverage;
/**
* The name of the input directory containing seed files.
*
* If not provided, then fuzzing starts with randomly generated
* initial inputs.
*/
@Parameter(property="in")
private String inputDirectory;
/**
* The name of the output directory where fuzzing results will
* be stored.
*
* The directory will be created inside the standard Maven
* project build directory.
*
* If not provided, defaults to
* jqf-fuzz/${testClassName}/${$testMethod}.
*/
@Parameter(property="out")
private String outputDirectory;
/**
* Whether to save ALL inputs generated during fuzzing, even
* the ones that do not have any unique code coverage.
*
* This setting leads to a very large number of files being
* created in the output directory, and could potentially
* reduce the overall performance of fuzzing.
*/
@Parameter(property="saveAll")
private boolean saveAll;
/**
* Weather to use libFuzzer like output instead of AFL like stats
* screen
*
* If this property is set to true, then output will look like libFuzzer output
* https://llvm.org/docs/LibFuzzer.html#output
* .
*/
@Parameter(property="libFuzzerCompatOutput")
private String libFuzzerCompatOutput;
/**
* Whether to avoid printing fuzzing statistics progress in the console.
*
* If not provided, defaults to {@code false}.
*/
@Parameter(property="quiet")
private boolean quiet;
/**
* Whether to stop fuzzing once a crash is found.
*
* If this property is set to true, then the fuzzing
* will exit on first crash. Useful for continuous fuzzing when you dont wont to consume resource
* once a crash is found. Also fuzzing will be more effective once the crash is fixed.
*/
@Parameter(property="exitOnCrash")
private String exitOnCrash;
/**
* The timeout for each individual trial, in milliseconds.
*
* If not provided, defaults to 0 (unlimited).
*/
@Parameter(property="runTimeout")
private int runTimeout;
/**
* Whether to bound size of inputs being mutated by the fuzzer.
*
* If this property is set to true, then the fuzzing engine
* will treat inputs as fixed-size arrays of bytes
* rather than as an infinite stream of pseudo-random choices. This option
* is appropriate when fuzzing test methods that take a single
* argument of type {@link java.io.InputStream} and that also provide a
* set of seed inputs via the `in' property.
*
* If not provided, defaults to {@code false}.
*/
@Parameter(property="fixedSize")
private boolean fixedSizeInputs;
@Override
public void execute() throws MojoExecutionException, MojoFailureException {
ClassLoader loader;
Log log = getLog();
PrintStream out = log.isDebugEnabled() ? System.out : null;
Result result;
// Configure classes to instrument
if (excludes != null) {
System.setProperty("janala.excludes", excludes);
}
if (includes != null) {
System.setProperty("janala.includes", includes);
}
// Configure Zest Guidance
if (saveAll) {
System.setProperty("jqf.ei.LOG_ALL_INPUTS", "true");
}
if (libFuzzerCompatOutput != null) {
System.setProperty("jqf.ei.LIBFUZZER_COMPAT_OUTPUT", libFuzzerCompatOutput);
}
if (quiet) {
System.setProperty("jqf.ei.QUIET_MODE", "true");
}
if (exitOnCrash != null) {
System.setProperty("jqf.ei.EXIT_ON_CRASH", exitOnCrash);
}
if (runTimeout > 0) {
System.setProperty("jqf.ei.TIMEOUT", String.valueOf(runTimeout));
}
if (fixedSizeInputs) {
System.setProperty("jqf.ei.GENERATE_EOF_WHEN_OUT", String.valueOf(true));
}
final Duration duration;
if (time != null && !time.isEmpty()) {
try {
duration = Duration.parse("PT"+time);
} catch (DateTimeParseException e) {
throw new MojoExecutionException("Invalid time duration: " + time);
}
} else {
duration = null;
}
try {
List classpathElements = project.getTestClasspathElements();
if (disableCoverage) {
loader = new URLClassLoader(
stringsToUrls(classpathElements.toArray(new String[0])),
getClass().getClassLoader());
} else {
loader = new InstrumentingClassLoader(
classpathElements.toArray(new String[0]),
getClass().getClassLoader());
}
} catch (DependencyResolutionRequiredException|MalformedURLException e) {
throw new MojoExecutionException("Could not get project classpath", e);
}
File outputDir = new File(target, "fuzz-results" + File.separator + testClassName);
File seedsDir = inputDirectory == null ? null : new File(inputDirectory);
try {
if ("*".equals(testMethod)) {
// Load the test class
Class testClass = loader.loadClass(testClassName);
// Create a guidance supplier that creates a new guidance for each method
GuidedFuzzing.GuidanceSupplier guidanceSupplier = (testMethod) -> {
// Create a unique random generator for each method
Random methodRnd = randomSeed != null ?
new Random(randomSeed ^ testMethod.hashCode()) : new Random();
// Create a fresh guidance instance for this method
return createGuidance(testClassName, testMethod, duration, trials, seedsDir, methodRnd);
};
// Run all @Fuzz methods with individual guidance instances
result = GuidedFuzzing.runAll(testClass, guidanceSupplier, out);
} else {
Random rnd = randomSeed != null ? new Random(randomSeed) : new Random();
// Create a single guidance instance for the specified method
Guidance guidance = createGuidance(testClassName, testMethod,
duration, trials, seedsDir, rnd);
// Run a specific test method
result = GuidedFuzzing.run(testClassName, testMethod, loader, guidance, out);
}
} catch (ClassNotFoundException e) {
throw new MojoExecutionException("Could not load test class", e);
} catch (IllegalArgumentException e) {
throw new MojoExecutionException("Bad request", e);
} catch (RuntimeException e) {
throw new MojoExecutionException("Internal error", e);
} catch (IOException e) {
throw new MojoExecutionException("I/O error", e);
}
if (!result.wasSuccessful()) {
Throwable e = result.getFailures().get(0).getException();
if (result.getFailureCount() == 1) {
if (e instanceof GuidanceException) {
throw new MojoExecutionException("Internal error", e);
}
}
throw new MojoFailureException(String.format("Fuzzing resulted in the test failing on " +
"%d input(s). Possible bugs found. " +
"Use mvn jqf:repro to reproduce failing test cases from %s/fuzz-results/%s/%s/failures. ",
result.getFailureCount(), target, testClassName, testMethod) +
"Sample exception included with this message.", e);
}
}
/**
* Helper method to create a guidance instance based on the configured engine
*/
private Guidance createGuidance(String testClassName, String testMethod, Duration duration, Long trials,
File seedsDir, Random rnd)
throws IOException {
// Store results in a folder with the target method name
File resultsDir = new File(target, "fuzz-results" + File.separator + testClassName + File.separator + testMethod);
String targetName = testClassName + "#" + testMethod;
switch (engine) {
case "zest":
ZestGuidance zest = new ZestGuidance(targetName, duration, trials, resultsDir, seedsDir, rnd);
zest.setBlind(blind);
return zest;
case "zeal":
System.setProperty("jqf.tracing.TRACE_GENERATORS", "true");
System.setProperty("jqf.tracing.MATCH_CALLEE_NAMES", "true");
ExecutionIndexingGuidance zeal =
new ExecutionIndexingGuidance(targetName, duration, trials, resultsDir, seedsDir, rnd);
zeal.setBlind(blind);
return zeal;
default:
throw new IllegalArgumentException("Unknown fuzzing engine: " + engine);
}
}
}