org.evosuite.junit.CoverageAnalysis Maven / Gradle / Ivy
/**
* Copyright (C) 2010-2018 Gordon Fraser, Andrea Arcuri and EvoSuite
* contributors
*
* This file is part of EvoSuite.
*
* EvoSuite is free software: you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published
* by the Free Software Foundation, either version 3.0 of the License, or
* (at your option) any later version.
*
* EvoSuite is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with EvoSuite. If not, see
* CoverageAnalysis class.
*
*
* @author Gordon Fraser
* @author José Campos
*/
public class CoverageAnalysis {
/**
* FIXME
*
* OUTPUT
* METHOD
* METHODNOEXCEPTION
*
* relies on Observers. to have coverage of these criteria, JUnit test cases
* must have to be converted to some format that EvoSuite can understand
*/
private final static Logger logger = LoggerFactory.getLogger(CoverageAnalysis.class);
private static int totalGoals = 0;
private static int totalCoveredGoals = 0;
private static Set targetClasses = new LinkedHashSet();
/**
* Identify all JUnit tests starting with the given name prefix, instrument
* and run tests
*/
public static void analyzeCoverage() {
Sandbox.goingToExecuteSUTCode();
TestGenerationContext.getInstance().goingToExecuteSUTCode();
Sandbox.goingToExecuteUnsafeCodeOnSameThread();
ExecutionTracer.setCheckCallerThread(false);
try {
String cp = ClassPathHandler.getInstance().getTargetProjectClasspath();
if (Properties.TARGET_CLASS.endsWith(".jar")
|| Properties.TARGET_CLASS.contains(File.separator)) {
targetClasses = DependencyAnalysis.analyzeTarget(Properties.TARGET_CLASS,
Arrays.asList(cp.split(File.pathSeparator)));
}
else {
targetClasses.add(Properties.TARGET_CLASS);
DependencyAnalysis.analyzeClass(Properties.TARGET_CLASS,
Arrays.asList(cp.split(File.pathSeparator)));
}
LoggingUtils.getEvoLogger().info("* " + ClientProcess.getPrettyPrintIdentifier() + "Finished analyzing classpath");
} catch (Throwable e) {
LoggingUtils.getEvoLogger().error("* " + ClientProcess.getPrettyPrintIdentifier()
+ "Error while initializing target class: "
+ (e.getMessage() != null ? e.getMessage() : e.toString()));
logger.error("Problem for " + Properties.TARGET_CLASS + ". Full stack:", e);
return;
} finally {
Sandbox.doneWithExecutingUnsafeCodeOnSameThread();
Sandbox.doneWithExecutingSUTCode();
TestGenerationContext.getInstance().doneWithExecutingSUTCode();
}
// TestCluster.getInstance();
List> testClasses = getTestClasses();
LoggingUtils.getEvoLogger().info("* " + ClientProcess.getPrettyPrintIdentifier() + "Found " + testClasses.size()
+ " test class(es)");
if (testClasses.isEmpty())
return;
/*
* sort them in a deterministic way, in case there are
* static state dependencies
*/
sortTestClasses(testClasses);
Class[] tests = testClasses.toArray(new Class[testClasses.size()]);
LoggingUtils.getEvoLogger().info("* " + ClientProcess.getPrettyPrintIdentifier() + "Executing test(s)");
if (Properties.SELECTED_JUNIT == null) {
boolean origUseAgent = EvoRunner.useAgent;
boolean origUseClassLoader = EvoRunner.useClassLoader;
try {
EvoRunner.useAgent = false; //avoid double instrumentation
EvoRunner.useClassLoader = false; //avoid double instrumentation
List results = executeTests(tests);
printReport(results);
} finally {
EvoRunner.useAgent = origUseAgent;
EvoRunner.useClassLoader = origUseClassLoader;
}
} else {
// instead of just running junit tests, carve them
JUnitTestCarvedChromosomeFactory carvedFactory = new JUnitTestCarvedChromosomeFactory(null);
TestSuiteChromosome testSuite = carvedFactory.getCarvedTestSuite();
int goals = 0;
for (Properties.Criterion pc : Properties.CRITERION) {
LoggingUtils.getEvoLogger().info("* " + ClientProcess.getPrettyPrintIdentifier()
+ "Coverage analysis for criterion " + pc);
TestFitnessFactory ffactory = FitnessFunctions.getFitnessFactory(pc);
goals += ffactory.getCoverageGoals().size();
FitnessFunction ffunction = FitnessFunctions.getFitnessFunction(pc);
ffunction.getFitness(testSuite);
CoverageCriteriaAnalyzer.analyzeCoverage(testSuite, pc);
}
// Generate test suite
TestSuiteGenerator.writeJUnitTestsAndCreateResult(testSuite);
StatisticsSender.executedAndThenSendIndividualToMaster(testSuite);
ClientServices.getInstance().getClientNode().trackOutputVariable(RuntimeVariable.Total_Goals, goals);
if (Properties.COVERAGE_MATRIX)
throw new IllegalArgumentException("Coverage matrix not yet available when measuring coverage of a carved test suite");
}
}
/**
* Return the number of covered goals
*
* @param testClass
* @param allGoals
* @return
*/
public static Set getCoveredGoals(Class testClass, List allGoals) {
// A dummy Chromosome
TestChromosome dummy = new TestChromosome();
dummy.setChanged(false);
// Execution result of a dummy Test Case
ExecutionResult executionResult = new ExecutionResult(dummy.getTestCase());
Set coveredGoals = new HashSet();
List results = executeTests(testClass);
for (JUnitResult testResult : results) {
executionResult.setTrace(testResult.getExecutionTrace());
dummy.setLastExecutionResult(executionResult);
for(TestFitnessFunction goal : allGoals) {
if(coveredGoals.contains(goal))
continue;
else if (goal.isCovered(dummy))
coveredGoals.add(goal);
}
}
return coveredGoals;
}
private static List> getTestClassesFromClasspath() {
List> classes = new ArrayList>();
for(String prefix : Properties.JUNIT.split(":")) {
Set suts = ResourceList.getInstance(TestGenerationContext.getInstance().getClassLoaderForSUT()).getAllClasses(
ClassPathHandler.getInstance().getTargetProjectClasspath(), prefix, false);
LoggingUtils.getEvoLogger().info("* " + ClientProcess.getPrettyPrintIdentifier() + "Found " + suts.size()
+ " classes with prefix '" + prefix + "'");
if (!suts.isEmpty()) {
for (String sut : suts) {
if (targetClasses.contains(sut)) {
continue ;
}
try {
Class clazz = Class.forName(
sut,true,TestGenerationContext.getInstance().getClassLoaderForSUT());
if (isTest(clazz)) {
classes.add(clazz);
}
} catch (ClassNotFoundException e2) {
logger.info("Could not find class "+sut);
} catch(Throwable t) {
logger.info("Error while initialising class "+sut);
}
}
}
}
return classes;
}
private static List> getTestClasses() {
List> testClasses = new ArrayList>();
logger.debug("JUNIT: "+Properties.JUNIT);
for(String prefix : Properties.JUNIT.split(":")) {
LoggingUtils.getEvoLogger().info("* " + ClientProcess.getPrettyPrintIdentifier() + "Analyzing entry: "+prefix);
// If the target name is a path analyze it
File path = new File(prefix);
if (path.exists()) {
if (Properties.JUNIT.endsWith(".jar"))
testClasses.addAll(getTestClassesJar(path));
else
testClasses.addAll(getTestClasses(path));
} else {
try {
Class clazz = Class.forName(prefix,
true,
TestGenerationContext.getInstance().getClassLoaderForSUT());
testClasses.add(clazz);
} catch (ClassNotFoundException e) {
// Second, try if the target name is a package name
testClasses.addAll(getTestClassesFromClasspath());
}
}
}
return testClasses;
}
/**
* Analyze all classes that can be found in a given directory
*
* @param directory
* a {@link java.io.File} object.
* @throws ClassNotFoundException
* if any.
* @return a {@link java.util.List} object.
*/
private static List> getTestClasses(File directory) {
List> testClasses = new ArrayList>();
if (directory.getName().endsWith(".class")) {
LoggingUtils.muteCurrentOutAndErrStream();
try {
File file = new File(directory.getPath());
byte[] array = new byte[(int) file.length()];
ByteArrayOutputStream out = new ByteArrayOutputStream(array.length);
InputStream in = new FileInputStream(file);
try {
int length = in.read(array);
while (length > 0) {
out.write(array, 0, length);
length = in.read(array);
}
} finally {
in.close();
}
ClassReader reader = new ClassReader(array);
String className = reader.getClassName();
// Use default classLoader
Class clazz = Class.forName(className.replace('/', '.'), true,
TestGenerationContext.getInstance().getClassLoaderForSUT());
LoggingUtils.restorePreviousOutAndErrStream();
//clazz = Class.forName(clazz.getName());
if (isTest(clazz))
testClasses.add(clazz);
} catch (IllegalAccessError e) {
LoggingUtils.restorePreviousOutAndErrStream();
System.out.println(" Cannot access class "
+ directory.getName().substring(0,
directory.getName().length() - 6)
+ ": " + e);
} catch (NoClassDefFoundError e) {
LoggingUtils.restorePreviousOutAndErrStream();
System.out.println(" Error while loading "
+ directory.getName().substring(0,
directory.getName().length() - 6)
+ ": Cannot find " + e.getMessage());
//e.printStackTrace();
} catch (ExceptionInInitializerError e) {
LoggingUtils.restorePreviousOutAndErrStream();
System.out.println(" Exception in initializer of "
+ directory.getName().substring(0,
directory.getName().length() - 6));
} catch (ClassNotFoundException e) {
LoggingUtils.restorePreviousOutAndErrStream();
System.out.println(" Class not found in classpath: "
+ directory.getName().substring(0,
directory.getName().length() - 6)
+ ": " + e);
} catch (Throwable e) {
LoggingUtils.restorePreviousOutAndErrStream();
System.out.println(" Unexpected error: "
+ directory.getName().substring(0,
directory.getName().length() - 6)
+ ": " + e);
}
} else if (directory.isDirectory()) {
for (File file : directory.listFiles()) {
testClasses.addAll(getTestClasses(file));
}
}
return testClasses;
}
/**
*
* getClassesJar
*
*
* @param file
* a {@link java.io.File} object.
* @return a {@link java.util.List} object.
*/
private static List> getTestClassesJar(File file) {
List> testClasses = new ArrayList>();
ZipFile zf;
try {
zf = new ZipFile(file);
} catch (final ZipException e) {
throw new Error(e);
} catch (final IOException e) {
throw new Error(e);
}
final Enumeration e = zf.entries();
while (e.hasMoreElements()) {
final ZipEntry ze = (ZipEntry) e.nextElement();
final String fileName = ze.getName();
if (!fileName.endsWith(".class"))
continue;
/*if (fileName.contains("$"))
continue;*/
PrintStream old_out = System.out;
PrintStream old_err = System.err;
//System.setOut(outStream);
//System.setErr(outStream);
try {
Class clazz = Class.forName(fileName.replace(".class", "").replace("/",
"."),
true,
TestGenerationContext.getInstance().getClassLoaderForSUT());
if (isTest(clazz))
testClasses.add(clazz);
} catch (IllegalAccessError ex) {
System.setOut(old_out);
System.setErr(old_err);
System.out.println("Cannot access class "
+ file.getName().substring(0, file.getName().length() - 6));
} catch (NoClassDefFoundError ex) {
System.setOut(old_out);
System.setErr(old_err);
System.out.println("Cannot find dependent class " + ex);
} catch (ExceptionInInitializerError ex) {
System.setOut(old_out);
System.setErr(old_err);
System.out.println("Exception in initializer of "
+ file.getName().substring(0, file.getName().length() - 6));
} catch (ClassNotFoundException ex) {
System.setOut(old_out);
System.setErr(old_err);
System.out.println("Cannot find class "
+ file.getName().substring(0, file.getName().length() - 6) + ": "
+ ex);
} catch (Throwable t) {
System.setOut(old_out);
System.setErr(old_err);
System.out.println(" Unexpected error: "
+ file.getName().substring(0, file.getName().length() - 6) + ": "
+ t);
} finally {
System.setOut(old_out);
System.setErr(old_err);
}
}
try {
zf.close();
} catch (final IOException e1) {
throw new Error(e1);
}
return testClasses;
}
private static void analyzeCoverageCriterion(List results, Properties.Criterion criterion) {
logger.info("analysing coverage of " + criterion);
// Factory
TestFitnessFactory factory = FitnessFunctions.getFitnessFactory(criterion);
// Goals
List goals = null;
if (criterion == Criterion.MUTATION
|| criterion == Criterion.STRONGMUTATION) {
goals = MutationPool.getMutants();
} else {
goals = factory.getCoverageGoals();
}
totalGoals += goals.size();
// A dummy Chromosome
TestChromosome dummy = new TestChromosome();
dummy.setChanged(false);
// Execution result of a dummy Test Case
ExecutionResult executionResult = new ExecutionResult(dummy.getTestCase());
// coverage matrix (each row represents the coverage of each test case
// and each column represents the coverage of each component (e.g., line)
// this coverage matrix is useful for Rho fitness
boolean[][] coverage_matrix = new boolean[results.size()][goals.size() + 1]; // +1 because we also want to include the test result
BitSet covered = new BitSet(goals.size());
for (int index_test = 0; index_test < results.size(); index_test++) {
JUnitResult tR = results.get(index_test);
ExecutionTrace trace = tR.getExecutionTrace();
executionResult.setTrace(trace);
dummy.getTestCase().clearCoveredGoals();
dummy.setLastExecutionResult(executionResult);
if (criterion == Criterion.MUTATION
|| criterion == Criterion.STRONGMUTATION) {
for (Integer mutationID : trace.getTouchedMutants()) {
Mutation mutation = MutationPool.getMutant(mutationID);
if (goals.contains(mutation)) {
MutationObserver.activateMutation(mutationID);
List mutationResults = executeTests(tR.getJUnitClass());
MutationObserver.deactivateMutation();
for (JUnitResult mR : mutationResults) {
if (mR.getFailureCount() != tR.getFailureCount()) {
logger.info("Mutation killed: " + mutationID);
covered.set(mutation.getId());
coverage_matrix[index_test][mutationID.intValue()] = true;
break;
}
}
}
}
} else {
if (criterion==Criterion.EXCEPTION) {
// TODO collect exception goals from execution results
}
for (int index_component = 0; index_component < goals.size(); index_component++) {
TestFitnessFunction goal = (TestFitnessFunction) goals.get(index_component);
if (goal.isCovered(dummy)) {
covered.set(index_component);
coverage_matrix[index_test][index_component] = true;
}
else {
coverage_matrix[index_test][index_component] = false;
}
}
}
coverage_matrix[index_test][goals.size()] = tR.wasSuccessful();
}
totalCoveredGoals += covered.cardinality();
if (Properties.COVERAGE_MATRIX) {
CoverageReportGenerator.writeCoverage(coverage_matrix, criterion);
}
StringBuilder str = new StringBuilder();
for (int index_component = 0; index_component < goals.size(); index_component++) {
str.append(covered.get(index_component) ? "1" : "0");
}
logger.info("* CoverageBitString " + str.toString());
RuntimeVariable bitStringVariable = CoverageCriteriaAnalyzer.getBitStringVariable(criterion);
if (goals.isEmpty()) {
LoggingUtils.getEvoLogger().info("* " + ClientProcess.getPrettyPrintIdentifier() + "Coverage of criterion "
+ criterion + ": 100% (no goals)");
ClientServices.getInstance().getClientNode().trackOutputVariable(CoverageCriteriaAnalyzer.getCoverageVariable(criterion), 1.0);
if (bitStringVariable != null) {
ClientServices.getInstance().getClientNode().trackOutputVariable(bitStringVariable, "1");
}
}
else {
double coverage = ((double) covered.cardinality()) / ((double) goals.size());
LoggingUtils.getEvoLogger().info("* " + ClientProcess.getPrettyPrintIdentifier() + "Coverage of criterion " + criterion
+ ": " + NumberFormat.getPercentInstance().format(coverage));
LoggingUtils.getEvoLogger().info("* " + ClientProcess.getPrettyPrintIdentifier() + "Number of covered goals: "
+ covered.cardinality() + " / " + goals.size());
ClientServices.getInstance().getClientNode().trackOutputVariable(CoverageCriteriaAnalyzer.getCoverageVariable(criterion), coverage);
if (bitStringVariable != null) {
ClientServices.getInstance().getClientNode().trackOutputVariable(bitStringVariable, str.toString());
}
}
}
private static void printReport(List results) {
Iterator it = targetClasses.iterator();
Criterion[] criterion = Properties.CRITERION;
while (it.hasNext()) {
String targetClass = it.next();
// restart variables
totalGoals = 0;
totalCoveredGoals = 0;
Properties.TARGET_CLASS = targetClass;
LoggingUtils.getEvoLogger().info("* " + ClientProcess.getPrettyPrintIdentifier() + "Target class "
+ Properties.TARGET_CLASS);
ClientServices.getInstance().getClientNode().updateProperty("TARGET_CLASS", Properties.TARGET_CLASS);
for (int criterion_index = 0; criterion_index < criterion.length; criterion_index++) {
Properties.Criterion c = criterion[criterion_index];
Properties.CRITERION = new Criterion[] { c };
analyzeCoverageCriterion(results, c);
}
// restore
Properties.CRITERION = criterion;
LoggingUtils.getEvoLogger().info("* " + ClientProcess.getPrettyPrintIdentifier() + "Total number of covered goals: "
+ totalCoveredGoals + " / " + "" + totalGoals);
ClientServices.getInstance().getClientNode().trackOutputVariable(RuntimeVariable.Total_Goals, totalGoals);
ClientServices.getInstance().getClientNode().trackOutputVariable(RuntimeVariable.Covered_Goals, totalCoveredGoals);
double coverage = totalGoals == 0 ? 1.0 : ((double) totalCoveredGoals) / ((double) totalGoals);
LoggingUtils.getEvoLogger().info("* " + ClientProcess.getPrettyPrintIdentifier() + "Total coverage: "
+ NumberFormat.getPercentInstance().format(coverage));
ClientServices.getInstance().getClientNode().trackOutputVariable(RuntimeVariable.Coverage, coverage);
// need to give some time for transmission before client is killed
try {
Thread.sleep(100);
} catch (InterruptedException e) {
e.printStackTrace();
}
// last element will be flush by master process
if (it.hasNext()) {
ClientServices.getInstance().getClientNode().flushStatisticsForClassChange();
}
}
}
private static List executeTests(Class... testClasses) {
ExecutionTracer.enable();
ExecutionTracer.setCheckCallerThread(false);
ExecutionTracer.getExecutionTracer().clear();
List results = new ArrayList();
for (Class testClass : testClasses) {
LoggingUtils.getEvoLogger().info(" Executing " + testClass.getSimpleName());
// Set the context classloader in case the SUT requests it
Thread.currentThread().setContextClassLoader(testClass.getClassLoader());
JUnitRunner jR = new JUnitRunner(testClass);
jR.run();
results.addAll(jR.getTestResults());
}
ExecutionTracer.disable();
LoggingUtils.getEvoLogger().info("* " + ClientProcess.getPrettyPrintIdentifier() + "Executed " + results.size() + " unit "
+ "test(s)");
ClientServices.getInstance().getClientNode().trackOutputVariable(RuntimeVariable.Tests_Executed, results.size());
return results;
}
/**
* Determine if a class contains JUnit tests
*
* @param cls
* @return
*/
public static boolean isTest(Class cls) {
if (Modifier.isAbstract(cls.getModifiers())) {
return false;
}
TestClass tc;
try {
tc = new TestClass(cls);
} catch (IllegalArgumentException e) {
return false;
} catch (RuntimeException e){
//this can happen if class has Annotations that are not available on classpath
throw new RuntimeException("Failed to analyze class "+cls.getName()+ " due to: " + e.toString());
}
// JUnit 4
try {
List methods = new ArrayList<>();
methods.addAll(tc.getAnnotatedMethods(Test.class));
methods.addAll(tc.getAnnotatedMethods(EvoSuiteTest.class));
for (FrameworkMethod method : methods) {
List errors = new ArrayList();
method.validatePublicVoidNoArg(false, errors);
if (errors.isEmpty()) {
return true;
}
}
} catch (IllegalArgumentException e) {
return false;
}
// JUnit 3
Class superClass = cls;
while ((superClass = superClass.getSuperclass()) != null) {
if (superClass.getCanonicalName().equals(Object.class.getCanonicalName())) {
break ;
}
else if (superClass.getCanonicalName().equals(TestCase.class.getCanonicalName())) {
return true;
}
}
// TODO add support for other frameworks, e.g., TestNG ?
return false;
}
/**
* re-order test classes
*
* @param tests
*/
private static void sortTestClasses(List> tests) {
Collections.sort(tests, new Comparator>() {
@Override
public int compare(Class t0, Class t1) {
return Integer.compare(t1.getName().length(), t0.getName().length());
}
});
}
/**
*
* run
*
*/
public void run() {
LoggingUtils.getEvoLogger().info("* Connecting to master process on port "
+ Properties.PROCESS_COMMUNICATION_PORT);
ExternalProcessUtilities util = new ExternalProcessUtilities();
if (!util.connectToMainProcess()) {
throw new RuntimeException("Could not connect to master process on port "
+ Properties.PROCESS_COMMUNICATION_PORT);
}
analyzeCoverage();
/*
* for now, we ignore the instruction (originally was meant to support several client in parallel and
* restarts, but that will be done in RMI)
*/
util.informSearchIsFinished(null);
}
// just for testing
protected static void reset() {
totalGoals = 0;
totalCoveredGoals = 0;
targetClasses.clear();
}
}