stioner.lib.2024.7.0.source-code.Reporting.kt Maven / Gradle / Ivy
Show all versions of lib Show documentation
@file:Suppress("SpellCheckingInspection")
package edu.illinois.cs.cs125.questioner.lib
import edu.illinois.cs.cs125.jeed.core.suppressionComment
import edu.illinois.cs.cs125.jenisol.core.TestResult
import org.apache.commons.text.StringEscapeUtils
private fun wrapDocument(question: Question, body: String) = """
${question.published.name}
${
if (!question.validated) {
"""
|
|Validation Failed
|Please see below for more details.
|
|$body
""".trimMargin()
} else {
""
}
}
${question.published.name}
${question.published.author}
${question.published.version}
${
if (question.hasKotlin) {
"""Kotlin"""
} else {
""
}
}
Descriptions and Solutions
Java
Description
Solution
${StringEscapeUtils.escapeHtml4(question.getCorrect(Language.java)!!)}
${
question.published.starters?.get(Language.java)?.let {
"""Starter Code
${
StringEscapeUtils.escapeHtml4(it)
}
"""
} ?: ""
}
${
if (question.hasKotlin) {
"""Kotlin
Description
Solution
${StringEscapeUtils.escapeHtml4(question.alternativeSolutions.find { it.language == Language.kotlin }!!.contents)}
""" +
(question.published.starters?.get(Language.kotlin)?.let {
"""Starter Code
${
StringEscapeUtils.escapeHtml4(it)
}
"""
} ?: "")
} else {
""
}
}
${
if (question.validated) {
body
} else {
""
}
}
"""
fun IncorrectResults.html(index: Int, question: Question): String {
val contents = if (incorrect.mutation != null) {
incorrect.mutation.marked().contents.deTemplate(question.getTemplate(Language.java))
} else {
incorrect.contents
}
return """
Incorrect $index
${
if (incorrect.mutation != null) {
"Mutation (${incorrect.mutation.mutations.first().mutation.mutationType.name})"
} else if (incorrect.starter) {
if (incorrect.path == null) {
"Starter (autogenerated)"
} else {
"@Starter annotated"
}
} else {
"@Incorrect annotated${
if (incorrect.reason != Question.IncorrectFile.Reason.TEST) {
": (Reason ${incorrect.reason})"
} else {
""
}
}"
}
}
${
if (results.tests() != null) {
val actualCount = if (question.fauxStatic) {
results.tests()!!.filter { it.jenisol!!.type != TestResult.Type.CONSTRUCTOR }.size
} else {
results.tests()!!.size
}
val alert = if (results.succeeded && incorrect.reason == Question.IncorrectFile.Reason.DEADCODE) {
"""Testing succeeded, but found ${results.complete.coverage!!.increase - results.complete.coverage!!.limit} too many lines of dead code as expected."""
} else if (results.succeeded && incorrect.reason == Question.IncorrectFile.Reason.LINECOUNT) {
"""Testing succeeded, but submission ran for ${results.complete.executionCount!!.submission - results.complete.executionCount!!.limit} too many line counts as expected."""
} else if (results.succeeded && incorrect.reason == Question.IncorrectFile.Reason.TOOLONG) {
"""Testing succeeded, but submission contained ${results.complete.lineCount!!.increase - results.complete.lineCount!!.limit} too many lines of code as expected."""
} else if (results.succeeded && incorrect.reason == Question.IncorrectFile.Reason.MEMORYLIMIT) {
"""Testing succeeded, but submission allocated ${results.complete.memoryAllocation!!.submission - results.complete.memoryAllocation!!.limit} too many bytes as expected."""
} else if (results.succeeded && incorrect.reason == Question.IncorrectFile.Reason.RECURSION) {
"""Testing succeeded, but submission did not implement a method recursively."""
} else if (results.succeeded && incorrect.reason == Question.IncorrectFile.Reason.COMPLEXITY) {
"""Testing succeeded, but submission was too complex."""
} else if (results.succeeded && incorrect.reason == Question.IncorrectFile.Reason.FEATURES) {
"""Testing succeeded, but submission used incorrect features."""
} else if (results.tests()!!.size > question.control.minTestCount!!) {
"""Slowly found a failing test. Consider adding this input to @FixedParameters."""
} else {
"""Quickly found a failing test."""
}
"""
$alert
Tests
Time
$actualCount
${results.taskResults!!.interval.length}
${results.tests()?.find { !it.passed }?.explanation ?: "No explanation available"}
"""
} else {
"""${results.summary}"""
}
}
${StringEscapeUtils.escapeHtml4(contents)}
"""
}
fun ValidationReport.report(): String {
val incorrectBody = incorrect
.sortedBy { it.results.tests()?.size }
.reversed()
.mapIndexed { i, it -> it.html(i, question) }
.joinToString("\n")
val testingSequence = if (solutionTestingSequence != null) {
"""
|Testing Sequence
|
|${solutionTestingSequence.joinToString("\n")}
|
""".trimMargin()
} else {
""
}
val body = """
|Incorrect Examples
|Used ${incorrect.size} incorrect examples to generate test cases.
|$incorrectBody
|$testingSequence
|""".trimMargin()
return wrapDocument(question, body)
}
fun ValidationFailed.report(question: Question): String {
val body = when (this) {
is SolutionFailed -> {
"""
|Solution Failed Testing
|The following solution failed testing:
| ${StringEscapeUtils.escapeHtml4(solution.contents)}
|${explanation}
|Please verify that this solution matches the reference solution.
""".trimMargin()
}
is SolutionReceiverGeneration -> {
"""
|Solution Failed Testing
|The following solution failed testing:
| ${StringEscapeUtils.escapeHtml4(solution.contents)}
|Couldn't generate enough receivers during testing.
|Examine any @FilterParameters methods you might be using, or exceptions thrown in your constructor.
|Consider adding parameter generation methods for your constructor.
""".trimMargin()
}
is SolutionThrew -> {
"""
|Solution Not Expected to Throw
|The solution was not expected to throw, but threw $threw
on parameters $parameters
.
| ${StringEscapeUtils.escapeHtml4(solution.contents)}
|
|- If it should throw, allow it using
@Correct(solutionThrows = true)
|- Otherwise filter the inputs using
@FixedParameters
, @RandomParameters
, or @FilterParameters
|
""".trimMargin()
}
is SolutionLacksEntropy -> {
"""
|Solution Results Lack Entropy
|Random inputs to the solution only generated $amount distinct return values.
| ${StringEscapeUtils.escapeHtml4(solution.contents)}
|You may need to add or adjust your @RandomParameters method.
""".trimMargin()
}
is NoIncorrect -> {
"""
|No Incorrect Examples Found
|No incorrect examples found or generated through mutation.
|Please add some using @Incorrect or by enabling suppressed mutations.
|
""".trimMargin()
}
is TooFewMutations -> {
"""
|Too Few Mutations Found
|Generated $found mutations but needed $needed.
|Please reduce the required number or remove mutation suppressions.
|
""".trimMargin()
}
is TooMuchOutput -> {
"""
|Too Much Output
|The following submission generated too much output ($size > $maxSize)
| ${StringEscapeUtils.escapeHtml4(contents)}
|Consider reducing the number of tests using @Correct(minTestCount = NUM)
.
""".trimMargin()
}
is IncorrectPassed -> {
val contents = incorrect.mutation?.marked()?.contents?.deTemplate(question.getTemplate(incorrect.language))
?: incorrect.contents
"""
|Incorrect Code Passed the Test Suite
|The following incorrect code passed the test suites:
| ${StringEscapeUtils.escapeHtml4(contents)}
|
|- If the code is in fact incorrect, you may need to add a failing input using
@FixedParameters
|${
if (incorrect.mutation != null) {
" - If the code is a mutation that should pass, you may need to disable the mutation using
${incorrect.mutation.mutations.first().mutation.mutationType.suppressionComment()}
"
} else {
""
}
}
| - You may also need to increase the test count using
@Correct(maxTestCount = NUM)
, or remove an existing limitation
|
""".trimMargin()
}
is IncorrectTooManyTests -> {
val contents = incorrect.mutation?.marked()?.contents?.deTemplate(question.getTemplate(incorrect.language))
?: incorrect.contents
"""
|Incorrect Code Required Too Many Tests
|Incorrect code eventually failed but required too many tests ($testsRequired > $testsLimit).
|${failingInput?.let { "We found failing inputs $failingInput" } ?: "We were unable to find a failing input"}
|
| ${StringEscapeUtils.escapeHtml4(contents)}
|
|- If the code is incorrect, add an input using
@FixedParameters
to handle this case
|${
if (incorrect.mutation != null) {
" - If the code is correct, you may need to disable this mutation using " +
"// ${incorrect.mutation.mutations.first().mutation.mutationType.suppressionComment()}
"
} else {
""
}
}- You may also need to increase the test count using
@Correct(maxTestCount = NUM)
|
""".trimMargin()
}
is IncorrectWrongReason -> {
"""
|Incorrect Code Failed for the Wrong Reason
|Incorrect code failed but not for the reason we expected: Expected: $expected, but found: $explanation.
|
| ${StringEscapeUtils.escapeHtml4(incorrect.contents)}
|Check the arguments to @Incorrect(reason = REASON)
""".trimMargin()
}
is SolutionDeadCode -> {
"""
|Solution Contains Dead Code
|Solution contains $amount lines of untested code, more than the maximum of $maximum.
|Either adjust the inputs, prune unused code paths, or increase the amount of allowed dead code.
|
""".trimMargin()
}
is SolutionFailedLinting -> {
"""
|Solution Failed Linting
|Solution failed linting with this message: $message
|
""".trimMargin()
}
else -> error("Invalid error: $this")
}
return wrapDocument(question, body)
}