ai.timefold.solver.benchmark.impl.report.benchmarkReport.html.ftl Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of timefold-solver-benchmark Show documentation
Timefold solves planning problems. This lightweight, embeddable planning engine implements powerful and scalable algorithms to optimize business resource scheduling and planning. This module contains the benchmarker toolkit.
There is a newer version: 1.16.0
Show newest version
<#ftl output_format="HTML"> <#-- So that Freemarker escapes automatically. -->



    
    
    ${benchmarkReport.plannerBenchmarkResult.name} Planner benchmark report

    
    

    
    
    
    
     <#-- Too large for a webjar. -->

    
     <#-- Contains functions called by chart.js; must go first. -->
    
     <#-- Includes Popper for dropdowns. -->
    
    
    
    

<#macro addSolverBenchmarkBadges solverBenchmarkResult>
    <#if solverBenchmarkResult.favorite>
        ${solverBenchmarkResult.ranking}
    <#elseif solverBenchmarkResult.ranking??>
        ${solverBenchmarkResult.ranking}
    

    <#if solverBenchmarkResult.hasAnyFailure()>
        F
    <#elseif solverBenchmarkResult.hasAnyUninitializedSolution()>
        !
    <#elseif solverBenchmarkResult.hasAnyInfeasibleScore()>
        !
    

<#macro addProblemBenchmarkBadges problemBenchmarkResult>
    <#if problemBenchmarkResult.hasAnyFailure()>
        F
    

<#macro addSolverProblemBenchmarkResultBadges solverProblemBenchmarkResult>
    <#if solverProblemBenchmarkResult.winner>
        ${solverProblemBenchmarkResult.ranking}
    <#elseif solverProblemBenchmarkResult.ranking??>
        ${solverProblemBenchmarkResult.ranking}
    

    <#if solverProblemBenchmarkResult.hasAnyFailure()>
        F
    <#elseif !solverProblemBenchmarkResult.initialized>
        !
    <#elseif !solverProblemBenchmarkResult.scoreFeasible>
        !
    

<#macro addChartList chartList idPrefix>
    
        
            <#assign scoreLevelIndex = 0>
            <#list chartList as chart>
                <#assign tabId = idPrefix + "_chart_" + scoreLevelIndex />
                
                    <@addChart chart=chart />
                
                <#assign scoreLevelIndex = scoreLevelIndex + 1>
            
        
        
            <#assign scoreLevelIndex = 0>
            <#list chartList as chart>
                <#assign tabId = idPrefix + "_chart_" + scoreLevelIndex />
                
                <#assign scoreLevelIndex = scoreLevelIndex + 1>
            
        
    

<#macro addChart chart>
    
        
        
    


    
        
            
                
                    
                
                

                Benchmark Report
            

            

            
                
                    
                        
                            Summary
                        
                        
                            Results
                            Performance
                        
                    
                    
                        
                            Problems
                        
                        
                            <#list benchmarkReport.plannerBenchmarkResult.unifiedProblemBenchmarkResultList as problemBenchmarkResult>
                                <@addProblemBenchmarkBadges problemBenchmarkResult=problemBenchmarkResult/> ${problemBenchmarkResult.name}
                            
                        
                    
                    Solvers
                    Environment
                
                
                    
                        Star
                        
                        
                        
                        Contact
                    
                
            
        
    

     
        
            Timefold Benchmark report
        

        
            Result summary
            <#if benchmarkReport.plannerBenchmarkResult.hasAnyFailure()>
                
                    ${benchmarkReport.plannerBenchmarkResult.failureCount} benchmarks have failed!
                
            
            <#list benchmarkReport.getWarningList() as warning>
                
                    ${warning}
                
            
            
                
                    
                        
                    
                    
                        
                    
                    
                        
                    
                    
                        
                    
                    
                        
                    
                
                
                    
                        Best score summary
                        Useful for visualizing the best solver configuration.
                        <@addChartList chartList=benchmarkReport.bestScoreSummaryChartList idPrefix="summary_bestScore" />
                        
                            
                                    <#list benchmarkReport.plannerBenchmarkResult.unifiedProblemBenchmarkResultList as problemBenchmarkResult>
                                        
                                    <#list benchmarkReport.plannerBenchmarkResult.solverBenchmarkResultList as solverBenchmarkResult>
                                         class="table-success">
                                            
                                            <#list benchmarkReport.plannerBenchmarkResult.unifiedProblemBenchmarkResultList as problemBenchmarkResult>
                                                <#if !solverBenchmarkResult.findSingleBenchmark(problemBenchmarkResult)??>
                                                    
                                                <#else>
                                                    <#assign singleBenchmarkResult = solverBenchmarkResult.findSingleBenchmark(problemBenchmarkResult)>
                                                    <#if !singleBenchmarkResult.hasAllSuccess()>
                                                        
                                                    <#else>
                                                        <#if solverBenchmarkResult.subSingleCount lte 1>
                                                            
                                                        <#else>
                                                            
                                
                                    
                                        Solver
                                        Total
                                        Average
                                        Standard Deviation
                                        Problem
                                    
                                    ${problemBenchmarkResult.name}
                                    
                                    
                                
                                ${solverBenchmarkResult.name} <@addSolverBenchmarkBadges solverBenchmarkResult=solverBenchmarkResult/>
                                            ${solverBenchmarkResult.totalScore!""}
                                            ${solverBenchmarkResult.averageScore!""}
                                            ${solverBenchmarkResult.standardDeviationString!""} Failed ${singleBenchmarkResult.averageScore!""} <@addSolverProblemBenchmarkResultBadges solverProblemBenchmarkResult=singleBenchmarkResult/> 
                                                                
                                                                    
                                                                    
                                                                        <#list singleBenchmarkResult.subSingleBenchmarkResultList as subSingleBenchmarkResult>
                                                                            Run #${subSingleBenchmarkResult.getSubSingleBenchmarkIndex()}
                                                                            ${subSingleBenchmarkResult.score!""} <@addSolverProblemBenchmarkResultBadges solverProblemBenchmarkResult=subSingleBenchmarkResult/>
                                                                        
                                                                        
                                                                        Median
                                                                        ${singleBenchmarkResult.median.score!""}
                                                                        Standard Deviation
                                                                        ${singleBenchmarkResult.standardDeviationString!""}
                                                                        Best
                                                                        ${singleBenchmarkResult.best.score!""}
                                                                        Average
                                                                        ${singleBenchmarkResult.averageScore!""}
                                                                        Worst
                                                                        ${singleBenchmarkResult.worst.score!""}
                                                                    
                                                                
                                                            
                                                        
                                                    
                                                
                                            
                                        
                                    
                                
                            
                        
                    
                    
                        Best score scalability summary
                        Useful for visualizing the scalability of each solver configuration.
                        <@addChartList chartList=benchmarkReport.bestScoreScalabilitySummaryChartList idPrefix="summary_bestScoreScalability" />
                    
                    
                        Best score distribution summary
                        Useful for visualizing the reliability of each solver configuration.
                        <#assign maximumSubSingleCount = benchmarkReport.plannerBenchmarkResult.getMaximumSubSingleCount()>
                        <#if maximumSubSingleCount lte 1>
                            
                                Benchmarker did not run multiple subSingles, so there is no distribution and therefore no reliability indication.
                            
                        
                        <@addChartList chartList=benchmarkReport.bestScoreDistributionSummaryChartList idPrefix="summary_bestScoreDistribution" />
                    
                    
                        Winning score difference summary
                        Useful for zooming in on the results of the best score summary.
                        <@addChartList chartList=benchmarkReport.winningScoreDifferenceSummaryChartList idPrefix="summary_winningScoreDifference" />
                        
                            
                                    <#list benchmarkReport.plannerBenchmarkResult.unifiedProblemBenchmarkResultList as problemBenchmarkResult>
                                        
                                    <#list benchmarkReport.plannerBenchmarkResult.solverBenchmarkResultList as solverBenchmarkResult>
                                         class="table-success">
                                            
                                            <#list benchmarkReport.plannerBenchmarkResult.unifiedProblemBenchmarkResultList as problemBenchmarkResult>
                                                <#if !solverBenchmarkResult.findSingleBenchmark(problemBenchmarkResult)??>
                                                    
                                                <#else>
                                                    <#assign singleBenchmarkResult = solverBenchmarkResult.findSingleBenchmark(problemBenchmarkResult)>
                                                    <#if !singleBenchmarkResult.hasAllSuccess()>
                                                        
                                                    <#else>
                                                        
                                
                                    
                                        Solver
                                        Total
                                        Average
                                        Problem
                                    
                                    ${problemBenchmarkResult.name}
                                    
                                    
                                
                                ${solverBenchmarkResult.name} <@addSolverBenchmarkBadges solverBenchmarkResult=solverBenchmarkResult/>
                                            ${solverBenchmarkResult.totalWinningScoreDifference!""}
                                            ${solverBenchmarkResult.averageWinningScoreDifference!""} Failed ${singleBenchmarkResult.winningScoreDifference} <@addSolverProblemBenchmarkResultBadges solverProblemBenchmarkResult=singleBenchmarkResult/>
                                                    
                                                
                                            
                                        
                                    
                                
                            
                        
                    
                    
                        Worst score difference percentage summary (ROI)
                        Useful for visualizing the return on investment (ROI) to decision makers.
                        <@addChartList chartList=benchmarkReport.worstScoreDifferencePercentageSummaryChartList idPrefix="summary_worstScoreDifferencePercentage" />
                        
                            
                                    <#list benchmarkReport.plannerBenchmarkResult.unifiedProblemBenchmarkResultList as problemBenchmarkResult>
                                        
                                    <#list benchmarkReport.plannerBenchmarkResult.solverBenchmarkResultList as solverBenchmarkResult>
                                         class="table-success">
                                            
                                            <#if !solverBenchmarkResult.averageWorstScoreDifferencePercentage??>
                                                
                                            <#else>
                                                
                                            <#list benchmarkReport.plannerBenchmarkResult.unifiedProblemBenchmarkResultList as problemBenchmarkResult>
                                                <#if !solverBenchmarkResult.findSingleBenchmark(problemBenchmarkResult)??>
                                                    
                                                <#else>
                                                    <#assign singleBenchmarkResult = solverBenchmarkResult.findSingleBenchmark(problemBenchmarkResult)>
                                                    <#if !singleBenchmarkResult.hasAllSuccess()>
                                                        
                                                    <#else>
                                                        
                                
                                    
                                        Solver
                                        Average
                                        Problem
                                    
                                    ${problemBenchmarkResult.name}
                                    
                                    
                                
                                ${solverBenchmarkResult.name} <@addSolverBenchmarkBadges solverBenchmarkResult=solverBenchmarkResult/> ${solverBenchmarkResult.averageWorstScoreDifferencePercentage.toString(.locale_object)}
                                            Failed ${singleBenchmarkResult.worstScoreDifferencePercentage.toString(.locale_object)} <@addSolverProblemBenchmarkResultBadges solverProblemBenchmarkResult=singleBenchmarkResult/>
                                                    
                                                
                                            
                                        
                                    
                                
                            
                        
                    
                
            
        

        
            Performance summary
            
                
                    
                        
                    
                    
                        
                    
                    
                        
                    
                    
                        
                    
                    
                        
                    
                
                
                    
                        Score calculation speed summary
                        
                            Useful for comparing different score calculators and/or constraint implementations
                            (presuming that the solver configurations do not differ otherwise).
                            Also useful to measure the scalability cost of an extra constraint.
                        
                        <@addChart chart=benchmarkReport.scoreCalculationSpeedSummaryChart />
                        
                            
                                        <#list benchmarkReport.plannerBenchmarkResult.unifiedProblemBenchmarkResultList as problemBenchmarkResult>
                                            
                                        <#list benchmarkReport.plannerBenchmarkResult.unifiedProblemBenchmarkResultList as problemBenchmarkResult>
                                            
                                    <#list benchmarkReport.plannerBenchmarkResult.solverBenchmarkResultList as solverBenchmarkResult>
                                         class="table-success">
                                            
                                            <#list benchmarkReport.plannerBenchmarkResult.unifiedProblemBenchmarkResultList as problemBenchmarkResult>
                                                <#if !solverBenchmarkResult.findSingleBenchmark(problemBenchmarkResult)??>
                                                    
                                                <#else>
                                                    <#assign singleBenchmarkResult = solverBenchmarkResult.findSingleBenchmark(problemBenchmarkResult)>
                                                    <#if !singleBenchmarkResult.hasAllSuccess()>
                                                        
                                                    <#else>
                                                        <#if solverBenchmarkResult.subSingleCount lte 1>
                                                            
                                                        <#else>
                                                            
                                
                                    
                                        Solver
                                        Average
                                        Problem
                                    
                                    ${problemBenchmarkResult.name}
                                        
                                    
                                    
                                        Problem scale
                                        ${benchmarkReport.plannerBenchmarkResult.averageProblemScale!""} ${problemBenchmarkResult.problemScale!""}
                                        
                                    
                                
                                ${solverBenchmarkResult.name} <@addSolverBenchmarkBadges solverBenchmarkResult=solverBenchmarkResult/>
                                            ${solverBenchmarkResult.averageScoreCalculationSpeed!""}/s Failed ${singleBenchmarkResult.scoreCalculationSpeed}/s 
                                                                
                                                                    
                                                                    
                                                                        <#list singleBenchmarkResult.subSingleBenchmarkResultList as subSingleBenchmarkResult>
                                                                            Run #${subSingleBenchmarkResult.getSubSingleBenchmarkIndex()}
                                                                            ${subSingleBenchmarkResult.scoreCalculationSpeed!""}/s
                                                                        
                                                                    
                                                                
                                                            
                                                        
                                                    
                                                
                                            
                                        
                                    
                                
                            
                        
                    
                    
                        Worst score calculation speed difference percentage
                        
                            Useful for comparing different score calculators and/or constraint implementations
                            (presuming that the solver configurations do not differ otherwise).
                            Also useful to measure the scalability cost of an extra constraint.
                        
                        <@addChart chart=benchmarkReport.worstScoreCalculationSpeedDifferencePercentageSummaryChart />
                        
                            
                                        <#list benchmarkReport.plannerBenchmarkResult.unifiedProblemBenchmarkResultList as problemBenchmarkResult>
                                            
                                    <#list benchmarkReport.plannerBenchmarkResult.solverBenchmarkResultList as solverBenchmarkResult>
                                         class="table-success">
                                            
                                            <#if solverBenchmarkResult.averageWorstScoreCalculationSpeedDifferencePercentage??>
                                                
                                            <#else>
                                                
                                            <#list benchmarkReport.plannerBenchmarkResult.unifiedProblemBenchmarkResultList as problemBenchmarkResult>
                                                <#if !solverBenchmarkResult.findSingleBenchmark(problemBenchmarkResult)??>
                                                    
                                                <#else>
                                                    <#assign singleBenchmarkResult = solverBenchmarkResult.findSingleBenchmark(problemBenchmarkResult)>
                                                    <#if !singleBenchmarkResult.hasAllSuccess()>
                                                        
                                                    <#else>
                                                        
                                
                                    
                                        Solver
                                        Average
                                        Problem
                                    
                                    ${problemBenchmarkResult.name}
                                        
                                    
                                
                                ${solverBenchmarkResult.name} <@addSolverBenchmarkBadges solverBenchmarkResult=solverBenchmarkResult/> ${solverBenchmarkResult.averageWorstScoreCalculationSpeedDifferencePercentage?string["0.00%"]!""} 
                                            Failed ${singleBenchmarkResult.worstScoreCalculationSpeedDifferencePercentage?string["0.00%"]!""}
                                                    
                                                
                                            
                                        
                                    
                                
                            
                        
                    
                    
                        Time spent summary
                        Useful for visualizing the performance of construction heuristics (presuming that no other solver phases are configured).
                        <@addChart chart=benchmarkReport.timeSpentSummaryChart />
                        
                            
                                        <#list benchmarkReport.plannerBenchmarkResult.unifiedProblemBenchmarkResultList as problemBenchmarkResult>
                                            
                                        <#list benchmarkReport.plannerBenchmarkResult.unifiedProblemBenchmarkResultList as problemBenchmarkResult>
                                            
                                    <#list benchmarkReport.plannerBenchmarkResult.solverBenchmarkResultList as solverBenchmarkResult>
                                         class="table-success">
                                            
                                            <#list benchmarkReport.plannerBenchmarkResult.unifiedProblemBenchmarkResultList as problemBenchmarkResult>
                                                <#if !solverBenchmarkResult.findSingleBenchmark(problemBenchmarkResult)??>
                                                    
                                                <#else>
                                                    <#assign singleBenchmarkResult = solverBenchmarkResult.findSingleBenchmark(problemBenchmarkResult)>
                                                    <#if !singleBenchmarkResult.hasAllSuccess()>
                                                        
                                                    <#else>
                                                        <#if solverBenchmarkResult.subSingleCount lte 1>
                                                            
                                                        <#else>
                                                            
                                
                                    
                                        Solver
                                        Average
                                        Problem
                                    
                                    ${problemBenchmarkResult.name}
                                        
                                    
                                    
                                        Problem scale
                                        ${benchmarkReport.plannerBenchmarkResult.averageProblemScale!""} ${problemBenchmarkResult.problemScale!""}
                                        
                                    
                                
                                ${solverBenchmarkResult.name} <@addSolverBenchmarkBadges solverBenchmarkResult=solverBenchmarkResult/>
                                            ${solverBenchmarkResult.averageTimeMillisSpent!0?string.@msDuration} Failed ${singleBenchmarkResult.timeMillisSpent?string.@msDuration} 
                                                                
                                                                    
                                                                    
                                                                        <#list singleBenchmarkResult.subSingleBenchmarkResultList as subSingleBenchmarkResult>
                                                                            Run #${subSingleBenchmarkResult.getSubSingleBenchmarkIndex()}
                                                                            ${subSingleBenchmarkResult.timeMillisSpent?string.@msDuration}
                                                                        
                                                                    
                                                                
                                                            
                                                        
                                                    
                                                
                                            
                                        
                                    
                                
                            
                        
                    
                    
                        Time spent scalability summary
                        Useful for extrapolating the scalability of construction heuristics (presuming that no other solver phases are configured).
                        <@addChart chart=benchmarkReport.timeSpentScalabilitySummaryChart />
                    
                    
                        Best score per time spent summary
                        Useful for visualizing trade-off between the best score versus the time spent for construction heuristics (presuming that no other solver phases are configured).
                        <@addChartList chartList=benchmarkReport.bestScorePerTimeSpentSummaryChartList idPrefix="summary_bestScorePerTimeSpent" />
                    
                
            
        

        
            Problem benchmarks
            <#list benchmarkReport.plannerBenchmarkResult.unifiedProblemBenchmarkResultList as problemBenchmarkResult>
                
                    ${problemBenchmarkResult.name}
                    <#if problemBenchmarkResult.hasAnyFailure()>
                        
                            ${problemBenchmarkResult.failureCount} benchmarks have failed!
                        
                    
                    
                            <#if problemBenchmarkResult.inputSolutionLoadingTimeMillisSpent??>
                                
                            <#if problemBenchmarkResult.averageUsedMemoryAfterInputSolution??>
                                
                        
                            
                                Entity count
                                ${problemBenchmarkResult.entityCount!""}
                            
                            
                                Variable count
                                ${problemBenchmarkResult.variableCount!""}
                            
                            
                                Maximum value count
                                ${problemBenchmarkResult.maximumValueCount!""}
                            
                            
                                Problem scale
                                ${problemBenchmarkResult.problemScale!""}
                            

                                    Time to load input solution
                                    <#if problemBenchmarkResult.inputSolutionLoadingTimeMillisSpent lt 1>< 1 ms<#else>${problemBenchmarkResult.inputSolutionLoadingTimeMillisSpent?string.@msDuration}
                                
                            
                                    Memory usage after loading input solution
                                    ${problemBenchmarkResult.averageUsedMemoryAfterInputSolution?string.number} bytes on average
                                
                            
                        
                    
                    <#if problemBenchmarkResult.hasAnySuccess() && problemBenchmarkResult.hasAnyStatistic()>
                        <#if problemBenchmarkResult.getMaximumSubSingleCount() gt 1>
                            
                                Only the median sub single run of each solver is shown in the statistics below.
                            
                        
                        
                            
                                <#assign firstRow = true>
                                <#list problemBenchmarkResult.problemStatisticList as problemStatistic>
                                    <#assign tabId = "problemStatistic_" + problemStatistic.anchorId />
                                    
                                        
                                    
                                    <#assign firstRow = false>
                                
                                <#list problemBenchmarkResult.extractSingleStatisticTypeList() as singleStatisticType>
                                    <#assign tabId = "singleStatistic_" + problemBenchmarkResult.anchorId + "_" + singleStatisticType.anchorId />
                                     class="active">
                                        
                                    
                                    <#assign firstRow = false>
                                
                            
                            
                                <#assign firstRow = true>
                                <#list problemBenchmarkResult.problemStatisticList as problemStatistic>
                                    <#assign tabId = "problemStatistic_" + problemStatistic.anchorId />
                                    
                                        <#-- TODO somehow figure out the solver name; sub single stats have it. -->
                                        <#list problemStatistic.getWarningList() as warning>
                                            
                                                ${warning}
                                            
                                        
                                        <#assign chartList = problemStatistic.getChartList()>
                                        <#if chartList?size != 0>
                                            <#if problemStatistic.problemStatisticType.hasScoreLevels()>
                                                <@addChartList chartList=chartList idPrefix="problemStatistic_" + problemStatistic.anchorId />
                                            <#else>
                                                <@addChart chart=chartList[0] />
                                            
                                        <#else>
                                            
                                                Graph not available. Either the statistic is not available for this solver configuration, or the benchmark failed.
                                            
                                        
                                        <#if !benchmarkReport.plannerBenchmarkResult.aggregation>
                                            CSV files per solver:
                                            
                                                <#list problemStatistic.subSingleStatisticList as subSingleStatistic>
                                                    
                                                
                                            
                                        
                                    
                                    <#assign firstRow = false>
                                
                                <#list problemBenchmarkResult.extractSingleStatisticTypeList() as singleStatisticType>
                                    <#assign tabId = "singleStatistic_" + problemBenchmarkResult.anchorId + "_" + singleStatisticType.anchorId />
                                    
                                        <#list problemBenchmarkResult.extractPureSubSingleStatisticList(singleStatisticType) as pureSubSingleStatistic>
                                            ${pureSubSingleStatistic.subSingleBenchmarkResult.singleBenchmarkResult.solverBenchmarkResult.name}
                                            <#assign chartList = pureSubSingleStatistic.getChartList()>
                                            <#if chartList?size != 0>
                                                <#if singleStatisticType.hasScoreLevels()>
                                                    <@addChartList chartList=chartList idPrefix="singleStatistic_" + problemBenchmarkResult.anchorId + "_" + singleStatisticType.anchorId />
                                                <#else>
                                                    <@addChart chart=chartList[0] />
                                                
                                            <#else>
                                                
                                                    Graph not available. Either the statistic is not available for this solver configuration, or the benchmark failed.
                                                
                                            
                                            <#if !benchmarkReport.plannerBenchmarkResult.aggregation>
                                                CSV file:
                                                
                                                    
                                                
                                            
                                        
                                    
                                    <#assign firstRow = false>
                                
                            
                        
                    
                    <#list problemBenchmarkResult.singleBenchmarkResultList as singleBenchmarkResult>
                        
                            ${singleBenchmarkResult.name}
                            <#if singleBenchmarkResult.hasAnyFailure()>
                                
                                    ${singleBenchmarkResult.failureCount} benchmarks have failed!
                                
                            <#else>
                                <#if singleBenchmarkResult.getScoreExplanationSummary()??>
                                    <#if singleBenchmarkResult.getSubSingleCount() gt 1 >
                                        
                                            Only the median sub single run of each solver is shown in the statistics below.
                                        
                                    
                                    ${singleBenchmarkResult.scoreExplanationSummary}
                                <#else>
                                    Score summary not provided.
                                
                            
                        
                    
                
            
        

        
            Solver benchmarks
            
                <#assign firstRow = true>
                <#list benchmarkReport.plannerBenchmarkResult.solverBenchmarkResultList as solverBenchmarkResult>
                    <#assign tabId="solverBenchmark_" + solverBenchmarkResult.anchorId + "_config" />
                    
                        
                    
                    <#assign firstRow = false>
                
            
            
                <#assign firstRow = true>
                <#list benchmarkReport.plannerBenchmarkResult.solverBenchmarkResultList as solverBenchmarkResult>
                    <#assign tabId="solverBenchmark_" + solverBenchmarkResult.anchorId + "_config" />
                    
                        
                            <#if solverBenchmarkResult.hasAnyFailure()>
                                
                                    ${solverBenchmarkResult.failureCount} benchmarks have failed!
                                
                            
                            ${solverBenchmarkResult.solverConfigAsString}
                        
                    
                    <#assign firstRow = false>
                
            
        

        
            Environment Information
            
                    <#if benchmarkReport.plannerBenchmarkResult.warmUpTimeMillisSpentLimit??>
                        
                    <#else>
                        
                    <#if benchmarkReport.plannerBenchmarkResult.parallelBenchmarkCount?? && benchmarkReport.plannerBenchmarkResult.availableProcessors??>
                        
                    <#else>
                        
                    <#if benchmarkReport.plannerBenchmarkResult.benchmarkTimeMillisSpent??>
                        
                    <#else>
                        
                    <#if (benchmarkReport.plannerBenchmarkResult.maxMemory?string.number)??>
                        
                    <#else>
                        
                
                    Name
                    ${benchmarkReport.plannerBenchmarkResult.name}
                
                
                    Aggregation
                    ${benchmarkReport.plannerBenchmarkResult.aggregation?string}
                
                
                    Failure count
                    ${benchmarkReport.plannerBenchmarkResult.failureCount}
                
                
                    Starting timestamp
                    ${(benchmarkReport.plannerBenchmarkResult.startingTimestampAsMediumString)!"Differs"}
                
                
                    Warm-up time spent ${benchmarkReport.plannerBenchmarkResult.warmUpTimeMillisSpentLimit?string.@msDuration} Differs
                    
                
                
                    Parallel benchmark count / available processors ${benchmarkReport.plannerBenchmarkResult.parallelBenchmarkCount} / ${benchmarkReport.plannerBenchmarkResult.availableProcessors} Differs
                    
                
                
                    Benchmark time spent ${benchmarkReport.plannerBenchmarkResult.benchmarkTimeMillisSpent?string.@msDuration} Differs
                    
                
                
                    Environment mode
                    ${benchmarkReport.plannerBenchmarkResult.environmentMode!"Differs"}
                
                
                    Logging level for ai.timefold.solver.core
                    ${benchmarkReport.plannerBenchmarkResult.loggingLevelTimefoldSolverCore!"Differs"}
                
                
                    Solver ranking class
                    
                        
                            ${benchmarkReport.solverRankingClassSimpleName!"Unknown"}
                        
                    
                
                
                    VM max memory (as in -Xmx but lower) ${benchmarkReport.plannerBenchmarkResult.maxMemory?string.number} bytes Differs
                    
                
                
                    Timefold Solver version
                    ${benchmarkReport.plannerBenchmarkResult.timefoldSolverVersion!"Differs"}
                
                
                    Java version
                    ${benchmarkReport.plannerBenchmarkResult.javaVersion!"Differs"}
                
                
                    Java VM
                    ${benchmarkReport.plannerBenchmarkResult.javaVM!"Differs"}
                
                
                    Operating system
                    ${benchmarkReport.plannerBenchmarkResult.operatingSystem!"Differs"}
                
                
                    Report locale
                    ${benchmarkReport.locale_object!"Unknown"}
                
                
                    Report timezone
                    ${benchmarkReport.timezoneId!"Unknown"}
                
            
        
    
    
        
            
                Timefold
                
                Documentation
                
                Code
                
                Support
Solver	Total	Average	Standard Deviation	Problem
Solver	Total	Average	Standard Deviation	${problemBenchmarkResult.name}
${solverBenchmarkResult.name} <@addSolverBenchmarkBadges solverBenchmarkResult=solverBenchmarkResult/>	${solverBenchmarkResult.totalScore!""}	${solverBenchmarkResult.averageScore!""}	${solverBenchmarkResult.standardDeviationString!""}		Failed	${singleBenchmarkResult.averageScore!""} <@addSolverProblemBenchmarkResultBadges solverProblemBenchmarkResult=singleBenchmarkResult/>	<#list singleBenchmarkResult.subSingleBenchmarkResultList as subSingleBenchmarkResult> Run #${subSingleBenchmarkResult.getSubSingleBenchmarkIndex()} ${subSingleBenchmarkResult.score!""} <@addSolverProblemBenchmarkResultBadges solverProblemBenchmarkResult=subSingleBenchmarkResult/> Median ${singleBenchmarkResult.median.score!""} Standard Deviation ${singleBenchmarkResult.standardDeviationString!""} Best ${singleBenchmarkResult.best.score!""} Average ${singleBenchmarkResult.averageScore!""} Worst ${singleBenchmarkResult.worst.score!""}
Entity count	${problemBenchmarkResult.entityCount!""}
Variable count	${problemBenchmarkResult.variableCount!""}
Maximum value count	${problemBenchmarkResult.maximumValueCount!""}
Problem scale	${problemBenchmarkResult.problemScale!""}
Time to load input solution	<#if problemBenchmarkResult.inputSolutionLoadingTimeMillisSpent lt 1>< 1 ms<#else>${problemBenchmarkResult.inputSolutionLoadingTimeMillisSpent?string.@msDuration}
Memory usage after loading input solution	${problemBenchmarkResult.averageUsedMemoryAfterInputSolution?string.number} bytes on average
Name	${benchmarkReport.plannerBenchmarkResult.name}
Aggregation	${benchmarkReport.plannerBenchmarkResult.aggregation?string}
Failure count	${benchmarkReport.plannerBenchmarkResult.failureCount}
Starting timestamp	${(benchmarkReport.plannerBenchmarkResult.startingTimestampAsMediumString)!"Differs"}
Warm-up time spent	${benchmarkReport.plannerBenchmarkResult.warmUpTimeMillisSpentLimit?string.@msDuration}	Differs
Parallel benchmark count / available processors	${benchmarkReport.plannerBenchmarkResult.parallelBenchmarkCount} / ${benchmarkReport.plannerBenchmarkResult.availableProcessors}	Differs
Benchmark time spent	${benchmarkReport.plannerBenchmarkResult.benchmarkTimeMillisSpent?string.@msDuration}	Differs
Environment mode	${benchmarkReport.plannerBenchmarkResult.environmentMode!"Differs"}
Logging level for ai.timefold.solver.core	${benchmarkReport.plannerBenchmarkResult.loggingLevelTimefoldSolverCore!"Differs"}
Solver ranking class	${benchmarkReport.solverRankingClassSimpleName!"Unknown"}
VM max memory (as in -Xmx but lower)	${benchmarkReport.plannerBenchmarkResult.maxMemory?string.number} bytes	Differs
Timefold Solver version	${benchmarkReport.plannerBenchmarkResult.timefoldSolverVersion!"Differs"}
Java version	${benchmarkReport.plannerBenchmarkResult.javaVersion!"Differs"}
Java VM	${benchmarkReport.plannerBenchmarkResult.javaVM!"Differs"}
Operating system	${benchmarkReport.plannerBenchmarkResult.operatingSystem!"Differs"}
Report locale	${benchmarkReport.locale_object!"Unknown"}
Report timezone	${benchmarkReport.timezoneId!"Unknown"}