smile.validation.ClassificationValidations Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of smile-core Show documentation
smile-core
The newest version!
/*
 * Copyright (c) 2010-2021 Haifeng Li. All rights reserved.
 *
 * Smile is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Smile is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Smile.  If not, see .
 */

package smile.validation;

import java.io.Serial;
import java.io.Serializable;
import java.util.List;
import smile.math.MathEx;

/**
 * Classification model validation results.
 *
 * @param  the model type.
 *
 * @author Haifeng Li
 */
public class ClassificationValidations implements Serializable {
    @Serial
    private static final long serialVersionUID = 2L;

    /** The multiple round validations. */
    public final List> rounds;

    /** The average of metrics. */
    public final ClassificationMetrics avg;

    /** The standard deviation of metrics. */
    public final ClassificationMetrics sd;

    /**
     * Constructor.
     * @param rounds the validation metrics of multipl rounds.
     */
    public ClassificationValidations(List> rounds) {
        this.rounds = rounds;

        int k = rounds.size();
        double[] fitTime = new double[k];
        double[] scoreTime = new double[k];
        int[] size = new int[k];
        int[] error = new int[k];
        double[] accuracy = new double[k];
        double[] sensitivity = new double[k];
        double[] specificity = new double[k];
        double[] precision = new double[k];
        double[] f1 = new double[k];
        double[] mcc = new double[k];
        double[] auc = new double[k];
        double[] logloss = new double[k];
        double[] crossentropy = new double[k];

        for (int i = 0; i < k; i++) {
            ClassificationMetrics metrics = rounds.get(i).metrics;
            fitTime[i] = metrics.fitTime();
            scoreTime[i] = metrics.scoreTime();
            size[i] = metrics.size();
            error[i] = metrics.error();
            accuracy[i] = metrics.accuracy();
            sensitivity[i] = metrics.sensitivity();
            specificity[i] = metrics.specificity();
            precision[i] = metrics.precision();
            f1[i] = metrics.f1();
            mcc[i] = metrics.mcc();
            auc[i] = metrics.auc();
            logloss[i] = metrics.logloss();
            crossentropy[i] = metrics.crossentropy();
        }

        avg = new ClassificationMetrics(
                MathEx.mean(fitTime),
                MathEx.mean(scoreTime),
                (int) Math.round(MathEx.mean(size)),
                (int) Math.round(MathEx.mean(error)),
                MathEx.mean(accuracy),
                MathEx.mean(sensitivity),
                MathEx.mean(specificity),
                MathEx.mean(precision),
                MathEx.mean(f1),
                MathEx.mean(mcc),
                MathEx.mean(auc),
                MathEx.mean(logloss),
                MathEx.mean(crossentropy)
        );
        sd = new ClassificationMetrics(
                MathEx.sd(fitTime),
                MathEx.sd(scoreTime),
                (int) Math.round(MathEx.sd(size)),
                (int) Math.round(MathEx.sd(error)),
                MathEx.sd(accuracy),
                MathEx.sd(sensitivity),
                MathEx.sd(specificity),
                MathEx.sd(precision),
                MathEx.sd(f1),
                MathEx.sd(mcc),
                MathEx.sd(auc),
                MathEx.sd(logloss),
                MathEx.sd(crossentropy)
        );
    }

    @Override
    public String toString() {
        StringBuilder sb = new StringBuilder("{\n");
        sb.append(String.format("  fit time: %.3f ms ± %.3f,\n", avg.fitTime(), sd.fitTime()));
        sb.append(String.format("  score time: %.3f ms ± %.3f,\n", avg.scoreTime(), sd.scoreTime()));
        sb.append(String.format("  validation data size: %d ± %d,\n", avg.size(), sd.size()));
        sb.append(String.format("  error: %d ± %d,\n", avg.error(), sd.error()));
        sb.append(String.format("  accuracy: %.2f%% ± %.2f", 100 * avg.accuracy(), 100 * sd.accuracy()));
        if (!Double.isNaN(avg.sensitivity())) sb.append(String.format(",\n  sensitivity: %.2f%% ± %.2f", 100 * avg.sensitivity(), 100 * sd.sensitivity()));
        if (!Double.isNaN(avg.specificity())) sb.append(String.format(",\n  specificity: %.2f%% ± %.2f", 100 * avg.specificity(), 100 * sd.specificity()));
        if (!Double.isNaN(avg.precision())) sb.append(String.format(",\n  precision: %.2f%% ± %.2f", 100 * avg.precision(), 100 * sd.precision()));
        if (!Double.isNaN(avg.f1())) sb.append(String.format(",\n  F1 score: %.2f%% ± %.2f", 100 * avg.f1(), 100 * sd.f1()));
        if (!Double.isNaN(avg.mcc())) sb.append(String.format(",\n  MCC: %.2f%% ± %.2f", 100 * avg.mcc(), 100 * sd.mcc()));
        if (!Double.isNaN(avg.auc())) sb.append(String.format(",\n  AUC: %.2f%% ± %.2f", 100 * avg.auc(), 100 * sd.auc()));
        if (!Double.isNaN(avg.logloss())) sb.append(String.format(",\n  log loss: %.4f ± %.4f", avg.logloss(), sd.logloss()));
        else if (!Double.isNaN(avg.crossentropy())) sb.append(String.format(",\n  cross entropy: %.4f ± %.4f", avg.crossentropy(), sd.crossentropy()));
        sb.append("\n}");
        return sb.toString();
    }
}