Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.nd4j.evaluation.classification.ROC Maven / Gradle / Ivy
/*
* ******************************************************************************
* *
* *
* * This program and the accompanying materials are made available under the
* * terms of the Apache License, Version 2.0 which is available at
* * https://www.apache.org/licenses/LICENSE-2.0.
* *
* * See the NOTICE file distributed with this work for additional
* * information regarding copyright ownership.
* * Unless required by applicable law or agreed to in writing, software
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* * License for the specific language governing permissions and limitations
* * under the License.
* *
* * SPDX-License-Identifier: Apache-2.0
* *****************************************************************************
*/
package org.nd4j.evaluation.classification;
import lombok.*;
import org.apache.commons.lang3.ArrayUtils;
import org.nd4j.common.base.Preconditions;
import org.nd4j.evaluation.BaseEvaluation;
import org.nd4j.evaluation.IEvaluation;
import org.nd4j.evaluation.IMetric;
import org.nd4j.evaluation.curves.PrecisionRecallCurve;
import org.nd4j.evaluation.curves.RocCurve;
import org.nd4j.evaluation.serde.ROCSerializer;
import org.nd4j.linalg.api.buffer.DataType;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.ops.Op;
import org.nd4j.linalg.api.ops.impl.reduce.longer.MatchCondition;
import org.nd4j.linalg.api.ops.impl.transforms.comparison.CompareAndSet;
import org.nd4j.linalg.api.ops.impl.transforms.pairwise.arithmetic.MulOp;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.indexing.INDArrayIndex;
import org.nd4j.linalg.indexing.NDArrayIndex;
import org.nd4j.linalg.indexing.conditions.Condition;
import org.nd4j.linalg.indexing.conditions.Conditions;
import org.nd4j.common.primitives.Pair;
import org.nd4j.common.primitives.Triple;
import org.nd4j.shade.jackson.annotation.JsonIgnoreProperties;
import org.nd4j.shade.jackson.annotation.JsonTypeInfo;
import org.nd4j.shade.jackson.databind.annotation.JsonSerialize;
import java.io.Serializable;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import static org.nd4j.linalg.indexing.NDArrayIndex.all;
import static org.nd4j.linalg.indexing.NDArrayIndex.interval;
@EqualsAndHashCode(callSuper = true,
exclude = {"auc", "auprc", "probAndLabel", "exactAllocBlockSize", "rocCurve", "prCurve", "axis"})
@Data
@JsonIgnoreProperties({"probAndLabel", "exactAllocBlockSize"})
@JsonSerialize(using = ROCSerializer.class)
@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY)
public class ROC extends BaseEvaluation {
/**
* AUROC: Area under ROC curve counts = new LinkedHashMap<>();
private int thresholdSteps;
private long countActualPositive;
private long countActualNegative;
private Double auc;
private Double auprc;
private RocCurve rocCurve;
private PrecisionRecallCurve prCurve;
private boolean isExact;
private INDArray probAndLabel;
private int exampleCount = 0;
private boolean rocRemoveRedundantPts;
private int exactAllocBlockSize;
protected int axis = 1;
public ROC(int thresholdSteps, boolean rocRemoveRedundantPts, int exactAllocBlockSize, int axis) {
this(thresholdSteps, rocRemoveRedundantPts, exactAllocBlockSize);
this.axis = axis;
}
public ROC() {
//Default to exact
this(0);
}
/**
* @param thresholdSteps Number of threshold steps to use for the ROC calculation. If set to 0: use exact calculation
*/
public ROC(int thresholdSteps) {
this(thresholdSteps, true);
}
/**
* @param thresholdSteps Number of threshold steps to use for the ROC calculation. If set to 0: use exact calculation
* @param rocRemoveRedundantPts Usually set to true. If true, remove any redundant points from ROC and P-R curves
*/
public ROC(int thresholdSteps, boolean rocRemoveRedundantPts) {
this(thresholdSteps, rocRemoveRedundantPts, DEFAULT_EXACT_ALLOC_BLOCK_SIZE);
}
/**
* @param thresholdSteps Number of threshold steps to use for the ROC calculation. If set to 0: use exact calculation
* @param rocRemoveRedundantPts Usually set to true. If true, remove any redundant points from ROC and P-R curves
* @param exactAllocBlockSize if using exact mode, the block size relocation. Users can likely use the default
* setting in almost all cases
*/
public ROC(int thresholdSteps, boolean rocRemoveRedundantPts, int exactAllocBlockSize) {
if (thresholdSteps > 0) {
this.thresholdSteps = thresholdSteps;
double step = 1.0 / thresholdSteps;
for (int i = 0; i <= thresholdSteps; i++) {
double currThreshold = i * step;
counts.put(currThreshold, new CountsForThreshold(currThreshold));
}
isExact = false;
} else {
//Exact
isExact = true;
}
this.rocRemoveRedundantPts = rocRemoveRedundantPts;
this.exactAllocBlockSize = exactAllocBlockSize;
}
public static ROC fromJson(String json) {
return fromJson(json, ROC.class);
}
/**
* Set the axis for evaluation - this should be a size 1 dimension
* For DL4J, this can be left as the default setting (axis = 1). p = removeRedundant(tOut, x_fpr_out, y_tpr_out, null, null, null);
double[][] temp = p.getFirst();
tOut = temp[0];
x_fpr_out = temp[1];
y_tpr_out = temp[2];
}
this.rocCurve = new RocCurve(tOut, x_fpr_out, y_tpr_out);
return rocCurve;
} else {
double[][] out = new double[3][thresholdSteps + 1];
int i = 0;
for (Map.Entry entry : counts.entrySet()) {
CountsForThreshold c = entry.getValue();
double tpr = c.getCountTruePositive() / ((double) countActualPositive);
double fpr = c.getCountFalsePositive() / ((double) countActualNegative);
out[0][i] = c.getThreshold();
out[1][i] = fpr;
out[2][i] = tpr;
i++;
}
return new RocCurve(out[0], out[1], out[2]);
}
}
protected INDArray getProbAndLabelUsed() {
if (probAndLabel == null || exampleCount == 0) {
return null;
}
return probAndLabel.get(interval(0, exampleCount), all());
}
private static Pair removeRedundant(double[] threshold, double[] x, double[] y, int[] tpCount,
int[] fpCount, int[] fnCount) {
double[] t_compacted = new double[threshold.length];
double[] x_compacted = new double[x.length];
double[] y_compacted = new double[y.length];
int[] tp_compacted = null;
int[] fp_compacted = null;
int[] fn_compacted = null;
boolean hasInts = false;
if (tpCount != null) {
tp_compacted = new int[tpCount.length];
fp_compacted = new int[fpCount.length];
fn_compacted = new int[fnCount.length];
hasInts = true;
}
int lastOutPos = -1;
for (int i = 0; i < threshold.length; i++) {
boolean keep;
if (i == 0 || i == threshold.length - 1) {
keep = true;
} else {
boolean ommitSameY = y[i - 1] == y[i] && y[i] == y[i + 1];
boolean ommitSameX = x[i - 1] == x[i] && x[i] == x[i + 1];
keep = !ommitSameX && !ommitSameY;
}
if (keep) {
lastOutPos++;
t_compacted[lastOutPos] = threshold[i];
y_compacted[lastOutPos] = y[i];
x_compacted[lastOutPos] = x[i];
if (hasInts) {
tp_compacted[lastOutPos] = tpCount[i];
fp_compacted[lastOutPos] = fpCount[i];
fn_compacted[lastOutPos] = fnCount[i];
}
}
}
if (lastOutPos < x.length - 1) {
t_compacted = Arrays.copyOfRange(t_compacted, 0, lastOutPos + 1);
x_compacted = Arrays.copyOfRange(x_compacted, 0, lastOutPos + 1);
y_compacted = Arrays.copyOfRange(y_compacted, 0, lastOutPos + 1);
if (hasInts) {
tp_compacted = Arrays.copyOfRange(tp_compacted, 0, lastOutPos + 1);
fp_compacted = Arrays.copyOfRange(fp_compacted, 0, lastOutPos + 1);
fn_compacted = Arrays.copyOfRange(fn_compacted, 0, lastOutPos + 1);
}
}
return new Pair<>(new double[][]{t_compacted, x_compacted, y_compacted},
hasInts ? new int[][]{tp_compacted, fp_compacted, fn_compacted} : null);
}
private double getAuprc() {
if (auprc != null) {
return auprc;
}
auprc = calculateAUCPR();
return auprc;
}
/**
* Calculate the area under the precision/recall curve - aka AUCPR
*
* @return
*/
public double calculateAUCPR() {
if (auprc != null) {
return auprc;
}
Preconditions.checkState(exampleCount > 0, "Unable to calculate AUPRC: no evaluation has been performed (no examples)");
auprc = getPrecisionRecallCurve().calculateAUPRC();
return auprc;
}
/**
* Get the precision recall curve as array.
* return[0] = threshold array pair = removeRedundant(thresholdOut, precisionOut, recallOut, tpCountOut,
fpCountOut, fnCountOut);
double[][] temp = pair.getFirst();
int[][] temp2 = pair.getSecond();
thresholdOut = temp[0];
precisionOut = temp[1];
recallOut = temp[2];
tpCountOut = temp2[0];
fpCountOut = temp2[1];
fnCountOut = temp2[2];
}
} else {
thresholdOut = new double[counts.size()];
precisionOut = new double[counts.size()];
recallOut = new double[counts.size()];
tpCountOut = new int[counts.size()];
fpCountOut = new int[counts.size()];
fnCountOut = new int[counts.size()];
int i = 0;
for (Map.Entry entry : counts.entrySet()) {
double t = entry.getKey();
CountsForThreshold c = entry.getValue();
long tpCount = c.getCountTruePositive();
long fpCount = c.getCountFalsePositive();
//For edge cases: http://stats.stackexchange.com/questions/1773/what-are-correct-values-for-precision-and-recall-in-edge-cases
//precision == 1 when FP = 0 -> no incorrect positive predictions
//recall == 1 when no dataset positives are present (got all 0 of 0 positives)
double precision;
if (tpCount == 0 && fpCount == 0) {
//At this threshold: no predicted positive cases
precision = 1.0;
} else {
precision = tpCount / (double) (tpCount + fpCount);
}
double recall;
if (countActualPositive == 0) {
recall = 1.0;
} else {
recall = tpCount / ((double) countActualPositive);
}
thresholdOut[i] = c.getThreshold();
precisionOut[i] = precision;
recallOut[i] = recall;
tpCountOut[i] = (int) tpCount;
fpCountOut[i] = (int) fpCount;
fnCountOut[i] = (int) (countActualPositive - tpCount);
i++;
}
}
prCurve = new PrecisionRecallCurve(thresholdOut, precisionOut, recallOut, tpCountOut, fpCountOut, fnCountOut,
exampleCount);
return prCurve;
}
@AllArgsConstructor
@Data
@NoArgsConstructor
public static class CountsForThreshold implements Serializable, Cloneable {
private double threshold;
private long countTruePositive;
private long countFalsePositive;
public CountsForThreshold(double threshold) {
this(threshold, 0, 0);
}
@Override
public CountsForThreshold clone() {
return new CountsForThreshold(threshold, countTruePositive, countFalsePositive);
}
public void incrementFalsePositive(long count) {
countFalsePositive += count;
}
public void incrementTruePositive(long count) {
countTruePositive += count;
}
}
/**
* Evaluate (collect statistics for) the given minibatch of data.
* For time series (3 dimensions) use {@link #evalTimeSeries(INDArray, INDArray)} or {@link #evalTimeSeries(INDArray, INDArray, INDArray)}
*
* @param labels Labels / true outcomes
* @param predictions Predictions
*/
@Override
public void eval(INDArray labels, INDArray predictions, INDArray mask, List recordMetaData) {
Triple p = BaseEvaluation.reshapeAndExtractNotMasked(labels, predictions, mask, axis);
if (p == null) {
//All values masked out; no-op
return;
}
INDArray labels2d = p.getFirst();
INDArray predictions2d = p.getSecond();
if (labels2d.rank() == 3 && predictions2d.rank() == 3) {
//Assume time series input -> reshape to 2d
evalTimeSeries(labels2d, predictions2d);
}
if (labels2d.rank() > 2 || predictions2d.rank() > 2 || labels2d.size(1) != predictions2d.size(1)
|| labels2d.size(1) > 2) {
throw new IllegalArgumentException("Invalid input data shape: labels shape = "
+ Arrays.toString(labels2d.shape()) + ", predictions shape = "
+ Arrays.toString(predictions2d.shape()) + "; require rank 2 array with size(1) == 1 or 2");
}
if (labels2d.dataType() != predictions2d.dataType())
labels2d = labels2d.castTo(predictions2d.dataType());
//Check for NaNs in predictions - without this, evaulation could silently be intepreted as class 0 prediction due to argmax
long count = Nd4j.getExecutioner().execAndReturn(new MatchCondition(predictions2d, Conditions.isNan())).getFinalResult().longValue();
Preconditions.checkState(count == 0, "Cannot perform evaluation with NaN(s) present:" +
" %s NaN(s) present in predictions INDArray", count);
double step = 1.0 / thresholdSteps;
boolean singleOutput = labels2d.size(1) == 1;
if (isExact) {
//Exact approach: simply add them to the storage for later computation/use
if (probAndLabel == null) {
//Do initial allocation
val initialSize = Math.max(labels2d.size(0), exactAllocBlockSize);
probAndLabel = Nd4j.create(DataType.DOUBLE, new long[]{initialSize, 2}, 'c'); //First col: probability of class 1. Second col: "is class 1"
}
//Allocate a larger array if necessary
if (exampleCount + labels2d.size(0) >= probAndLabel.size(0)) {
val newSize = probAndLabel.size(0) + Math.max(exactAllocBlockSize, labels2d.size(0));
INDArray newProbAndLabel = Nd4j.create(DataType.DOUBLE, new long[]{newSize, 2}, 'c');
if (exampleCount > 0) {
//If statement to handle edge case: no examples, but we need to re-allocate right away
newProbAndLabel.get(interval(0, exampleCount), all()).assign(
probAndLabel.get(interval(0, exampleCount), all()));
}
probAndLabel = newProbAndLabel;
}
//put values
INDArray probClass1;
INDArray labelClass1;
if (singleOutput) {
probClass1 = predictions2d;
labelClass1 = labels2d;
} else {
probClass1 = predictions2d.getColumn(1,true);
labelClass1 = labels2d.getColumn(1,true);
}
val currMinibatchSize = labels2d.size(0);
probAndLabel.get(interval(exampleCount, exampleCount + currMinibatchSize),
NDArrayIndex.point(0)).assign(probClass1);
probAndLabel.get(interval(exampleCount, exampleCount + currMinibatchSize),
NDArrayIndex.point(1)).assign(labelClass1);
int countClass1CurrMinibatch = labelClass1.sumNumber().intValue();
countActualPositive += countClass1CurrMinibatch;
countActualNegative += labels2d.size(0) - countClass1CurrMinibatch;
} else {
//Thresholded approach
INDArray positivePredictedClassColumn;
INDArray positiveActualClassColumn;
INDArray negativeActualClassColumn;
if (singleOutput) {
//Single binary variable case
positiveActualClassColumn = labels2d;
negativeActualClassColumn = labels2d.rsub(1.0); //1.0 - label
positivePredictedClassColumn = predictions2d;
} else {
//Standard case - 2 output variables (probability distribution)
positiveActualClassColumn = labels2d.getColumn(1,true);
negativeActualClassColumn = labels2d.getColumn(0,true);
positivePredictedClassColumn = predictions2d.getColumn(1,true);
}
//Increment global counts - actual positive/negative observed
countActualPositive += positiveActualClassColumn.sumNumber().intValue();
countActualNegative += negativeActualClassColumn.sumNumber().intValue();
//Here: calculate true positive rate (TPR) vs. false positive rate (FPR) at different threshold
INDArray ppc = null;
INDArray itp = null;
INDArray ifp = null;
for (int i = 0; i <= thresholdSteps; i++) {
double currThreshold = i * step;
//Work out true/false positives - do this by replacing probabilities (predictions) with 1 or 0 based on threshold
Condition condGeq = Conditions.greaterThanOrEqual(currThreshold);
Condition condLeq = Conditions.lessThanOrEqual(currThreshold);
if (ppc == null) {
ppc = positivePredictedClassColumn.dup(positiveActualClassColumn.ordering());
} else {
ppc.assign(positivePredictedClassColumn);
}
Op op = new CompareAndSet(ppc, 1.0, condGeq);
INDArray predictedClass1 = Nd4j.getExecutioner().exec(op);
op = new CompareAndSet(predictedClass1, 0.0, condLeq);
predictedClass1 = Nd4j.getExecutioner().exec(op);
//True positives: occur when positive predicted class and actual positive actual class...
//False positive occurs when positive predicted class, but negative actual class
INDArray isTruePositive; // = predictedClass1.mul(positiveActualClassColumn); //If predicted == 1 and actual == 1 at this threshold: 1x1 = 1. 0 otherwise
INDArray isFalsePositive; // = predictedClass1.mul(negativeActualClassColumn); //If predicted == 1 and actual == 0 at this threshold: 1x1 = 1. 0 otherwise
if (i == 0) {
isTruePositive = predictedClass1.mul(positiveActualClassColumn);
isFalsePositive = predictedClass1.mul(negativeActualClassColumn);
itp = isTruePositive;
ifp = isFalsePositive;
} else {
isTruePositive = Nd4j.getExecutioner().exec(new MulOp(predictedClass1, positiveActualClassColumn, itp))[0];
isFalsePositive = Nd4j.getExecutioner().exec(new MulOp(predictedClass1, negativeActualClassColumn, ifp))[0];
}
//Counts for this batch:
int truePositiveCount = isTruePositive.sumNumber().intValue();
int falsePositiveCount = isFalsePositive.sumNumber().intValue();
//Increment counts for this thold
CountsForThreshold thresholdCounts = counts.get(currThreshold);
thresholdCounts.incrementTruePositive(truePositiveCount);
thresholdCounts.incrementFalsePositive(falsePositiveCount);
}
}
exampleCount += labels2d.size(0);
auc = null;
auprc = null;
rocCurve = null;
prCurve = null;
}
/**
* Merge this ROC instance with another.
* This ROC instance is modified, by adding the stats from the other instance.
*
* @param other ROC instance to combine with this one
*/
@Override
public void merge(ROC other) {
if (this.thresholdSteps != other.thresholdSteps) {
throw new UnsupportedOperationException(
"Cannot merge ROC instances with different numbers of threshold steps ("
+ this.thresholdSteps + " vs. " + other.thresholdSteps + ")");
}
this.countActualPositive += other.countActualPositive;
this.countActualNegative += other.countActualNegative;
this.auc = null;
this.auprc = null;
this.rocCurve = null;
this.prCurve = null;
if (isExact) {
if (other.exampleCount == 0) {
return;
}
if (this.exampleCount == 0) {
this.exampleCount = other.exampleCount;
this.probAndLabel = other.probAndLabel;
return;
}
if (this.exampleCount + other.exampleCount > this.probAndLabel.size(0)) {
//Allocate new array
val newSize = this.probAndLabel.size(0) + Math.max(other.probAndLabel.size(0), exactAllocBlockSize);
INDArray newProbAndLabel = Nd4j.create(DataType.DOUBLE, newSize, 2);
newProbAndLabel.put(new INDArrayIndex[]{interval(0, exampleCount), all()}, probAndLabel.get(interval(0, exampleCount), all()));
probAndLabel = newProbAndLabel;
}
INDArray toPut = other.probAndLabel.get(interval(0, other.exampleCount), all());
probAndLabel.put(new INDArrayIndex[]{
interval(exampleCount, exampleCount + other.exampleCount), all()},
toPut);
} else {
for (Double d : this.counts.keySet()) {
CountsForThreshold cft = this.counts.get(d);
CountsForThreshold otherCft = other.counts.get(d);
cft.countTruePositive += otherCft.countTruePositive;
cft.countFalsePositive += otherCft.countFalsePositive;
}
}
this.exampleCount += other.exampleCount;
}
@Override
public void reset() {
countActualPositive = 0L;
countActualNegative = 0L;
counts.clear();
if (isExact) {
probAndLabel = null;
} else {
double step = 1.0 / thresholdSteps;
for (int i = 0; i <= thresholdSteps; i++) {
double currThreshold = i * step;
counts.put(currThreshold, new CountsForThreshold(currThreshold));
}
}
exampleCount = 0;
auc = null;
auprc = null;
}
@Override
public String stats() {
if(this.exampleCount == 0){
return "ROC: No data available (no data has been performed)";
}
StringBuilder sb = new StringBuilder();
sb.append("AUC (Area under ROC Curve): ").append(calculateAUC()).append("\n");
sb.append("AUPRC (Area under Precision/Recall Curve): ").append(calculateAUCPR());
if (!isExact) {
sb.append("\n");
sb.append("[Note: Thresholded AUC/AUPRC calculation used with ").append(thresholdSteps)
.append(" steps); accuracy may reduced compared to exact mode]");
}
return sb.toString();
}
@Override
public String toString(){
return stats();
}
public double scoreForMetric(Metric metric){
switch (metric){
case AUROC:
return calculateAUC();
case AUPRC:
return calculateAUCPR();
default:
throw new IllegalStateException("Unknown metric: " + metric);
}
}
@Override
public double getValue(IMetric metric){
if(metric instanceof Metric){
return scoreForMetric((Metric) metric);
} else
throw new IllegalStateException("Can't get value for non-ROC Metric " + metric);
}
@Override
public ROC newInstance() {
return new ROC(thresholdSteps, rocRemoveRedundantPts, exactAllocBlockSize, axis);
}
}
