tensorflow_metadata.proto.v0.metric.proto Maven / Gradle / Ivy
The newest version!
// Copyright 2017 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// =============================================================================
syntax = "proto3";
package tensorflow.metadata.v0;
import "google/protobuf/any.proto";
import "google/protobuf/wrappers.proto";
import "google/protobuf/descriptor.proto";
import "tensorflow_metadata/proto/v0/path.proto";
option cc_enable_arenas = true;
option java_package = "org.tensorflow.metadata.v0";
option java_multiple_files = true;
// Metric type indicates which direction of a real-valued metric is "better".
// For most message types, this is invariant. For custom message types,
// is_maximized == true is like MAXIMIZE, and otherwise MINIMIZE.
enum MetricType {
UNKNOWN = 0;
// Maximize the metric (i.e. a utility).
MAXIMIZE = 1;
// Minimize the metric (i.e. a loss).
MINIMIZE = 2;
// Look for a field is_maximized.
CUSTOM = 3;
}
extend google.protobuf.MessageOptions {
MetricType metric_type = 227673489;
}
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/binary_accuracy
message BinaryAccuracy {
option (metric_type) = MAXIMIZE;
}
// categorical_accuracy(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/categorical_accuracy
message CategoricalAccuracy {
option (metric_type) = MAXIMIZE;
}
// categorical_crossentropy(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/categorical_crossentropy
message CategoricalCrossEntropy {
option (metric_type) = MINIMIZE;
}
// cosine(...)
// cosine_proximity(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/cosine_proximity
// DEPRECATED
message Cosine {
option (metric_type) = MINIMIZE;
}
// Linear Hinge Loss
// hinge(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/hinge
// DEPRECATED
message Hinge {
option (metric_type) = MINIMIZE;
}
// kld(...)
// kullback_leibler_divergence(...)
// KLD(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/kullback_leibler_divergence
// DEPRECATED
message KullbackLeiblerDivergence {
option (metric_type) = MINIMIZE;
}
// MAE(...)
// mae(...)
// mean_absolute_error(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/mean_absolute_error
message MeanAbsoluteError {
option (metric_type) = MINIMIZE;
}
// MAPE(...)
// mape(...)
// mean_absolute_percentage_error(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/mean_absolute_percentage_error
message MeanAbsolutePercentageError {
option (metric_type) = MINIMIZE;
}
// MSE(...)
// mse(...)
// mean_squared_error(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/mean_squared_error
message MeanSquaredError {
option (metric_type) = MINIMIZE;
}
// msle(...)
// MSLE(...)
// mean_squared_logarithmic_error(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/mean_squared_logarithmic_error
message MeanSquaredLogarithmicError {
option (metric_type) = MINIMIZE;
}
// poisson(...)
// DEPRECATED
message Poisson {
option (metric_type) = MINIMIZE;
}
// squared_hinge(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/squared_hinge
// DEPRECATED
message SquaredHinge {
option (metric_type) = MINIMIZE;
}
// top_k_categorical_accuracy(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/top_k_categorical_accuracy
message TopKCategoricalAccuracy {
option (metric_type) = MAXIMIZE;
}
// sparse_top_k_categorical_accuracy(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/sparse_top_k_categorical_accuracy
// DEPRECATED
message SparseTopKCategoricalAccuracy {
option (metric_type) = MAXIMIZE;
}
// Binary cross entropy as a metric is equal to the negative log likelihood
// (see logistic regression).
// In addition, when used to solve a binary classification task, binary cross
// entropy implies that the binary label will maximize binary accuracy.
// binary_crossentropy(...)
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/binary_crossentropy
message BinaryCrossEntropy {
option (metric_type) = MINIMIZE;
}
// AKA the negative log likelihood or log loss.
// Given a label y\in {0,1} and a predicted probability p in [0,1]:
// -yln(p)-(1-y)ln(1-p)
// TODO(martinz): if this is interpreted the same as binary_cross_entropy,
// we may need to revisit the semantics.
// DEPRECATED
message LogisticRegression {
option (metric_type) = MINIMIZE;
}
// Area under curve for the ROC-curve.
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/AUC
message AUC {
option (metric_type) = MAXIMIZE;
}
// Area under curve for the precision-recall-curve.
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/AUC
message AUCPrecisionRecall {
option (metric_type) = MAXIMIZE;
}
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/SensitivityAtSpecificity
message SensitivityAtSpecificity {
option (metric_type) = MAXIMIZE;
// Minimal required specificity, (0.0, 1.0).
google.protobuf.DoubleValue specificity = 1;
}
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/SpecificityAtSensitivity
message SpecificityAtSensitivity {
option (metric_type) = MAXIMIZE;
// Minimal required sensitivity, (0.0, 1.0).
google.protobuf.DoubleValue sensitivity = 1;
}
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/PrecisionAtRecall
message PrecisionAtRecall {
option (metric_type) = MAXIMIZE;
// Minimal required recall, (0.0, 1.0).
google.protobuf.DoubleValue recall = 1;
}
// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/RecallAtPrecision
message RecallAtPrecision {
option (metric_type) = MAXIMIZE;
// Minimal required precision, (0.0, 1.0).
google.protobuf.DoubleValue precision = 1;
}
message FalseNegativeRateAtThreshold {
option (metric_type) = MAXIMIZE;
// Threshold to apply to a prediction to determine positive vs negative.
// Note: if the model is calibrated, the threshold can be thought of as a
// probability so the threshold has a stable, intuitive semantic.
// However, not all solutions may be calibrated, and not all computations of
// the metric may operate on a calibrated score. In AutoTFX, the final model
// metrics are computed on a calibrated score, but the metrics computed within
// the model selection process are uncalibrated. Be aware of this possible
// skew in the metrics between model selection and final model evaluation.
google.protobuf.DoubleValue threshold = 1;
}
message FalsePositiveRateAtThreshold {
option (metric_type) = MAXIMIZE;
// Threshold to apply to a prediction to determine positive vs negative.
// Note: if the model is calibrated, the threshold can be thought of as a
// probability so the threshold has a stable, intuitive semantic.
// However, not all solutions may be calibrated, and not all computations of
// the metric may operate on a calibrated score. In AutoTFX, the final model
// metrics are computed on a calibrated score, but the metrics computed within
// the model selection process are uncalibrated. Be aware of this possible
// skew in the metrics between model selection and final model evaluation.
google.protobuf.DoubleValue threshold = 1;
}
message PrecisionAtK {
option (metric_type) = MAXIMIZE;
}
message MeanReciprocalRank {}
// https://www.tensorflow.org/responsible_ai/model_remediation/api_docs/python/model_remediation/min_diff/losses/MMDLoss
message MaximumMeanDiscrepancy {
option (metric_type) = MINIMIZE;
// Kernel to apply to the predictions. Currently supported values are
// 'gaussian' and 'laplace'. Defaults to 'gaussian'.
string kernel = 1;
}
// The mean of the prediction across the dataset.
message PredictionMean {}
// Area under ROC-curve calculated globally for MultiClassClassification (model
// predicts a single label) or MultiLabelClassification (model predicts class
// probabilities). The area is calculated by treating the entire set of data as
// an aggregate result, and computing a single metric rather than k metrics
// (one for each target label) that get averaged together. For example, the FPR
// and TPR at a given point on the AUC curve for k targer labels are:
// FPR = (FP1 + FP2 + ... + FPk) / ((FP1 + FP2 + ... + FPk) +
// (TN1 + TN2 + ... + TNk))
// TPR = (TP1 + TP2 + ... +TPk) / ((TP1 + TP2 + ... + TPk) +
// (FN1 + FN2 + ... + FNk))
message MicroAUC {
option (metric_type) = MAXIMIZE;
}
// Cross entropy for MultiLabelClassification where each target and
// prediction is the probabily of belonging to that class independent of other
// classes.
message MultilabelCrossEntropy {
option (metric_type) = MINIMIZE;
}
// DEPRECATED
message BlockUtility {
option (metric_type) = MAXIMIZE;
repeated double weight = 1;
}
// A custom metric.
// Prefer using or adding an explicit metric message
// and only use this generic message as a last resort.
// NEXT_TAG: 4
message CustomMetric {
option (metric_type) = CUSTOM;
// The display name of a metric computed by the model. The name should match
// ^[a-zA-Z0-9\s]{1,25}$ and must be unique across all performance metrics.
// Trailing and leading spaces will be truncated before matching.
string name = 1;
// True if the metric is maximized: false if it is minimized.
// Must be specified if the CustomMetric is used as an objective.
bool is_maximized = 2;
// RegistrySpec is a full specification of the custom metric and its
// construction based on the binary’s metric registry. New custom metrics must
// be linked to the binary and registered in its metric registry to be
// identifiable via this specification.
message RegistrySpec {
// Identifier of the metric class in the metric registry of the binary.
string key = 1;
// Generic proto describing the configuration for the metric to be computed.
// It's upto the implementer of the metric to parse this configuration.
google.protobuf.Any config = 2;
}
// Specification of the metric in the binary’s metric registry.
RegistrySpec registry_spec = 3;
}
// Performance metrics measure the quality of a model. They need not be
// differentiable.
message PerformanceMetric {
oneof performance_metric {
AUC auc = 1;
AUCPrecisionRecall auc_precision_recall = 26;
BinaryAccuracy binary_accuracy = 2;
BinaryCrossEntropy binary_cross_entropy = 3;
BlockUtility block_utility = 4 [deprecated = true];
CategoricalAccuracy categorical_accuracy = 5;
CategoricalCrossEntropy categorical_cross_entropy = 6;
Cosine cosine = 7 [deprecated = true];
Hinge hinge = 8 [deprecated = true];
KullbackLeiblerDivergence kullback_leibler_divergence = 9
[deprecated = true];
LogisticRegression logistic_regression = 10 [deprecated = true];
MeanAbsoluteError mean_absolute_error = 11;
MeanAbsolutePercentageError mean_absolute_percentage_error = 12;
MeanSquaredError squared_error = 13;
MeanSquaredLogarithmicError mean_squared_logarithmic_error = 14;
MeanReciprocalRank mean_reciprocal_rank = 15;
MicroAUC micro_auc = 27;
MultilabelCrossEntropy multi_label_cross_entropy = 28;
Poisson poisson = 16 [deprecated = true];
PrecisionAtK precision_at_k = 17;
SquaredHinge squared_hinge = 18 [deprecated = true];
SparseTopKCategoricalAccuracy sparse_top_k_categorical_accuracy = 19
[deprecated = true];
TopKCategoricalAccuracy top_k_categorical_accuracy = 20;
CustomMetric custom_metric = 21;
SensitivityAtSpecificity sensitivity_at_specificity = 22;
SpecificityAtSensitivity specificity_at_sensitivity = 23;
PrecisionAtRecall precision_at_recall = 24;
RecallAtPrecision recall_at_precision = 25;
}
// NEXT_TAG: 37;
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy