All Downloads are FREE. Search and download functionalities are using the official Maven repository.

weka.gui.explorer.ClassifierErrorsPlotInstances Maven / Gradle / Ivy

Go to download

The Waikato Environment for Knowledge Analysis (WEKA), a machine learning workbench. This is the stable version. Apart from bugfixes, this version does not receive any other updates.

There is a newer version: 3.8.6
Show newest version
/*
 *   This program is free software: you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation, either version 3 of the License, or
 *   (at your option) any later version.
 *
 *   This program is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with this program.  If not, see .
 */

/*
 * ClassifierErrorsPlotInstances.java
 * Copyright (C) 2009-2012 University of Waikato, Hamilton, New Zealand
 */

package weka.gui.explorer;

import java.util.ArrayList;

import weka.classifiers.Classifier;
import weka.classifiers.Evaluation;
import weka.classifiers.IntervalEstimator;
import weka.classifiers.evaluation.NumericPrediction;
import weka.classifiers.evaluation.Prediction;
import weka.core.Attribute;
import weka.core.DenseInstance;
import weka.core.Instance;
import weka.core.Instances;
import weka.core.Utils;
import weka.gui.visualize.Plot2D;
import weka.gui.visualize.PlotData2D;

/**
 * A class for generating plottable visualization errors.
 * 

 * Example usage:
 * 
 * 
 * Instances train = ... // from somewhere
 * Instances test = ... // from somewhere
 * Classifier cls = ... // from somewhere
 * // build classifier
 * cls.buildClassifier(train);
 * // evaluate classifier and generate plot instances
 * ClassifierPlotInstances plotInstances = new ClassifierPlotInstances();
 * plotInstances.setClassifier(cls);
 * plotInstances.setInstances(train);
 * plotInstances.setClassIndex(train.classIndex());
 * plotInstances.setUp();
 * Evaluation eval = new Evaluation(train);
 * for (int i = 0; i < test.numInstances(); i++)
 *   plotInstances.process(test.instance(i), cls, eval);
 * // generate visualization
 * VisualizePanel visPanel = new VisualizePanel();
 * visPanel.addPlot(plotInstances.getPlotData("plot name"));
 * visPanel.setColourIndex(plotInstances.getPlotInstances().classIndex()+1);
 * // clean up
 * plotInstances.cleanUp();
 * 

 * 
 * @author fracpete (fracpete at waikato dot ac dot nz)
 * @version $Revision: 10220 $
 */
public class ClassifierErrorsPlotInstances extends AbstractPlotInstances {

  /** for serialization. */
  private static final long serialVersionUID = -3941976365792013279L;

  /** the minimum plot size for numeric errors. */
  protected int m_MinimumPlotSizeNumeric;

  /** the maximum plot size for numeric errors. */
  protected int m_MaximumPlotSizeNumeric;

  /**
   * whether to save the instances for visualization or just evaluate the
   * instance.
   */
  protected boolean m_SaveForVisualization;

  protected boolean m_pointSizeProportionalToMargin;

  /** for storing the plot shapes. */
  protected ArrayList m_PlotShapes;

  /** for storing the plot sizes. */
  protected ArrayList m_PlotSizes;

  /** the classifier being used. */
  protected Classifier m_Classifier;

  /** the class index. */
  protected int m_ClassIndex;

  /** the Evaluation object to use. */
  protected Evaluation m_Evaluation;

  /**
   * Initializes the members.
   */
  @Override
  protected void initialize() {
    super.initialize();

    m_PlotShapes = new ArrayList();
    m_PlotSizes = new ArrayList();
    m_Classifier = null;
    m_ClassIndex = -1;
    m_Evaluation = null;
    m_SaveForVisualization = true;
    m_MinimumPlotSizeNumeric = ExplorerDefaults
      .getClassifierErrorsMinimumPlotSizeNumeric();
    m_MaximumPlotSizeNumeric = ExplorerDefaults
      .getClassifierErrorsMaximumPlotSizeNumeric();
  }

  /**
   * Get the vector of plot shapes (see weka.gui.visualize.Plot2D).
   * 
   * @return the vector of plot shapes.
   */
  public ArrayList getPlotShapes() {
    return m_PlotShapes;
  }

  /**
   * Get the vector of plot sizes (see weka.gui.visualize.Plot2D).
   * 
   * @return the vector of plot sizes.
   */
  public ArrayList getPlotSizes() {
    return m_PlotSizes;
  }

  /**
   * Set the vector of plot shapes to use;
   * 
   * @param plotShapes
   */
  public void setPlotShapes(ArrayList plotShapes) {
    m_PlotShapes = plotShapes;
  }

  /**
   * Set the vector of plot sizes to use
   * 
   * @param plotSizes the plot sizes to use
   */
  public void setPlotSizes(ArrayList plotSizes) {
    m_PlotSizes = plotSizes;
  }

  /**
   * Sets the classifier used for making the predictions.
   * 
   * @param value the classifier to use
   */
  public void setClassifier(Classifier value) {
    m_Classifier = value;
  }

  /**
   * Returns the currently set classifier.
   * 
   * @return the classifier in use
   */
  public Classifier getClassifier() {
    return m_Classifier;
  }

  /**
   * Sets the 0-based class index.
   * 
   * @param index the class index
   */
  public void setClassIndex(int index) {
    m_ClassIndex = index;
  }

  /**
   * Returns the 0-based class index.
   * 
   * @return the class index
   */
  public int getClassIndex() {
    return m_ClassIndex;
  }

  /**
   * Sets the Evaluation object to use.
   * 
   * @param value the evaluation to use
   */
  public void setEvaluation(Evaluation value) {
    m_Evaluation = value;
  }

  /**
   * Returns the Evaluation object in use.
   * 
   * @return the evaluation object
   */
  public Evaluation getEvaluation() {
    return m_Evaluation;
  }

  /**
   * Sets whether the instances are saved for visualization or only evaluation
   * of the prediction is to happen.
   * 
   * @param value if true then the instances will be saved
   */
  public void setSaveForVisualization(boolean value) {
    m_SaveForVisualization = value;
  }

  /**
   * Returns whether the instances are saved for visualization for only
   * evaluation of the prediction is to happen.
   * 
   * @return true if the instances are saved
   */
  public boolean getSaveForVisualization() {
    return m_SaveForVisualization;
  }

  /**
   * Set whether the point size should be proportional to the prediction margin
   * (classification only).
   * 
   * @param b true if the point size should be proportional to the margin
   */
  public void setPointSizeProportionalToMargin(boolean b) {
    m_pointSizeProportionalToMargin = b;
  }

  /**
   * Get whether the point size should be proportional to the prediction margin
   * (classification only).
   * 
   * @return true if the point size should be proportional to the margin
   */
  public boolean getPointSizeProportionalToMargin() {
    return m_pointSizeProportionalToMargin;
  }

  /**
   * Checks whether classifier, class index and evaluation are provided.
   */
  @Override
  protected void check() {
    super.check();

    if (m_Classifier == null) {
      throw new IllegalStateException("No classifier set!");
    }

    if (m_ClassIndex == -1) {
      throw new IllegalStateException("No class index set!");
    }

    if (m_Evaluation == null) {
      throw new IllegalStateException("No evaluation set");
    }
  }

  /**
   * Sets up the structure for the plot instances. Sets m_PlotInstances to null
   * if instances are not saved for visualization.
   * 
   * @see #getSaveForVisualization()
   */
  @Override
  protected void determineFormat() {
    ArrayList hv;
    Attribute predictedClass;
    Attribute classAt;
    Attribute margin = null;
    ArrayList attVals;
    int i;

    if (!m_SaveForVisualization) {
      m_PlotInstances = null;
      return;
    }

    hv = new ArrayList();

    classAt = m_Instances.attribute(m_ClassIndex);
    if (classAt.isNominal()) {
      attVals = new ArrayList();
      for (i = 0; i < classAt.numValues(); i++) {
        attVals.add(classAt.value(i));
      }
      predictedClass = new Attribute("predicted " + classAt.name(), attVals);
      margin = new Attribute("prediction margin");
    } else {
      predictedClass = new Attribute("predicted" + classAt.name());
    }

    for (i = 0; i < m_Instances.numAttributes(); i++) {
      if (i == m_Instances.classIndex()) {
        if (classAt.isNominal()) {
          hv.add(margin);
        }
        hv.add(predictedClass);
      }
      hv.add((Attribute) m_Instances.attribute(i).copy());
    }

    m_PlotInstances = new Instances(m_Instances.relationName() + "_predicted",
      hv, m_Instances.numInstances());
    if (classAt.isNominal()) {
      m_PlotInstances.setClassIndex(m_ClassIndex + 2);
    } else {
      m_PlotInstances.setClassIndex(m_ClassIndex + 1);
    }
  }

  public void process(Instances batch, double[][] predictions, Evaluation eval) {
    try {

      for (int j = 0; j < batch.numInstances(); j++) {
        Instance toPredict = batch.instance(j);
        double[] preds = predictions[j];
        double probActual = 0;
        double probNext = 0;

        double pred = 0;
        if (batch.classAttribute().isNominal()) {
          pred = (Utils.sum(preds) == 0) ? Utils.missingValue() : Utils
            .maxIndex(preds);

          probActual = (Utils.sum(preds) == 0) ? Utils.missingValue() : (!Utils
            .isMissingValue(toPredict.classIndex()) ? preds[(int) toPredict
            .classValue()] : preds[Utils.maxIndex(preds)]);

          for (int i = 0; i < toPredict.classAttribute().numValues(); i++) {
            if (i != (int) toPredict.classValue() && preds[i] > probNext) {
              probNext = preds[i];
            }
          }
        } else {
          pred = preds[0];
        }

        eval.evaluationForSingleInstance(preds, toPredict, true);

        if (!m_SaveForVisualization) {
          continue;
        }

        if (m_PlotInstances != null) {
          double[] values = new double[m_PlotInstances.numAttributes()];
          boolean isNominal = toPredict.classAttribute().isNominal();
          for (int i = 0; i < m_PlotInstances.numAttributes(); i++) {
            if (i < toPredict.classIndex()) {
              values[i] = toPredict.value(i);
            } else if (i == toPredict.classIndex()) {
              if (isNominal) {
                values[i] = probActual - probNext;
                values[i + 1] = pred;
                values[i + 2] = toPredict.value(i);
                i += 2;
              } else {
                values[i] = pred;
                values[i + 1] = toPredict.value(i);
                i++;
              }
            } else {
              if (isNominal) {
                values[i] = toPredict.value(i - 2);
              } else {
                values[i] = toPredict.value(i - 1);
              }
            }
          }

          m_PlotInstances.add(new DenseInstance(1.0, values));

          if (toPredict.classAttribute().isNominal()) {
            if (toPredict.isMissing(toPredict.classIndex())
              || Utils.isMissingValue(pred)) {
              m_PlotShapes.add(new Integer(Plot2D.MISSING_SHAPE));
            } else if (pred != toPredict.classValue()) {
              // set to default error point shape
              m_PlotShapes.add(new Integer(Plot2D.ERROR_SHAPE));
            } else {
              // otherwise set to constant (automatically assigned) point shape
              m_PlotShapes.add(new Integer(Plot2D.CONST_AUTOMATIC_SHAPE));
            }

            if (m_pointSizeProportionalToMargin) {
              // margin
              m_PlotSizes.add(new Double(probActual - probNext));
            } else {
              int sizeAdj = 0;
              if (pred != toPredict.classValue()) {
                sizeAdj = 1;
              }
              m_PlotSizes.add(new Integer(Plot2D.DEFAULT_SHAPE_SIZE + sizeAdj));
            }
          } else {
            // store the error (to be converted to a point size later)
            Double errd = null;
            if (!toPredict.isMissing(toPredict.classIndex())
              && !Utils.isMissingValue(pred)) {
              errd = new Double(pred - toPredict.classValue());
              m_PlotShapes.add(new Integer(Plot2D.CONST_AUTOMATIC_SHAPE));
            } else {
              // missing shape if actual class not present or prediction is
              // missing
              m_PlotShapes.add(new Integer(Plot2D.MISSING_SHAPE));
            }
            m_PlotSizes.add(errd);
          }
        }
      }
    } catch (Exception ex) {
      ex.printStackTrace();
    }
  }

  /**
   * Process a classifier's prediction for an instance and update a set of
   * plotting instances and additional plotting info. m_PlotShape for nominal
   * class datasets holds shape types (actual data points have automatic shape
   * type assignment; classifier error data points have box shape type). For
   * numeric class datasets, the actual data points are stored in
   * m_PlotInstances and m_PlotSize stores the error (which is later converted
   * to shape size values).
   * 
   * @param toPredict the actual data point
   * @param classifier the classifier
   * @param eval the evaluation object to use for evaluating the classifier on
   *          the instance to predict
   * @see #m_PlotShapes
   * @see #m_PlotSizes
   * @see #m_PlotInstances
   */
  public void process(Instance toPredict, Classifier classifier, Evaluation eval) {
    double pred;
    double[] values;
    int i;

    try {
      pred = 0;

      double[] preds = null;
      double probActual = 0;
      double probNext = 0;
      int mappedClass = -1;

      Instance classMissing = (Instance) toPredict.copy();
      classMissing.setDataset(toPredict.dataset());

      // Only need to do this if the class is nominal, since we call
      // evalForSingleInstance()
      // which only takes a prob array
      if (classifier instanceof weka.classifiers.misc.InputMappedClassifier
        && toPredict.classAttribute().isNominal()) {
        toPredict = (Instance) toPredict.copy();
        toPredict = ((weka.classifiers.misc.InputMappedClassifier) classifier)
          .constructMappedInstance(toPredict);
        mappedClass = ((weka.classifiers.misc.InputMappedClassifier) classifier)
          .getMappedClassIndex();
        classMissing.setMissing(mappedClass);
      } else {
        classMissing.setClassMissing();
      }

      if (toPredict.classAttribute().isNominal()) {
        preds = classifier.distributionForInstance(classMissing);

        pred = (Utils.sum(preds) == 0) ? Utils.missingValue() : Utils
          .maxIndex(preds);

        probActual = (Utils.sum(preds) == 0) ? Utils.missingValue() : (!Utils
          .isMissingValue(toPredict.classIndex()) ? preds[(int) toPredict
          .classValue()] : preds[Utils.maxIndex(preds)]);

        for (i = 0; i < toPredict.classAttribute().numValues(); i++) {
          if (i != (int) toPredict.classValue() && preds[i] > probNext) {
            probNext = preds[i];
          }
        }

        eval.evaluationForSingleInstance(preds, toPredict, true);
      } else {
        // Numeric class. evalModelOnceAndRecordPrediciton() does the
        // InputMappedClassifier
        // transformation for us.
        pred = eval.evaluateModelOnceAndRecordPrediction(classifier, toPredict);
      }

      //

      if (!m_SaveForVisualization) {
        return;
      }

      if (m_PlotInstances != null) {
        boolean isNominal = toPredict.classAttribute().isNominal();
        values = new double[m_PlotInstances.numAttributes()];
        for (i = 0; i < m_PlotInstances.numAttributes(); i++) {
          if (i < toPredict.classIndex()) {
            values[i] = toPredict.value(i);
          } else if (i == toPredict.classIndex()) {
            if (isNominal) {
              values[i] = probActual - probNext;
              values[i + 1] = pred;
              values[i + 2] = toPredict.value(i);
              i += 2;
            } else {
              values[i] = pred;
              values[i + 1] = toPredict.value(i);
              i++;
            }
          } else {
            if (isNominal) {
              values[i] = toPredict.value(i - 2);
            } else {
              values[i] = toPredict.value(i - 1);
            }
          }
        }

        m_PlotInstances.add(new DenseInstance(1.0, values));

        if (toPredict.classAttribute().isNominal()) {
          if (toPredict.isMissing(toPredict.classIndex())
            || Utils.isMissingValue(pred)) {
            m_PlotShapes.add(new Integer(Plot2D.MISSING_SHAPE));
          } else if (pred != toPredict.classValue()) {
            // set to default error point shape
            m_PlotShapes.add(new Integer(Plot2D.ERROR_SHAPE));
          } else {
            // otherwise set to constant (automatically assigned) point shape
            m_PlotShapes.add(new Integer(Plot2D.CONST_AUTOMATIC_SHAPE));
          }
          if (m_pointSizeProportionalToMargin) {
            // margin
            m_PlotSizes.add(new Double(probActual - probNext));
          } else {
            int sizeAdj = 0;
            if (pred != toPredict.classValue()) {
              sizeAdj = 1;
            }
            m_PlotSizes.add(new Integer(Plot2D.DEFAULT_SHAPE_SIZE + sizeAdj));
          }
        } else {
          // store the error (to be converted to a point size later)
          Double errd = null;
          if (!toPredict.isMissing(toPredict.classIndex())
            && !Utils.isMissingValue(pred)) {

            errd = new Double(pred - toPredict.classValue());
            m_PlotShapes.add(new Integer(Plot2D.CONST_AUTOMATIC_SHAPE));
          } else {
            // missing shape if actual class not present or prediction is
            // missing
            m_PlotShapes.add(new Integer(Plot2D.MISSING_SHAPE));
          }
          m_PlotSizes.add(errd);
        }
      }
    } catch (Exception ex) {
      ex.printStackTrace();
    }
  }

  /**
   * Scales numeric class predictions into shape sizes for plotting in the
   * visualize panel.
   */
  protected void scaleNumericPredictions() {
    double maxErr;
    double minErr;
    double err;
    int i;
    Double errd;
    double temp;

    maxErr = Double.NEGATIVE_INFINITY;
    minErr = Double.POSITIVE_INFINITY;

    if (m_Instances.classAttribute().isNominal()) {
      maxErr = 1;
      minErr = 0;
    } else {

      // find min/max errors
      for (i = 0; i < m_PlotSizes.size(); i++) {
        errd = (Double) m_PlotSizes.get(i);
        if (errd != null) {
          err = Math.abs(errd.doubleValue());
          if (err < minErr) {
            minErr = err;
          }
          if (err > maxErr) {
            maxErr = err;
          }
        }
      }
    }

    // scale errors
    for (i = 0; i < m_PlotSizes.size(); i++) {
      errd = (Double) m_PlotSizes.get(i);
      if (errd != null) {
        err = Math.abs(errd.doubleValue());
        if (maxErr - minErr > 0) {
          temp = (((err - minErr) / (maxErr - minErr)) * (m_MaximumPlotSizeNumeric
            - m_MinimumPlotSizeNumeric + 1));
          m_PlotSizes
            .set(i, new Integer((int) temp) + m_MinimumPlotSizeNumeric);
        } else {
          m_PlotSizes.set(i, new Integer(m_MinimumPlotSizeNumeric));
        }
      } else {
        m_PlotSizes.set(i, new Integer(m_MinimumPlotSizeNumeric));
      }
    }
  }

  /**
   * Adds the prediction intervals as additional attributes at the end. Since
   * classifiers can returns varying number of intervals per instance, the
   * dataset is filled with missing values for non-existing intervals.
   */
  protected void addPredictionIntervals() {
    int maxNum;
    int num;
    int i;
    int n;
    ArrayList preds;
    ArrayList atts;
    Instances data;
    Instance inst;
    Instance newInst;
    double[] values;
    double[][] predInt;

    // determine the maximum number of intervals
    maxNum = 0;
    preds = m_Evaluation.predictions();
    for (i = 0; i < preds.size(); i++) {
      num = ((NumericPrediction) preds.get(i)).predictionIntervals().length;
      if (num > maxNum) {
        maxNum = num;
      }
    }

    // create new header
    atts = new ArrayList();
    for (i = 0; i < m_PlotInstances.numAttributes(); i++) {
      atts.add(m_PlotInstances.attribute(i));
    }
    for (i = 0; i < maxNum; i++) {
      atts
        .add(new Attribute("predictionInterval_" + (i + 1) + "-lowerBoundary"));
      atts
        .add(new Attribute("predictionInterval_" + (i + 1) + "-upperBoundary"));
      atts.add(new Attribute("predictionInterval_" + (i + 1) + "-width"));
    }
    data = new Instances(m_PlotInstances.relationName(), atts,
      m_PlotInstances.numInstances());
    data.setClassIndex(m_PlotInstances.classIndex());

    // update data
    for (i = 0; i < m_PlotInstances.numInstances(); i++) {
      inst = m_PlotInstances.instance(i);
      // copy old values
      values = new double[data.numAttributes()];
      System
        .arraycopy(inst.toDoubleArray(), 0, values, 0, inst.numAttributes());
      // add interval data
      predInt = ((NumericPrediction) preds.get(i)).predictionIntervals();
      for (n = 0; n < maxNum; n++) {
        if (n < predInt.length) {
          values[m_PlotInstances.numAttributes() + n * 3 + 0] = predInt[n][0];
          values[m_PlotInstances.numAttributes() + n * 3 + 1] = predInt[n][1];
          values[m_PlotInstances.numAttributes() + n * 3 + 2] = predInt[n][1]
            - predInt[n][0];
        } else {
          values[m_PlotInstances.numAttributes() + n * 3 + 0] = Utils
            .missingValue();
          values[m_PlotInstances.numAttributes() + n * 3 + 1] = Utils
            .missingValue();
          values[m_PlotInstances.numAttributes() + n * 3 + 2] = Utils
            .missingValue();
        }
      }
      // create new Instance
      newInst = new DenseInstance(inst.weight(), values);
      data.add(newInst);
    }

    m_PlotInstances = data;
  }

  /**
   * Performs optional post-processing.
   * 
   * @see #scaleNumericPredictions()
   * @see #addPredictionIntervals()
   */
  @Override
  protected void finishUp() {
    super.finishUp();

    if (!m_SaveForVisualization) {
      return;
    }

    if (m_Instances.classAttribute().isNumeric()
      || m_pointSizeProportionalToMargin) {
      scaleNumericPredictions(); // now handles point sizes based on the margin
                                 // too
    }

    if (m_Instances.attribute(m_ClassIndex).isNumeric()) {
      if (m_Classifier instanceof IntervalEstimator) {
        addPredictionIntervals();
      }
    }
  }

  /**
   * Assembles and returns the plot. The relation name of the dataset gets added
   * automatically.
   * 
   * @param name the name of the plot
   * @return the plot or null if plot instances weren't saved for visualization
   * @throws Exception if plot generation fails
   */
  @Override
  protected PlotData2D createPlotData(String name) throws Exception {
    PlotData2D result;
    if (!m_SaveForVisualization) {
      return null;
    }

    result = new PlotData2D(m_PlotInstances);
    result.setShapeSize(m_PlotSizes);
    result.setShapeType(m_PlotShapes);
    result.setPlotName(name + " (" + m_Instances.relationName() + ")");
    // result.addInstanceNumberAttribute();

    return result;
  }

  /**
   * For freeing up memory. Plot data cannot be generated after this call!
   */
  @Override
  public void cleanUp() {
    super.cleanUp();

    m_Classifier = null;
    m_PlotShapes = null;
    m_PlotSizes = null;
    m_Evaluation = null;
  }
}
 















© 2015 - 2024 Weber Informatics LLC | Privacy Policy