• Home
• nz.ac.waikato.cms.weka
• weka-dev
• 3.9.3
• source code
• LinearRegression.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

weka.classifiers.functions.LinearRegression Maven / Gradle / Ivy

/*
 *   This program is free software: you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation, either version 3 of the License, or
 *   (at your option) any later version.
 *
 *   This program is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with this program.  If not, see .
 */

/*
 *    LinearRegression.java
 *    Copyright (C) 1999-2012 University of Waikato, Hamilton, New Zealand
 *
 */

package weka.classifiers.functions;

import java.util.Arrays;
import java.util.Collections;
import java.util.Enumeration;

import no.uib.cipr.matrix.*;
import no.uib.cipr.matrix.Matrix;

import weka.classifiers.AbstractClassifier;
import weka.classifiers.evaluation.RegressionAnalysis;
import weka.core.*;
import weka.core.Capabilities.Capability;
import weka.core.matrix.QRDecomposition;
import weka.filters.Filter;
import weka.filters.supervised.attribute.NominalToBinary;
import weka.filters.unsupervised.attribute.ReplaceMissingValues;

/**
 
 * Class for using linear regression for prediction.
 * Uses the Akaike criterion for model selection, and is able to deal with
 * weighted instances.
 * 

 
 * 
 
 * Valid options are:
 * 

 * 
 * 
 * -S <number of selection method>
 *  Set the attribute selection method to use. 1 = None, 2 = Greedy.
 *  (default 0 = M5' method)
 * 
 * 
 * 
 * -C
 *  Do not try to eliminate colinear attributes.
 * 
 * 
 * 
 * -R <double>
 *  Set ridge parameter (default 1.0e-8).
 * 
 * 
 * 
 * -minimal
 *  Conserve memory, don't keep dataset header and means/stdevs.
 *  Model cannot be printed out if this option is enabled. (default: keep data)
 * 
 * 
 * 
 * -additional-stats
 *  Output additional statistics.
 * 
 * 
 * 
 * -output-debug-info
 *  If set, classifier is run in debug mode and
 *  may output additional info to the console
 * 
 * 
 * 
 * -do-not-check-capabilities
 *  If set, classifier capabilities are not checked before classifier is built
 *  (use with caution).
 * 
 *
 * 
 * -do-not-check-capabilities
 *  If set, classifier capabilities are not checked before classifier is built
 *  (use with caution).
 * 
 
 * 
 * @author Eibe Frank ([email protected])
 * @author Len Trigg ([email protected])
 * @version $Revision: 14872 $
 */
public class LinearRegression extends AbstractClassifier implements
  OptionHandler, WeightedInstancesHandler {

  /** Attribute selection method: M5 method */
  public static final int SELECTION_M5 = 0;
  /** Attribute selection method: No attribute selection */
  public static final int SELECTION_NONE = 1;
  /** Attribute selection method: Greedy method */
  public static final int SELECTION_GREEDY = 2;
  /** Attribute selection methods */
  public static final Tag[] TAGS_SELECTION = {
    new Tag(SELECTION_NONE, "No attribute selection"),
    new Tag(SELECTION_M5, "M5 method"),
    new Tag(SELECTION_GREEDY, "Greedy method") };
  /** for serialization */
  static final long serialVersionUID = -3364580862046573747L;
  /** Array for storing coefficients of linear regression. */
  protected double[] m_Coefficients;
  /** Which attributes are relevant? */
  protected boolean[] m_SelectedAttributes;
  /** Variable for storing transformed training data. */
  protected Instances m_TransformedData;
  /** The filter for removing missing values. */
  protected ReplaceMissingValues m_MissingFilter;
  /**
   * The filter storing the transformation from nominal to binary attributes.
   */
  protected NominalToBinary m_TransformFilter;
  /** The standard deviations of the class attribute */
  protected double m_ClassStdDev;
  /** The mean of the class attribute */
  protected double m_ClassMean;
  /** The index of the class attribute */
  protected int m_ClassIndex;
  /** The attributes means */
  protected double[] m_Means;
  /** The attribute standard deviations */
  protected double[] m_StdDevs;
  /**
   * Whether to output additional statistics such as std. dev. of coefficients
   * and t-stats
   */
  protected boolean m_outputAdditionalStats;
  /** The current attribute selection method */
  protected int m_AttributeSelection;
  /** Try to eliminate correlated attributes? */
  protected boolean m_EliminateColinearAttributes = true;
  /** Turn off all checks and conversions? */
  protected boolean m_checksTurnedOff = false;
  /** Use QR decomposition */
  protected boolean m_useQRDecomposition = false;
  /** The ridge parameter */
  protected double m_Ridge = 1.0e-8;
  /** Conserve memory? */
  protected boolean m_Minimal = false;
  /** Model already built? */
  protected boolean m_ModelBuilt = false;
  /** True if the model is a zero R one */
  protected boolean m_isZeroR;
  /** The degrees of freedom of the regression model */
  private int m_df;
  /** The R-squared value of the regression model */
  private double m_RSquared;
  /** The adjusted R-squared value of the regression model */
  private double m_RSquaredAdj;
  /** The F-statistic of the regression model */
  private double m_FStat;
  /** Array for storing the standard error of each coefficient */
  private double[] m_StdErrorOfCoef;
  /** Array for storing the t-statistic of each coefficient */
  private double[] m_TStats;

  public LinearRegression() {
    m_numDecimalPlaces = 4;
  }

  /**
   * Generates a linear regression function predictor.
   *
   * @param argv the options
   */
  public static void main(String argv[]) {
    runClassifier(new LinearRegression(), argv);
  }

  /**
   * Returns a string describing this classifier
   *
   * @return a description of the classifier suitable for displaying in the
   *         explorer/experimenter gui
   */
  public String globalInfo() {
    return "Class for using linear regression for prediction. Uses the Akaike "
      + "criterion for model selection, and is able to deal with weighted "
      + "instances.";
  }

  /**
   * Returns default capabilities of the classifier.
   *
   * @return the capabilities of this classifier
   */
  @Override
  public Capabilities getCapabilities() {
    Capabilities result = super.getCapabilities();
    result.disableAll();

    // attributes
    result.enable(Capability.NOMINAL_ATTRIBUTES);
    result.enable(Capability.NUMERIC_ATTRIBUTES);
    result.enable(Capability.DATE_ATTRIBUTES);
    result.enable(Capability.MISSING_VALUES);

    // class
    result.enable(Capability.NUMERIC_CLASS);
    result.enable(Capability.DATE_CLASS);
    result.enable(Capability.MISSING_CLASS_VALUES);

    return result;
  }

  /**
   * Builds a regression model for the given data.
   *
   * @param data the training data to be used for generating the linear
   *          regression function
   * @throws Exception if the classifier could not be built successfully
   */
  @Override
  public void buildClassifier(Instances data) throws Exception {
    m_ModelBuilt = false;
    m_isZeroR = false;

    if (data.numInstances() == 1) {
      m_Coefficients = new double[1];
      m_Coefficients[0] = data.instance(0).classValue();
      m_SelectedAttributes = new boolean[data.numAttributes()];
      m_isZeroR = true;
      return;
    }

    if (!m_checksTurnedOff) {
      // can classifier handle the data?
      getCapabilities().testWithFail(data);

      if (m_outputAdditionalStats) {
        // check that the instances weights are all 1
        // because the RegressionAnalysis class does
        // not handle weights
        boolean ok = true;
        for (int i = 0; i < data.numInstances(); i++) {
          if (data.instance(i).weight() != 1) {
            ok = false;
            break;
          }
        }
        if (!ok) {
          throw new Exception(
            "Can only compute additional statistics on unweighted data");
        }
      }

      // remove instances with missing class
      data = new Instances(data);
      data.deleteWithMissingClass();

      m_TransformFilter = new NominalToBinary();
      m_TransformFilter.setInputFormat(data);
      data = Filter.useFilter(data, m_TransformFilter);
      m_MissingFilter = new ReplaceMissingValues();
      m_MissingFilter.setInputFormat(data);
      data = Filter.useFilter(data, m_MissingFilter);
      data.deleteWithMissingClass();
    } else {
      m_TransformFilter = null;
      m_MissingFilter = null;
    }

    m_ClassIndex = data.classIndex();
    m_TransformedData = data;

    // Turn all attributes on for a start
    m_Coefficients = null;

    // Compute means and standard deviations
    m_SelectedAttributes = new boolean[data.numAttributes()];
    m_Means = new double[data.numAttributes()];
    m_StdDevs = new double[data.numAttributes()];
    for (int j = 0; j < data.numAttributes(); j++) {
      if (j != m_ClassIndex) {
        m_SelectedAttributes[j] = true; // Turn attributes on for a start
        m_Means[j] = data.meanOrMode(j);
        m_StdDevs[j] = Math.sqrt(data.variance(j));
        if (m_StdDevs[j] == 0) {
          m_SelectedAttributes[j] = false;
        }
      }
    }

    m_ClassStdDev = Math.sqrt(data.variance(m_TransformedData.classIndex()));
    m_ClassMean = data.meanOrMode(m_TransformedData.classIndex());

    // Perform the regression
    findBestModel();

    if (m_outputAdditionalStats) {
      // find number of coefficients, degrees of freedom
      int k = 1;
      for (int i = 0; i < data.numAttributes(); i++) {
        if (i != data.classIndex()) {
          if (m_SelectedAttributes[i]) {
            k++;
          }
        }
      }
      m_df = m_TransformedData.numInstances() - k;

      // calculate R^2 and F-stat
      double se = calculateSE(m_SelectedAttributes, m_Coefficients);
      m_RSquared = RegressionAnalysis.calculateRSquared(m_TransformedData, se);
      m_RSquaredAdj =
        RegressionAnalysis.calculateAdjRSquared(m_RSquared,
          m_TransformedData.numInstances(), k);
      m_FStat =
        RegressionAnalysis.calculateFStat(m_RSquared,
          m_TransformedData.numInstances(), k);
      // calculate std error of coefficients and t-stats
      m_StdErrorOfCoef =
        RegressionAnalysis.calculateStdErrorOfCoef(m_TransformedData,
          m_SelectedAttributes, se, m_TransformedData.numInstances(), k);
      m_TStats =
        RegressionAnalysis.calculateTStats(m_Coefficients, m_StdErrorOfCoef, k);
    }

    // Save memory
    if (m_Minimal) {
      m_TransformedData = null;
      m_Means = null;
      m_StdDevs = null;
    } else {
      m_TransformedData = new Instances(data, 0);
    }

    m_ModelBuilt = true;
  }

  /**
   * Classifies the given instance using the linear regression function.
   *
   * @param instance the test instance
   * @return the classification
   * @throws Exception if classification can't be done successfully
   */
  @Override
  public double classifyInstance(Instance instance) throws Exception {

    // Transform the input instance
    Instance transformedInstance = instance;
    if (!m_checksTurnedOff && !m_isZeroR) {
      m_TransformFilter.input(transformedInstance);
      m_TransformFilter.batchFinished();
      transformedInstance = m_TransformFilter.output();
      m_MissingFilter.input(transformedInstance);
      m_MissingFilter.batchFinished();
      transformedInstance = m_MissingFilter.output();
    }

    // Calculate the dependent variable from the regression model
    return regressionPrediction(transformedInstance, m_SelectedAttributes,
      m_Coefficients);
  }

  /**
   * Outputs the linear regression model as a string.
   *
   * @return the model as string
   */
  @Override
  public String toString() {
    if (!m_ModelBuilt) {
      return "Linear Regression: No model built yet.";
    }

    if (m_Minimal) {
      return "Linear Regression: Model built.";
    }

    try {
      StringBuilder text = new StringBuilder();
      int column = 0;
      boolean first = true;

      text.append("\nLinear Regression Model\n\n");

      text.append(m_TransformedData.classAttribute().name() + " =\n\n");
      for (int i = 0; i < m_TransformedData.numAttributes(); i++) {
        if ((i != m_ClassIndex) && (m_SelectedAttributes[i])) {
          if (!first) {
            text.append(" +\n");
          } else {
            first = false;
          }
          text.append(Utils.doubleToString(m_Coefficients[column], 12,
            m_numDecimalPlaces) + " * ");
          text.append(m_TransformedData.attribute(i).name());
          column++;
        }
      }
      text.append(" +\n"
        + Utils.doubleToString(m_Coefficients[column], 12, m_numDecimalPlaces));

      if (m_outputAdditionalStats) {
        int maxAttLength = 0;
        for (int i = 0; i < m_TransformedData.numAttributes(); i++) {
          if ((i != m_ClassIndex) && (m_SelectedAttributes[i])) {
            if (m_TransformedData.attribute(i).name().length() > maxAttLength) {
              maxAttLength = m_TransformedData.attribute(i).name().length();
            }
          }
        }
        maxAttLength += 3;
        if (maxAttLength < "Variable".length() + 3) {
          maxAttLength = "Variable".length() + 3;
        }

        text.append("\n\nRegression Analysis:\n\n"
          + Utils.padRight("Variable", maxAttLength)
          + "  Coefficient     SE of Coef        t-Stat");
        column = 0;
        for (int i = 0; i < m_TransformedData.numAttributes(); i++) {
          if ((i != m_ClassIndex) && (m_SelectedAttributes[i])) {
            text.append("\n"
              + Utils.padRight(m_TransformedData.attribute(i).name(),
                maxAttLength));
            text.append(Utils.doubleToString(m_Coefficients[column], 12,
              m_numDecimalPlaces));
            text.append("   "
              + Utils.doubleToString(m_StdErrorOfCoef[column], 12,
                m_numDecimalPlaces));
            text.append("   "
              + Utils.doubleToString(m_TStats[column], 12, m_numDecimalPlaces));
            column++;
          }
        }
        text.append(Utils.padRight("\nconst", maxAttLength + 1)
          + Utils
            .doubleToString(m_Coefficients[column], 12, m_numDecimalPlaces));
        text.append("   "
          + Utils.doubleToString(m_StdErrorOfCoef[column], 12,
            m_numDecimalPlaces));
        text.append("   "
          + Utils.doubleToString(m_TStats[column], 12, m_numDecimalPlaces));

        text.append("\n\nDegrees of freedom = " + Integer.toString(m_df));
        text.append("\nR^2 value = "
          + Utils.doubleToString(m_RSquared, m_numDecimalPlaces));
        text.append("\nAdjusted R^2 = "
          + Utils.doubleToString(m_RSquaredAdj, 5));
        text.append("\nF-statistic = "
          + Utils.doubleToString(m_FStat, m_numDecimalPlaces));
      }

      return text.toString();
    } catch (Exception e) {
      return "Can't print Linear Regression!";
    }
  }

  /**
   * Returns an enumeration describing the available options.
   *
   * @return an enumeration of all the available options.
   */
  @Override
  public Enumeration listOptions() {
    java.util.Vector newVector = new java.util.Vector();

    newVector.addElement(new Option("\tSet the attribute selection method"
      + " to use. 1 = None, 2 = Greedy.\n" + "\t(default 0 = M5' method)", "S",
      1, "-S "));

    newVector.addElement(new Option("\tDo not try to eliminate colinear"
      + " attributes.\n", "C", 0, "-C"));

    newVector.addElement(new Option(
      "\tSet ridge parameter (default 1.0e-8).\n", "R", 1, "-R "));

    newVector.addElement(new Option(
      "\tConserve memory, don't keep dataset header and means/stdevs.\n"
        + "\tModel cannot be printed out if this option is enabled."
        + "\t(default: keep data)", "minimal", 0, "-minimal"));

    newVector.addElement(new Option("\tOutput additional statistics.",
      "additional-stats", 0, "-additional-stats"));

    newVector.addElement(new Option("\tUse QR decomposition to find coefficients",
            "use-qr", 0, "-use-qr"));

    newVector.addAll(Collections.list(super.listOptions()));

    return newVector.elements();
  }

  /**
   * Returns the coefficients for this linear model.
   * 
   * @return the coefficients for this linear model
   */
  public double[] coefficients() {

    double[] coefficients = new double[m_SelectedAttributes.length + 1];
    int counter = 0;
    for (int i = 0; i < m_SelectedAttributes.length; i++) {
      if ((m_SelectedAttributes[i]) && ((i != m_ClassIndex))) {
        coefficients[i] = m_Coefficients[counter++];
      }
    }
    coefficients[m_SelectedAttributes.length] = m_Coefficients[counter];
    return coefficients;
  }

  /**
   * Gets the current settings of the classifier.
   * 
   * @return an array of strings suitable for passing to setOptions
   */
  @Override
  public String[] getOptions() {
    java.util.Vector result = new java.util.Vector();

    result.add("-S");
    result.add("" + getAttributeSelectionMethod().getSelectedTag().getID());

    if (!getEliminateColinearAttributes()) {
      result.add("-C");
    }

    result.add("-R");
    result.add("" + getRidge());

    if (getMinimal()) {
      result.add("-minimal");
    }

    if (getOutputAdditionalStats()) {
      result.add("-additional-stats");
    }

    if (getUseQRDecomposition()) {
      result.add("-use-qr");
    }

    Collections.addAll(result, super.getOptions());

    return result.toArray(new String[result.size()]);
  }

  /**
   * Parses a given list of options.
   * 

   *
   
   * Valid options are:
   * 

   *
   * 
   * -S <number of selection method>
   *  Set the attribute selection method to use. 1 = None, 2 = Greedy.
   *  (default 0 = M5' method)
   * 
   *
   * 
   * -C
   *  Do not try to eliminate colinear attributes.
   * 
   *
   * 
   * -R <double>
   *  Set ridge parameter (default 1.0e-8).
   * 
   *
   * 
   * -minimal
   *  Conserve memory, don't keep dataset header and means/stdevs.
   *  Model cannot be printed out if this option is enabled. (default: keep data)
   * 
   *
   * 
   * -additional-stats
   *  Output additional statistics.
   * 
   *
   * 
   * -output-debug-info
   *  If set, classifier is run in debug mode and
   *  may output additional info to the console
   * 
   *
   * 
   * -do-not-check-capabilities
   *  If set, classifier capabilities are not checked before classifier is built
   *  (use with caution).
   * 
   *
   * 
   * -use-qr
   *  If set, QR decomposition will be used to find coefficients.
   * 
   
   *
   * @param options the list of options as an array of strings
   * @throws Exception if an option is not supported
   */
  @Override
  public void setOptions(String[] options) throws Exception {

    String selectionString = Utils.getOption('S', options);
    if (selectionString.length() != 0) {
      setAttributeSelectionMethod(new SelectedTag(
        Integer.parseInt(selectionString), TAGS_SELECTION));
    } else {
      setAttributeSelectionMethod(new SelectedTag(SELECTION_M5, TAGS_SELECTION));
    }
    String ridgeString = Utils.getOption('R', options);
    if (ridgeString.length() != 0) {
      setRidge(new Double(ridgeString).doubleValue());
    } else {
      setRidge(1.0e-8);
    }
    setEliminateColinearAttributes(!Utils.getFlag('C', options));
    setMinimal(Utils.getFlag("minimal", options));

    setOutputAdditionalStats(Utils.getFlag("additional-stats", options));

    setUseQRDecomposition(Utils.getFlag("use-qr", options));

    super.setOptions(options);
    Utils.checkForRemainingOptions(options);
  }

  /**
   * Returns the tip text for this property
   *
   * @return tip text for this property suitable for displaying in the
   *         explorer/experimenter gui
   */
  public String ridgeTipText() {
    return "The value of the Ridge parameter.";
  }

  /**
   * Get the value of Ridge.
   *
   * @return Value of Ridge.
   */
  public double getRidge() {

    return m_Ridge;
  }

  /**
   * Set the value of Ridge.
   *
   * @param newRidge Value to assign to Ridge.
   */
  public void setRidge(double newRidge) {

    m_Ridge = newRidge;
  }

  /**
   * Returns the tip text for this property
   *
   * @return tip text for this property suitable for displaying in the
   *         explorer/experimenter gui
   */
  public String eliminateColinearAttributesTipText() {
    return "Eliminate colinear attributes.";
  }

  /**
   * Get the value of EliminateColinearAttributes.
   *
   * @return Value of EliminateColinearAttributes.
   */
  public boolean getEliminateColinearAttributes() {

    return m_EliminateColinearAttributes;
  }

  /**
   * Set the value of EliminateColinearAttributes.
   *
   * @param newEliminateColinearAttributes Value to assign to
   *          EliminateColinearAttributes.
   */
  public void setEliminateColinearAttributes(
    boolean newEliminateColinearAttributes) {

    m_EliminateColinearAttributes = newEliminateColinearAttributes;
  }

  /**
   * Get the number of coefficients used in the model
   *
   * @return the number of coefficients
   */
  public int numParameters() {
    return m_Coefficients.length - 1;
  }

  /**
   * Returns the tip text for this property
   *
   * @return tip text for this property suitable for displaying in the
   *         explorer/experimenter gui
   */
  public String attributeSelectionMethodTipText() {
    return "Set the method used to select attributes for use in the linear "
      + "regression. Available methods are: no attribute selection, attribute "
      + "selection using M5's method (step through the attributes removing the one "
      + "with the smallest standardised coefficient until no improvement is observed "
      + "in the estimate of the error given by the Akaike "
      + "information criterion), and a greedy selection using the Akaike information "
      + "metric.";
  }

  /**
   * Gets the method used to select attributes for use in the linear regression.
   * 
   * @return the method to use.
   */
  public SelectedTag getAttributeSelectionMethod() {

    return new SelectedTag(m_AttributeSelection, TAGS_SELECTION);
  }

  /**
   * Sets the method used to select attributes for use in the linear regression.
   *
   * @param method the attribute selection method to use.
   */
  public void setAttributeSelectionMethod(SelectedTag method) {

    if (method.getTags() == TAGS_SELECTION) {
      m_AttributeSelection = method.getSelectedTag().getID();
    }
  }

  /**
   * Returns the tip text for this property.
   *
   * @return tip text for this property suitable for displaying in the
   *         explorer/experimenter gui
   */
  public String minimalTipText() {
    return "If enabled, dataset header, means and stdevs get discarded to conserve memory; also, the model cannot be printed out.";
  }

  /**
   * Returns whether to be more memory conservative or being able to output the
   * model as string.
   * 
   * @return true if memory conservation is preferred over outputting model
   *         description
   */
  public boolean getMinimal() {
    return m_Minimal;
  }

  /**
   * Sets whether to be more memory conservative or being able to output the
   * model as string.
   *
   * @param value if true memory will be conserved
   */
  public void setMinimal(boolean value) {
    m_Minimal = value;
  }

  /**
   * Returns the tip text for this property.
   *
   * @return tip text for this property suitable for displaying in the
   *         explorer/experimenter gui
   */
  public String outputAdditionalStatsTipText() {
    return "Output additional statistics (such as "
      + "std deviation of coefficients and t-statistics)";
  }

  /**
   * Get whether to output additional statistics (such as std. deviation of
   * coefficients and t-statistics
   * 
   * @return true if additional stats are to be output
   */
  public boolean getOutputAdditionalStats() {
    return m_outputAdditionalStats;
  }

  /**
   * Set whether to output additional statistics (such as std. deviation of
   * coefficients and t-statistics
   *
   * @param additional true if additional stats are to be output
   */
  public void setOutputAdditionalStats(boolean additional) {
    m_outputAdditionalStats = additional;
  }

  /**
   * Returns the tip text for this property.
   *
   * @return tip text for this property suitable for displaying in the
   *         explorer/experimenter gui
   */
  public String useQRDecompositionTipText() {
    return "Use QR decomposition to find the coefficients";
  }

  /**
   * Get whether to use QR decomposition.
   *
   * @return true if QR decomposition is to be used
   */
  public boolean getUseQRDecomposition() {
    return m_useQRDecomposition;
  }

  /**
   * Set whether to use QR decomposition.
   *
   * @param useQR true if QR decomposition is to be used
   */
  public void setUseQRDecomposition(boolean useQR) {
    m_useQRDecomposition = useQR;
  }

  /**
   * Turns off checks for missing values, etc. Use with caution. Also turns off
   * scaling.
   */
  public void turnChecksOff() {
    m_checksTurnedOff = true;
  }

  /**
   * Turns on checks for missing values, etc. Also turns on scaling.
   */
  public void turnChecksOn() {
    m_checksTurnedOff = false;
  }

  /**
   * Removes the attribute with the highest standardised coefficient greater
   * than 1.5 from the selected attributes.
   *
   * @param selectedAttributes an array of flags indicating which attributes are
   *          included in the regression model
   * @param coefficients an array of coefficients for the regression model
   * @return true if an attribute was removed
   */
  protected boolean deselectColinearAttributes(boolean[] selectedAttributes,
    double[] coefficients) {

    double maxSC = 1.5;
    int maxAttr = -1, coeff = 0;
    for (int i = 0; i < selectedAttributes.length; i++) {
      if (selectedAttributes[i]) {
        double SC =
          Math.abs(coefficients[coeff] * m_StdDevs[i] / m_ClassStdDev);
        if (SC > maxSC) {
          maxSC = SC;
          maxAttr = i;
        }
        coeff++;
      }
    }
    if (maxAttr >= 0) {
      selectedAttributes[maxAttr] = false;
      if (m_Debug) {
        System.out.println("Deselected colinear attribute:" + (maxAttr + 1)
          + " with standardised coefficient: " + maxSC);
      }
      return true;
    }
    return false;
  }

  /**
   * Performs a greedy search for the best regression model using Akaike's
   * criterion.
   *
   * @throws Exception if regression can't be done
   */
  protected void findBestModel() throws Exception {

    // For the weighted case we still use numInstances in
    // the calculation of the Akaike criterion.
    int numInstances = m_TransformedData.numInstances();

    if (m_Debug) {
      System.out.println((new Instances(m_TransformedData, 0)).toString());
    }

    // Perform a regression for the full model, and remove colinear attributes
    do {
      m_Coefficients = doRegression(m_SelectedAttributes);
    } while (m_EliminateColinearAttributes
      && deselectColinearAttributes(m_SelectedAttributes, m_Coefficients));

    // Figure out current number of attributes + 1. (We treat this model
    // as the full model for the Akaike-based methods.)
    int numAttributes = 1;
    for (boolean m_SelectedAttribute : m_SelectedAttributes) {
      if (m_SelectedAttribute) {
        numAttributes++;
      }
    }

    double fullMSE = calculateSE(m_SelectedAttributes, m_Coefficients);
    double akaike = (numInstances - numAttributes) + 2 * numAttributes;
    if (m_Debug) {
      System.out.println("Initial Akaike value: " + akaike);
    }

    boolean improved;
    int currentNumAttributes = numAttributes;
    switch (m_AttributeSelection) {

    case SELECTION_GREEDY:

      // Greedy attribute removal
      do {
        boolean[] currentSelected = m_SelectedAttributes.clone();
        improved = false;
        currentNumAttributes--;

        for (int i = 0; i < m_SelectedAttributes.length; i++) {
          if (currentSelected[i]) {

            // Calculate the akaike rating without this attribute
            currentSelected[i] = false;
            double[] currentCoeffs = doRegression(currentSelected);
            double currentMSE = calculateSE(currentSelected, currentCoeffs);
            double currentAkaike =
              currentMSE / fullMSE * (numInstances - numAttributes) + 2
                * currentNumAttributes;
            if (m_Debug) {
              System.out.println("(akaike: " + currentAkaike);
            }

            // If it is better than the current best
            if (currentAkaike < akaike) {
              if (m_Debug) {
                System.err.println("Removing attribute " + (i + 1)
                  + " improved Akaike: " + currentAkaike);
              }
              improved = true;
              akaike = currentAkaike;
              System.arraycopy(currentSelected, 0, m_SelectedAttributes, 0,
                m_SelectedAttributes.length);
              m_Coefficients = currentCoeffs;
            }
            currentSelected[i] = true;
          }
        }
      } while (improved);
      break;

    case SELECTION_M5:

      // Step through the attributes removing the one with the smallest
      // standardised coefficient until no improvement in Akaike
      do {
        improved = false;
        currentNumAttributes--;

        // Find attribute with smallest SC
        double minSC = 0;
        int minAttr = -1, coeff = 0;
        for (int i = 0; i < m_SelectedAttributes.length; i++) {
          if (m_SelectedAttributes[i]) {
            double SC =
              Math.abs(m_Coefficients[coeff] * m_StdDevs[i] / m_ClassStdDev);
            if ((coeff == 0) || (SC < minSC)) {
              minSC = SC;
              minAttr = i;
            }
            coeff++;
          }
        }

        // See whether removing it improves the Akaike score
        if (minAttr >= 0) {
          m_SelectedAttributes[minAttr] = false;
          double[] currentCoeffs = doRegression(m_SelectedAttributes);
          double currentMSE = calculateSE(m_SelectedAttributes, currentCoeffs);
          double currentAkaike =
            currentMSE / fullMSE * (numInstances - numAttributes) + 2
              * currentNumAttributes;
          if (m_Debug) {
            System.out.println("(akaike: " + currentAkaike);
          }

          // If it is better than the current best
          if (currentAkaike < akaike) {
            if (m_Debug) {
              System.err.println("Removing attribute " + (minAttr + 1)
                + " improved Akaike: " + currentAkaike);
            }
            improved = true;
            akaike = currentAkaike;
            m_Coefficients = currentCoeffs;
          } else {
            m_SelectedAttributes[minAttr] = true;
          }
        }
      } while (improved);
      break;

    case SELECTION_NONE:
      break;
    }
  }

  /**
   * Calculate the squared error of a regression model on the training data
   *
   * @param selectedAttributes an array of flags indicating which attributes are
   *          included in the regression model
   * @param coefficients an array of coefficients for the regression model
   * @return the mean squared error on the training data
   * @throws Exception if there is a missing class value in the training data
   */
  protected double calculateSE(boolean[] selectedAttributes,
    double[] coefficients) throws Exception {

    double mse = 0;
    for (int i = 0; i < m_TransformedData.numInstances(); i++) {
      double prediction =
        regressionPrediction(m_TransformedData.instance(i), selectedAttributes,
          coefficients);
      double error = prediction - m_TransformedData.instance(i).classValue();
      mse += error * error;
    }
    return mse;
  }

  /**
   * Calculate the dependent value for a given instance for a given regression
   * model.
   *
   * @param transformedInstance the input instance
   * @param selectedAttributes an array of flags indicating which attributes are
   *          included in the regression model
   * @param coefficients an array of coefficients for the regression model
   * @return the regression value for the instance.
   * @throws Exception if the class attribute of the input instance is not
   *           assigned
   */
  protected double regressionPrediction(Instance transformedInstance,
    boolean[] selectedAttributes, double[] coefficients) throws Exception {

    double result = 0;
    int column = 0;
    for (int j = 0; j < transformedInstance.numAttributes(); j++) {
      if ((m_ClassIndex != j) && (selectedAttributes[j])) {
        result += coefficients[column] * transformedInstance.value(j);
        column++;
      }
    }
    result += coefficients[column];

    return result;
  }

  /**
   * Calculate a linear regression using the selected attributes
   *
   * @param selectedAttributes an array of booleans where each element is true
   *          if the corresponding attribute should be included in the
   *          regression.
   * @return an array of coefficients for the linear regression model.
   * @throws Exception if an error occurred during the regression.
   */
  protected double[] doRegression(boolean[] selectedAttributes)
    throws Exception {

    if (m_Debug) {
      System.out.print("doRegression(");
      for (boolean selectedAttribute : selectedAttributes) {
        System.out.print(" " + selectedAttribute);
      }
      System.out.println(" )");
    }
    int numAttributes = 0;
    for (boolean selectedAttribute : selectedAttributes) {
      if (selectedAttribute) {
        numAttributes++;
      }
    }

    // Check whether there are still attributes left
    Matrix independentTransposed = null;
    Matrix independent = null;
    Vector dependent = null;
    if (numAttributes > 0) {
      if (!m_useQRDecomposition) {
        independentTransposed = new DenseMatrix(numAttributes, m_TransformedData.numInstances());
        dependent = new DenseVector(m_TransformedData.numInstances());
      } else {
        if (m_Ridge <= 0) {
          independent = new DenseMatrix(m_TransformedData.numInstances(), numAttributes);
          dependent = new DenseVector(m_TransformedData.numInstances());
        } else {
          independent = new DenseMatrix(m_TransformedData.numInstances() + numAttributes, numAttributes);
          dependent = new DenseVector(m_TransformedData.numInstances() + numAttributes);
        }
      }
      for (int i = 0; i < m_TransformedData.numInstances(); i++) {
        Instance inst = m_TransformedData.instance(i);
        double sqrt_weight = Math.sqrt(inst.weight());
        int index = 0;
        for (int j = 0; j < m_TransformedData.numAttributes(); j++) {
          if (j == m_ClassIndex) {
            dependent.set(i, inst.classValue() * sqrt_weight);
          } else {
            if (selectedAttributes[j]) {
              double value = inst.value(j) - m_Means[j];

              // We only need to do this if we want to
              // scale the input
              if (!m_checksTurnedOff) {
                value /= m_StdDevs[j];
              }
              if (!m_useQRDecomposition) {
                independentTransposed.set(index, i, value * sqrt_weight);
              } else {
                independent.set(i, index, value * sqrt_weight);
              }
              index++;
            }
          }
        }
      }
    }
    if (m_useQRDecomposition && m_Ridge > 0) {
      double sqrtRidge = Math.sqrt(m_Ridge);
      for (int i = 0; i < numAttributes; i++) {
        independent.set(m_TransformedData.numInstances() + i, i, sqrtRidge);
      }
    }

    // Compute coefficients (note that we have to treat the
    // intercept separately so that it doesn't get affected
    // by the ridge constant.)
    double[] coefficients = new double[numAttributes + 1];
    if (numAttributes > 0) {

      if (!m_useQRDecomposition) { // Use Cholesky based on covariance matrix
        Vector aTy = independentTransposed.mult(dependent, new DenseVector(numAttributes));
        Matrix aTa = new UpperSPDDenseMatrix(numAttributes).rank1(independentTransposed);
        independentTransposed = null;
        dependent = null;

        double ridge = getRidge();
        for (int i = 0; i < numAttributes; i++) {
          aTa.add(i, i, ridge);
        }
        Vector coeffsWithoutIntercept = aTa.solve(aTy, new DenseVector(numAttributes));
        System.arraycopy(((DenseVector) coeffsWithoutIntercept).getData(), 0, coefficients, 0, numAttributes);
      } else { // Use QR decomposition
        QRP qrp = QRP.factorize(independent);
        independent = null;
        Matrix Q = qrp.getQ();
        Matrix R = new UpperTriangDenseMatrix(qrp.getR(), false);
        Matrix P = qrp.getP();
        DenseVector cPlusd = (DenseVector)Q.transMult(dependent, new DenseVector(dependent.size()));
        dependent = null;
        Vector c = new DenseVector(Arrays.copyOf(cPlusd.getData(), numAttributes));
        Vector y = R.solve(c, new DenseVector(numAttributes));
        Vector coeffsWithoutIntercept = P.mult(y, new DenseVector(numAttributes));
        System.arraycopy(((DenseVector) coeffsWithoutIntercept).getData(), 0, coefficients, 0, numAttributes);
      }
  }
    coefficients[numAttributes] = m_ClassMean;

    // Convert coefficients into original scale
    int column = 0;
    for (int i = 0; i < m_TransformedData.numAttributes(); i++) {
      if ((i != m_TransformedData.classIndex()) && (selectedAttributes[i])) {

        // We only need to do this if we have scaled the
        // input.
        if (!m_checksTurnedOff) {
          coefficients[column] /= m_StdDevs[i];
        }

        // We have centred the input
        coefficients[coefficients.length - 1] -=
          coefficients[column] * m_Means[i];
        column++;
      }
    }

    return coefficients;
  }

  /**
   * Returns the revision string.
   *
   * @return the revision
   */
  @Override
  public String getRevision() {
    return RevisionUtils.extract("$Revision: 14872 $");
  }
}