• Home
• ai.h2o
• h2o-algos
• 3.6.0.4
• source code
• DeepLearningModel.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

hex.deeplearning.DeepLearningModel Maven / Gradle / Ivy

package hex.deeplearning;

import hex.*;
import hex.quantile.Quantile;
import hex.quantile.QuantileModel;
import hex.schemas.DeepLearningModelV3;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import water.*;
import water.api.ModelSchema;
import water.codegen.CodeGenerator;
import water.codegen.CodeGeneratorPipeline;
import water.exceptions.H2OIllegalArgumentException;
import water.exceptions.JCodeSB;
import water.fvec.Chunk;
import water.fvec.Frame;
import water.fvec.NewChunk;
import water.fvec.Vec;
import water.util.*;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import static hex.ModelMetrics.calcVarImp;
import static hex.deeplearning.DeepLearning.makeDataInfo;
import static water.H2O.technote;

/**
 * The Deep Learning model
 * It contains a DeepLearningModelInfo with the most up-to-date model,
 * a scoring history, as well as some helpers to indicate the progress
 */

public class DeepLearningModel extends Model implements Model.DeepFeatures {

  /**
   * The Deep Learning model output contains a few extra fields in addition to the metrics in Model.Output
   * 1) Scoring history (raw data)
   * 2) weights/biases (raw data)
   * 3) variable importances (TwoDimTable)
   */
  public static class DeepLearningModelOutput extends Model.Output {
    public DeepLearningModelOutput() { super(); autoencoder = false;}
    public DeepLearningModelOutput(DeepLearning b) {
      super(b);
      autoencoder = b._parms._autoencoder;
      assert b.isSupervised() == !autoencoder;
    }
    final boolean autoencoder;

    DeepLearningScoring errors;
    Key[] weights;
    Key[] biases;
    double[] normmul;
    double[] normsub;
    double[] normrespmul;
    double[] normrespsub;
    int[] catoffsets;
    public TwoDimTable _variable_importances;

    @Override public ModelCategory getModelCategory() {
      return autoencoder ? ModelCategory.AutoEncoder : super.getModelCategory();
    }

    @Override public boolean isSupervised() {
      return !autoencoder;
    }
  }

  /**
   * Deviance of given distribution function at predicted value f
   * @param w observation weight
   * @param y (actual) response
   * @param f (predicted) response in original response space
   * @return value of gradient
   */
  @Override
  public double deviance(double w, double y, double f) {
    // Note: Must use sanitized parameters via get_params() as this._params can still have defaults AUTO, etc.)
    assert(get_params()._distribution != Distribution.Family.AUTO);
    return new Distribution(get_params()._distribution, get_params()._tweedie_power).deviance(w,y,f);
  }

  // Default publicly visible Schema is V2
  public ModelSchema schema() { return new DeepLearningModelV3(); }

  void set_model_info(DeepLearningModelInfo mi) { assert(mi != null); model_info = mi; }
  final public DeepLearningModelInfo model_info() { return model_info; }
  final public VarImp varImp() { return _output.errors.variable_importances; }

  private volatile DeepLearningModelInfo model_info;

  public long total_run_time;
  public long total_scoring_time;
  private long time_of_start;

  public long actual_train_samples_per_iteration;
  public long tspiGuess;
  public double time_for_communication_us; //helper for auto-tuning: time in microseconds for collective bcast/reduce of the model

  public double epoch_counter;
  public boolean stopped_early;

  public long training_rows;

  public long validation_rows;

  private DeepLearningScoring[] errors;
  public DeepLearningScoring[] scoring_history() { return errors; }
  public ScoreKeeper[] scoreKeepers() {
    ScoreKeeper[] sk = new ScoreKeeper[scoring_history().length];
    for (int i=0;i o.loss() ? 1 : 0);
  }

  public static class DeepLearningScoring extends Iced {
    public double epoch_counter;
    public double training_samples;
    public long time_stamp;
    public long training_time_ms;
    boolean validation;
    public long score_training_samples;
    public long score_validation_samples;
    public boolean classification;
    VarImp variable_importances;
    ScoreKeeper scored_train = new ScoreKeeper();
    ScoreKeeper scored_valid = new ScoreKeeper();
    public AUC2 training_AUC;
    public AUC2 validation_AUC;
    public long scoring_time;

    DeepLearningScoring deep_clone() {
      AutoBuffer ab = new AutoBuffer();
      this.write(ab);
      ab.flipForReading();
      return (DeepLearningScoring) new DeepLearningScoring().read(ab);
    }
  }

  public double classification_error() {
    if (errors == null) return Double.NaN;
    return last_scored().validation ? last_scored().scored_valid._classError : last_scored().scored_train._classError;
  }

  public double mse() {
    if (errors == null) return Double.NaN;
    return last_scored().validation ? last_scored().scored_valid._mse : last_scored().scored_train._mse;
  }

  public double auc() {
    if (errors == null) return Double.NaN;
    return last_scored().validation ? last_scored().scored_valid._AUC : last_scored().scored_train._AUC;
  }

  public double deviance() {
    if (errors == null) return Double.NaN;
    return last_scored().validation ? last_scored().scored_valid._mean_residual_deviance : last_scored().scored_train._mean_residual_deviance;
  }

  public double logloss() {
    if (errors == null) return Double.NaN;
    return last_scored().validation ? last_scored().scored_valid._logloss : last_scored().scored_train._logloss;
  }

  private TwoDimTable createScoringHistoryTable(DeepLearningScoring[] errors) {
    List colHeaders = new ArrayList<>();
    List colTypes = new ArrayList<>();
    List colFormat = new ArrayList<>();
    colHeaders.add("Timestamp"); colTypes.add("string"); colFormat.add("%s");
    colHeaders.add("Duration"); colTypes.add("string"); colFormat.add("%s");
    colHeaders.add("Training Speed"); colTypes.add("string"); colFormat.add("%s");
    colHeaders.add("Epochs"); colTypes.add("double"); colFormat.add("%.5f");
    colHeaders.add("Samples"); colTypes.add("double"); colFormat.add("%f");
    colHeaders.add("Training MSE"); colTypes.add("double"); colFormat.add("%.5f");

    if (_output.getModelCategory() == ModelCategory.Regression) {
      colHeaders.add("Training Deviance"); colTypes.add("double"); colFormat.add("%.5f");
    }
    if (!_output.autoencoder) {
      colHeaders.add("Training R^2"); colTypes.add("double"); colFormat.add("%.5f");
    }
    if (_output.isClassifier()) {
      colHeaders.add("Training LogLoss"); colTypes.add("double"); colFormat.add("%.5f");
    }
    if (_output.getModelCategory() == ModelCategory.Binomial) {
      colHeaders.add("Training AUC"); colTypes.add("double"); colFormat.add("%.5f");
    }
    if (_output.getModelCategory() == ModelCategory.Binomial || _output.getModelCategory() == ModelCategory.Multinomial) {
      colHeaders.add("Training Classification Error"); colTypes.add("double"); colFormat.add("%.5f");
    }
    if (get_params()._valid != null) {
      colHeaders.add("Validation MSE"); colTypes.add("double"); colFormat.add("%.5f");
      if (_output.getModelCategory() == ModelCategory.Regression) {
        colHeaders.add("Validation Deviance"); colTypes.add("double"); colFormat.add("%.5f");
      }
      if (!_output.autoencoder) {
        colHeaders.add("Validation R^2"); colTypes.add("double"); colFormat.add("%.5f");
      }
      if (_output.isClassifier()) {
        colHeaders.add("Validation LogLoss"); colTypes.add("double"); colFormat.add("%.5f");
      }
      if (_output.getModelCategory() == ModelCategory.Binomial) {
        colHeaders.add("Validation AUC"); colTypes.add("double"); colFormat.add("%.5f");
      }
      if (_output.isClassifier()) {
        colHeaders.add("Validation Classification Error"); colTypes.add("double"); colFormat.add("%.5f");
      }
    }

    final int rows = errors.length;
    String[] s = new String[0];
    TwoDimTable table = new TwoDimTable(
            "Scoring History", null,
            new String[rows],
            colHeaders.toArray(s),
            colTypes.toArray(s),
            colFormat.toArray(s),
            "");
    int row = 0;
    long scoring_time = 0;
    for (final DeepLearningScoring e : errors) {
      scoring_time += e.scoring_time;
      int col = 0;
      assert (row < table.getRowDim());
      assert (col < table.getColDim());
      DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss");
      table.set(row, col++, fmt.print(e.time_stamp));
      table.set(row, col++, PrettyPrint.msecs(e.training_time_ms, true));
      int speed = (int)(e.training_samples / ((e.training_time_ms - scoring_time)/ 1e3));
//      assert(speed >= 0) : "Speed should not be negative! " + speed + " = (int)(" + e.training_samples + "/((" + e.training_time_ms + "-" + scoring_time + ")/1e3)";
      table.set(row, col++, e.training_time_ms == 0 ? null : (String.format("%d", speed) + " rows/sec"));
      table.set(row, col++, e.epoch_counter);
      table.set(row, col++, e.training_samples);
      table.set(row, col++, e.scored_train != null ? e.scored_train._mse : Double.NaN);
      if (_output.getModelCategory() == ModelCategory.Regression) {
        table.set(row, col++, e.scored_train != null ? e.scored_train._mean_residual_deviance : Double.NaN);
      }
      if (!_output.autoencoder) {
        table.set(row, col++, e.scored_train != null ? e.scored_train._r2 : Double.NaN);
      }
      if (_output.isClassifier()) {
        table.set(row, col++, e.scored_train != null ? e.scored_train._logloss : Double.NaN);
      }
      if (_output.getModelCategory() == ModelCategory.Binomial) {
        table.set(row, col++, e.training_AUC != null ? e.training_AUC._auc : Double.NaN);
      }
      if (_output.isClassifier()) {
        table.set(row, col++, e.scored_train != null ? e.scored_train._classError : Double.NaN);
      }
      if (get_params()._valid != null) {
        table.set(row, col++, e.scored_valid != null ? e.scored_valid._mse : Double.NaN);
        if (_output.getModelCategory() == ModelCategory.Regression) {
          table.set(row, col++, e.scored_valid != null ? e.scored_valid._mean_residual_deviance : Double.NaN);
        }
        if (!_output.autoencoder) {
          table.set(row, col++, e.scored_valid != null ? e.scored_valid._r2 : Double.NaN);
        }
        if (_output.isClassifier()) {
          table.set(row, col++, e.scored_valid != null ? e.scored_valid._logloss : Double.NaN);
        }
        if (_output.getModelCategory() == ModelCategory.Binomial) {
          table.set(row, col++, e.validation_AUC != null ? e.validation_AUC._auc : Double.NaN);
        }
        if (_output.isClassifier()) {
          table.set(row, col, e.scored_valid != null ? e.scored_valid._classError : Double.NaN);
        }
      }
      row++;
    }
    return table;
  }

  /**
   * Helper to allocate keys for output frames for weights and biases
   * @param destKey Base destination key for output frames
   */
  private void makeWeightsBiases(Key destKey) {
    if (!model_info.get_params()._export_weights_and_biases) {
      _output.weights = null;
      _output.biases = null;
      _output.normmul = null;
      _output.normsub = null;
      _output.normrespmul = null;
      _output.normrespsub = null;
      _output.catoffsets = null;
    } else {
      _output.weights = new Key[get_params()._hidden.length + 1];
      for (int i = 0; i < _output.weights.length; ++i) {
        _output.weights[i] = Key.makeUserHidden(Key.make(destKey + ".weights." + i));
      }
      _output.biases = new Key[get_params()._hidden.length + 1];
      for (int i = 0; i < _output.biases.length; ++i) {
        _output.biases[i] = Key.makeUserHidden(Key.make(destKey + ".biases." + i));
      }
      _output.normmul = model_info.data_info._normMul;
      _output.normsub = model_info.data_info._normSub;
      _output.normrespmul = model_info.data_info._normRespMul;
      _output.normrespsub = model_info.data_info._normRespSub;
      _output.catoffsets = model_info.data_info._catOffsets;
    }
  }

  /** Constructor to restart from a checkpointed model
   * @param destKey New destination key for the model
   *  @param parms User-given parameters for checkpoint restart
   *  @param cp Checkpoint to restart from
   * @param store_best_model Store only the best model instead of the latest one  */
  public DeepLearningModel(final Key destKey, final DeepLearningParameters parms, final DeepLearningModel cp, final boolean store_best_model, final DataInfo dataInfo) {
    super(destKey, parms == null ? (DeepLearningParameters)cp._parms.clone() : parms, (DeepLearningModelOutput)cp._output.clone());
    assert(_parms != cp._parms); //make sure we have a clone
    model_info = cp.model_info.deep_clone(); //don't want to interfere with model being built, just make a deep copy and store that
    if (store_best_model) {
      model_info.data_info = dataInfo.deep_clone(); //replace previous data_info with updated version that's passed in (contains enum for classification)
    } else {
      model_info.data_info = dataInfo; //shallow clone is ok
      if (parms != null) {
        assert (_parms == parms);
        assert (_parms._checkpoint == parms._checkpoint);
        assert (_parms._checkpoint == cp._key);
      }
//      _parms._checkpoint = cp._key; //it's only a "real" checkpoint if job != null, otherwise a best model copy
    }
    DKV.put(dataInfo);
    assert(get_params() != cp.model_info().get_params()); //make sure we have a clone
    actual_best_model_key = cp.actual_best_model_key;
    time_of_start = cp.time_of_start;
    total_run_time = cp.total_run_time;
    total_scoring_time = cp.total_scoring_time;
    training_rows = cp.training_rows; //copy the value to display the right number on the model page before training has started
    validation_rows = cp.validation_rows; //copy the value to display the right number on the model page before training has started
    _bestLoss = cp._bestLoss;
    epoch_counter = cp.epoch_counter;

    // deep clone scoring history
    errors = cp.errors.clone();
    for (int i=0; i Value.MAX || fail)
      throw new IllegalArgumentException(technote(5, "Model is too large"));
  }

  public long _timeLastIterationEnter;
  public long _timeLastScoreStart; //start actual scoring
  private long _timeLastScoreEnd;  //finished actual scoring
  private long _timeLastPrintStart;

  /**
   * Score this DeepLearning model
   * @param ftrain potentially downsampled training data for scoring
   * @param ftest  potentially downsampled validation data for scoring
   * @param job_key key of the owning job
   * @param progressKey key of the progress
   * @param iteration Map/Reduce iteration count
   * @return true if model building is ongoing
   */
  boolean doScoring(Frame ftrain, Frame ftest, Key job_key, Key progressKey, int iteration, boolean finalScoring) {
    final long now = System.currentTimeMillis();
    final double time_since_last_iter = now - _timeLastIterationEnter;
    total_run_time +=  time_since_last_iter;
    _timeLastIterationEnter = now;
    epoch_counter = (double)model_info().get_processed_total()/training_rows;


    boolean keep_running;
    // Auto-tuning
    // if multi-node and auto-tuning and at least 10 ms for communication (to avoid doing thins on multi-JVM on same node),
    // then adjust the auto-tuning parameter 'actual_train_samples_per_iteration' such that the targeted ratio of comm to comp is achieved
    // Note: actual communication time is estimated by the NetworkTest's collective test.
    if (H2O.CLOUD.size() > 1 && get_params()._train_samples_per_iteration == -2 && iteration > 1) {
      Log.info("Auto-tuning train_samples_per_iteration.");
      if (time_for_communication_us > 1e4) {
        Log.info("  Time taken for communication: " + PrettyPrint.usecs((long) time_for_communication_us));
        Log.info("  Time taken for Map/Reduce iteration: " + PrettyPrint.msecs((long) time_since_last_iter, true));
        final double comm_to_work_ratio = (time_for_communication_us * 1e-3) / time_since_last_iter;
        Log.info("  Ratio of network communication to computation: " + String.format("%.5f", comm_to_work_ratio));
        Log.info("  target_comm_to_work: " + get_params()._target_ratio_comm_to_comp);
        Log.info("Old value of train_samples_per_iteration: " + actual_train_samples_per_iteration);
        double correction = get_params()._target_ratio_comm_to_comp / comm_to_work_ratio;
        correction = Math.max(0.5,Math.min(2, correction)); //it's ok to train up to 2x more training rows per iteration, but not fewer than half.
        if (Math.abs(correction) < 0.8 || Math.abs(correction) > 1.2) { //don't correct unless it's significant (avoid slow drift)
          actual_train_samples_per_iteration /= correction;
          actual_train_samples_per_iteration = Math.max(1, actual_train_samples_per_iteration);
          Log.info("New value of train_samples_per_iteration: " + actual_train_samples_per_iteration);
        } else {
          Log.info("Keeping value of train_samples_per_iteration the same (would deviate too little from previous value): " + actual_train_samples_per_iteration);
        }
      } else {
        Log.info("Communication is faster than 10 ms. Not modifying train_samples_per_iteration: " + actual_train_samples_per_iteration);
      }
    }

    keep_running = (epoch_counter < get_params()._epochs) && !stopped_early;
    final long sinceLastScore = now -_timeLastScoreStart;

    // this is potentially slow - only do every so often
    if( !keep_running ||
        (sinceLastScore > get_params()._score_interval *1000 //don't score too often
            &&(double)(_timeLastScoreEnd-_timeLastScoreStart)/sinceLastScore < get_params()._score_duty_cycle) ) { //duty cycle
      if (progressKey != null) {
        new Job.ProgressUpdate("Scoring on " + ftrain.numRows() + " training samples" +
            (ftest != null ? (", " + ftest.numRows() + " validation samples") : "")
        ).fork(progressKey);
      }
      final boolean printme = !get_params()._quiet_mode;
      _timeLastScoreStart = System.currentTimeMillis();
      model_info().computeStats(); //might not be necessary, but is done to be certain that numbers are good
      DeepLearningScoring err = new DeepLearningScoring();
      err.time_stamp = _timeLastScoreStart;
      err.training_time_ms = total_run_time;
      err.epoch_counter = epoch_counter;
      err.training_samples = (double)model_info().get_processed_total();
      err.validation = ftest != null;
      err.score_training_samples = ftrain.numRows();
      err.classification = _output.isClassifier();

      if (get_params()._autoencoder) {
        if (printme) Log.info("Scoring the auto-encoder.");
        // training
        {
          final Frame mse_frame = scoreAutoEncoder(ftrain, Key.make(), false);
          mse_frame.delete();
          ModelMetrics mtrain = ModelMetrics.getFromDKV(this,ftrain); //updated by model.score
          _output._training_metrics = mtrain;
          err.scored_train = new ScoreKeeper(mtrain);
        }
        if (ftest != null) {
          final Frame mse_frame = scoreAutoEncoder(ftest, Key.make(), false);
          mse_frame.delete();
          ModelMetrics mtest = ModelMetrics.getFromDKV(this,ftest); //updated by model.score
          _output._validation_metrics = mtest;
          err.scored_valid = new ScoreKeeper(mtest);
        }
      } else {
        if (printme) Log.info("Scoring the model.");
        // compute errors
        final String m = model_info().toString();
        if (m.length() > 0) Log.info(m);
        final Frame trainPredict = score(ftrain);
        trainPredict.delete();

        hex.ModelMetrics mtrain = ModelMetrics.getFromDKV(this, ftrain);
        _output._training_metrics = mtrain;
        err.scored_train = new ScoreKeeper(mtrain);
        hex.ModelMetrics mtest;

        hex.ModelMetricsSupervised mm1 = (ModelMetricsSupervised)ModelMetrics.getFromDKV(this,ftrain);
        if (mm1 instanceof ModelMetricsBinomial) {
          ModelMetricsBinomial mm = (ModelMetricsBinomial)(mm1);
          err.training_AUC = mm._auc;
        }
        if (ftrain.numRows() != training_rows) {
          _output._training_metrics._description = "Metrics reported on temporary training frame with " + ftrain.numRows() + " samples";
        } else if (ftrain._key != null && ftrain._key.toString().contains("chunks")){
          _output._training_metrics._description = "Metrics reported on temporary (load-balanced) training frame";
        } else {
          _output._training_metrics._description = "Metrics reported on full training frame";
        }

        if (ftest != null) {
          Frame validPred = score(ftest);
          validPred.delete();
          mtest = ModelMetrics.getFromDKV(this, ftest);
          _output._validation_metrics = mtest;
          err.scored_valid = new ScoreKeeper(mtest);
          if (mtest != null) {
            if (mtest instanceof ModelMetricsBinomial) {
              ModelMetricsBinomial mm = (ModelMetricsBinomial)mtest;
              err.validation_AUC = mm._auc;
            }
            if (ftest.numRows() != validation_rows) {
              _output._validation_metrics._description = "Metrics reported on temporary validation frame with " + ftest.numRows() + " samples";
              if (get_params()._score_validation_sampling == DeepLearningParameters.ClassSamplingMethod.Stratified) {
                _output._validation_metrics._description += " (stratified sampling)";
              }
            } else if (ftest._key != null && ftest._key.toString().contains("chunks")){
              _output._validation_metrics._description = "Metrics reported on temporary (load-balanced) validation frame";
            } else {
              _output._validation_metrics._description = "Metrics reported on full validation frame";
            }
          }
        }
      }
      if (get_params()._variable_importances) {
        if (!get_params()._quiet_mode) Log.info("Computing variable importances.");
        final float[] vi = model_info().computeVariableImportances();
        err.variable_importances = new VarImp(vi, Arrays.copyOfRange(model_info().data_info().coefNames(), 0, vi.length));
      }

      _timeLastScoreEnd = System.currentTimeMillis();
      err.scoring_time = _timeLastScoreEnd - _timeLastScoreStart;
      err.training_time_ms += err.scoring_time; //training_time_was recorded above based on time of entry into this function, but we need to add the time for scoring to this to get the total time right
      total_scoring_time += err.scoring_time;
      // enlarge the error array by one, push latest score back
      if (errors == null) {
        errors = new DeepLearningScoring[]{err};
      } else {
        DeepLearningScoring[] err2 = new DeepLearningScoring[errors.length + 1];
        System.arraycopy(errors, 0, err2, 0, errors.length);
        err2[err2.length - 1] = err;
        errors = err2;
      }
      _output.errors = last_scored();
      makeWeightsBiases(_key);
      water.util.Timer t = new Timer();
      // store weights and matrices to Frames
      if (_output.weights != null && _output.biases != null) {
        for (int i = 0; i < _output.weights.length; ++i) {
          Frame f = model_info.get_weights(i).toFrame(_output.weights[i]);
          if (i==0) {
            f._names = model_info.data_info.coefNames();
            DKV.put(f);
          }
        }
        for (int i = 0; i < _output.biases.length; ++i) {
          model_info.get_biases(i).toFrame(_output.biases[i]);
        }
        if (!_parms._quiet_mode)
          Log.info("Writing weights and biases to Frames took " + t.time()/1000. + " seconds.");
      }
      _output._scoring_history = createScoringHistoryTable(errors);
      _output._variable_importances = calcVarImp(last_scored().variable_importances);
      _output._model_summary = model_info.createSummaryTable();

      // always keep a copy of the best model so far (based on the following criterion)
      if (!finalScoring) {
        if (actual_best_model_key != null && get_params()._overwrite_with_best_model && (
                // if we have a best_model in DKV, then compare against its error() (unless it's a different model as judged by the network size)
                (DKV.get(actual_best_model_key) != null && (loss() < DKV.get(actual_best_model_key).get().loss() || !Arrays.equals(model_info().units, DKV.get(actual_best_model_key).get().model_info().units)))
                        ||
                        // otherwise, compare against our own _bestError
                        (DKV.get(actual_best_model_key) == null && loss() < _bestLoss)
        ) ) {
          _bestLoss = loss();
          putMeAsBestModel(actual_best_model_key);
        }
        // print the freshly scored model to ASCII
        if (keep_running && printme)
          Log.info(toString());
        if ((_output.isClassifier() && last_scored().scored_train._classError <= get_params()._classification_stop)
                || (!_output.isClassifier() && last_scored().scored_train._mse <= get_params()._regression_stop)) {
          Log.info("Achieved requested predictive accuracy on the training data. Model building completed.");
          stopped_early = true;
        }
        if (ScoreKeeper.earlyStopping(scoreKeepers(),
                get_params()._stopping_rounds, _output.isClassifier(), get_params()._stopping_metric, get_params()._stopping_tolerance
        )) {
          Log.info("Convergence detected based on simple moving average of the loss function for the past " + get_params()._stopping_rounds + " scoring events. Model building completed.");
          stopped_early = true;
        }
        if (printme) Log.info("Time taken for scoring and diagnostics: " + PrettyPrint.msecs(err.scoring_time, true));
      }
    }
    if (stopped_early) {
      // pretend as if we finished all epochs to get the progress bar pretty (especially for N-fold and grid-search)
      ((Job) DKV.getGet(job_key)).update((long) (_parms._epochs * training_rows));
      update(job_key);
      return false;
    }
    progressUpdate(progressKey, job_key, iteration, keep_running);
    update(job_key);
    return keep_running;
  }

  private void progressUpdate(Key progressKey, Key job_key, int iteration, boolean keep_running) {
    long now = System.currentTimeMillis();
    long timeSinceEntering = now - _timeLastIterationEnter;
    Job.Progress prog = DKV.getGet(progressKey);
    double progress = prog == null ? 0 : prog.progress();
    int speed = (int)(model_info().get_processed_total() * 1000. / ((total_run_time + timeSinceEntering) - total_scoring_time));
//    assert(speed >= 0);
    String msg = "Map/Reduce Iterations: " + String.format("%,d", iteration) + ". Speed: " + String.format("%,d", speed) + " samples/sec."
            + (progress == 0 ? "" : " Estimated time left: " + PrettyPrint.msecs((long) (total_run_time * (1. - progress) / progress), true));
    ((Job) DKV.getGet(job_key)).update(actual_train_samples_per_iteration); //mark the amount of work done for the progress bar
    if (progressKey != null) new Job.ProgressUpdate(msg).fork(progressKey); //update the message for the progress bar
    long sinceLastPrint = now -_timeLastPrintStart;
    if (!keep_running || sinceLastPrint > get_params()._score_interval * 1000) { //print this after every score_interval, not considering duty cycle
      _timeLastPrintStart = now;
      if (!get_params()._quiet_mode) {
        Log.info(
                "Training time: " + PrettyPrint.msecs((total_run_time + timeSinceEntering), true) + " (scoring: " + PrettyPrint.msecs(total_scoring_time, true) + "). "
                + "Processed " + String.format("%,d", model_info().get_processed_total()) + " samples" + " (" + String.format("%.3f", epoch_counter) + " epochs).\n");
        Log.info(msg);
      }
    }
  }

  /** Make either a prediction or a reconstruction.
   * @param orig Test dataset
   * @param adaptedFr Test dataset, adapted to the model
   * @return A frame containing the prediction or reconstruction
   */
  @Override protected Frame predictScoreImpl(Frame orig, Frame adaptedFr, String destination_key) {
    if (!get_params()._autoencoder) {
      return super.predictScoreImpl(orig, adaptedFr, destination_key);
    } else {
      // Reconstruction
      final int len = model_info().data_info().fullN();
      assert(model_info().data_info()._responses == 0);
      String[] coefnames = model_info().data_info().coefNames();
      assert(len == coefnames.length);
      String[] names = new String[len];
      for(int i = 0; i < names.length; ++i) {
        names[i] = "reconstr_" + coefnames[i];
      }
      Frame f = new MRTask() {
        @Override public void map( Chunk chks[], NewChunk recon[] ) {
          double tmp [] = new double[_output._names.length];
          double preds[] = new double [len];
          final Neurons[] neurons = DeepLearningTask.makeNeuronsForTesting(model_info);
          for( int row=0; row