hex.tree.ScoreBuildHistogram Maven / Gradle / Ivy
package hex.tree;
import hex.genmodel.utils.DistributionFamily;
import water.H2O.H2OCountedCompleter;
import water.MRTask;
import water.fvec.C0DChunk;
import water.fvec.Chunk;
import water.fvec.Frame;
import water.util.ArrayUtils;
/** Score and Build Histogram
*
* Fuse 2 conceptual passes into one:
*
*
*
* - Pass 1:
- Score a prior partially-built tree model, and make new Node assignments to
* every row. This involves pulling out the current assigned DecidedNode,
* "scoring" the row against that Node's decision criteria, and assigning the
* row to a new child UndecidedNode (and giving it an improved prediction).
*
* - Pass 2:
- Build new summary DHistograms on the new child UndecidedNodes
* every row got assigned into. Collect counts, mean, variance, min,
* max per bin, per column.
*
*
* The result is a set of DHistogram arrays; one DHistogram array for each
* unique 'leaf' in the tree being histogramed in parallel. These have node
* ID's (nids) from 'leaf' to 'tree._len'. Each DHistogram array is for all
* the columns in that 'leaf'.
*
*
The other result is a prediction "score" for the whole dataset, based on
* the previous passes' DHistograms.
*/
public class ScoreBuildHistogram extends MRTask {
final int _k; // Which tree
final int _ncols;// Active feature columns
final int _nbins;// Numerical columns: Number of bins in each histogram
final int _nbins_cats;// Categorical columns: Number of bins in each histogram
final DTree _tree; // Read-only, shared (except at the histograms in the Nodes)
final int _leaf; // Number of active leaves (per tree)
// Histograms for every tree, split & active column
DHistogram _hcs[/*tree-relative node-id*/][/*column*/];
final DistributionFamily _family;
final int _weightIdx;
final int _workIdx;
final int _nidIdx;
public ScoreBuildHistogram(H2OCountedCompleter cc, int k, int ncols, int nbins, int nbins_cats, DTree tree, int leaf, DHistogram hcs[][], DistributionFamily family, int weightIdx, int workIdx, int nidIdx) {
super(cc);
_k = k;
_ncols= ncols;
_nbins= nbins;
_nbins_cats= nbins_cats;
_tree = tree;
_leaf = leaf;
_hcs = hcs;
_family = family;
_weightIdx = weightIdx;
_workIdx = workIdx;
_nidIdx = nidIdx;
}
public ScoreBuildHistogram dfork2(byte[] types, Frame fr, boolean run_local) {
return dfork(types,fr,run_local);
}
/** Marker for already decided row. */
static public final int DECIDED_ROW = -1;
/** Marker for sampled out rows */
static public final int OUT_OF_BAG = -2;
/** Marker for rows without a response */
static public final int MISSING_RESPONSE = -1;
/** Marker for a fresh tree */
static public final int UNDECIDED_CHILD_NODE_ID = -1; //Integer.MIN_VALUE;
static public final int FRESH = 0;
static public boolean isOOBRow(int nid) { return nid <= OUT_OF_BAG; }
static public boolean isDecidedRow(int nid) { return nid == DECIDED_ROW; }
static public int oob2Nid(int oobNid) { return -oobNid + OUT_OF_BAG; }
static public int nid2Oob(int nid) { return -nid + OUT_OF_BAG; }
// Once-per-node shared init
@Override public void setupLocal( ) {
// Init all the internal tree fields after shipping over the wire
_tree.init_tree();
// Allocate local shared memory histograms
for( int l=_leaf; l<_tree._len; l++ ) {
DTree.UndecidedNode udn = _tree.undecided(l);
DHistogram hs[] = _hcs[l-_leaf];
int sCols[] = udn._scoreCols;
if( sCols != null ) { // Sub-selecting just some columns?
for( int col : sCols ) // For tracked cols
hs[col].init();
} else { // Else all columns
for( int j=0; j<_ncols; j++) // For all columns
if( hs[j] != null ) // Tracking this column?
hs[j].init();
}
}
}
@Override
public void map(Chunk[] chks) {
final Chunk wrks = chks[_workIdx];
final Chunk nids = chks[_nidIdx];
final Chunk weight = _weightIdx>=0 ? chks[_weightIdx] : new C0DChunk(1, chks[0].len());
// Pass 1: Score a prior partially-built tree model, and make new Node
// assignments to every row. This involves pulling out the current
// assigned DecidedNode, "scoring" the row against that Node's decision
// criteria, and assigning the row to a new child UndecidedNode (and
// giving it an improved prediction).
int nnids[] = new int[nids._len];
if( _leaf > 0) // Prior pass exists?
score_decide(chks,nids,nnids);
else // Just flag all the NA rows
for( int row=0; row= 0 ) { // row already predicts perfectly or OOB
double w = weight.atd(row);
if (w == 0) continue;
double resp = wrks.atd(row);
assert !Double.isNaN(wrks.atd(row)); // Already marked as sampled-away
DHistogram nhs[] = _hcs[nid];
int sCols[] = _tree.undecided(nid+_leaf)._scoreCols; // Columns to score (null, or a list of selected cols)
if (sCols == null) {
for(int col=0; col= 0 )
nh[i+1]++;
// Rollup the histogram of rows-per-NID in this chunk
for( int i=0; i<_hcs.length; i++ ) nh[i+1] += nh[i];
// Splat the rows into NID-groups
int rows[] = new int[nnids.length];
for( int row=0; row= 0 )
rows[nh[nnids[row]]++] = row;
// rows[] has Chunk-local ROW-numbers now, in-order, grouped by NID.
// nh[] lists the start of each new NID, and is indexed by NID+1.
final DHistogram hcs[][] = _hcs;
if( hcs.length==0 ) return; // Unlikely fast cutout
// Local temp arrays, no atomic updates.
LocalHisto lh = new LocalHisto(Math.max(_nbins,_nbins_cats));
final int cols = _ncols;
final int hcslen = hcs.length;
// these arrays will be re-used for all cols and nodes
double[] ws = new double[chks[0]._len];
double[] cs = new double[chks[0]._len];
double[] ys = new double[chks[0]._len];
weight.getDoubles(ws,0,ws.length);
wrks.getDoubles(ys,0,ys.length);
for (int c = 0; c < cols; c++) {
boolean extracted = false;
for (int n = 0; n < hcslen; n++) {
int sCols[] = _tree.undecided(n + _leaf)._scoreCols; // Columns to score (null, or a list of selected cols)
if (sCols == null || ArrayUtils.find(sCols,c) >= 0) {
if (!extracted) {
chks[c].getDoubles(cs, 0, cs.length);
extracted = true;
}
DHistogram h = hcs[n][c];
if( h==null ) continue; // Ignore untracked columns in this split
lh.resizeIfNeeded(h._nbin);
h.updateSharedHistosAndReset(lh, ws, cs, ys, rows, nh[n], n == 0 ? 0 : nh[n - 1]);
}
}
}
}
/**
* Helper class to store the thread-local histograms
* Can now change the internal memory layout without affecting the calling code
*/
static class LocalHisto {
public void wAdd(int b, double val) { bins[b]+=val; }
public void wYAdd(int b, double val) { sums[b]+=val; }
public void wYYAdd(int b, double val) { ssqs[b]+=val; }
public void wClear(int b) { bins[b]=0; }
public void wYClear(int b) { sums[b]=0; }
public void wYYClear(int b) { ssqs[b]=0; }
public double w(int b) { return bins[b]; }
public double wY(int b) { return sums[b]; }
public double wYY(int b) { return ssqs[b]; }
private double bins[];
private double sums[];
private double ssqs[];
LocalHisto(int len) {
bins = new double[len];
sums = new double[len];
ssqs = new double[len];
}
void resizeIfNeeded(int len) {
if( len > bins.length) {
bins = new double[len];
sums = new double[len];
ssqs = new double[len];
}
}
}
}