boofcv.alg.tracker.tld.TldDetection Maven / Gradle / Ivy

Go to download
/*
 * Copyright (c) 2011-2013, Peter Abeles. All Rights Reserved.
 *
 * This file is part of BoofCV (http://boofcv.org).
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package boofcv.alg.tracker.tld;

import boofcv.struct.ImageRectangle;
import boofcv.struct.image.ImageSingleBand;
import org.ddogleg.sorting.QuickSelectArray;
import org.ddogleg.struct.FastQueue;
import org.ddogleg.struct.GrowQueue_F64;
import org.ddogleg.struct.GrowQueue_I32;

import java.util.ArrayList;
import java.util.List;

/**
 * Runs a detection cascade for each region.  The cascade is composed of a variance test, Fern classifier, and finally
 * the template classifier.  The next test in the cascade is only considered if the previous passes.  Several changes
 * have been made in how the cascade operates compared to the original paper.  See code for comments.
 *
 * @author Peter Abeles
 */
public class TldDetection {

	// Detects rectangles: Removes candidates don't match the fern descriptors
	private TldFernClassifier fern;
	// Detects rectangles: Removes candidates don't match NCC descriptors
	protected TldTemplateMatching template;
	// Detects rectangles: Removes candidates which lack texture
	private TldVarianceFilter variance;

	// Storage for results of the fern test on individual regions
	protected FastQueue fernInfo = new FastQueue(TldRegionFernInfo.class,true);

	protected TldParameters config;

	// Storage for sorting of results
	private GrowQueue_F64 storageMetric = new GrowQueue_F64();
	private GrowQueue_I32 storageIndexes = new GrowQueue_I32();
	private List storageRect = new ArrayList();

	// storage for regions which pass the fern test
	protected List fernRegions = new ArrayList();

	// list all regions which had the template test run on them
	protected FastQueue candidateDetections = new FastQueue(TldRegion.class,true);
	// results from non-maximum suppression
	private FastQueue localMaximums = new FastQueue(TldRegion.class,true);

	// list of regions which have almost the same confidence as the maximum
	private List ambiguousRegions = new ArrayList();

	private TldHelperFunctions helper = new TldHelperFunctions();

	// the most likely region
	private TldRegion best;
	// is it ambiguous which region is the best?
	protected boolean ambiguous;
	// was it successful at selecting a single region?
	private boolean success;

	// Removes all but the best rectangles.
	private TldNonMaximalSuppression nonmax;

	public TldDetection(TldFernClassifier fern, TldTemplateMatching template, TldVarianceFilter variance, TldParameters config) {
		this.fern = fern;
		this.template = template;
		this.variance = variance;
		this.config = config;

		nonmax = new TldNonMaximalSuppression(config.regionConnect);
	}

	protected TldDetection() {
	}

	/**
	 * Detects the object inside the image.  Eliminates candidate regions using a cascade of tests
	 */
	protected void detectionCascade( FastQueue cascadeRegions ) {

		// initialize data structures
		success = false;
		ambiguous = false;
		best = null;
		candidateDetections.reset();
		localMaximums.reset();
		ambiguousRegions.clear();

		storageMetric.reset();
		storageIndexes.reset();
		storageRect.clear();
		fernRegions.clear();

		fernInfo.reset();

		int totalP = 0;
		int totalN = 0;

		// Run through all candidate regions, ignore ones without enough variance, compute
		// the fern for each one
		TldRegionFernInfo info = fernInfo.grow();
		for( int i = 0; i < cascadeRegions.size; i++ ) {
			ImageRectangle region = cascadeRegions.get(i);

			if( !variance.checkVariance(region)) {
				continue;
			}

			info.r = region;

			if( fern.lookupFernPN(info)) {
				totalP += info.sumP;
				totalN += info.sumN;
				info = fernInfo.grow();
			}
		}
		fernInfo.removeTail();

		// avoid overflow errors in the future by re-normalizing the Fern detector
		if( totalP > 0x0fffffff)
			fern.renormalizeP();
		if( totalN > 0x0fffffff)
			fern.renormalizeN();

		// Select the ferns with the highest likelihood
		selectBestRegionsFern(totalP, totalN);

		// From the remaining regions, score using the template algorithm
		computeTemplateConfidence();

		if( candidateDetections.size == 0 ) {
			return;
		}

		// use non-maximum suppression to reduce the number of candidates
		nonmax.process(candidateDetections, localMaximums);

		best = selectBest();
		if( best != null ) {
			ambiguous = checkAmbiguous(best);

			success = true;
		}
	}

	/**
	 * Computes the confidence for all the regions which pass the fern test
	 */
	protected void computeTemplateConfidence() {
		double max = 0;
		for( int i = 0; i < fernRegions.size(); i++ ) {
			ImageRectangle region = fernRegions.get(i);

			double confidence = template.computeConfidence(region);

			max = Math.max(max,confidence);

			if( confidence < config.confidenceThresholdUpper)
				continue;
			TldRegion r = candidateDetections.grow();
			r.connections = 0;
			r.rect.set(region);
			r.confidence = confidence;
		}
	}

	/**
	 * compute the probability that each region is the target conditional upon this image
	 * the sumP and sumN are needed for image conditional probability
	 *
	 * NOTE: This is a big change from the original paper
	 */
	protected void selectBestRegionsFern(double totalP, double totalN) {

		for( int i = 0; i < fernInfo.size; i++ ) {
			TldRegionFernInfo info = fernInfo.get(i);

			double probP = info.sumP/totalP;
			double probN = info.sumN/totalN;

			// only consider regions with a higher P likelihood
			if( probP > probN ) {
				// reward regions with a large difference between the P and N values
				storageMetric.add(-(probP-probN));
				storageRect.add( info.r );
			}
		}

		// Select the N regions with the highest fern probability
		if( config.maximumCascadeConsider < storageMetric.size ) {
			int N = Math.min(config.maximumCascadeConsider, storageMetric.size);
			storageIndexes.resize(storageMetric.size);

			QuickSelectArray.selectIndex(storageMetric.data, N - 1, storageMetric.size, storageIndexes.data);
			for( int i = 0; i < N; i++ ) {
				fernRegions.add(storageRect.get(storageIndexes.get(i)));
			}
		} else {
			fernRegions.addAll(storageRect);
		}
	}

	public TldRegion selectBest() {
		TldRegion best = null;
		double bestConfidence = 0;

		for( int i = 0; i < localMaximums.size; i++ ) {
			TldRegion r = localMaximums.get(i);

			if( r.confidence > bestConfidence ) {
				bestConfidence = r.confidence;
				best = r;
			}
		}

		return best;
	}

	private boolean checkAmbiguous( TldRegion best ) {
		double thresh = best.confidence*0.9;

		for( int i = 0; i < localMaximums.size; i++ ) {
			TldRegion r = localMaximums.get(i);

			if( r.confidence >= thresh ) {
				double overlap = helper.computeOverlap(r.rect,best.rect);

				if( overlap <= config.overlapLower )  {
					ambiguousRegions.add(r.rect);
				}
			}
		}

		return !ambiguousRegions.isEmpty();
	}

	public TldRegion getBest() {
		return best;
	}

	public boolean isAmbiguous() {
		return ambiguous;
	}

	public TldNonMaximalSuppression getNonmax() {
		return nonmax;
	}

	public GrowQueue_F64 getStorageMetric() {
		return storageMetric;
	}

	public List getStorageRect() {
		return storageRect;
	}

	public FastQueue getCandidateDetections() {
		return candidateDetections;
	}

	public FastQueue getLocalMaximums() {
		return localMaximums;
	}

	public List getAmbiguousRegions() {
		return ambiguousRegions;
	}

	public FastQueue getFernInfo() {
		return fernInfo;
	}

	/**
	 * Rectangles selected by the fern classifier as candidates
	 * @return
	 */
	public List getSelectedFernRectangles() {
		return fernRegions;
	}

	public boolean isSuccess() {
		return success;
	}
}