boofcv.alg.tracker.tld.TldDetection Maven / Gradle / Ivy

Go to download
/*
 * Copyright (c) 2021, Peter Abeles. All Rights Reserved.
 *
 * This file is part of BoofCV (http://boofcv.org).
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package boofcv.alg.tracker.tld;

import boofcv.struct.ImageRectangle;
import boofcv.struct.image.ImageGray;
import org.ddogleg.sorting.QuickSelect;
import org.ddogleg.struct.DogArray;
import org.ddogleg.struct.DogArray_F64;
import org.ddogleg.struct.DogArray_I32;
import org.jetbrains.annotations.Nullable;

import java.util.ArrayList;
import java.util.List;

/**
 * Runs a detection cascade for each region. The cascade is composed of a variance test, Fern classifier, and finally
 * the template classifier. The next test in the cascade is only considered if the previous passes. Several changes
 * have been made in how the cascade operates compared to the original paper. See code for comments.
 *
 * @author Peter Abeles
 */
@SuppressWarnings({"NullAway.Init"})
public class TldDetection> {

	// Detects rectangles: Removes candidates don't match the fern descriptors
	private TldFernClassifier fern;
	// Detects rectangles: Removes candidates don't match NCC descriptors
	protected TldTemplateMatching template;
	// Detects rectangles: Removes candidates which lack texture
	private TldVarianceFilter variance;

	// Storage for results of the fern test on individual regions
	protected DogArray fernInfo = new DogArray<>(TldRegionFernInfo::new);

	protected ConfigTld config;

	// Storage for sorting of results
	private final DogArray_F64 storageMetric = new DogArray_F64();
	private final DogArray_I32 storageIndexes = new DogArray_I32();
	private final List storageRect = new ArrayList<>();

	// storage for regions which pass the fern test
	protected List fernRegions = new ArrayList<>();

	// list all regions which had the template test run on them
	protected DogArray candidateDetections = new DogArray<>(TldRegion::new);
	// results from non-maximum suppression
	private final DogArray localMaximums = new DogArray<>(TldRegion::new);

	// list of regions which have almost the same confidence as the maximum
	private final List ambiguousRegions = new ArrayList<>();

	private final TldHelperFunctions helper = new TldHelperFunctions();

	// the most likely region
	private @Nullable TldRegion best;
	// is it ambiguous which region is the best?
	protected boolean ambiguous;
	// was it successful at selecting a single region?
	private boolean success;

	// Removes all but the best rectangles.
	private TldNonMaximalSuppression nonmax;

	public TldDetection( TldFernClassifier fern, TldTemplateMatching template, TldVarianceFilter variance, ConfigTld config ) {
		this.fern = fern;
		this.template = template;
		this.variance = variance;
		this.config = config;

		nonmax = new TldNonMaximalSuppression(config.regionConnect);
	}

	protected TldDetection() {}

	/**
	 * Detects the object inside the image. Eliminates candidate regions using a cascade of tests
	 */
	protected void detectionCascade( DogArray cascadeRegions ) {

		// initialize data structures
		success = false;
		ambiguous = false;
		best = null;
		candidateDetections.reset();
		localMaximums.reset();
		ambiguousRegions.clear();

		storageMetric.reset();
		storageIndexes.reset();
		storageRect.clear();
		fernRegions.clear();

		fernInfo.reset();

		int totalP = 0;
		int totalN = 0;

		// Run through all candidate regions, ignore ones without enough variance, compute
		// the fern for each one
		TldRegionFernInfo info = fernInfo.grow();
		for (int i = 0; i < cascadeRegions.size; i++) {
			ImageRectangle region = cascadeRegions.get(i);

			if (!variance.checkVariance(region)) {
				continue;
			}

			info.r = region;

			if (fern.lookupFernPN(info)) {
				totalP += info.sumP;
				totalN += info.sumN;
				info = fernInfo.grow();
			}
		}
		fernInfo.removeTail();

		// avoid overflow errors in the future by re-normalizing the Fern detector
		if (totalP > 0x0fffffff)
			fern.renormalizeP();
		if (totalN > 0x0fffffff)
			fern.renormalizeN();

		// Select the ferns with the highest likelihood
		selectBestRegionsFern(totalP, totalN);

		// From the remaining regions, score using the template algorithm
		computeTemplateConfidence();

		if (candidateDetections.size == 0) {
			return;
		}

		// use non-maximum suppression to reduce the number of candidates
		nonmax.process(candidateDetections, localMaximums);

		best = selectBest();
		if (best != null) {
			ambiguous = checkAmbiguous(best);

			success = true;
		}
	}

	/**
	 * Computes the confidence for all the regions which pass the fern test
	 */
	protected void computeTemplateConfidence() {
		double max = 0;
		for (int i = 0; i < fernRegions.size(); i++) {
			ImageRectangle region = fernRegions.get(i);

			double confidence = template.computeConfidence(region);

			max = Math.max(max, confidence);

			if (confidence < config.confidenceThresholdUpper)
				continue;
			TldRegion r = candidateDetections.grow();
			r.connections = 0;
			r.rect.setTo(region);
			r.confidence = confidence;
		}
	}

	/**
	 * compute the probability that each region is the target conditional upon this image
	 * the sumP and sumN are needed for image conditional probability
	 *
	 * NOTE: This is a big change from the original paper
	 */
	protected void selectBestRegionsFern( double totalP, double totalN ) {

		for (int i = 0; i < fernInfo.size; i++) {
			TldRegionFernInfo info = fernInfo.get(i);

			double probP = info.sumP/totalP;
			double probN = info.sumN/totalN;

			// only consider regions with a higher P likelihood
			if (probP > probN) {
				// reward regions with a large difference between the P and N values
				storageMetric.add(-(probP - probN));
				storageRect.add(info.r);
			}
		}

		// Select the N regions with the highest fern probability
		if (config.maximumCascadeConsider < storageMetric.size) {
			int N = Math.min(config.maximumCascadeConsider, storageMetric.size);
			storageIndexes.resize(storageMetric.size);

			QuickSelect.selectIndex(storageMetric.data, N - 1, storageMetric.size, storageIndexes.data);
			for (int i = 0; i < N; i++) {
				fernRegions.add(storageRect.get(storageIndexes.get(i)));
			}
		} else {
			fernRegions.addAll(storageRect);
		}
	}

	public @Nullable TldRegion selectBest() {
		TldRegion best = null;
		double bestConfidence = 0;

		for (int i = 0; i < localMaximums.size; i++) {
			TldRegion r = localMaximums.get(i);

			if (r.confidence > bestConfidence) {
				bestConfidence = r.confidence;
				best = r;
			}
		}

		return best;
	}

	private boolean checkAmbiguous( TldRegion best ) {
		double thresh = best.confidence*0.9;

		for (int i = 0; i < localMaximums.size; i++) {
			TldRegion r = localMaximums.get(i);

			if (r.confidence >= thresh) {
				double overlap = helper.computeOverlap(r.rect, best.rect);

				if (overlap <= config.overlapLower) {
					ambiguousRegions.add(r.rect);
				}
			}
		}

		return !ambiguousRegions.isEmpty();
	}

	public @Nullable TldRegion getBest() {
		return best;
	}

	public boolean isAmbiguous() {
		return ambiguous;
	}

	public TldNonMaximalSuppression getNonmax() {
		return nonmax;
	}

	public DogArray_F64 getStorageMetric() {
		return storageMetric;
	}

	public List getStorageRect() {
		return storageRect;
	}

	public DogArray getCandidateDetections() {
		return candidateDetections;
	}

	public DogArray getLocalMaximums() {
		return localMaximums;
	}

	public List getAmbiguousRegions() {
		return ambiguousRegions;
	}

	public DogArray getFernInfo() {
		return fernInfo;
	}

	/**
	 * Rectangles selected by the fern classifier as candidates
	 *
	 * @return List of rectangles
	 */
	public List getSelectedFernRectangles() {
		return fernRegions;
	}

	public boolean isSuccess() {
		return success;
	}
}