boofcv.alg.tracker.tld.TldDetection Maven / Gradle / Ivy
/*
* Copyright (c) 2021, Peter Abeles. All Rights Reserved.
*
* This file is part of BoofCV (http://boofcv.org).
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package boofcv.alg.tracker.tld;
import boofcv.struct.ImageRectangle;
import boofcv.struct.image.ImageGray;
import org.ddogleg.sorting.QuickSelect;
import org.ddogleg.struct.DogArray;
import org.ddogleg.struct.DogArray_F64;
import org.ddogleg.struct.DogArray_I32;
import org.jetbrains.annotations.Nullable;
import java.util.ArrayList;
import java.util.List;
/**
* Runs a detection cascade for each region. The cascade is composed of a variance test, Fern classifier, and finally
* the template classifier. The next test in the cascade is only considered if the previous passes. Several changes
* have been made in how the cascade operates compared to the original paper. See code for comments.
*
* @author Peter Abeles
*/
@SuppressWarnings({"NullAway.Init"})
public class TldDetection> {
// Detects rectangles: Removes candidates don't match the fern descriptors
private TldFernClassifier fern;
// Detects rectangles: Removes candidates don't match NCC descriptors
protected TldTemplateMatching template;
// Detects rectangles: Removes candidates which lack texture
private TldVarianceFilter variance;
// Storage for results of the fern test on individual regions
protected DogArray fernInfo = new DogArray<>(TldRegionFernInfo::new);
protected ConfigTld config;
// Storage for sorting of results
private final DogArray_F64 storageMetric = new DogArray_F64();
private final DogArray_I32 storageIndexes = new DogArray_I32();
private final List storageRect = new ArrayList<>();
// storage for regions which pass the fern test
protected List fernRegions = new ArrayList<>();
// list all regions which had the template test run on them
protected DogArray candidateDetections = new DogArray<>(TldRegion::new);
// results from non-maximum suppression
private final DogArray localMaximums = new DogArray<>(TldRegion::new);
// list of regions which have almost the same confidence as the maximum
private final List ambiguousRegions = new ArrayList<>();
private final TldHelperFunctions helper = new TldHelperFunctions();
// the most likely region
private @Nullable TldRegion best;
// is it ambiguous which region is the best?
protected boolean ambiguous;
// was it successful at selecting a single region?
private boolean success;
// Removes all but the best rectangles.
private TldNonMaximalSuppression nonmax;
public TldDetection( TldFernClassifier fern, TldTemplateMatching template, TldVarianceFilter variance, ConfigTld config ) {
this.fern = fern;
this.template = template;
this.variance = variance;
this.config = config;
nonmax = new TldNonMaximalSuppression(config.regionConnect);
}
protected TldDetection() {}
/**
* Detects the object inside the image. Eliminates candidate regions using a cascade of tests
*/
protected void detectionCascade( DogArray cascadeRegions ) {
// initialize data structures
success = false;
ambiguous = false;
best = null;
candidateDetections.reset();
localMaximums.reset();
ambiguousRegions.clear();
storageMetric.reset();
storageIndexes.reset();
storageRect.clear();
fernRegions.clear();
fernInfo.reset();
int totalP = 0;
int totalN = 0;
// Run through all candidate regions, ignore ones without enough variance, compute
// the fern for each one
TldRegionFernInfo info = fernInfo.grow();
for (int i = 0; i < cascadeRegions.size; i++) {
ImageRectangle region = cascadeRegions.get(i);
if (!variance.checkVariance(region)) {
continue;
}
info.r = region;
if (fern.lookupFernPN(info)) {
totalP += info.sumP;
totalN += info.sumN;
info = fernInfo.grow();
}
}
fernInfo.removeTail();
// avoid overflow errors in the future by re-normalizing the Fern detector
if (totalP > 0x0fffffff)
fern.renormalizeP();
if (totalN > 0x0fffffff)
fern.renormalizeN();
// Select the ferns with the highest likelihood
selectBestRegionsFern(totalP, totalN);
// From the remaining regions, score using the template algorithm
computeTemplateConfidence();
if (candidateDetections.size == 0) {
return;
}
// use non-maximum suppression to reduce the number of candidates
nonmax.process(candidateDetections, localMaximums);
best = selectBest();
if (best != null) {
ambiguous = checkAmbiguous(best);
success = true;
}
}
/**
* Computes the confidence for all the regions which pass the fern test
*/
protected void computeTemplateConfidence() {
double max = 0;
for (int i = 0; i < fernRegions.size(); i++) {
ImageRectangle region = fernRegions.get(i);
double confidence = template.computeConfidence(region);
max = Math.max(max, confidence);
if (confidence < config.confidenceThresholdUpper)
continue;
TldRegion r = candidateDetections.grow();
r.connections = 0;
r.rect.setTo(region);
r.confidence = confidence;
}
}
/**
* compute the probability that each region is the target conditional upon this image
* the sumP and sumN are needed for image conditional probability
*
* NOTE: This is a big change from the original paper
*/
protected void selectBestRegionsFern( double totalP, double totalN ) {
for (int i = 0; i < fernInfo.size; i++) {
TldRegionFernInfo info = fernInfo.get(i);
double probP = info.sumP/totalP;
double probN = info.sumN/totalN;
// only consider regions with a higher P likelihood
if (probP > probN) {
// reward regions with a large difference between the P and N values
storageMetric.add(-(probP - probN));
storageRect.add(info.r);
}
}
// Select the N regions with the highest fern probability
if (config.maximumCascadeConsider < storageMetric.size) {
int N = Math.min(config.maximumCascadeConsider, storageMetric.size);
storageIndexes.resize(storageMetric.size);
QuickSelect.selectIndex(storageMetric.data, N - 1, storageMetric.size, storageIndexes.data);
for (int i = 0; i < N; i++) {
fernRegions.add(storageRect.get(storageIndexes.get(i)));
}
} else {
fernRegions.addAll(storageRect);
}
}
public @Nullable TldRegion selectBest() {
TldRegion best = null;
double bestConfidence = 0;
for (int i = 0; i < localMaximums.size; i++) {
TldRegion r = localMaximums.get(i);
if (r.confidence > bestConfidence) {
bestConfidence = r.confidence;
best = r;
}
}
return best;
}
private boolean checkAmbiguous( TldRegion best ) {
double thresh = best.confidence*0.9;
for (int i = 0; i < localMaximums.size; i++) {
TldRegion r = localMaximums.get(i);
if (r.confidence >= thresh) {
double overlap = helper.computeOverlap(r.rect, best.rect);
if (overlap <= config.overlapLower) {
ambiguousRegions.add(r.rect);
}
}
}
return !ambiguousRegions.isEmpty();
}
public @Nullable TldRegion getBest() {
return best;
}
public boolean isAmbiguous() {
return ambiguous;
}
public TldNonMaximalSuppression getNonmax() {
return nonmax;
}
public DogArray_F64 getStorageMetric() {
return storageMetric;
}
public List getStorageRect() {
return storageRect;
}
public DogArray getCandidateDetections() {
return candidateDetections;
}
public DogArray getLocalMaximums() {
return localMaximums;
}
public List getAmbiguousRegions() {
return ambiguousRegions;
}
public DogArray getFernInfo() {
return fernInfo;
}
/**
* Rectangles selected by the fern classifier as candidates
*
* @return List of rectangles
*/
public List getSelectedFernRectangles() {
return fernRegions;
}
public boolean isSuccess() {
return success;
}
}