boofcv.alg.tracker.meanshift.TrackerMeanShiftComaniciu2003 Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of boofcv-recognition Show documentation
Show all versions of boofcv-recognition Show documentation
BoofCV is an open source Java library for real-time computer vision and robotics applications.
/*
* Copyright (c) 2021, Peter Abeles. All Rights Reserved.
*
* This file is part of BoofCV (http://boofcv.org).
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package boofcv.alg.tracker.meanshift;
import boofcv.struct.RectangleRotate_F32;
import boofcv.struct.image.ImageBase;
import georegression.struct.point.Point2D_F32;
import java.util.List;
/**
*
* Mean shift tracker which adjusts the scale (or bandwidth) to account for changes in scale of the target
* and is based off of [1]. The tracker seeks to minimize the histogram error within the sampled region.
* The mean-shift region is sampled using an oriented rectangle and weighted using a 2D gaussian. The target
* is modeled using a color histogram of the input image, which can be optionally updated after each frame
* is processed. It can also be configured to not allow scale changes, which can improve stability.
*
*
*
* Scale selection is done using sum-of-absolute-difference (SAD) error instead of Bhattacharyya as the paper
* suggests. Situations where found that two errors counteracted each other when using Bhattacharyya
* and the incorrect scale would be selected even with perfect data.
*
*
*
* Another difference from the paper is that mean shift records which hypothesis has the best SAD error. After
* mean-shift stops iterating it selects the best solution. This is primarily helpful in situations where mean-shift
* doesn't converge in time and jumped away from the solution.
*
*
*
* [1] Dorin Comaniciu, Visvanathan Ramesh, and Peter Meer,"Kernel-Based Object Tracking." IEEE Transactions on
* Pattern Analysis and Machine Intelligence 25.4 (2003): 1.
*
*
* @author Peter Abeles
*/
@SuppressWarnings({"NullAway.Init"})
public class TrackerMeanShiftComaniciu2003> {
// computes the histogram inside a rotated rectangle
private final LocalWeightedHistogramRotRect calcHistogram;
// the key-frame histogram which is being compared again
protected float[] keyHistogram;
// weight each element contributes
protected float[] weightHistogram;
// the amount the scale can change
protected float scaleChange;
// most recently select target region
private final RectangleRotate_F32 region = new RectangleRotate_F32();
// maximum allowed mean-shift iterations
private final int maxIterations;
// minimum change stopping condition
private final float minimumChange;
// storage for the track region at different sizes
private final RectangleRotate_F32 region0 = new RectangleRotate_F32();
private final RectangleRotate_F32 region1 = new RectangleRotate_F32();
private final RectangleRotate_F32 region2 = new RectangleRotate_F32();
private float[] histogram0;
private float[] histogram1;
private float[] histogram2;
// weighting factor for change in scale. 0 to 1. 0 is 100% selected region
private final float gamma;
// should it update the histogram after tracking?
private final boolean updateHistogram;
// if true assume the scale is constant
private final boolean constantScale;
// ratio of the original object size that the track can become
private final float minimumSizeRatio;
// stores the minimum width that the object can be
private float minimumWidth;
/**
* Configures tracker.
*
* @param updateHistogram If true the histogram will be updated using the most recent image. Try true.
* @param maxIterations Maximum number of mean-shift iterations. Try 30
* @param minimumChange Mean-shift will stop when the change is below this threshold. Try 1e-4f
* @param gamma Scale weighting factor. Value from 0 to 1. Closer to 0 the more it will prefer
* the most recent estimate. Try 0.1
* @param minimumSizeRatio Fraction of the original region that the track is allowed to shrink to. Try 0.25
* @param scaleChange The scale can be changed by this much between frames. 0 to 1. 0 = no scale change. 0.1 is
* recommended value in paper. no scale change is more stable.
* @param calcHistogram Calculates the histogram
*/
public TrackerMeanShiftComaniciu2003( boolean updateHistogram,
int maxIterations,
float minimumChange,
float gamma,
float minimumSizeRatio,
float scaleChange,
LocalWeightedHistogramRotRect calcHistogram ) {
if (scaleChange < 0 || scaleChange > 1)
throw new IllegalArgumentException("Scale change must be >= 0 and <= 1");
this.updateHistogram = updateHistogram;
this.maxIterations = maxIterations;
this.minimumChange = minimumChange;
this.gamma = gamma;
this.scaleChange = scaleChange;
this.constantScale = scaleChange == 0;
this.minimumSizeRatio = minimumSizeRatio;
this.calcHistogram = calcHistogram;
keyHistogram = new float[calcHistogram.getHistogram().length];
weightHistogram = new float[keyHistogram.length];
if (updateHistogram) {
histogram0 = new float[calcHistogram.getHistogram().length];
histogram1 = new float[calcHistogram.getHistogram().length];
histogram2 = new float[calcHistogram.getHistogram().length];
}
}
/**
* Specifies the initial image to learn the target description
*
* @param image Image
* @param initial Initial image which contains the target
*/
public void initialize( T image, RectangleRotate_F32 initial ) {
this.region.set(initial);
calcHistogram.computeHistogram(image, initial);
System.arraycopy(calcHistogram.getHistogram(), 0, keyHistogram, 0, keyHistogram.length);
this.minimumWidth = initial.width*minimumSizeRatio;
}
/**
* Used to set the location of the track without changing any appearance history.
*
* @param location new location
*/
public void setTrackLocation( RectangleRotate_F32 location ) {
this.region.set(location);
this.minimumWidth = location.width*minimumSizeRatio;
}
/**
* Searches for the target in the most recent image.
*
* @param image Most recent image in the sequence
*/
public void track( T image ) {
// configure the different regions based on size
region0.set(region);
region1.set(region);
region2.set(region);
region0.width *= 1 - scaleChange;
region0.height *= 1 - scaleChange;
region2.width *= 1 + scaleChange;
region2.height *= 1 + scaleChange;
// distance from histogram
double distance0 = 1, distance1, distance2 = 1;
// perform mean-shift at the different sizes and compute their distance
if (!constantScale) {
if (region0.width >= minimumWidth) {
updateLocation(image, region0);
distance0 = distanceHistogram(keyHistogram, calcHistogram.getHistogram());
if (updateHistogram)
System.arraycopy(calcHistogram.getHistogram(), 0, histogram0, 0, histogram0.length);
}
updateLocation(image, region2);
distance2 = distanceHistogram(keyHistogram, calcHistogram.getHistogram());
if (updateHistogram) System.arraycopy(calcHistogram.getHistogram(), 0, histogram2, 0, histogram2.length);
}
// update the no scale change hypothesis
updateLocation(image, region1);
if (!constantScale) {
distance1 = distanceHistogram(keyHistogram, calcHistogram.getHistogram());
} else {
// force it to select
distance1 = 0;
}
if (updateHistogram) System.arraycopy(calcHistogram.getHistogram(), 0, histogram1, 0, histogram1.length);
RectangleRotate_F32 selected;
float[] selectedHist;
switch (selectBest(distance0, distance1, distance2)) {
case 0 -> {
selected = region0;
selectedHist = histogram0;
}
case 1 -> {
selected = region1;
selectedHist = histogram1;
}
case 2 -> {
selected = region2;
selectedHist = histogram2;
}
default -> throw new RuntimeException("Bug in selectBest");
}
// Set region to the best scale, but reduce sensitivity by weighting it against the original size
// equation 14
float w = selected.width*(1 - gamma) + gamma*region.width;
float h = selected.height*(1 - gamma) + gamma*region.height;
region.set(selected);
region.width = w;
region.height = h;
if (updateHistogram) {
System.arraycopy(selectedHist, 0, keyHistogram, 0, keyHistogram.length);
}
}
/**
* Given the 3 scores return the index of the best
*/
private int selectBest( double a, double b, double c ) {
if (a < b) {
if (a < c)
return 0;
else
return 2;
} else if (b <= c) {
return 1;
} else {
return 2;
}
}
/**
* Updates the region's location using the standard mean-shift algorithm
*/
protected void updateLocation( T image, RectangleRotate_F32 region ) {
double bestHistScore = Double.MAX_VALUE;
float bestX = -1, bestY = -1;
for (int i = 0; i < maxIterations; i++) {
calcHistogram.computeHistogram(image, region);
float[] histogram = calcHistogram.getHistogram();
updateWeights(histogram);
// the histogram fit doesn't always improve with each mean-shift iteration
// save the best one and use it later on
double histScore = distanceHistogram(keyHistogram, histogram);
if (histScore < bestHistScore) {
bestHistScore = histScore;
bestX = region.cx;
bestY = region.cy;
}
List samples = calcHistogram.getSamplePts();
int[] sampleHistIndex = calcHistogram.getSampleHistIndex();
// Compute equation 13
float meanX = 0;
float meanY = 0;
float totalWeight = 0;
for (int j = 0; j < samples.size(); j++) {
Point2D_F32 samplePt = samples.get(j);
int histIndex = sampleHistIndex[j];
if (histIndex < 0)
continue;
// compute the weight derived from the Bhattacharyya coefficient. Equation 10.
float w = weightHistogram[histIndex];
meanX += w*samplePt.x;
meanY += w*samplePt.y;
totalWeight += w;
}
meanX /= totalWeight;
meanY /= totalWeight;
// convert to image pixels
calcHistogram.squareToImageSample(meanX, meanY, region);
meanX = calcHistogram.imageX;
meanY = calcHistogram.imageY;
// see if the change is below the threshold
boolean done = Math.abs(meanX - region.cx) <= minimumChange && Math.abs(meanY - region.cy) <= minimumChange;
region.cx = meanX;
region.cy = meanY;
if (done) {
break;
}
}
// use the best location found
region.cx = bestX;
region.cy = bestY;
}
/**
* Update the weights for each element in the histogram. Weights are used to favor colors which are
* less than expected.
*/
private void updateWeights( float[] histogram ) {
for (int j = 0; j < weightHistogram.length; j++) {
float h = histogram[j];
if (h != 0) {
weightHistogram[j] = (float)Math.sqrt(keyHistogram[j]/h);
}
}
}
/**
* Computes the difference between two histograms using SAD.
*
* This is a change from the paper, which uses Bhattacharyya. Bhattacharyya could give poor performance
* even with perfect data since two errors can cancel each other out. For example, part of the histogram
* is too small and another part is too large.
*/
protected double distanceHistogram( float[] histogramA, float[] histogramB ) {
double sumP = 0;
for (int i = 0; i < histogramA.length; i++) {
float q = histogramA[i];
float p = histogramB[i];
sumP += Math.abs(q - p);
}
return sumP;
}
public RectangleRotate_F32 getRegion() {
return region;
}
}