boofcv.alg.feature.describe.llah.LlahOperations Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of boofcv-feature Show documentation
Show all versions of boofcv-feature Show documentation
BoofCV is an open source Java library for real-time computer vision and robotics applications.
/*
* Copyright (c) 2011-2020, Peter Abeles. All Rights Reserved.
*
* This file is part of BoofCV (http://boofcv.org).
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package boofcv.alg.feature.describe.llah;
import boofcv.alg.nn.KdTreePoint2D_F64;
import boofcv.struct.geo.PointIndex2D_F64;
import georegression.struct.point.Point2D_F64;
import gnu.trove.map.hash.TIntObjectHashMap;
import lombok.Getter;
import org.ddogleg.combinatorics.Combinations;
import org.ddogleg.nn.FactoryNearestNeighbor;
import org.ddogleg.nn.NearestNeighbor;
import org.ddogleg.nn.NnData;
import org.ddogleg.sorting.QuickSort_F64;
import org.ddogleg.struct.FastQueue;
import org.ddogleg.struct.GrowQueue_I32;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.IntStream;
/**
* Locally Likely Arrangement Hashing (LLAH) [1] computes a descriptor for a landmark based on the local geometry of
* neighboring landmarks on the image plane. Originally proposed for document retrieval. These features are either
* invariant to perspective or affine transforms.
*
* Works by sampling the N neighbors around a point. These ports are sorted in clockwise order. However,
* it is not known which points should be first so all cyclical permutations of set-N are now found.
* It is assumed that at least M points in set M are a member
* of the set used to compute the feature, so all M combinations of points in set-N are found. Then the geometric
* invariants are computed using set-M.
*
* When describing the documents the hash and invariant values of each point in a document is saved. When
* looking up documents these features are again computed for all points in view, but then the document
* type is voted upon and returned.
*
*
* - Nakai, Tomohiro, Koichi Kise, and Masakazu Iwamura.
* "Use of affine invariants in locally likely arrangement hashing for camera-based document image retrieval."
* International Workshop on Document Analysis Systems. Springer, Berlin, Heidelberg, 2006.
*
*
* @author Peter Abeles
*/
public class LlahOperations {
// Number of nearest neighbors it will search for
@Getter final int numberOfNeighborsN;
// Size of combination set from the set of neighbors
@Getter final int sizeOfCombinationM;
// Number of invariants in the feature. Determined by the type and M
@Getter final int numberOfInvariants;
final List setM = new ArrayList<>();
final List permuteM = new ArrayList<>();
// Computes the hash value for each feature
@Getter LlahHasher hasher;
// Used to look up features/documents
@Getter final LlahHashTable hashTable = new LlahHashTable();
// List of all documents
@Getter final FastQueue documents = new FastQueue<>(LlahDocument::new);
//========================== Internal working variables
final NearestNeighbor nn = FactoryNearestNeighbor.kdtree(new KdTreePoint2D_F64());
private final NearestNeighbor.Search search = nn.createSearch();
private final FastQueue> resultsNN = new FastQueue<>(NnData::new);
final List neighbors = new ArrayList<>();
private final double[] angles;
private final QuickSort_F64 sorter = new QuickSort_F64();
private final FastQueue resultsStorage = new FastQueue<>(FoundDocument::new);
private final TIntObjectHashMap foundMap = new TIntObjectHashMap<>();
private final FastQueue allFeatures;
// Used to compute all the combinations of a set
private final Combinations combinator = new Combinations<>();
// recycle to avoid garbage collector
FastQueue storageD2L = new FastQueue<>(DotCount::new); // dot to landmark
/**
* Configures the LLAH feature computation
*
* @param numberOfNeighborsN Number of neighbors to be considered
* @param sizeOfCombinationM Number of different combinations within the neighbors
* @param hasher Computes the hash code
*/
public LlahOperations( int numberOfNeighborsN , int sizeOfCombinationM,
LlahHasher hasher ) {
this.numberOfNeighborsN = numberOfNeighborsN;
this.sizeOfCombinationM = sizeOfCombinationM;
this.numberOfInvariants = hasher.getNumberOfInvariants(sizeOfCombinationM);
this.hasher = hasher;
angles = new double[numberOfNeighborsN];
allFeatures = new FastQueue<>(()->new LlahFeature(numberOfInvariants));
}
/**
* Forgets all the documents and recycles data
*/
public void clearDocuments() {
documents.reset();
allFeatures.reset();
hashTable.reset();
}
/**
* Learns the hashing function from the set of point sets
* @param pointSets Point sets. Each set represents one document
* @param numDiscrete Number of discrete values the invariant is converted to
* @param histogramLength Number of elements in the histogram. 100,000 is recommended
* @param maxInvariantValue The maximum number of value an invariant is assumed to have.
* For affine ~25. Cross Ratio
*/
public void learnHashing(Iterable> pointSets , int numDiscrete ,
int histogramLength,double maxInvariantValue ) {
// to make the math faster use a fine grained array with more extreme values than expected
int[] histogram = new int[histogramLength];
// Storage for computed invariants
double[] invariants = new double[numberOfInvariants];
// Go through each point and compute some invariants from it
for( var locations2D : pointSets ) {
nn.setPoints(locations2D,false);
computeAllFeatures(locations2D, (idx,l)-> {
hasher.computeInvariants(l,invariants,0);
for (int i = 0; i < invariants.length; i++) {
int j = Math.min(histogram.length-1,(int)(histogram.length*invariants[i]/maxInvariantValue));
histogram[j]++;
}
});
}
// Sanity check
double endFraction = histogram[histogram.length-1]/(double)IntStream.of(histogram).sum();
double maxAllowed = 0.5/numDiscrete;
if( endFraction > maxAllowed )
System.err.println("WARNING: last element in histogram has a significant count. " +endFraction+" > "+maxAllowed+
" maxInvariantValue should be increased");
hasher.learnDiscretization(histogram,histogram.length,maxInvariantValue,numDiscrete);
}
/**
* Creates a new document from the 2D points. The document and points are added to the hash table
* for later retrieval.
*
* @param locations2D Location of points inside the document
* @return The document which was added to the hash table.
*/
public LlahDocument createDocument(List locations2D ) {
checkListSize(locations2D);
LlahDocument doc = documents.grow();
doc.reset();
doc.documentID = documents.size()-1;
// copy the points
doc.landmarks.copyAll(locations2D,(src,dst)->dst.set(src));
computeAllFeatures(locations2D, (idx,l) -> createProcessor(doc, idx));
return doc;
}
/**
* Computes the maximum number of unique hash code a point can have.
*/
public long computeMaxUniqueHashPerPoint() {
long comboHash = Combinations.computeTotalCombinations(numberOfNeighborsN,sizeOfCombinationM);
return comboHash*sizeOfCombinationM;
}
private void createProcessor(LlahDocument doc, int idx) {
// Given this set compute the feature
LlahFeature feature = allFeatures.grow();
feature.reset();
hasher.computeHash(permuteM,feature);
// save the results
feature.landmarkID = idx;
feature.documentID = doc.documentID;
doc.features.add(feature);
hashTable.add(feature);
}
/**
* Given the set of observed locations, compute all the features for each point. Have processor handle
* the results as they are found
*/
void computeAllFeatures(List dots, ProcessPermutation processor ) {
// set up nn search
nn.setPoints(dots,false);
// Compute the features for all points in this document
for (int dotIdx = 0; dotIdx < dots.size(); dotIdx++) {
// System.out.println("================ pointID "+dotIdx);
findNeighbors(dots.get(dotIdx));
// All combinations of size M from neighbors
combinator.init(neighbors, sizeOfCombinationM);
do {
setM.clear();
for (int i = 0; i < sizeOfCombinationM; i++) {
setM.add( combinator.get(i) );
}
// Cyclical permutations of 'setM'
// When you look it up you won't know the order points are observed in
for (int i = 0; i < sizeOfCombinationM; i++) {
permuteM.clear();
for (int j = 0; j < sizeOfCombinationM; j++) {
int idx = (i+j)%sizeOfCombinationM;
permuteM.add(setM.get(idx));
}
processor.process(dotIdx,permuteM);
}
} while( combinator.next() );
}
}
/**
* Finds all the neighbors
*/
void findNeighbors(Point2D_F64 target) {
// Find N nearest-neighbors of p0
search.findNearest(target,-1, numberOfNeighborsN+1,resultsNN);
// Find the neighbors, removing p0
neighbors.clear();
for (int i = 0; i < resultsNN.size; i++) {
Point2D_F64 n = resultsNN.get(i).point;
if( n == target ) // it will always find the p0 point
continue;
neighbors.add(n);
}
// Compute the angle of each neighbor
for (int i = 0; i < neighbors.size(); i++) {
Point2D_F64 n = neighbors.get(i);
angles[i] = Math.atan2(n.y-target.y, n.x-target.x);
}
// sort the neighbors in clockwise order
sorter.sort(angles,angles.length,neighbors);
// System.out.println("tgt"+target);
// for (int i = 0; i < neighbors.size(); i++) {
// System.out.println(" "+neighbors.get(i));
// }
}
/**
* Looks up all the documents which match observed features.
* @param dots Observed feature locations
* @param minLandmarks Minimum number of landmarks that are assigned to a document for it to be accepted
* @param output Storage for results. WARNING: Results are recycled on next call!
*/
public void lookupDocuments( List dots , int minLandmarks, List output ) {
output.clear();
// It needs to have a minimum of this number of points to work
if (dots.size() < numberOfNeighborsN + 1)
return;
storageD2L.reset();
foundMap.clear();
resultsStorage.reset();
// Used to keep track of what has been seen and what has not been seen
FastQueue votingBooths = new FastQueue<>(DotVotingBooth::new);
votingBooths.resize(dots.size());
var featureComputed = new LlahFeature(numberOfInvariants);
// Compute features, look up matching known features, then vote
computeAllFeatures(dots, (dotIdx,pointSet)->
lookupProcessor(pointSet,dotIdx, featureComputed, votingBooths));
for (int dotIdx = 0; dotIdx < dots.size(); dotIdx++) {
DotVotingBooth booth = votingBooths.get(dotIdx);
if( booth.votes.size == 0 )
continue;
DotToLandmark best = booth.votes.get(0);
for (int i = 1; i < booth.votes.size; i++) {
DotToLandmark b = booth.votes.get(i);
if( b.count > best.count ) {
best = b;
}
}
FoundDocument doc = foundMap.get(best.documentID);
if( doc == null ){
doc = resultsStorage.grow();
doc.init(documents.get(best.documentID));
foundMap.put(best.documentID,doc);
}
if( doc.landmarkHits.get(best.landmarkID) < best.count ) {
doc.landmarkHits.set(best.landmarkID,best.count);
doc.landmarkToDots.set(best.landmarkID, dotIdx);
}
}
foundMap.forEachEntry((docID,doc)->{
if( doc.countSeenLandmarks() >= minLandmarks ) {
output.add(doc);
}
return true;
});
}
/**
* Place holder function for the document retrieval in the LLAH paper. Just throws an exception for now.
* @param dots observed dots
* @param output storage for found document
* @return true if successful
*/
public boolean lookupocument( List dots , FoundDocument output ) {
throw new RuntimeException("Implement");
}
/**
* Ensures that the points passed in is an acceptable size
*/
void checkListSize(List locations2D) {
if (locations2D.size() < numberOfNeighborsN + 1)
throw new IllegalArgumentException("There needs to be at least " + (numberOfNeighborsN + 1) + " points");
}
/**
* Computes the feature for the set of points and see if they match anything in the dictionary. If they do vote.
*/
private void lookupProcessor( List pointSet, int dotIdx, LlahFeature featureComputed,
FastQueue votingBooths )
{
DotVotingBooth booth = votingBooths.get(dotIdx);
// Compute the feature for this set
hasher.computeHash(pointSet,featureComputed);
// Find the set of features which match this has code
LlahFeature foundFeat = hashTable.lookup(featureComputed.hashCode);
while( foundFeat != null ) {
// Condition 1: See if the invariant's match
if( featureComputed.doInvariantsMatch(foundFeat) ) {
DotToLandmark vote = booth.lookup(foundFeat.documentID, foundFeat.landmarkID);
vote.count += 1;
}
foundFeat = foundFeat.next;
}
}
/**
* Abstracts the inner most step when computing features
*/
interface ProcessPermutation
{
void process( int dotIdx, List points );
}
public static class DotVotingBooth {
final FastQueue votes = new FastQueue<>(DotToLandmark::new);
final TIntObjectHashMap> map = new TIntObjectHashMap<>();
public void reset() {
votes.reset();
map.clear();
}
public DotToLandmark lookup( int documentID , int landmarkID ) {
TIntObjectHashMap voteDoc = map.get(documentID);
if( voteDoc == null ) {
voteDoc = new TIntObjectHashMap<>();
map.put(documentID, voteDoc);
}
DotToLandmark vote = voteDoc.get(landmarkID);
if( vote == null ) {
vote = votes.grow();
vote.documentID = documentID;
vote.landmarkID = landmarkID;
vote.count = 0;
voteDoc.put(landmarkID,vote);
}
return vote;
}
}
public static class DotToLandmark {
public int documentID;
public int landmarkID;
public int count;
}
/**
* Used to relate observed dots to landmarks in a document
*/
public static class DotCount
{
// index of dot in input array
public int dotIdx;
// how many times this dot was matched to this landmark
public int counts;
public void reset() {
dotIdx = -1;
counts = 0;
}
@Override
public int hashCode() {
return dotIdx;
}
}
/**
* Documents that were found to match observed dots
*/
public static class FoundDocument {
/** Which document */
public LlahDocument document;
/**
* Indicates the number of times a particular point was matched
*/
public final GrowQueue_I32 landmarkHits = new GrowQueue_I32();
public final GrowQueue_I32 landmarkToDots = new GrowQueue_I32();
public void init( LlahDocument document) {
this.document = document;
final int totalLandmarks = document.landmarks.size;
landmarkHits.resize(totalLandmarks);
landmarkHits.fill(0);
landmarkToDots.resize(totalLandmarks);
landmarkToDots.fill(-1);
}
public boolean seenLandmark( int which ) {
return landmarkHits.get(which) > 0;
}
public void lookupMatches(FastQueue matches ) {
matches.reset();
for (int i = 0; i < landmarkHits.size; i++) {
if( landmarkHits.get(i) > 0 ) {
var p = document.landmarks.get(i);
matches.grow().set(p.x,p.y,i);
}
}
}
public int countSeenLandmarks() {
int total = 0;
for (int i = 0; i < landmarkHits.size; i++) {
if( landmarkHits.get(i) > 0 )
total++;
}
return total;
}
public int countHits() {
int total = 0;
for (int i = 0; i < landmarkHits.size; i++) {
total += landmarkHits.get(i);
}
return total;
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy