boofcv.abst.scene.nister2006.FeatureSceneRecognitionNister2006 Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of boofcv-recognition Show documentation
BoofCV is an open source Java library for real-time computer vision and robotics applications.
There is a newer version: 1.1.6
/*
 * Copyright (c) 2021, Peter Abeles. All Rights Reserved.
 *
 * This file is part of BoofCV (http://boofcv.org).
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package boofcv.abst.scene.nister2006;

import boofcv.abst.scene.FeatureSceneRecognition;
import boofcv.abst.scene.SceneRecognition;
import boofcv.alg.scene.bow.BowMatch;
import boofcv.alg.scene.nister2006.LearnNodeWeights;
import boofcv.alg.scene.nister2006.RecognitionVocabularyTreeNister2006;
import boofcv.alg.scene.vocabtree.HierarchicalVocabularyTree;
import boofcv.alg.scene.vocabtree.LearnHierarchicalTree;
import boofcv.factory.struct.FactoryTupleDesc;
import boofcv.misc.BoofLambdas;
import boofcv.misc.BoofMiscOps;
import boofcv.struct.PackedArray;
import boofcv.struct.feature.TupleDesc;
import boofcv.struct.kmeans.FactoryTupleCluster;
import lombok.Getter;
import lombok.Setter;
import org.ddogleg.clustering.kmeans.StandardKMeans;
import org.ddogleg.struct.DogArray;
import org.ddogleg.struct.DogArray_I32;
import org.ddogleg.struct.Factory;
import org.jetbrains.annotations.Nullable;

import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Set;

/**
 * High level implementation of {@link RecognitionVocabularyTreeNister2006} for {@link FeatureSceneRecognition}.
 *
 * @author Peter Abeles
 */
@SuppressWarnings({"NullAway.Init"})
public class FeatureSceneRecognitionNister2006> implements FeatureSceneRecognition {
	/** Configuration for this class */
	@Getter ConfigRecognitionNister2006 config;

	/** Tree representation of image features */
	@Getter HierarchicalVocabularyTree tree;

	/** Manages saving and locating images */
	@Getter RecognitionVocabularyTreeNister2006 database;

	/** Stores features found in one image */
	@Getter @Setter DogArray imageFeatures;

	/** List of all the images in the dataset */
	@Getter List imageIds = new ArrayList<>();

	/** Performance tuning. If less than this number of features a single thread algorithm will be used */
	@Getter @Setter public int minimumForThread = 500; // This value has not been proven to be optimal

	// Describes how to store the feature descriptor
	Class tupleType;
	int tupleDOF;

	// If not null then print verbose information
	@Nullable PrintStream verbose;

	// Internal Profiling. All times in milliseconds
	@Getter long timeLearnDescribeMS;
	@Getter long timeLearnClusterMS;
	@Getter long timeLearnWeightsMS;

	public FeatureSceneRecognitionNister2006( ConfigRecognitionNister2006 config, Factory factory ) {
		this.config = config;
		this.imageFeatures = new DogArray<>(factory);
		this.database = new RecognitionVocabularyTreeNister2006<>();

		database.setDistanceType(config.distanceNorm);
		database.minimumDepthFromRoot = config.minimumDepthFromRoot;
		database.maximumQueryImagesInNode.setTo(config.queryMaximumImagesInNode);

		tupleDOF = imageFeatures.grow().size();
		tupleType = (Class)imageFeatures.get(0).getClass();
	}

	public void setDatabase( RecognitionVocabularyTreeNister2006 db ) {
		database = db;
		tree = db.getTree();
	}

	@Override public void learnModel( Iterator> images ) {
		PackedArray packedFeatures = FactoryTupleDesc.createPackedBig(tupleDOF, tupleType);

		// Keep track of where features from one image begins/ends
		DogArray_I32 startIndex = new DogArray_I32();

		// Detect features in all the images and save into a single array
		long time0 = System.currentTimeMillis();
		while (images.hasNext()) {
			Features image = images.next();
			startIndex.add(packedFeatures.size());
			int N = image.size();
			packedFeatures.reserve(N);
			for (int i = 0; i < N; i++) {
				packedFeatures.append(image.getDescription(i));
			}
			if (verbose != null)
				verbose.println("described.size=" + startIndex.size + " features=" + N + " packed.size=" + packedFeatures.size());
		}
		startIndex.add(packedFeatures.size());
		if (verbose != null) verbose.println("packedFeatures.size=" + packedFeatures.size());
		long time1 = System.currentTimeMillis();

		// Create the tree data structure
		PackedArray packedArray = FactoryTupleDesc.createPackedBig(tupleDOF, tupleType);

		tree = new HierarchicalVocabularyTree<>(FactoryTupleCluster.createDistance(tupleType), packedArray);
		tree.branchFactor = config.tree.branchFactor;
		tree.maximumLevel = config.tree.maximumLevel;

		// Learn the tree's structure
		if (verbose != null) verbose.println("learning the tree");

		BoofLambdas.Factory> factoryKMeans = () ->
				FactoryTupleCluster.kmeans(config.kmeans, minimumForThread, tupleDOF, tupleType);

		LearnHierarchicalTree learnTree = new LearnHierarchicalTree<>(
				() -> FactoryTupleDesc.createPackedBig(tupleDOF, tupleType), factoryKMeans, config.randSeed);
		learnTree.minimumPointsForChildren.setTo(config.learningMinimumPointsForChildren);
		if (verbose != null)
			BoofMiscOps.verboseChildren(verbose, null, learnTree);
		learnTree.process(packedFeatures, tree);
		long time2 = System.currentTimeMillis();

		if (verbose != null) {
			verbose.println("Tree {bf=" + tree.branchFactor + " ml=" + tree.maximumLevel +
					" nodes.size=" + tree.nodes.size + "}");
		}

		// Learn the weight for each node in the tree
		if (verbose != null) verbose.println("learning the weights");
		if (config.learnNodeWeights) {
			LearnNodeWeights learnWeights = new LearnNodeWeights<>();
			learnWeights.maximumNumberImagesInNode.setTo(config.learningMaximumImagesInNode);
			learnWeights.reset(tree);

			for (int imgIdx = 1; imgIdx < startIndex.size; imgIdx++) {
				imageFeatures.reset();
				int idx0 = startIndex.get(imgIdx - 1);
				int idx1 = startIndex.get(imgIdx);
				for (int i = idx0; i < idx1; i++) {
					imageFeatures.grow().setTo(packedFeatures.getTemp(i));
				}
				learnWeights.addImage(imageFeatures.toList());
			}
			learnWeights.fixate();
		} else {
			// Set all weights to zero, except for the root which has to contain everything
			tree.nodes.forEach(n -> n.weight = 1.0);
			tree.nodes.get(0).weight = 0.0;
		}
		long time3 = System.currentTimeMillis();

		// Initialize the database
		database.initializeTree(tree);

		// Compute internal profiling
		timeLearnDescribeMS = time1 - time0;
		timeLearnClusterMS = time2 - time1;
		timeLearnWeightsMS = time3 - time2;

		if (verbose != null)
			verbose.printf("Time (s): describe=%.1f cluster=%.1f weights=%.1f\n",
					timeLearnDescribeMS*1e-3, timeLearnClusterMS*1e-3, timeLearnWeightsMS*1e-3);
	}

	@Override public void clearDatabase() {
		imageIds.clear();
		database.clearImages();
	}

	@Override public void addImage( String id, Features features ) {
		// Copy image features into an array
		imageFeatures.resize(features.size());
		for (int i = 0; i < imageFeatures.size; i++) {
			imageFeatures.get(i).setTo(features.getDescription(i));
		}

		// Save the ID and convert into a format the database understands
		int imageIndex = imageIds.size();
		imageIds.add(id);

		if (verbose != null)
			verbose.println("added[" + imageIndex + "].size=" + features.size() + " id=" + id);

		// Add the image
		database.addImage(imageIndex, imageFeatures.toList());
	}

	@Override public List getImageIds( @Nullable List storage ) {
		if (storage == null)
			storage = new ArrayList<>();
		else
			storage.clear();

		storage.addAll(imageIds);
		return storage;
	}

	@Override public boolean query( Features query, @Nullable BoofLambdas.Filter filter,
									int limit, DogArray matches ) {
		// Default is no matches
		matches.resize(0);

		// Handle the case where the limit is unlimited
		limit = limit <= 0 ? Integer.MAX_VALUE : limit;

		// Detect image features then copy features into an array
		imageFeatures.resize(query.size());
		for (int i = 0; i < imageFeatures.size; i++) {
			imageFeatures.get(i).setTo(query.getDescription(i));
		}

		// Wrap the user provided filter by converting the int ID into a String ID
		BoofLambdas.FilterInt filterInt = filter == null ? null : ( index ) -> filter.keep(imageIds.get(index));

		// Look up the closest matches
		if (!database.query(imageFeatures.toList(), filterInt, limit))
			return false;

		DogArray found = database.getMatches();

		if (verbose != null) verbose.println("matches.size=" + found.size + " best.error=" + found.get(0).error);

		// Copy results into output format
		matches.resize(found.size);
		for (int i = 0; i < matches.size; i++) {
			BowMatch f = found.get(i);
			matches.get(i).id = imageIds.get(f.identification);
			matches.get(i).error = f.error;
		}

		return !matches.isEmpty();
	}

	@Override public int getQueryWord( int featureIdx ) {
		return traverseUpGetID(database.getFeatureIdxToLeafID().get(featureIdx),
				config.featureSingleWordHops);
	}

	@Override public void getQueryWords( int featureIdx, DogArray_I32 words ) {
		int leafID = getQueryWord(featureIdx);
		lookupWordsFromLeafID(leafID, words);
	}

	@Override public int lookupWord( TD description ) {
		return traverseUpGetID(database.tree.searchPathToLeaf(description, ( idx, n ) -> {}),
				config.featureSingleWordHops);
	}

	@Override public void lookupWords( TD description, DogArray_I32 word ) {
		lookupWordsFromLeafID(lookupWord(description), word);
	}

	/**
	 * Given the leafID lookup the other words in the tree leading to the leaf. Words is reset on each call
	 */
	protected void lookupWordsFromLeafID( int leafID, DogArray_I32 word ) {
		word.reset();
		HierarchicalVocabularyTree.Node node = database.tree.nodes.get(leafID);
		while (node != null) {
			// the root node has a parent of -1 and we don't want to use that as a word since every feature has it
			if (node.parent == -1)
				break;
			word.add(node.index);
			node = database.tree.nodes.get(node.parent);
		}
	}

	/**
	 * Traverses up the tree to find a parent node X hops away
	 */
	protected int traverseUpGetID( int id, int maxHops ) {
		int hops = maxHops;
		HierarchicalVocabularyTree.Node node = database.tree.nodes.get(id);
		while (hops-- > 0) {
			node = database.tree.nodes.get(node.parent);
			// the root node has a parent of -1 and we don't want to use that as a word since every feature has it
			if (node.parent == -1)
				break;
			id = node.index;
		}
		return id;
	}

	@Override public int getTotalWords() {
		return database.tree.nodes.size;
	}

	@Override public Class getDescriptorType() {
		return tupleType;
	}

	@Override public void setVerbose( @Nullable PrintStream out, @Nullable Set set ) {
		this.verbose = BoofMiscOps.addPrefix(this, out);
	}
}