org.openimaj.knn.pq.LongADCNearestNeighbours Maven / Gradle / Ivy
/*
AUTOMATICALLY GENERATED BY jTemp FROM
/Users/jsh2/Work/openimaj/target/checkout/machine-learning/nearest-neighbour/src/main/jtemp/org/openimaj/knn/pq/#T#ADCNearestNeighbours.jtemp
*/
/**
* Copyright (c) 2011, The University of Southampton and the individual contributors.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* * Neither the name of the University of Southampton nor the names of its
* contributors may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.openimaj.knn.pq;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.openimaj.citation.annotation.Reference;
import org.openimaj.citation.annotation.ReferenceType;
import org.openimaj.knn.LongNearestNeighbours;
import org.openimaj.util.pair.IntDoublePair;
import org.openimaj.util.queue.BoundedPriorityQueue;
/**
* Nearest-neighbours using Asymmetric Distance Computation (ADC) on Product
* Quantised vectors. In ADC, only the database points are quantised. The
* queries themselves are not quantised. The overall distance is computed as the
* summed distance of each subvector of the query to each corresponding
* centroids of each database vector.
*
* For efficiency, the distance of each sub-vector of a query is computed to
* every centroid (for the sub-vector under consideration) only once, and is
* then cached for the lookup during the computation of the distance to each
* database vector.
*
* @author Jonathon Hare ([email protected])
*/
@Reference(
type = ReferenceType.Article,
author = { "Jegou, Herve", "Douze, Matthijs", "Schmid, Cordelia" },
title = "Product Quantization for Nearest Neighbor Search",
year = "2011",
journal = "IEEE Trans. Pattern Anal. Mach. Intell.",
pages = { "117", "", "128" },
url = "http://dx.doi.org/10.1109/TPAMI.2010.57",
month = "January",
number = "1",
publisher = "IEEE Computer Society",
volume = "33",
customData = {
"issn", "0162-8828",
"numpages", "12",
"doi", "10.1109/TPAMI.2010.57",
"acmid", "1916695",
"address", "Washington, DC, USA",
"keywords", "High-dimensional indexing, High-dimensional indexing, image indexing, very large databases, approximate search., approximate search., image indexing, very large databases"
})
public class LongADCNearestNeighbours extends LongNearestNeighbours {
protected final LongProductQuantiser pq;
protected final int ndims;
protected final byte[][] data;
/**
* Construct the ADC with the given quantiser and data points.
*
* @param pq
* the Product Quantiser
* @param dataPoints
* the data points to index
*/
public LongADCNearestNeighbours(LongProductQuantiser pq, long[][] dataPoints) {
this.pq = pq;
this.ndims = dataPoints[0].length;
this.data = new byte[dataPoints.length][];
for (int i = 0; i < dataPoints.length; i++) {
data[i] = pq.quantise(dataPoints[i]);
}
}
/**
* Construct the ADC with the given quantiser and pre-quantised data .
*
* @param pq
* the Product Quantiser
* @param pqData
* the pre-quantised data (i.e. vectors already quantised with
* the given pq)
* @param ndims
* the dimensionality of the indexed data
*/
public LongADCNearestNeighbours(LongProductQuantiser pq, byte[][] pqData, int ndims) {
this.ndims = ndims;
this.pq = pq;
this.data = pqData;
}
@Override
public void searchNN(final long [][] qus, int [] indices, double [] distances) {
final int N = qus.length;
final BoundedPriorityQueue queue =
new BoundedPriorityQueue(1, IntDoublePair.SECOND_ITEM_ASCENDING_COMPARATOR);
//prepare working data
List list = new ArrayList(2);
list.add(new IntDoublePair());
list.add(new IntDoublePair());
for (int n=0; n < N; ++n) {
List result = search(qus[n], queue, list);
final IntDoublePair p = result.get(0);
indices[n] = p.first;
distances[n] = p.second;
}
}
@Override
public void searchKNN(final long [][] qus, int K, int [][] indices, double [][] distances) {
// Fix for when the user asks for too many points.
K = Math.min(K, data.length);
final int N = qus.length;
final BoundedPriorityQueue queue =
new BoundedPriorityQueue(K, IntDoublePair.SECOND_ITEM_ASCENDING_COMPARATOR);
//prepare working data
List list = new ArrayList(K + 1);
for (int i = 0; i < K + 1; i++) {
list.add(new IntDoublePair());
}
// search on each query
for (int n = 0; n < N; ++n) {
List result = search(qus[n], queue, list);
for (int k = 0; k < K; ++k) {
final IntDoublePair p = result.get(k);
indices[n][k] = p.first;
distances[n][k] = p.second;
}
}
}
@Override
public void searchNN(final List qus, int [] indices, double [] distances) {
final int N = qus.size();
final BoundedPriorityQueue queue =
new BoundedPriorityQueue(1, IntDoublePair.SECOND_ITEM_ASCENDING_COMPARATOR);
//prepare working data
List list = new ArrayList(2);
list.add(new IntDoublePair());
list.add(new IntDoublePair());
for (int n=0; n < N; ++n) {
List result = search(qus.get(n), queue, list);
final IntDoublePair p = result.get(0);
indices[n] = p.first;
distances[n] = p.second;
}
}
@Override
public void searchKNN(final List qus, int K, int [][] indices, double [][] distances) {
// Fix for when the user asks for too many points.
K = Math.min(K, data.length);
final int N = qus.size();
final BoundedPriorityQueue queue =
new BoundedPriorityQueue(K, IntDoublePair.SECOND_ITEM_ASCENDING_COMPARATOR);
//prepare working data
List list = new ArrayList(K + 1);
for (int i = 0; i < K + 1; i++) {
list.add(new IntDoublePair());
}
// search on each query
for (int n = 0; n < N; ++n) {
List result = search(qus.get(n), queue, list);
for (int k = 0; k < K; ++k) {
final IntDoublePair p = result.get(k);
indices[n][k] = p.first;
distances[n][k] = p.second;
}
}
}
@Override
public List searchKNN(long[] query, int K) {
// Fix for when the user asks for too many points.
K = Math.min(K, data.length);
final BoundedPriorityQueue queue =
new BoundedPriorityQueue(K, IntDoublePair.SECOND_ITEM_ASCENDING_COMPARATOR);
//prepare working data
List list = new ArrayList(K + 1);
for (int i = 0; i < K + 1; i++) {
list.add(new IntDoublePair());
}
// search
return search(query, queue, list);
}
@Override
public IntDoublePair searchNN(final long[] query) {
final BoundedPriorityQueue queue =
new BoundedPriorityQueue(1, IntDoublePair.SECOND_ITEM_ASCENDING_COMPARATOR);
//prepare working data
List list = new ArrayList(2);
list.add(new IntDoublePair());
list.add(new IntDoublePair());
return search(query, queue, list).get(0);
}
private List search(long[] query, BoundedPriorityQueue queue, List results) {
IntDoublePair wp = null;
// reset all values in the queue to MAX, -1
for (final IntDoublePair p : results) {
p.second = Float.MAX_VALUE;
p.first = -1;
wp = queue.offerItem(p);
}
// perform the search
computeDistances(query, queue, wp);
return queue.toOrderedListDestructive();
}
protected void computeDistances(long[] fullQuery, BoundedPriorityQueue queue, IntDoublePair wp) {
final double[][] distances = new double[pq.assigners.length][];
for (int j = 0, from = 0; j < this.pq.assigners.length; j++) {
final LongNearestNeighbours nn = this.pq.assigners[j];
final int to = nn.numDimensions();
final int K = nn.size();
final long[][] qus = { Arrays.copyOfRange(fullQuery, from, from + to) };
final int[][] idx = new int[1][K];
final double[][] dst = new double[1][K];
nn.searchKNN(qus, K, idx, dst);
distances[j] = new double[K];
for (int k = 0; k < K; k++) {
distances[j][idx[0][k]] = dst[0][k];
}
from += to;
}
for (int i = 0; i < data.length; i++) {
wp.first = i;
wp.second = 0;
for (int j = 0; j < this.pq.assigners.length; j++) {
final int centroid = this.data[i][j] + 128;
wp.second += distances[j][centroid];
}
wp = queue.offerItem(wp);
}
}
@Override
public int numDimensions() {
return ndims;
}
@Override
public int size() {
return data.length;
}
}