org.apache.lucene.geo.BaseGeoPointTestCase Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of lucene-test-framework Show documentation
Show all versions of lucene-test-framework Show documentation
Apache Lucene (module: test-framework)
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.geo;
import java.io.IOException;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.util.Arrays;
import java.util.BitSet;
import java.util.HashSet;
import java.util.Locale;
import java.util.Set;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.PointsFormat;
import org.apache.lucene.codecs.PointsReader;
import org.apache.lucene.codecs.PointsWriter;
import org.apache.lucene.codecs.lucene60.Lucene60PointsReader;
import org.apache.lucene.codecs.lucene60.Lucene60PointsWriter;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.geo.Rectangle;
import org.apache.lucene.geo.GeoUtils;
import org.apache.lucene.geo.Polygon;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SerialMergeScheduler;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.SimpleCollector;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.SloppyMath;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.bkd.BKDWriter;
/**
* Abstract class to do basic tests for a geospatial impl (high level
* fields and queries)
* NOTE: This test focuses on geospatial (distance queries, polygon
* queries, etc) indexing and search, not any underlying storage
* format or encoding: it merely supplies two hooks for the encoding
* so that tests can be exact. The [stretch] goal is for this test to be
* so thorough in testing a new geo impl that if this
* test passes, then all Lucene/Solr tests should also pass. Ie,
* if there is some bug in a given geo impl that this
* test fails to catch then this test needs to be improved! */
public abstract class BaseGeoPointTestCase extends LuceneTestCase {
protected static final String FIELD_NAME = "point";
// TODO: remove these hooks once all subclasses can pass with new random!
protected double nextLongitude() {
return org.apache.lucene.geo.GeoTestUtil.nextLongitude();
}
protected double nextLatitude() {
return org.apache.lucene.geo.GeoTestUtil.nextLatitude();
}
protected Rectangle nextBox() {
return org.apache.lucene.geo.GeoTestUtil.nextBox();
}
protected Polygon nextPolygon() {
return org.apache.lucene.geo.GeoTestUtil.nextPolygon();
}
/** Valid values that should not cause exception */
public void testIndexExtremeValues() {
Document document = new Document();
addPointToDoc("foo", document, 90.0, 180.0);
addPointToDoc("foo", document, 90.0, -180.0);
addPointToDoc("foo", document, -90.0, 180.0);
addPointToDoc("foo", document, -90.0, -180.0);
}
/** Invalid values */
public void testIndexOutOfRangeValues() {
Document document = new Document();
IllegalArgumentException expected;
expected = expectThrows(IllegalArgumentException.class, () -> {
addPointToDoc("foo", document, Math.nextUp(90.0), 50.0);
});
assertTrue(expected.getMessage().contains("invalid latitude"));
expected = expectThrows(IllegalArgumentException.class, () -> {
addPointToDoc("foo", document, Math.nextDown(-90.0), 50.0);
});
assertTrue(expected.getMessage().contains("invalid latitude"));
expected = expectThrows(IllegalArgumentException.class, () -> {
addPointToDoc("foo", document, 90.0, Math.nextUp(180.0));
});
assertTrue(expected.getMessage().contains("invalid longitude"));
expected = expectThrows(IllegalArgumentException.class, () -> {
addPointToDoc("foo", document, 90.0, Math.nextDown(-180.0));
});
assertTrue(expected.getMessage().contains("invalid longitude"));
}
/** NaN: illegal */
public void testIndexNaNValues() {
Document document = new Document();
IllegalArgumentException expected;
expected = expectThrows(IllegalArgumentException.class, () -> {
addPointToDoc("foo", document, Double.NaN, 50.0);
});
assertTrue(expected.getMessage().contains("invalid latitude"));
expected = expectThrows(IllegalArgumentException.class, () -> {
addPointToDoc("foo", document, 50.0, Double.NaN);
});
assertTrue(expected.getMessage().contains("invalid longitude"));
}
/** Inf: illegal */
public void testIndexInfValues() {
Document document = new Document();
IllegalArgumentException expected;
expected = expectThrows(IllegalArgumentException.class, () -> {
addPointToDoc("foo", document, Double.POSITIVE_INFINITY, 50.0);
});
assertTrue(expected.getMessage().contains("invalid latitude"));
expected = expectThrows(IllegalArgumentException.class, () -> {
addPointToDoc("foo", document, Double.NEGATIVE_INFINITY, 50.0);
});
assertTrue(expected.getMessage().contains("invalid latitude"));
expected = expectThrows(IllegalArgumentException.class, () -> {
addPointToDoc("foo", document, 50.0, Double.POSITIVE_INFINITY);
});
assertTrue(expected.getMessage().contains("invalid longitude"));
expected = expectThrows(IllegalArgumentException.class, () -> {
addPointToDoc("foo", document, 50.0, Double.NEGATIVE_INFINITY);
});
assertTrue(expected.getMessage().contains("invalid longitude"));
}
/** Add a single point and search for it in a box */
// NOTE: we don't currently supply an exact search, only ranges, because of the lossiness...
public void testBoxBasics() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
// add a doc with a point
Document document = new Document();
addPointToDoc("field", document, 18.313694, -65.227444);
writer.addDocument(document);
// search and verify we found our doc
IndexReader reader = writer.getReader();
IndexSearcher searcher = newSearcher(reader);
assertEquals(1, searcher.count(newRectQuery("field", 18, 19, -66, -65)));
reader.close();
writer.close();
dir.close();
}
/** null field name not allowed */
public void testBoxNull() {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
newRectQuery(null, 18, 19, -66, -65);
});
assertTrue(expected.getMessage().contains("field must not be null"));
}
// box should not accept invalid lat/lon
public void testBoxInvalidCoordinates() throws Exception {
expectThrows(Exception.class, () -> {
newRectQuery("field", -92.0, -91.0, 179.0, 181.0);
});
}
/** test we can search for a point */
public void testDistanceBasics() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
// add a doc with a location
Document document = new Document();
addPointToDoc("field", document, 18.313694, -65.227444);
writer.addDocument(document);
// search within 50km and verify we found our doc
IndexReader reader = writer.getReader();
IndexSearcher searcher = newSearcher(reader);
assertEquals(1, searcher.count(newDistanceQuery("field", 18, -65, 50_000)));
reader.close();
writer.close();
dir.close();
}
/** null field name not allowed */
public void testDistanceNull() {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
newDistanceQuery(null, 18, -65, 50_000);
});
assertTrue(expected.getMessage().contains("field must not be null"));
}
/** distance query should not accept invalid lat/lon as origin */
public void testDistanceIllegal() throws Exception {
expectThrows(Exception.class, () -> {
newDistanceQuery("field", 92.0, 181.0, 120000);
});
}
/** negative distance queries are not allowed */
public void testDistanceNegative() {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
newDistanceQuery("field", 18, 19, -1);
});
assertTrue(expected.getMessage().contains("radiusMeters"));
assertTrue(expected.getMessage().contains("invalid"));
}
/** NaN distance queries are not allowed */
public void testDistanceNaN() {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
newDistanceQuery("field", 18, 19, Double.NaN);
});
assertTrue(expected.getMessage().contains("radiusMeters"));
assertTrue(expected.getMessage().contains("invalid"));
}
/** Inf distance queries are not allowed */
public void testDistanceInf() {
IllegalArgumentException expected;
expected = expectThrows(IllegalArgumentException.class, () -> {
newDistanceQuery("field", 18, 19, Double.POSITIVE_INFINITY);
});
assertTrue(expected.getMessage().contains("radiusMeters"));
assertTrue(expected.getMessage().contains("invalid"));
expected = expectThrows(IllegalArgumentException.class, () -> {
newDistanceQuery("field", 18, 19, Double.NEGATIVE_INFINITY);
});
assertTrue(expected.getMessage(), expected.getMessage().contains("radiusMeters"));
assertTrue(expected.getMessage().contains("invalid"));
}
/** test we can search for a polygon */
public void testPolygonBasics() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
// add a doc with a point
Document document = new Document();
addPointToDoc("field", document, 18.313694, -65.227444);
writer.addDocument(document);
// search and verify we found our doc
IndexReader reader = writer.getReader();
IndexSearcher searcher = newSearcher(reader);
assertEquals(1, searcher.count(newPolygonQuery("field", new Polygon(
new double[] { 18, 18, 19, 19, 18 },
new double[] { -66, -65, -65, -66, -66 }))));
reader.close();
writer.close();
dir.close();
}
/** test we can search for a polygon with a hole (but still includes the doc) */
public void testPolygonHole() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
// add a doc with a point
Document document = new Document();
addPointToDoc("field", document, 18.313694, -65.227444);
writer.addDocument(document);
// search and verify we found our doc
IndexReader reader = writer.getReader();
IndexSearcher searcher = newSearcher(reader);
Polygon inner = new Polygon(new double[] { 18.5, 18.5, 18.7, 18.7, 18.5 },
new double[] { -65.7, -65.4, -65.4, -65.7, -65.7 });
Polygon outer = new Polygon(new double[] { 18, 18, 19, 19, 18 },
new double[] { -66, -65, -65, -66, -66 }, inner);
assertEquals(1, searcher.count(newPolygonQuery("field", outer)));
reader.close();
writer.close();
dir.close();
}
/** test we can search for a polygon with a hole (that excludes the doc) */
public void testPolygonHoleExcludes() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
// add a doc with a point
Document document = new Document();
addPointToDoc("field", document, 18.313694, -65.227444);
writer.addDocument(document);
// search and verify we found our doc
IndexReader reader = writer.getReader();
IndexSearcher searcher = newSearcher(reader);
Polygon inner = new Polygon(new double[] { 18.2, 18.2, 18.4, 18.4, 18.2 },
new double[] { -65.3, -65.2, -65.2, -65.3, -65.3 });
Polygon outer = new Polygon(new double[] { 18, 18, 19, 19, 18 },
new double[] { -66, -65, -65, -66, -66 }, inner);
assertEquals(0, searcher.count(newPolygonQuery("field", outer)));
reader.close();
writer.close();
dir.close();
}
/** test we can search for a multi-polygon */
public void testMultiPolygonBasics() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
// add a doc with a point
Document document = new Document();
addPointToDoc("field", document, 18.313694, -65.227444);
writer.addDocument(document);
// search and verify we found our doc
IndexReader reader = writer.getReader();
IndexSearcher searcher = newSearcher(reader);
Polygon a = new Polygon(new double[] { 28, 28, 29, 29, 28 },
new double[] { -56, -55, -55, -56, -56 });
Polygon b = new Polygon(new double[] { 18, 18, 19, 19, 18 },
new double[] { -66, -65, -65, -66, -66 });
assertEquals(1, searcher.count(newPolygonQuery("field", a, b)));
reader.close();
writer.close();
dir.close();
}
/** null field name not allowed */
public void testPolygonNullField() {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
newPolygonQuery(null, new Polygon(
new double[] { 18, 18, 19, 19, 18 },
new double[] { -66, -65, -65, -66, -66 }));
});
assertTrue(expected.getMessage().contains("field must not be null"));
}
// A particularly tricky adversary for BKD tree:
public void testSamePointManyTimes() throws Exception {
int numPoints = atLeast(1000);
// Every doc has 2 points:
double theLat = nextLatitude();
double theLon = nextLongitude();
double[] lats = new double[numPoints];
Arrays.fill(lats, theLat);
double[] lons = new double[numPoints];
Arrays.fill(lons, theLon);
verify(lats, lons);
}
public void testAllLatEqual() throws Exception {
int numPoints = atLeast(10000);
double lat = nextLatitude();
double[] lats = new double[numPoints];
double[] lons = new double[numPoints];
boolean haveRealDoc = false;
for(int docID=0;docID 0 && x == 14 && haveRealDoc) {
int oldDocID;
while (true) {
oldDocID = random().nextInt(docID);
if (Double.isNaN(lats[oldDocID]) == false) {
break;
}
}
// Fully identical point:
lons[docID] = lons[oldDocID];
if (VERBOSE) {
System.out.println(" doc=" + docID + " lat=" + lat + " lon=" + lons[docID] + " (same lat/lon as doc=" + oldDocID + ")");
}
} else {
lons[docID] = nextLongitude();
haveRealDoc = true;
if (VERBOSE) {
System.out.println(" doc=" + docID + " lat=" + lat + " lon=" + lons[docID]);
}
}
lats[docID] = lat;
}
verify(lats, lons);
}
public void testAllLonEqual() throws Exception {
int numPoints = atLeast(10000);
double theLon = nextLongitude();
double[] lats = new double[numPoints];
double[] lons = new double[numPoints];
boolean haveRealDoc = false;
//System.out.println("theLon=" + theLon);
for(int docID=0;docID 0 && x == 14 && haveRealDoc) {
int oldDocID;
while (true) {
oldDocID = random().nextInt(docID);
if (Double.isNaN(lats[oldDocID]) == false) {
break;
}
}
// Fully identical point:
lats[docID] = lats[oldDocID];
if (VERBOSE) {
System.out.println(" doc=" + docID + " lat=" + lats[docID] + " lon=" + theLon + " (same lat/lon as doc=" + oldDocID + ")");
}
} else {
lats[docID] = nextLatitude();
haveRealDoc = true;
if (VERBOSE) {
System.out.println(" doc=" + docID + " lat=" + lats[docID] + " lon=" + theLon);
}
}
lons[docID] = theLon;
}
verify(lats, lons);
}
public void testMultiValued() throws Exception {
int numPoints = atLeast(10000);
// Every doc has 2 points:
double[] lats = new double[2*numPoints];
double[] lons = new double[2*numPoints];
Directory dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig();
// We rely on docID order:
iwc.setMergePolicy(newLogMergePolicy());
// and on seeds being able to reproduce:
iwc.setMergeScheduler(new SerialMergeScheduler());
RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
for (int id=0;id 0 && x < 3 && haveRealDoc) {
int oldID;
while (true) {
oldID = random().nextInt(id);
if (Double.isNaN(lats[oldID]) == false) {
break;
}
}
if (x == 0) {
// Identical lat to old point
lats[id] = lats[oldID];
lons[id] = nextLongitude();
if (VERBOSE) {
System.out.println(" id=" + id + " lat=" + lats[id] + " lon=" + lons[id] + " (same lat as doc=" + oldID + ")");
}
} else if (x == 1) {
// Identical lon to old point
lats[id] = nextLatitude();
lons[id] = lons[oldID];
if (VERBOSE) {
System.out.println(" id=" + id + " lat=" + lats[id] + " lon=" + lons[id] + " (same lon as doc=" + oldID + ")");
}
} else {
assert x == 2;
// Fully identical point:
lats[id] = lats[oldID];
lons[id] = lons[oldID];
if (VERBOSE) {
System.out.println(" id=" + id + " lat=" + lats[id] + " lon=" + lons[id] + " (same lat/lon as doc=" + oldID + ")");
}
}
} else {
lats[id] = nextLatitude();
lons[id] = nextLongitude();
haveRealDoc = true;
if (VERBOSE) {
System.out.println(" id=" + id + " lat=" + lats[id] + " lon=" + lons[id]);
}
}
}
verify(lats, lons);
}
/** Override this to quantize randomly generated lat, so the test won't fail due to quantization errors, which are 1) annoying to debug,
* and 2) should never affect "real" usage terribly. */
protected double quantizeLat(double lat) {
return lat;
}
/** Override this to quantize randomly generated lon, so the test won't fail due to quantization errors, which are 1) annoying to debug,
* and 2) should never affect "real" usage terribly. */
protected double quantizeLon(double lon) {
return lon;
}
protected abstract void addPointToDoc(String field, Document doc, double lat, double lon);
protected abstract Query newRectQuery(String field, double minLat, double maxLat, double minLon, double maxLon);
protected abstract Query newDistanceQuery(String field, double centerLat, double centerLon, double radiusMeters);
protected abstract Query newPolygonQuery(String field, Polygon... polygon);
static final boolean rectContainsPoint(Rectangle rect, double pointLat, double pointLon) {
assert Double.isNaN(pointLat) == false;
if (pointLat < rect.minLat || pointLat > rect.maxLat) {
return false;
}
if (rect.minLon <= rect.maxLon) {
return pointLon >= rect.minLon && pointLon <= rect.maxLon;
} else {
// Rect crosses dateline:
return pointLon <= rect.maxLon || pointLon >= rect.minLon;
}
}
private void verify(double[] lats, double[] lons) throws Exception {
// quantize each value the same way the index does
// NaN means missing for the doc!!!!!
for (int i = 0; i < lats.length; i++) {
if (!Double.isNaN(lats[i])) {
lats[i] = quantizeLat(lats[i]);
}
}
for (int i = 0; i < lons.length; i++) {
if (!Double.isNaN(lons[i])) {
lons[i] = quantizeLon(lons[i]);
}
}
verifyRandomRectangles(lats, lons);
verifyRandomDistances(lats, lons);
verifyRandomPolygons(lats, lons);
}
protected void verifyRandomRectangles(double[] lats, double[] lons) throws Exception {
IndexWriterConfig iwc = newIndexWriterConfig();
// Else seeds may not reproduce:
iwc.setMergeScheduler(new SerialMergeScheduler());
// Else we can get O(N^2) merging:
int mbd = iwc.getMaxBufferedDocs();
if (mbd != -1 && mbd < lats.length/100) {
iwc.setMaxBufferedDocs(lats.length/100);
}
Directory dir;
if (lats.length > 100000) {
dir = newFSDirectory(createTempDir(getClass().getSimpleName()));
} else {
dir = newDirectory();
}
Set deleted = new HashSet<>();
// RandomIndexWriter is too slow here:
IndexWriter w = new IndexWriter(dir, iwc);
for(int id=0;id 0 && random().nextInt(100) == 42) {
int idToDelete = random().nextInt(id);
w.deleteDocuments(new Term("id", ""+idToDelete));
deleted.add(idToDelete);
if (VERBOSE) {
System.out.println(" delete id=" + idToDelete);
}
}
}
if (random().nextBoolean()) {
w.forceMerge(1);
}
final IndexReader r = DirectoryReader.open(w);
w.close();
IndexSearcher s = newSearcher(r);
int iters = atLeast(25);
NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
Bits liveDocs = MultiFields.getLiveDocs(s.getIndexReader());
int maxDoc = s.getIndexReader().maxDoc();
for (int iter=0;iter 100000) {
dir = newFSDirectory(createTempDir(getClass().getSimpleName()));
} else {
dir = newDirectory();
}
Set deleted = new HashSet<>();
// RandomIndexWriter is too slow here:
IndexWriter w = new IndexWriter(dir, iwc);
for(int id=0;id 0 && random().nextInt(100) == 42) {
int idToDelete = random().nextInt(id);
w.deleteDocuments(new Term("id", ""+idToDelete));
deleted.add(idToDelete);
if (VERBOSE) {
System.out.println(" delete id=" + idToDelete);
}
}
}
if (random().nextBoolean()) {
w.forceMerge(1);
}
final IndexReader r = DirectoryReader.open(w);
w.close();
IndexSearcher s = newSearcher(r);
int iters = atLeast(25);
NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
Bits liveDocs = MultiFields.getLiveDocs(s.getIndexReader());
int maxDoc = s.getIndexReader().maxDoc();
for (int iter=0;iter 100000) {
dir = newFSDirectory(createTempDir(getClass().getSimpleName()));
} else {
dir = newDirectory();
}
Set deleted = new HashSet<>();
// RandomIndexWriter is too slow here:
IndexWriter w = new IndexWriter(dir, iwc);
for(int id=0;id 0 && random().nextInt(100) == 42) {
int idToDelete = random().nextInt(id);
w.deleteDocuments(new Term("id", ""+idToDelete));
deleted.add(idToDelete);
if (VERBOSE) {
System.out.println(" delete id=" + idToDelete);
}
}
}
if (random().nextBoolean()) {
w.forceMerge(1);
}
final IndexReader r = DirectoryReader.open(w);
w.close();
// We can't wrap with "exotic" readers because points needs to work:
IndexSearcher s = newSearcher(r);
final int iters = atLeast(75);
NumericDocValues docIDToID = MultiDocValues.getNumericValues(r, "id");
Bits liveDocs = MultiFields.getLiveDocs(s.getIndexReader());
int maxDoc = s.getIndexReader().maxDoc();
for (int iter=0;iter © 2015 - 2025 Weber Informatics LLC | Privacy Policy