org.heigit.ohsome.ohsomeapi.executor.ExecutionUtils Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of ohsome-api Show documentation
Show all versions of ohsome-api Show documentation
A public Web-RESTful-API for "ohsome" OpenStreetMap history data.
package org.heigit.ohsome.ohsomeapi.executor;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.core.JsonEncoding;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.util.DefaultIndenter;
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.opencsv.CSVWriter;
import com.zaxxer.hikari.HikariDataSource;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.sql.SQLException;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.util.EnumSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Stream;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.geojson.GeoJsonObject;
import org.heigit.bigspatialdata.oshdb.api.generic.OSHDBCombinedIndex;
import org.heigit.bigspatialdata.oshdb.api.generic.function.SerializableFunction;
import org.heigit.bigspatialdata.oshdb.api.generic.function.SerializableSupplier;
import org.heigit.bigspatialdata.oshdb.api.mapreducer.MapAggregator;
import org.heigit.bigspatialdata.oshdb.api.mapreducer.MapReducer;
import org.heigit.bigspatialdata.oshdb.api.object.OSMContribution;
import org.heigit.bigspatialdata.oshdb.api.object.OSMEntitySnapshot;
import org.heigit.bigspatialdata.oshdb.osm.OSMEntity;
import org.heigit.bigspatialdata.oshdb.osm.OSMType;
import org.heigit.bigspatialdata.oshdb.util.OSHDBBoundingBox;
import org.heigit.bigspatialdata.oshdb.util.OSHDBTag;
import org.heigit.bigspatialdata.oshdb.util.OSHDBTimestamp;
import org.heigit.bigspatialdata.oshdb.util.celliterator.ContributionType;
import org.heigit.bigspatialdata.oshdb.util.exceptions.OSHDBKeytablesNotFoundException;
import org.heigit.bigspatialdata.oshdb.util.geometry.Geo;
import org.heigit.bigspatialdata.oshdb.util.geometry.OSHDBGeometryBuilder;
import org.heigit.bigspatialdata.oshdb.util.tagtranslator.OSMTag;
import org.heigit.bigspatialdata.oshdb.util.tagtranslator.TagTranslator;
import org.heigit.bigspatialdata.oshdb.util.time.TimestampFormatter;
import org.heigit.ohsome.filter.FilterExpression;
import org.heigit.ohsome.ohsomeapi.Application;
import org.heigit.ohsome.ohsomeapi.controller.rawdata.ElementsGeometry;
import org.heigit.ohsome.ohsomeapi.exception.DatabaseAccessException;
import org.heigit.ohsome.ohsomeapi.exception.ExceptionMessages;
import org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessor;
import org.heigit.ohsome.ohsomeapi.inputprocessing.ProcessingData;
import org.heigit.ohsome.ohsomeapi.inputprocessing.SimpleFeatureType;
import org.heigit.ohsome.ohsomeapi.oshdb.DbConnData;
import org.heigit.ohsome.ohsomeapi.oshdb.ExtractMetadata;
import org.heigit.ohsome.ohsomeapi.output.Description;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.Attribution;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.Metadata;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.RatioResponse;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.RatioResult;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.Response;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.Result;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.elements.ElementsResult;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.groupbyresponse.GroupByObject;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.groupbyresponse.GroupByResult;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.groupbyresponse.RatioGroupByBoundaryResponse;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.groupbyresponse.RatioGroupByResult;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.users.UsersResult;
import org.heigit.ohsome.ohsomeapi.output.rawdataresponse.DataResponse;
import org.heigit.ohsome.ohsomeapi.utils.GroupByBoundaryGeoJsonGenerator;
import org.heigit.ohsome.ohsomeapi.utils.RequestUtils;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.Lineal;
import org.locationtech.jts.geom.Polygonal;
import org.locationtech.jts.geom.Puntal;
import org.wololo.geojson.LineString;
import org.wololo.geojson.Point;
import org.wololo.geojson.Polygon;
import org.wololo.jts2geojson.GeoJSONWriter;
/** Holds helper methods that are used by the executor classes. */
@RequiredArgsConstructor
public class ExecutionUtils {
private AtomicReference isFirst;
private final ProcessingData processingData;
private final DecimalFormat ratioDf = defineDecimalFormat("#.######");
private static final Point emptyPoint = new Point(new double[0]);
private static final LineString emptyLine = new LineString(new double[0][0]);
private static final Polygon emptyPolygon = new Polygon(new double[0][0][0]);
/** Applies a filter on the given MapReducer object using the given parameters. */
public MapReducer snapshotFilter(MapReducer mapRed,
Set osmTypes1, Set osmTypes2, Set simpleFeatureTypes1,
Set simpleFeatureTypes2, Integer[] keysInt1, Integer[] keysInt2,
Integer[] valuesInt1, Integer[] valuesInt2) {
return mapRed.filter(
snapshot -> snapshotMatches(snapshot, osmTypes1, simpleFeatureTypes1, keysInt1, valuesInt1)
|| snapshotMatches(snapshot, osmTypes2, simpleFeatureTypes2, keysInt2, valuesInt2));
}
/**
* Applies a filter on the given MapReducer object using the given parameters. Used in
* /ratio/groupBy/boundary requests.
*/
public MapAggregator, OSMEntitySnapshot> snapshotFilter(
MapAggregator, OSMEntitySnapshot> mapRed,
Set osmTypes1, Set osmTypes2, Set simpleFeatureTypes1,
Set simpleFeatureTypes2, Integer[] keysInt1, Integer[] keysInt2,
Integer[] valuesInt1, Integer[] valuesInt2) {
return mapRed.filter(
snapshot -> snapshotMatches(snapshot, osmTypes1, simpleFeatureTypes1, keysInt1, valuesInt1)
|| snapshotMatches(snapshot, osmTypes2, simpleFeatureTypes2, keysInt2, valuesInt2));
}
/** Applies a filter on the given MapReducer object using the given filter expressions. */
public MapReducer newSnapshotFilter(MapReducer mapRed,
FilterExpression filterExpr1, FilterExpression filterExpr2) {
return mapRed.filter(snapshot -> {
OSMEntity entity = snapshot.getEntity();
return filterExpr1.applyOSMGeometry(entity, snapshot.getGeometry())
|| filterExpr2.applyOSMGeometry(entity, snapshot.getGeometry());
});
}
/**
* Applies a filter on the given MapReducer object using the given filter expressions. Used in
* /ratio/groupBy/boundary requests.
*/
public MapAggregator, OSMEntitySnapshot> newSnapshotFilter(
MapAggregator, OSMEntitySnapshot> mapRed,
FilterExpression filterExpr1, FilterExpression filterExpr2) {
return mapRed.filter(snapshot -> {
OSMEntity entity = snapshot.getEntity();
return filterExpr1.applyOSMGeometry(entity, snapshot.getGeometry())
|| filterExpr2.applyOSMGeometry(entity, snapshot.getGeometry());
});
}
/** Compares the type(s) and tag(s) of the given snapshot to the given types|tags. */
public boolean snapshotMatches(OSMEntitySnapshot snapshot, Set osmTypes,
Set simpleFeatureTypes, Integer[] keysInt, Integer[] valuesInt) {
boolean matchesTags = true;
OSMEntity entity = snapshot.getEntity();
if (osmTypes.contains(entity.getType())) {
for (int i = 0; i < keysInt.length; i++) {
boolean matchesTag;
if (i < valuesInt.length) {
matchesTag = entity.hasTagValue(keysInt[i], valuesInt[i]);
} else {
matchesTag = entity.hasTagKey(keysInt[i]);
}
if (!matchesTag) {
matchesTags = false;
break;
}
}
} else {
matchesTags = false;
}
if (!simpleFeatureTypes.isEmpty()) {
boolean[] simpleFeatures = setRequestedSimpleFeatures(simpleFeatureTypes);
return matchesTags && (simpleFeatures[0] && snapshot.getGeometry() instanceof Puntal
|| simpleFeatures[1] && snapshot.getGeometry() instanceof Lineal
|| simpleFeatures[2] && snapshot.getGeometry() instanceof Polygonal || simpleFeatures[3]
&& "GeometryCollection".equalsIgnoreCase(snapshot.getGeometry().getGeometryType()));
}
return matchesTags;
}
/**
* Defines a certain decimal format.
*
* @param format String
defining the format (e.g.: "#.####" for getting 4 digits
* after the comma)
* @return DecimalFormat
object with the defined format.
*/
public static DecimalFormat defineDecimalFormat(String format) {
DecimalFormatSymbols otherSymbols = new DecimalFormatSymbols(Locale.getDefault());
otherSymbols.setDecimalSeparator('.');
return new DecimalFormat(format, otherSymbols);
}
/**
* Caches the given mapper value in the user data of the Geometry
object.
*
* @param geom Geometry
of an OSMEntitySnapshot object
* @param mapper arbitrary function that returns a time-independent value from a snapshot object,
* for example lenght, area, perimeter
* @return evaluated mapper function or cached value stored in the user data of the
* Geometry
object
*/
public static Double cacheInUserData(Geometry geom, SerializableSupplier mapper) {
if (geom.getUserData() == null) {
geom.setUserData(mapper.get());
}
return (Double) geom.getUserData();
}
/**
* Adapted helper function, which works like
* {@link org.heigit.bigspatialdata.oshdb.api.generic.OSHDBCombinedIndex#nest(Map) nest} but has
* switched <U> and <V> parameters.
*
* @param result the "flat" result data structure that should be converted to a nested structure
* @param an arbitrary data type, used for the data value items
* @param an arbitrary data type, used for the index'es key items
* @param an arbitrary data type, used for the index'es key items
* @return a nested data structure: for each index part there is a separate level of nested maps
*/
public static & Serializable, V extends Comparable & Serializable> SortedMap> nest(
Map, A> result) {
TreeMap> ret = new TreeMap<>();
result.forEach((index, data) -> {
if (!ret.containsKey(index.getSecondIndex())) {
ret.put(index.getSecondIndex(), new TreeMap<>());
}
ret.get(index.getSecondIndex()).put(index.getFirstIndex(), data);
});
return ret;
}
/**
* Streams the result of /elements, /elementsFullHistory and /contributions endpoints as an
* outputstream.
*
* @throws RuntimeException which only wraps {@link java.io.IOException IOException}
* @throws IOException thrown by {@link JsonGenerator
* com.fasterxml.jackson.core.JsonFactory#createGenerator(java.io.OutputStream,
* JsonEncoding) createGenerator},
* {@link com.fasterxml.jackson.core.JsonGenerator#writeObject(Object) writeObject},
* {@link javax.servlet.ServletResponse#getOutputStream() getOutputStream},
* {@link java.io.OutputStream#write(byte[]) write},
* {@link org.heigit.ohsome.ohsomeapi.executor.ExecutionUtils#writeStreamResponse(ThreadLocal, Stream, ThreadLocal, ServletOutputStream)
* writeStreamResponse}, {@link javax.servlet.ServletOutputStream#print(String) print},
* and {@link javax.servlet.ServletResponse#flushBuffer() flushBuffer}
* @throws ExecutionException thrown by
* {@link org.heigit.ohsome.ohsomeapi.executor.ExecutionUtils#writeStreamResponse(ThreadLocal, Stream, ThreadLocal, ServletOutputStream)
* writeStreamResponse}
* @throws InterruptedException thrown by
* {@link org.heigit.ohsome.ohsomeapi.executor.ExecutionUtils#writeStreamResponse(ThreadLocal, Stream, ThreadLocal, ServletOutputStream)
* writeStreamResponse}
*/
public void streamResponse(HttpServletResponse servletResponse, DataResponse osmData,
Stream resultStream) throws Exception {
JsonFactory jsonFactory = new JsonFactory();
ByteArrayOutputStream tempStream = new ByteArrayOutputStream();
ObjectMapper objMapper = new ObjectMapper();
objMapper.enable(SerializationFeature.INDENT_OUTPUT);
objMapper.setSerializationInclusion(Include.NON_NULL);
jsonFactory.createGenerator(tempStream, JsonEncoding.UTF8).setCodec(objMapper)
.writeObject(osmData);
String scaffold =
tempStream.toString(StandardCharsets.UTF_8).replaceFirst("\\s*]\\s*}\\s*$", "");
servletResponse.setContentType("application/geo+json; charset=utf-8");
ServletOutputStream outputStream = servletResponse.getOutputStream();
outputStream.write(scaffold.getBytes(StandardCharsets.UTF_8));
ThreadLocal outputBuffers =
ThreadLocal.withInitial(ByteArrayOutputStream::new);
ThreadLocal outputJsonGen = ThreadLocal.withInitial(() -> {
try {
DefaultPrettyPrinter.Indenter indenter = new DefaultIndenter() {
@Override
public void writeIndentation(JsonGenerator g, int level) throws IOException {
super.writeIndentation(g, level + 1);
}
};
DefaultPrettyPrinter printer =
new DefaultPrettyPrinter("").withArrayIndenter(indenter).withObjectIndenter(indenter);
return jsonFactory.createGenerator(outputBuffers.get(), JsonEncoding.UTF8)
.setCodec(objMapper).setPrettyPrinter(printer);
} catch (IOException e) {
throw new RuntimeException(e);
}
});
isFirst = new AtomicReference<>(true);
writeStreamResponse(outputJsonGen, resultStream, outputBuffers, outputStream);
outputStream.print("]\n}\n");
servletResponse.flushBuffer();
}
/** Writes a response in the csv format for /groupBy requests. */
public void writeCsvResponse(GroupByObject[] resultSet, HttpServletResponse servletResponse,
List comments) {
try {
servletResponse = setCsvSettingsInServletResponse(servletResponse);
CSVWriter writer = writeComments(servletResponse, comments);
Pair, List> rows;
if (resultSet instanceof GroupByResult[]) {
if (resultSet.length == 0) {
writer.writeNext(new String[] {"timestamp"}, false);
writer.close();
return;
} else {
GroupByResult result = (GroupByResult) resultSet[0];
if (result.getResult() instanceof UsersResult[]) {
rows = createCsvResponseForUsersGroupBy(resultSet);
} else {
rows = createCsvResponseForElementsGroupBy(resultSet);
}
}
} else {
rows = createCsvResponseForElementsRatioGroupBy(resultSet);
}
writer.writeNext(rows.getLeft().toArray(new String[rows.getLeft().size()]), false);
writer.writeAll(rows.getRight(), false);
writer.close();
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* Writes a response in the csv format for /count|length|perimeter|area(/density)(/ratio)
* requests.
*/
public void writeCsvResponse(Result[] resultSet, HttpServletResponse servletResponse,
List comments) {
try {
servletResponse = setCsvSettingsInServletResponse(servletResponse);
CSVWriter writer = writeComments(servletResponse, comments);
if (resultSet instanceof ElementsResult[]) {
writer.writeNext(new String[] {"timestamp", "value"}, false);
for (Result result : resultSet) {
ElementsResult elementsResult = (ElementsResult) result;
writer.writeNext(new String[] {elementsResult.getTimestamp(),
String.valueOf(elementsResult.getValue())});
}
} else if (resultSet instanceof UsersResult[]) {
writer.writeNext(new String[] {"fromTimestamp", "toTimestamp", "value"}, false);
for (Result result : resultSet) {
UsersResult usersResult = (UsersResult) result;
writer.writeNext(new String[] {usersResult.getFromTimestamp(),
usersResult.getToTimestamp(), String.valueOf(usersResult.getValue())});
}
} else if (resultSet instanceof RatioResult[]) {
writer.writeNext(new String[] {"timestamp", "value", "value2", "ratio"}, false);
for (Result result : resultSet) {
RatioResult ratioResult = (RatioResult) result;
writer.writeNext(
new String[] {ratioResult.getTimestamp(), String.valueOf(ratioResult.getValue()),
String.valueOf(ratioResult.getValue2()), String.valueOf(ratioResult.getRatio())});
}
}
writer.close();
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* Sets the boolean values for the respective simple feature type: 0 = hasPoint, 1 = hasLine, 2 =
* hasPolygon, 3 = hasOther.
*/
public boolean[] setRequestedSimpleFeatures(Set simpleFeatureTypes) {
boolean[] simpleFeatureArray = new boolean[] {false, false, false, false};
for (SimpleFeatureType type : simpleFeatureTypes) {
if (type.equals(SimpleFeatureType.POINT)) {
simpleFeatureArray[0] = true;
} else if (type.equals(SimpleFeatureType.LINE)) {
simpleFeatureArray[1] = true;
} else if (type.equals(SimpleFeatureType.POLYGON)) {
simpleFeatureArray[2] = true;
} else if (type.equals(SimpleFeatureType.OTHER)) {
simpleFeatureArray[3] = true;
}
}
return simpleFeatureArray;
}
/** Creates the comments of the csv response (Attribution, API-Version and optional Metadata). */
public List createCsvTopComments(String url, String text, String apiVersion,
Metadata metadata) {
List comments = new LinkedList<>();
comments.add(new String[] {"# Copyright URL: " + url});
comments.add(new String[] {"# Copyright Text: " + text});
comments.add(new String[] {"# API Version: " + apiVersion});
if (metadata != null) {
comments.add(new String[] {"# Execution Time: " + metadata.getExecutionTime()});
comments.add(new String[] {"# Description: " + metadata.getDescription()});
if (metadata.getRequestUrl() != null) {
comments.add(new String[] {"# Request URL: " + metadata.getRequestUrl()});
}
}
return comments;
}
/** Creates the Feature
objects in the OSM data response. */
public org.wololo.geojson.Feature createOSMFeature(OSMEntity entity, Geometry geometry,
Map properties, int[] keysInt, boolean includeTags,
boolean includeOSMMetadata, boolean isContributionsEndpoint, ElementsGeometry elemGeom,
EnumSet contributionTypes) {
if (geometry.isEmpty() && !contributionTypes.contains(ContributionType.DELETION)) {
// skip invalid geometries (e.g. ways with 0 nodes)
return null;
}
if (includeTags) {
for (OSHDBTag oshdbTag : entity.getTags()) {
properties.put(String.valueOf(oshdbTag.getKey()), oshdbTag);
}
} else if (keysInt.length != 0) {
int[] tags = entity.getRawTags();
for (int i = 0; i < tags.length; i += 2) {
int tagKeyId = tags[i];
int tagValueId = tags[i + 1];
for (int key : keysInt) {
if (tagKeyId == key) {
properties.put(String.valueOf(tagKeyId), new OSHDBTag(tagKeyId, tagValueId));
break;
}
}
}
}
if (includeOSMMetadata) {
properties =
addAdditionalProperties(entity, properties, isContributionsEndpoint, contributionTypes);
}
properties.put("@osmId", entity.getType().toString().toLowerCase() + "/" + entity.getId());
GeoJSONWriter gjw = new GeoJSONWriter();
boolean deletionHandling =
isContributionsEndpoint && contributionTypes.contains(ContributionType.DELETION);
Geometry outputGeometry;
switch (elemGeom) {
case BBOX:
if (deletionHandling) {
return new org.wololo.geojson.Feature(emptyPolygon, properties);
}
Envelope envelope = geometry.getEnvelopeInternal();
OSHDBBoundingBox bbox = OSHDBGeometryBuilder.boundingBoxOf(envelope);
outputGeometry = OSHDBGeometryBuilder.getGeometry(bbox);
break;
case CENTROID:
if (deletionHandling) {
return new org.wololo.geojson.Feature(emptyPoint, properties);
}
outputGeometry = geometry.getCentroid();
break;
case RAW:
default:
if (deletionHandling && geometry.getGeometryType().contains("Polygon")) {
return new org.wololo.geojson.Feature(emptyPolygon, properties);
}
if (deletionHandling && geometry.getGeometryType().contains("LineString")) {
return new org.wololo.geojson.Feature(emptyLine, properties);
}
if (deletionHandling && geometry.getGeometryType().contains("Point")) {
return new org.wololo.geojson.Feature(emptyPoint, properties);
}
outputGeometry = geometry;
}
return new org.wololo.geojson.Feature(gjw.write(outputGeometry), properties);
}
/**
* Computes the result depending on the RequestResource
using a
* MapAggregator
object as input and returning a SortedMap
.
*
* @throws Exception thrown by
* {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapAggregator#count() count}, and
* {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapAggregator#sum() sum}
*/
@SuppressWarnings({"unchecked"}) // intentionally suppressed as type format is valid
public & Serializable, V extends Number> SortedMap, V> computeResult(
RequestResource requestResource,
MapAggregator, OSMEntitySnapshot> preResult)
throws Exception {
switch (requestResource) {
case COUNT:
return (SortedMap, V>) preResult.count();
case LENGTH:
return (SortedMap, V>) preResult
.sum((SerializableFunction) snapshot -> cacheInUserData(
snapshot.getGeometry(), () -> Geo.lengthOf(snapshot.getGeometry())));
case PERIMETER:
return (SortedMap, V>) preResult
.sum((SerializableFunction) snapshot -> {
if (snapshot.getGeometry() instanceof Polygonal) {
return cacheInUserData(snapshot.getGeometry(),
() -> Geo.lengthOf(snapshot.getGeometry().getBoundary()));
}
return 0.0;
});
case AREA:
return (SortedMap, V>) preResult
.sum((SerializableFunction) snapshot -> cacheInUserData(
snapshot.getGeometry(), () -> Geo.areaOf(snapshot.getGeometry())));
default:
return null;
}
}
/**
* Computes the result depending on the RequestResource
using a
* MapAggregator
object as input and returning a SortedMap
.
*/
@SuppressWarnings({"unchecked"}) // intentionally suppressed as type format is valid
public & Serializable, V extends Number> SortedMap, OSHDBTimestamp>, V> computeNestedResult(
RequestResource requestResource,
MapAggregator, OSHDBTimestamp>, Geometry> preResult)
throws Exception {
switch (requestResource) {
case COUNT:
return (SortedMap, OSHDBTimestamp>, V>) preResult
.count();
case PERIMETER:
return (SortedMap, OSHDBTimestamp>, V>) preResult
.sum(geom -> {
if (!(geom instanceof Polygonal)) {
return 0.0;
}
return cacheInUserData(geom, () -> Geo.lengthOf(geom.getBoundary()));
});
case LENGTH:
return (SortedMap, OSHDBTimestamp>, V>) preResult
.sum(geom -> cacheInUserData(geom, () -> Geo.lengthOf(geom)));
case AREA:
return (SortedMap, OSHDBTimestamp>, V>) preResult
.sum(geom -> cacheInUserData(geom, () -> Geo.areaOf(geom)));
default:
return null;
}
}
/**
* Extracts the tags from the given OSMContribution
depending on the
* ContributionType
.
*/
public int[] extractContributionTags(OSMContribution contrib) {
int[] tags;
if (contrib.getContributionTypes().contains(ContributionType.DELETION)) {
tags = contrib.getEntityBefore().getRawTags();
} else if (contrib.getContributionTypes().contains(ContributionType.CREATION)) {
tags = contrib.getEntityAfter().getRawTags();
} else {
int[] tagsBefore = contrib.getEntityBefore().getRawTags();
int[] tagsAfter = contrib.getEntityAfter().getRawTags();
tags = new int[tagsBefore.length + tagsAfter.length];
System.arraycopy(tagsBefore, 0, tags, 0, tagsBefore.length);
System.arraycopy(tagsAfter, 0, tags, tagsBefore.length, tagsAfter.length);
}
return tags;
}
/** Fills the ElementsResult array with respective ElementsResult objects. */
public ElementsResult[] fillElementsResult(SortedMap entryVal,
boolean isDensity, DecimalFormat df, Geometry geom) {
ElementsResult[] results = new ElementsResult[entryVal.entrySet().size()];
int count = 0;
for (Entry entry : entryVal.entrySet()) {
if (isDensity) {
results[count] = new ElementsResult(
TimestampFormatter.getInstance().isoDateTime(entry.getKey()), Double.parseDouble(
df.format(entry.getValue().doubleValue() / (Geo.areaOf(geom) * 0.000001))));
} else {
results[count] =
new ElementsResult(TimestampFormatter.getInstance().isoDateTime(entry.getKey()),
Double.parseDouble(df.format(entry.getValue().doubleValue())));
}
count++;
}
return results;
}
/** Fills the UsersResult array with respective UsersResult objects. */
public UsersResult[] fillUsersResult(SortedMap entryVal,
boolean isDensity, InputProcessor inputProcessor, DecimalFormat df, Geometry geom) {
UsersResult[] results = new UsersResult[entryVal.entrySet().size()];
int count = 0;
String[] toTimestamps = inputProcessor.getUtils().getToTimestamps();
for (Entry entry : entryVal.entrySet()) {
if (isDensity) {
results[count] =
new UsersResult(TimestampFormatter.getInstance().isoDateTime(entry.getKey()),
toTimestamps[count + 1], Double.parseDouble(
df.format(entry.getValue().doubleValue() / (Geo.areaOf(geom) / 1000000))));
} else {
results[count] = new UsersResult(
TimestampFormatter.getInstance().isoDateTime(entry.getKey()), toTimestamps[count + 1],
Double.parseDouble(df.format(entry.getValue().doubleValue())));
}
count++;
}
return results;
}
/**
* Fills the result value arrays for the ratio/groupBy/boundary response.
*
* @param resultSet Set
containing the result values
* @param df DecimalFormat
defining the number of digits of the result values
* @return Double[]
containing the formatted result values
*/
public Double[] fillElementsRatioGroupByBoundaryResultValues(
Set extends Entry, ? extends Number>> resultSet,
DecimalFormat df) {
Double[] resultValues = new Double[resultSet.size()];
int valueCount = 0;
for (Entry, ? extends Number> innerEntry : resultSet) {
resultValues[valueCount] = Double.parseDouble(df.format(innerEntry.getValue().doubleValue()));
valueCount++;
}
return resultValues;
}
/**
* Maps the given OSMEntitySnapshot
to a given tag, or to the remainder (having -1,
* -1 as identifier) if none of the given tags is included.
*
* @param keysInt int value of the groupByKey parameter
* @param valuesInt Integer[] of the groupByValues parameter
* @param f OSMEntitySnapshot
* @return nested Pair
containing the integer values of the tag category and the
* OSMEntitySnapshot
*/
public Pair, OSMEntitySnapshot> mapSnapshotToTags(int keysInt,
Integer[] valuesInt, OSMEntitySnapshot f) {
int[] tags = f.getEntity().getRawTags();
for (int i = 0; i < tags.length; i += 2) {
int tagKeyId = tags[i];
int tagValueId = tags[i + 1];
if (tagKeyId == keysInt) {
if (valuesInt.length == 0) {
return new ImmutablePair<>(new ImmutablePair<>(tagKeyId, tagValueId), f);
}
for (int value : valuesInt) {
if (tagValueId == value) {
return new ImmutablePair<>(new ImmutablePair<>(tagKeyId, tagValueId), f);
}
}
}
}
return new ImmutablePair<>(new ImmutablePair<>(-1, -1), f);
}
/** Creates a RatioResponse. */
public Response createRatioResponse(String[] timeArray, Double[] value1, Double[] value2,
long startTime, RequestResource reqRes, String requestUrl,
HttpServletResponse servletResponse) {
RatioResult[] resultSet = new RatioResult[timeArray.length];
for (int i = 0; i < timeArray.length; i++) {
double ratio = value2[i] / value1[i];
// in case ratio has the values "NaN", "Infinity", etc.
try {
ratio = Double.parseDouble(ratioDf.format(ratio));
} catch (Exception e) {
// do nothing --> just return ratio without rounding (trimming)
}
resultSet[i] = new RatioResult(timeArray[i], value1[i], value2[i], ratio);
}
Metadata metadata = null;
if (processingData.isShowMetadata()) {
long duration = System.currentTimeMillis() - startTime;
metadata = new Metadata(duration,
Description.aggregateRatio(reqRes.getDescription(), reqRes.getUnit()), requestUrl);
}
RequestParameters requestParameters = processingData.getRequestParameters();
if ("csv".equalsIgnoreCase(requestParameters.getFormat())) {
writeCsvResponse(resultSet, servletResponse, createCsvTopComments(ElementsRequestExecutor.URL,
ElementsRequestExecutor.TEXT, Application.API_VERSION, metadata));
return null;
}
return new RatioResponse(
new Attribution(ExtractMetadata.attributionUrl, ExtractMetadata.attributionShort),
Application.API_VERSION, metadata, resultSet);
}
/** Creates a RatioGroupByBoundaryResponse. */
public Response createRatioGroupByBoundaryResponse(Object[] boundaryIds, String[] timeArray,
Double[] resultValues1, Double[] resultValues2, long startTime, RequestResource reqRes,
String requestUrl, HttpServletResponse servletResponse) {
Metadata metadata = null;
int boundaryIdsLength = boundaryIds.length;
int timeArrayLenth = timeArray.length;
RatioGroupByResult[] groupByResultSet = new RatioGroupByResult[boundaryIdsLength];
for (int i = 0; i < boundaryIdsLength; i++) {
Object groupByName = boundaryIds[i];
RatioResult[] ratioResultSet = new RatioResult[timeArrayLenth];
int innerCount = 0;
for (int j = i; j < timeArrayLenth * boundaryIdsLength; j += boundaryIdsLength) {
double ratio = resultValues2[j] / resultValues1[j];
// in case ratio has the values "NaN", "Infinity", etc.
try {
ratio = Double.parseDouble(ratioDf.format(ratio));
} catch (Exception e) {
// do nothing --> just return ratio without rounding (trimming)
}
ratioResultSet[innerCount] =
new RatioResult(timeArray[innerCount], resultValues1[j], resultValues2[j], ratio);
innerCount++;
}
groupByResultSet[i] = new RatioGroupByResult(groupByName, ratioResultSet);
}
if (processingData.isShowMetadata()) {
long duration = System.currentTimeMillis() - startTime;
metadata = new Metadata(duration,
Description.aggregateRatioGroupByBoundary(reqRes.getDescription(), reqRes.getUnit()),
requestUrl);
}
RequestParameters requestParameters = processingData.getRequestParameters();
Attribution attribution =
new Attribution(ExtractMetadata.attributionUrl, ExtractMetadata.attributionShort);
if ("geojson".equalsIgnoreCase(requestParameters.getFormat())) {
GeoJsonObject[] geoJsonGeoms = processingData.getGeoJsonGeoms();
return RatioGroupByBoundaryResponse.of(attribution, Application.API_VERSION, metadata,
"FeatureCollection",
GroupByBoundaryGeoJsonGenerator.createGeoJsonFeatures(groupByResultSet, geoJsonGeoms));
} else if ("csv".equalsIgnoreCase(requestParameters.getFormat())) {
writeCsvResponse(groupByResultSet, servletResponse,
createCsvTopComments(ElementsRequestExecutor.URL, ElementsRequestExecutor.TEXT,
Application.API_VERSION, metadata));
return null;
}
return new RatioGroupByBoundaryResponse(attribution, Application.API_VERSION, metadata,
groupByResultSet);
}
/**
* Extracts and returns a geometry out of the given contribution. The boolean values specify if it
* should be clipped/unclipped and if the geometry before/after a contribution should be taken.
*/
public Geometry getGeometry(OSMContribution contribution, boolean clipGeometries,
boolean before) {
Geometry geom = null;
if (clipGeometries) {
if (before) {
geom = contribution.getGeometryBefore();
} else {
geom = contribution.getGeometryAfter();
}
} else {
if (before) {
geom = contribution.getGeometryUnclippedBefore();
} else {
geom = contribution.getGeometryUnclippedAfter();
}
}
return geom;
}
/** Combines the two given filters with an OR operation. Used in /ratio computation. */
public String combineFiltersWithOr(String firstFilter, String secondFilter) {
if (firstFilter.isBlank() || secondFilter.isBlank()) {
// definition of an empty combined filter if filter1 or filter2 is empty
return "";
}
return "(" + firstFilter + ") or (" + secondFilter + ")";
}
/**
* Creates the csv response for /elements/_/groupBy requests.
*
* @param resultSet GroupByObject
array containing GroupByResult
objects
* containing ElementsResult
objects
* @return Pair
containing the column names (left) and the data rows (right)
*/
private ImmutablePair, List> createCsvResponseForElementsGroupBy(
GroupByObject[] resultSet) {
List columnNames = new LinkedList<>();
columnNames.add("timestamp");
List rows = new LinkedList<>();
for (int i = 0; i < resultSet.length; i++) {
GroupByResult groupByResult = (GroupByResult) resultSet[i];
Object groupByObject = groupByResult.getGroupByObjectId();
if (groupByObject instanceof Object[]) {
Object[] groupByObjectArr = (Object[]) groupByObject;
columnNames.add(groupByObjectArr[0].toString() + "_" + groupByObjectArr[1].toString());
} else {
columnNames.add(groupByObject.toString());
}
for (int j = 0; j < groupByResult.getResult().length; j++) {
ElementsResult elemResult = (ElementsResult) groupByResult.getResult()[j];
if (i == 0) {
String[] row = new String[resultSet.length + 1];
row[0] = elemResult.getTimestamp();
row[1] = String.valueOf(elemResult.getValue());
rows.add(row);
} else {
rows.get(j)[i + 1] = String.valueOf(elemResult.getValue());
}
}
}
return new ImmutablePair<>(columnNames, rows);
}
/**
* Creates the csv response for /elements/_/ratio/groupBy requests.
*
* @param resultSet GroupByObject
array containing RatioGroupByResult
* objects containing RatioResult
objects
* @return Pair
containing the column names (left) and the data rows (right)
*/
private ImmutablePair, List> createCsvResponseForElementsRatioGroupBy(
GroupByObject[] resultSet) {
List columnNames = new LinkedList<>();
columnNames.add("timestamp");
List rows = new LinkedList<>();
for (int i = 0; i < resultSet.length; i++) {
RatioGroupByResult ratioGroupByResult = (RatioGroupByResult) resultSet[i];
columnNames.add(ratioGroupByResult.getGroupByObjectId() + "_value");
columnNames.add(ratioGroupByResult.getGroupByObjectId() + "_value2");
columnNames.add(ratioGroupByResult.getGroupByObjectId() + "_ratio");
for (int j = 0; j < ratioGroupByResult.getRatioResult().length; j++) {
RatioResult ratioResult = ratioGroupByResult.getRatioResult()[j];
if (i == 0) {
String[] row = new String[resultSet.length * 3 + 1];
row[0] = ratioResult.getTimestamp();
row[1] = String.valueOf(ratioResult.getValue());
row[2] = String.valueOf(ratioResult.getValue2());
row[3] = String.valueOf(ratioResult.getRatio());
rows.add(row);
} else {
int count = i * 3 + 1;
rows.get(j)[count] = String.valueOf(ratioResult.getValue());
rows.get(j)[count + 1] = String.valueOf(ratioResult.getValue2());
rows.get(j)[count + 2] = String.valueOf(ratioResult.getRatio());
}
}
}
return new ImmutablePair<>(columnNames, rows);
}
/**
* Creates the csv response for /users/_/groupBy requests.
*
* @param resultSet GroupByObject
array containing GroupByResult
objects
* containing UsersResult
objects
* @return Pair
containing the column names (left) and the data rows (right)
*/
private ImmutablePair, List> createCsvResponseForUsersGroupBy(
GroupByObject[] resultSet) {
List columnNames = new LinkedList<>();
columnNames.add("fromTimestamp");
columnNames.add("toTimestamp");
List rows = new LinkedList<>();
for (int i = 0; i < resultSet.length; i++) {
GroupByResult groupByResult = (GroupByResult) resultSet[i];
columnNames.add(groupByResult.getGroupByObjectId().toString());
for (int j = 0; j < groupByResult.getResult().length; j++) {
UsersResult usersResult = (UsersResult) groupByResult.getResult()[j];
if (i == 0) {
String[] row = new String[resultSet.length + 2];
row[0] = usersResult.getFromTimestamp();
row[1] = usersResult.getToTimestamp();
row[2] = String.valueOf(usersResult.getValue());
rows.add(row);
} else {
int count = i + 2;
rows.get(j)[count] = String.valueOf(usersResult.getValue());
}
}
}
return new ImmutablePair<>(columnNames, rows);
}
/**
* Fills the given stream with output data using multiple parallel threads.
*
* @throws RuntimeException if any one thread experiences an exception, or it only wraps
* {@link IOException}
* @throws DatabaseAccessException if the access to keytables or database is not possible
* @throws ExecutionException thrown by {@link java.util.concurrent.ForkJoinTask#get() get}
* @throws InterruptedException thrown by {@link java.util.concurrent.ForkJoinTask#get() get}
* @throws IOException thrown by {@link java.io.OutputStream#flush() flush}
*/
private void writeStreamResponse(ThreadLocal outputJsonGen,
Stream stream, ThreadLocal outputBuffers,
final ServletOutputStream outputStream)
throws ExecutionException, InterruptedException, IOException {
ThreadLocal tts;
HikariDataSource keytablesConnectionPool;
if (DbConnData.keytablesDbPoolConfig != null) {
keytablesConnectionPool = new HikariDataSource(DbConnData.keytablesDbPoolConfig);
tts = ThreadLocal.withInitial(() -> {
try {
return new TagTranslator(keytablesConnectionPool.getConnection());
} catch (OSHDBKeytablesNotFoundException | SQLException e) {
throw new DatabaseAccessException(ExceptionMessages.DATABASE_ACCESS);
}
});
} else {
keytablesConnectionPool = null;
tts = ThreadLocal.withInitial(() -> DbConnData.tagTranslator);
}
ReentrantLock lock = new ReentrantLock();
AtomicBoolean errored = new AtomicBoolean(false);
ForkJoinPool threadPool = new ForkJoinPool(ProcessingData.getNumberOfDataExtractionThreads());
try {
threadPool.submit(() -> stream.parallel().map(data -> {
// 0. resolve tags
Map tags = data.getProperties();
List keysToDelete = new LinkedList<>();
List tagsToAdd = new LinkedList<>();
for (Entry tag : tags.entrySet()) {
String key = tag.getKey();
if (key.charAt(0) != '@') {
keysToDelete.add(key);
tagsToAdd.add(tts.get().getOSMTagOf((OSHDBTag) tag.getValue()));
}
}
tags.keySet().removeAll(keysToDelete);
for (OSMTag tag : tagsToAdd) {
tags.put(tag.getKey(), tag.getValue());
}
// 1. convert features to geojson
try {
outputBuffers.get().reset();
outputJsonGen.get().writeObject(data);
return outputBuffers.get().toByteArray();
} catch (IOException e) {
throw new RuntimeException(e);
}
}).forEach(data -> {
// 2. write data out to client
// only 1 thread is allowed to write at once!
lock.lock();
if (errored.get()) {
// when any one thread experienced an exception (e.g. a client disconnects):
// the "errored" flag is set and all threads abort themselves by throwing an exception
lock.unlock();
throw new RuntimeException();
}
try {
// separate features in the result by a comma, except for the very first one
if (isFirst.get()) {
isFirst.set(false);
} else {
outputStream.print(", ");
}
// write the feature
outputStream.write(data);
} catch (IOException e) {
errored.set(true);
throw new RuntimeException(e);
} finally {
lock.unlock();
}
})).get();
} finally {
threadPool.shutdown();
if (keytablesConnectionPool != null) {
keytablesConnectionPool.close();
}
outputStream.flush();
}
}
/** Defines character encoding, content type and cache header in given servlet response object. */
private HttpServletResponse setCsvSettingsInServletResponse(HttpServletResponse servletResponse) {
servletResponse.setCharacterEncoding("UTF-8");
servletResponse.setContentType("text/csv");
if (!RequestUtils.cacheNotAllowed(processingData.getRequestUrl(),
processingData.getRequestParameters().getTime())) {
servletResponse.setHeader("Cache-Control", "no-transform, public, max-age=31556926");
}
return servletResponse;
}
/**
* Creates a new CSVWriter, writes the given comments and returns the writer object.
*
* @throws IOException thrown by {@link javax.servlet.ServletResponse#getWriter() getWriter}
*/
private CSVWriter writeComments(HttpServletResponse servletResponse, List comments)
throws IOException {
CSVWriter writer =
new CSVWriter(servletResponse.getWriter(), ';', CSVWriter.DEFAULT_QUOTE_CHARACTER,
CSVWriter.DEFAULT_ESCAPE_CHARACTER, CSVWriter.DEFAULT_LINE_END);
writer.writeAll(comments, false);
return writer;
}
/** Adds additional properties like the version or the changeset ID to the feature. */
private Map addAdditionalProperties(OSMEntity entity,
Map properties, boolean isContributionsEndpoint,
EnumSet contributionTypes) {
properties.put("@version", entity.getVersion());
properties.put("@osmType", entity.getType());
properties.put("@changesetId", entity.getChangesetId());
if (isContributionsEndpoint) {
if (contributionTypes.contains(ContributionType.CREATION)) {
properties.put("@creation", "true");
}
if (contributionTypes.contains(ContributionType.DELETION)) {
properties.put("@deletion", "true");
}
if (contributionTypes.contains(ContributionType.TAG_CHANGE)) {
properties.put("@tagChange", "true");
}
if (contributionTypes.contains(ContributionType.GEOMETRY_CHANGE)) {
properties.put("@geometryChange", "true");
}
}
return properties;
}
/** Enum type used in /ratio computation. */
public enum MatchType {
MATCHES1, MATCHES2, MATCHESBOTH, MATCHESNONE
}
}