All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.heigit.ohsome.ohsomeapi.executor.ExecutionUtils Maven / Gradle / Ivy

There is a newer version: 1.10.4
Show newest version
package org.heigit.ohsome.ohsomeapi.executor;

import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.core.JsonEncoding;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.util.DefaultIndenter;
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.opencsv.CSVWriter;
import com.zaxxer.hikari.HikariDataSource;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.sql.SQLException;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.util.EnumSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Stream;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.geojson.GeoJsonObject;
import org.heigit.bigspatialdata.oshdb.api.generic.OSHDBCombinedIndex;
import org.heigit.bigspatialdata.oshdb.api.generic.function.SerializableFunction;
import org.heigit.bigspatialdata.oshdb.api.generic.function.SerializableSupplier;
import org.heigit.bigspatialdata.oshdb.api.mapreducer.MapAggregator;
import org.heigit.bigspatialdata.oshdb.api.mapreducer.MapReducer;
import org.heigit.bigspatialdata.oshdb.api.object.OSMContribution;
import org.heigit.bigspatialdata.oshdb.api.object.OSMEntitySnapshot;
import org.heigit.bigspatialdata.oshdb.osm.OSMEntity;
import org.heigit.bigspatialdata.oshdb.osm.OSMType;
import org.heigit.bigspatialdata.oshdb.util.OSHDBBoundingBox;
import org.heigit.bigspatialdata.oshdb.util.OSHDBTag;
import org.heigit.bigspatialdata.oshdb.util.OSHDBTimestamp;
import org.heigit.bigspatialdata.oshdb.util.celliterator.ContributionType;
import org.heigit.bigspatialdata.oshdb.util.exceptions.OSHDBKeytablesNotFoundException;
import org.heigit.bigspatialdata.oshdb.util.geometry.Geo;
import org.heigit.bigspatialdata.oshdb.util.geometry.OSHDBGeometryBuilder;
import org.heigit.bigspatialdata.oshdb.util.tagtranslator.OSMTag;
import org.heigit.bigspatialdata.oshdb.util.tagtranslator.TagTranslator;
import org.heigit.bigspatialdata.oshdb.util.time.TimestampFormatter;
import org.heigit.ohsome.filter.FilterExpression;
import org.heigit.ohsome.ohsomeapi.Application;
import org.heigit.ohsome.ohsomeapi.controller.rawdata.ElementsGeometry;
import org.heigit.ohsome.ohsomeapi.exception.DatabaseAccessException;
import org.heigit.ohsome.ohsomeapi.exception.ExceptionMessages;
import org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessor;
import org.heigit.ohsome.ohsomeapi.inputprocessing.ProcessingData;
import org.heigit.ohsome.ohsomeapi.inputprocessing.SimpleFeatureType;
import org.heigit.ohsome.ohsomeapi.oshdb.DbConnData;
import org.heigit.ohsome.ohsomeapi.oshdb.ExtractMetadata;
import org.heigit.ohsome.ohsomeapi.output.Description;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.Attribution;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.Metadata;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.RatioResponse;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.RatioResult;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.Response;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.Result;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.elements.ElementsResult;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.groupbyresponse.GroupByObject;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.groupbyresponse.GroupByResult;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.groupbyresponse.RatioGroupByBoundaryResponse;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.groupbyresponse.RatioGroupByResult;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.users.UsersResult;
import org.heigit.ohsome.ohsomeapi.output.rawdataresponse.DataResponse;
import org.heigit.ohsome.ohsomeapi.utils.GroupByBoundaryGeoJsonGenerator;
import org.heigit.ohsome.ohsomeapi.utils.RequestUtils;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.Lineal;
import org.locationtech.jts.geom.Polygonal;
import org.locationtech.jts.geom.Puntal;
import org.wololo.geojson.LineString;
import org.wololo.geojson.Point;
import org.wololo.geojson.Polygon;
import org.wololo.jts2geojson.GeoJSONWriter;

/** Holds helper methods that are used by the executor classes. */
@RequiredArgsConstructor
public class ExecutionUtils {
  private AtomicReference isFirst;
  private final ProcessingData processingData;
  private final DecimalFormat ratioDf = defineDecimalFormat("#.######");
  private static final Point emptyPoint = new Point(new double[0]);
  private static final LineString emptyLine = new LineString(new double[0][0]);
  private static final Polygon emptyPolygon = new Polygon(new double[0][0][0]);

  /** Applies a filter on the given MapReducer object using the given parameters. */
  public MapReducer snapshotFilter(MapReducer mapRed,
      Set osmTypes1, Set osmTypes2, Set simpleFeatureTypes1,
      Set simpleFeatureTypes2, Integer[] keysInt1, Integer[] keysInt2,
      Integer[] valuesInt1, Integer[] valuesInt2) {
    return mapRed.filter(
        snapshot -> snapshotMatches(snapshot, osmTypes1, simpleFeatureTypes1, keysInt1, valuesInt1)
            || snapshotMatches(snapshot, osmTypes2, simpleFeatureTypes2, keysInt2, valuesInt2));
  }

  /**
   * Applies a filter on the given MapReducer object using the given parameters. Used in
   * /ratio/groupBy/boundary requests.
   */
  public MapAggregator, OSMEntitySnapshot> snapshotFilter(
      MapAggregator, OSMEntitySnapshot> mapRed,
      Set osmTypes1, Set osmTypes2, Set simpleFeatureTypes1,
      Set simpleFeatureTypes2, Integer[] keysInt1, Integer[] keysInt2,
      Integer[] valuesInt1, Integer[] valuesInt2) {
    return mapRed.filter(
        snapshot -> snapshotMatches(snapshot, osmTypes1, simpleFeatureTypes1, keysInt1, valuesInt1)
            || snapshotMatches(snapshot, osmTypes2, simpleFeatureTypes2, keysInt2, valuesInt2));
  }

  /** Applies a filter on the given MapReducer object using the given filter expressions. */
  public MapReducer newSnapshotFilter(MapReducer mapRed,
      FilterExpression filterExpr1, FilterExpression filterExpr2) {
    return mapRed.filter(snapshot -> {
      OSMEntity entity = snapshot.getEntity();
      return filterExpr1.applyOSMGeometry(entity, snapshot.getGeometry())
          || filterExpr2.applyOSMGeometry(entity, snapshot.getGeometry());
    });
  }

  /**
   * Applies a filter on the given MapReducer object using the given filter expressions. Used in
   * /ratio/groupBy/boundary requests.
   */
  public MapAggregator, OSMEntitySnapshot> newSnapshotFilter(
      MapAggregator, OSMEntitySnapshot> mapRed,
      FilterExpression filterExpr1, FilterExpression filterExpr2) {
    return mapRed.filter(snapshot -> {
      OSMEntity entity = snapshot.getEntity();
      return filterExpr1.applyOSMGeometry(entity, snapshot.getGeometry())
          || filterExpr2.applyOSMGeometry(entity, snapshot.getGeometry());
    });
  }

  /** Compares the type(s) and tag(s) of the given snapshot to the given types|tags. */
  public boolean snapshotMatches(OSMEntitySnapshot snapshot, Set osmTypes,
      Set simpleFeatureTypes, Integer[] keysInt, Integer[] valuesInt) {
    boolean matchesTags = true;
    OSMEntity entity = snapshot.getEntity();
    if (osmTypes.contains(entity.getType())) {
      for (int i = 0; i < keysInt.length; i++) {
        boolean matchesTag;
        if (i < valuesInt.length) {
          matchesTag = entity.hasTagValue(keysInt[i], valuesInt[i]);
        } else {
          matchesTag = entity.hasTagKey(keysInt[i]);
        }
        if (!matchesTag) {
          matchesTags = false;
          break;
        }
      }
    } else {
      matchesTags = false;
    }
    if (!simpleFeatureTypes.isEmpty()) {
      boolean[] simpleFeatures = setRequestedSimpleFeatures(simpleFeatureTypes);
      return matchesTags && (simpleFeatures[0] && snapshot.getGeometry() instanceof Puntal
          || simpleFeatures[1] && snapshot.getGeometry() instanceof Lineal
          || simpleFeatures[2] && snapshot.getGeometry() instanceof Polygonal || simpleFeatures[3]
              && "GeometryCollection".equalsIgnoreCase(snapshot.getGeometry().getGeometryType()));
    }
    return matchesTags;
  }

  /**
   * Defines a certain decimal format.
   * 
   * @param format String defining the format (e.g.: "#.####" for getting 4 digits
   *        after the comma)
   * @return DecimalFormat object with the defined format.
   */
  public static DecimalFormat defineDecimalFormat(String format) {
    DecimalFormatSymbols otherSymbols = new DecimalFormatSymbols(Locale.getDefault());
    otherSymbols.setDecimalSeparator('.');
    return new DecimalFormat(format, otherSymbols);
  }

  /**
   * Caches the given mapper value in the user data of the Geometry object.
   * 
   * @param geom Geometry of an OSMEntitySnapshot object
   * @param mapper arbitrary function that returns a time-independent value from a snapshot object,
   *        for example lenght, area, perimeter
   * @return evaluated mapper function or cached value stored in the user data of the
   *         Geometry object
   */
  public static Double cacheInUserData(Geometry geom, SerializableSupplier mapper) {
    if (geom.getUserData() == null) {
      geom.setUserData(mapper.get());
    }
    return (Double) geom.getUserData();
  }

  /**
   * Adapted helper function, which works like
   * {@link org.heigit.bigspatialdata.oshdb.api.generic.OSHDBCombinedIndex#nest(Map) nest} but has
   * switched <U> and <V> parameters.
   *
   * @param result the "flat" result data structure that should be converted to a nested structure
   * @param  an arbitrary data type, used for the data value items
   * @param  an arbitrary data type, used for the index'es key items
   * @param  an arbitrary data type, used for the index'es key items
   * @return a nested data structure: for each index part there is a separate level of nested maps
   */
  public static  & Serializable, V extends Comparable & Serializable> SortedMap> nest(
      Map, A> result) {
    TreeMap> ret = new TreeMap<>();
    result.forEach((index, data) -> {
      if (!ret.containsKey(index.getSecondIndex())) {
        ret.put(index.getSecondIndex(), new TreeMap<>());
      }
      ret.get(index.getSecondIndex()).put(index.getFirstIndex(), data);
    });
    return ret;
  }

  /**
   * Streams the result of /elements, /elementsFullHistory and /contributions endpoints as an
   * outputstream.
   * 
   * @throws RuntimeException which only wraps {@link java.io.IOException IOException}
   * @throws IOException thrown by {@link JsonGenerator
   *         com.fasterxml.jackson.core.JsonFactory#createGenerator(java.io.OutputStream,
   *         JsonEncoding) createGenerator},
   *         {@link com.fasterxml.jackson.core.JsonGenerator#writeObject(Object) writeObject},
   *         {@link javax.servlet.ServletResponse#getOutputStream() getOutputStream},
   *         {@link java.io.OutputStream#write(byte[]) write},
   *         {@link org.heigit.ohsome.ohsomeapi.executor.ExecutionUtils#writeStreamResponse(ThreadLocal, Stream, ThreadLocal, ServletOutputStream)
   *         writeStreamResponse}, {@link javax.servlet.ServletOutputStream#print(String) print},
   *         and {@link javax.servlet.ServletResponse#flushBuffer() flushBuffer}
   * @throws ExecutionException thrown by
   *         {@link org.heigit.ohsome.ohsomeapi.executor.ExecutionUtils#writeStreamResponse(ThreadLocal, Stream, ThreadLocal, ServletOutputStream)
   *         writeStreamResponse}
   * @throws InterruptedException thrown by
   *         {@link org.heigit.ohsome.ohsomeapi.executor.ExecutionUtils#writeStreamResponse(ThreadLocal, Stream, ThreadLocal, ServletOutputStream)
   *         writeStreamResponse}
   */
  public void streamResponse(HttpServletResponse servletResponse, DataResponse osmData,
      Stream resultStream) throws Exception {
    JsonFactory jsonFactory = new JsonFactory();
    ByteArrayOutputStream tempStream = new ByteArrayOutputStream();

    ObjectMapper objMapper = new ObjectMapper();
    objMapper.enable(SerializationFeature.INDENT_OUTPUT);
    objMapper.setSerializationInclusion(Include.NON_NULL);
    jsonFactory.createGenerator(tempStream, JsonEncoding.UTF8).setCodec(objMapper)
        .writeObject(osmData);

    String scaffold =
        tempStream.toString(StandardCharsets.UTF_8).replaceFirst("\\s*]\\s*}\\s*$", "");

    servletResponse.setContentType("application/geo+json; charset=utf-8");
    ServletOutputStream outputStream = servletResponse.getOutputStream();
    outputStream.write(scaffold.getBytes(StandardCharsets.UTF_8));

    ThreadLocal outputBuffers =
        ThreadLocal.withInitial(ByteArrayOutputStream::new);
    ThreadLocal outputJsonGen = ThreadLocal.withInitial(() -> {
      try {
        DefaultPrettyPrinter.Indenter indenter = new DefaultIndenter() {
          @Override
          public void writeIndentation(JsonGenerator g, int level) throws IOException {
            super.writeIndentation(g, level + 1);
          }
        };
        DefaultPrettyPrinter printer =
            new DefaultPrettyPrinter("").withArrayIndenter(indenter).withObjectIndenter(indenter);
        return jsonFactory.createGenerator(outputBuffers.get(), JsonEncoding.UTF8)
            .setCodec(objMapper).setPrettyPrinter(printer);
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
    });
    isFirst = new AtomicReference<>(true);
    writeStreamResponse(outputJsonGen, resultStream, outputBuffers, outputStream);
    outputStream.print("]\n}\n");
    servletResponse.flushBuffer();
  }

  /** Writes a response in the csv format for /groupBy requests. */
  public void writeCsvResponse(GroupByObject[] resultSet, HttpServletResponse servletResponse,
      List comments) {
    try {
      servletResponse = setCsvSettingsInServletResponse(servletResponse);
      CSVWriter writer = writeComments(servletResponse, comments);
      Pair, List> rows;
      if (resultSet instanceof GroupByResult[]) {
        if (resultSet.length == 0) {
          writer.writeNext(new String[] {"timestamp"}, false);
          writer.close();
          return;
        } else {
          GroupByResult result = (GroupByResult) resultSet[0];
          if (result.getResult() instanceof UsersResult[]) {
            rows = createCsvResponseForUsersGroupBy(resultSet);
          } else {
            rows = createCsvResponseForElementsGroupBy(resultSet);
          }
        }
      } else {
        rows = createCsvResponseForElementsRatioGroupBy(resultSet);
      }
      writer.writeNext(rows.getLeft().toArray(new String[rows.getLeft().size()]), false);
      writer.writeAll(rows.getRight(), false);
      writer.close();
    } catch (IOException e) {
      e.printStackTrace();
    }
  }

  /**
   * Writes a response in the csv format for /count|length|perimeter|area(/density)(/ratio)
   * requests.
   */
  public void writeCsvResponse(Result[] resultSet, HttpServletResponse servletResponse,
      List comments) {
    try {
      servletResponse = setCsvSettingsInServletResponse(servletResponse);
      CSVWriter writer = writeComments(servletResponse, comments);
      if (resultSet instanceof ElementsResult[]) {
        writer.writeNext(new String[] {"timestamp", "value"}, false);
        for (Result result : resultSet) {
          ElementsResult elementsResult = (ElementsResult) result;
          writer.writeNext(new String[] {elementsResult.getTimestamp(),
              String.valueOf(elementsResult.getValue())});
        }
      } else if (resultSet instanceof UsersResult[]) {
        writer.writeNext(new String[] {"fromTimestamp", "toTimestamp", "value"}, false);
        for (Result result : resultSet) {
          UsersResult usersResult = (UsersResult) result;
          writer.writeNext(new String[] {usersResult.getFromTimestamp(),
              usersResult.getToTimestamp(), String.valueOf(usersResult.getValue())});
        }
      } else if (resultSet instanceof RatioResult[]) {
        writer.writeNext(new String[] {"timestamp", "value", "value2", "ratio"}, false);
        for (Result result : resultSet) {
          RatioResult ratioResult = (RatioResult) result;
          writer.writeNext(
              new String[] {ratioResult.getTimestamp(), String.valueOf(ratioResult.getValue()),
                  String.valueOf(ratioResult.getValue2()), String.valueOf(ratioResult.getRatio())});
        }
      }
      writer.close();
    } catch (IOException e) {
      e.printStackTrace();
    }
  }

  /**
   * Sets the boolean values for the respective simple feature type: 0 = hasPoint, 1 = hasLine, 2 =
   * hasPolygon, 3 = hasOther.
   */
  public boolean[] setRequestedSimpleFeatures(Set simpleFeatureTypes) {
    boolean[] simpleFeatureArray = new boolean[] {false, false, false, false};
    for (SimpleFeatureType type : simpleFeatureTypes) {
      if (type.equals(SimpleFeatureType.POINT)) {
        simpleFeatureArray[0] = true;
      } else if (type.equals(SimpleFeatureType.LINE)) {
        simpleFeatureArray[1] = true;
      } else if (type.equals(SimpleFeatureType.POLYGON)) {
        simpleFeatureArray[2] = true;
      } else if (type.equals(SimpleFeatureType.OTHER)) {
        simpleFeatureArray[3] = true;
      }
    }
    return simpleFeatureArray;
  }

  /** Creates the comments of the csv response (Attribution, API-Version and optional Metadata). */
  public List createCsvTopComments(String url, String text, String apiVersion,
      Metadata metadata) {
    List comments = new LinkedList<>();
    comments.add(new String[] {"# Copyright URL: " + url});
    comments.add(new String[] {"# Copyright Text: " + text});
    comments.add(new String[] {"# API Version: " + apiVersion});
    if (metadata != null) {
      comments.add(new String[] {"# Execution Time: " + metadata.getExecutionTime()});
      comments.add(new String[] {"# Description: " + metadata.getDescription()});
      if (metadata.getRequestUrl() != null) {
        comments.add(new String[] {"# Request URL: " + metadata.getRequestUrl()});
      }
    }
    return comments;
  }

  /** Creates the Feature objects in the OSM data response. */
  public org.wololo.geojson.Feature createOSMFeature(OSMEntity entity, Geometry geometry,
      Map properties, int[] keysInt, boolean includeTags,
      boolean includeOSMMetadata, boolean isContributionsEndpoint, ElementsGeometry elemGeom,
      EnumSet contributionTypes) {
    if (geometry.isEmpty() && !contributionTypes.contains(ContributionType.DELETION)) {
      // skip invalid geometries (e.g. ways with 0 nodes)
      return null;
    }
    if (includeTags) {
      for (OSHDBTag oshdbTag : entity.getTags()) {
        properties.put(String.valueOf(oshdbTag.getKey()), oshdbTag);
      }
    } else if (keysInt.length != 0) {
      int[] tags = entity.getRawTags();
      for (int i = 0; i < tags.length; i += 2) {
        int tagKeyId = tags[i];
        int tagValueId = tags[i + 1];
        for (int key : keysInt) {
          if (tagKeyId == key) {
            properties.put(String.valueOf(tagKeyId), new OSHDBTag(tagKeyId, tagValueId));
            break;
          }
        }
      }
    }
    if (includeOSMMetadata) {
      properties =
          addAdditionalProperties(entity, properties, isContributionsEndpoint, contributionTypes);
    }
    properties.put("@osmId", entity.getType().toString().toLowerCase() + "/" + entity.getId());
    GeoJSONWriter gjw = new GeoJSONWriter();
    boolean deletionHandling =
        isContributionsEndpoint && contributionTypes.contains(ContributionType.DELETION);
    Geometry outputGeometry;
    switch (elemGeom) {
      case BBOX:
        if (deletionHandling) {
          return new org.wololo.geojson.Feature(emptyPolygon, properties);
        }
        Envelope envelope = geometry.getEnvelopeInternal();
        OSHDBBoundingBox bbox = OSHDBGeometryBuilder.boundingBoxOf(envelope);
        outputGeometry = OSHDBGeometryBuilder.getGeometry(bbox);
        break;
      case CENTROID:
        if (deletionHandling) {
          return new org.wololo.geojson.Feature(emptyPoint, properties);
        }
        outputGeometry = geometry.getCentroid();
        break;
      case RAW:
      default:
        if (deletionHandling && geometry.getGeometryType().contains("Polygon")) {
          return new org.wololo.geojson.Feature(emptyPolygon, properties);
        }
        if (deletionHandling && geometry.getGeometryType().contains("LineString")) {
          return new org.wololo.geojson.Feature(emptyLine, properties);
        }
        if (deletionHandling && geometry.getGeometryType().contains("Point")) {
          return new org.wololo.geojson.Feature(emptyPoint, properties);
        }
        outputGeometry = geometry;
    }
    return new org.wololo.geojson.Feature(gjw.write(outputGeometry), properties);
  }

  /**
   * Computes the result depending on the RequestResource using a
   * MapAggregator object as input and returning a SortedMap.
   * 
   * @throws Exception thrown by
   *         {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapAggregator#count() count}, and
   *         {@link org.heigit.bigspatialdata.oshdb.api.mapreducer.MapAggregator#sum() sum}
   */
  @SuppressWarnings({"unchecked"}) // intentionally suppressed as type format is valid
  public  & Serializable, V extends Number> SortedMap, V> computeResult(
      RequestResource requestResource,
      MapAggregator, OSMEntitySnapshot> preResult)
      throws Exception {
    switch (requestResource) {
      case COUNT:
        return (SortedMap, V>) preResult.count();
      case LENGTH:
        return (SortedMap, V>) preResult
            .sum((SerializableFunction) snapshot -> cacheInUserData(
                snapshot.getGeometry(), () -> Geo.lengthOf(snapshot.getGeometry())));
      case PERIMETER:
        return (SortedMap, V>) preResult
            .sum((SerializableFunction) snapshot -> {
              if (snapshot.getGeometry() instanceof Polygonal) {
                return cacheInUserData(snapshot.getGeometry(),
                    () -> Geo.lengthOf(snapshot.getGeometry().getBoundary()));
              }
              return 0.0;
            });
      case AREA:
        return (SortedMap, V>) preResult
            .sum((SerializableFunction) snapshot -> cacheInUserData(
                snapshot.getGeometry(), () -> Geo.areaOf(snapshot.getGeometry())));
      default:
        return null;
    }
  }

  /**
   * Computes the result depending on the RequestResource using a
   * MapAggregator object as input and returning a SortedMap.
   */
  @SuppressWarnings({"unchecked"}) // intentionally suppressed as type format is valid
  public  & Serializable, V extends Number> SortedMap, OSHDBTimestamp>, V> computeNestedResult(
      RequestResource requestResource,
      MapAggregator, OSHDBTimestamp>, Geometry> preResult)
      throws Exception {
    switch (requestResource) {
      case COUNT:
        return (SortedMap, OSHDBTimestamp>, V>) preResult
            .count();
      case PERIMETER:
        return (SortedMap, OSHDBTimestamp>, V>) preResult
            .sum(geom -> {
              if (!(geom instanceof Polygonal)) {
                return 0.0;
              }
              return cacheInUserData(geom, () -> Geo.lengthOf(geom.getBoundary()));
            });
      case LENGTH:
        return (SortedMap, OSHDBTimestamp>, V>) preResult
            .sum(geom -> cacheInUserData(geom, () -> Geo.lengthOf(geom)));
      case AREA:
        return (SortedMap, OSHDBTimestamp>, V>) preResult
            .sum(geom -> cacheInUserData(geom, () -> Geo.areaOf(geom)));
      default:
        return null;
    }
  }

  /**
   * Extracts the tags from the given OSMContribution depending on the
   * ContributionType.
   */
  public int[] extractContributionTags(OSMContribution contrib) {
    int[] tags;
    if (contrib.getContributionTypes().contains(ContributionType.DELETION)) {
      tags = contrib.getEntityBefore().getRawTags();
    } else if (contrib.getContributionTypes().contains(ContributionType.CREATION)) {
      tags = contrib.getEntityAfter().getRawTags();
    } else {
      int[] tagsBefore = contrib.getEntityBefore().getRawTags();
      int[] tagsAfter = contrib.getEntityAfter().getRawTags();
      tags = new int[tagsBefore.length + tagsAfter.length];
      System.arraycopy(tagsBefore, 0, tags, 0, tagsBefore.length);
      System.arraycopy(tagsAfter, 0, tags, tagsBefore.length, tagsAfter.length);
    }
    return tags;
  }

  /** Fills the ElementsResult array with respective ElementsResult objects. */
  public ElementsResult[] fillElementsResult(SortedMap entryVal,
      boolean isDensity, DecimalFormat df, Geometry geom) {
    ElementsResult[] results = new ElementsResult[entryVal.entrySet().size()];
    int count = 0;
    for (Entry entry : entryVal.entrySet()) {
      if (isDensity) {
        results[count] = new ElementsResult(
            TimestampFormatter.getInstance().isoDateTime(entry.getKey()), Double.parseDouble(
                df.format(entry.getValue().doubleValue() / (Geo.areaOf(geom) * 0.000001))));
      } else {
        results[count] =
            new ElementsResult(TimestampFormatter.getInstance().isoDateTime(entry.getKey()),
                Double.parseDouble(df.format(entry.getValue().doubleValue())));
      }
      count++;
    }
    return results;
  }

  /** Fills the UsersResult array with respective UsersResult objects. */
  public UsersResult[] fillUsersResult(SortedMap entryVal,
      boolean isDensity, InputProcessor inputProcessor, DecimalFormat df, Geometry geom) {
    UsersResult[] results = new UsersResult[entryVal.entrySet().size()];
    int count = 0;
    String[] toTimestamps = inputProcessor.getUtils().getToTimestamps();
    for (Entry entry : entryVal.entrySet()) {
      if (isDensity) {
        results[count] =
            new UsersResult(TimestampFormatter.getInstance().isoDateTime(entry.getKey()),
                toTimestamps[count + 1], Double.parseDouble(
                    df.format(entry.getValue().doubleValue() / (Geo.areaOf(geom) / 1000000))));
      } else {
        results[count] = new UsersResult(
            TimestampFormatter.getInstance().isoDateTime(entry.getKey()), toTimestamps[count + 1],
            Double.parseDouble(df.format(entry.getValue().doubleValue())));
      }
      count++;
    }
    return results;
  }

  /**
   * Fills the result value arrays for the ratio/groupBy/boundary response.
   * 
   * @param resultSet Set containing the result values
   * @param df DecimalFormat defining the number of digits of the result values
   * @return Double[] containing the formatted result values
   */
  public Double[] fillElementsRatioGroupByBoundaryResultValues(
      Set, ? extends Number>> resultSet,
      DecimalFormat df) {
    Double[] resultValues = new Double[resultSet.size()];
    int valueCount = 0;
    for (Entry, ? extends Number> innerEntry : resultSet) {
      resultValues[valueCount] = Double.parseDouble(df.format(innerEntry.getValue().doubleValue()));
      valueCount++;
    }
    return resultValues;
  }

  /**
   * Maps the given OSMEntitySnapshot to a given tag, or to the remainder (having -1,
   * -1 as identifier) if none of the given tags is included.
   * 
   * @param keysInt int value of the groupByKey parameter
   * @param valuesInt Integer[] of the groupByValues parameter
   * @param f OSMEntitySnapshot
   * @return nested Pair containing the integer values of the tag category and the
   *         OSMEntitySnapshot
   */
  public Pair, OSMEntitySnapshot> mapSnapshotToTags(int keysInt,
      Integer[] valuesInt, OSMEntitySnapshot f) {
    int[] tags = f.getEntity().getRawTags();
    for (int i = 0; i < tags.length; i += 2) {
      int tagKeyId = tags[i];
      int tagValueId = tags[i + 1];
      if (tagKeyId == keysInt) {
        if (valuesInt.length == 0) {
          return new ImmutablePair<>(new ImmutablePair<>(tagKeyId, tagValueId), f);
        }
        for (int value : valuesInt) {
          if (tagValueId == value) {
            return new ImmutablePair<>(new ImmutablePair<>(tagKeyId, tagValueId), f);
          }
        }
      }
    }
    return new ImmutablePair<>(new ImmutablePair<>(-1, -1), f);
  }

  /** Creates a RatioResponse. */
  public Response createRatioResponse(String[] timeArray, Double[] value1, Double[] value2,
      long startTime, RequestResource reqRes, String requestUrl,
      HttpServletResponse servletResponse) {
    RatioResult[] resultSet = new RatioResult[timeArray.length];
    for (int i = 0; i < timeArray.length; i++) {
      double ratio = value2[i] / value1[i];
      // in case ratio has the values "NaN", "Infinity", etc.
      try {
        ratio = Double.parseDouble(ratioDf.format(ratio));
      } catch (Exception e) {
        // do nothing --> just return ratio without rounding (trimming)
      }
      resultSet[i] = new RatioResult(timeArray[i], value1[i], value2[i], ratio);
    }
    Metadata metadata = null;
    if (processingData.isShowMetadata()) {
      long duration = System.currentTimeMillis() - startTime;
      metadata = new Metadata(duration,
          Description.aggregateRatio(reqRes.getDescription(), reqRes.getUnit()), requestUrl);
    }
    RequestParameters requestParameters = processingData.getRequestParameters();
    if ("csv".equalsIgnoreCase(requestParameters.getFormat())) {
      writeCsvResponse(resultSet, servletResponse, createCsvTopComments(ElementsRequestExecutor.URL,
          ElementsRequestExecutor.TEXT, Application.API_VERSION, metadata));
      return null;
    }
    return new RatioResponse(
        new Attribution(ExtractMetadata.attributionUrl, ExtractMetadata.attributionShort),
        Application.API_VERSION, metadata, resultSet);
  }

  /** Creates a RatioGroupByBoundaryResponse. */
  public Response createRatioGroupByBoundaryResponse(Object[] boundaryIds, String[] timeArray,
      Double[] resultValues1, Double[] resultValues2, long startTime, RequestResource reqRes,
      String requestUrl, HttpServletResponse servletResponse) {
    Metadata metadata = null;
    int boundaryIdsLength = boundaryIds.length;
    int timeArrayLenth = timeArray.length;
    RatioGroupByResult[] groupByResultSet = new RatioGroupByResult[boundaryIdsLength];
    for (int i = 0; i < boundaryIdsLength; i++) {
      Object groupByName = boundaryIds[i];
      RatioResult[] ratioResultSet = new RatioResult[timeArrayLenth];
      int innerCount = 0;
      for (int j = i; j < timeArrayLenth * boundaryIdsLength; j += boundaryIdsLength) {
        double ratio = resultValues2[j] / resultValues1[j];
        // in case ratio has the values "NaN", "Infinity", etc.
        try {
          ratio = Double.parseDouble(ratioDf.format(ratio));
        } catch (Exception e) {
          // do nothing --> just return ratio without rounding (trimming)
        }
        ratioResultSet[innerCount] =
            new RatioResult(timeArray[innerCount], resultValues1[j], resultValues2[j], ratio);
        innerCount++;
      }
      groupByResultSet[i] = new RatioGroupByResult(groupByName, ratioResultSet);
    }
    if (processingData.isShowMetadata()) {
      long duration = System.currentTimeMillis() - startTime;
      metadata = new Metadata(duration,
          Description.aggregateRatioGroupByBoundary(reqRes.getDescription(), reqRes.getUnit()),
          requestUrl);
    }
    RequestParameters requestParameters = processingData.getRequestParameters();
    Attribution attribution =
        new Attribution(ExtractMetadata.attributionUrl, ExtractMetadata.attributionShort);
    if ("geojson".equalsIgnoreCase(requestParameters.getFormat())) {
      GeoJsonObject[] geoJsonGeoms = processingData.getGeoJsonGeoms();
      return RatioGroupByBoundaryResponse.of(attribution, Application.API_VERSION, metadata,
          "FeatureCollection",
          GroupByBoundaryGeoJsonGenerator.createGeoJsonFeatures(groupByResultSet, geoJsonGeoms));
    } else if ("csv".equalsIgnoreCase(requestParameters.getFormat())) {
      writeCsvResponse(groupByResultSet, servletResponse,
          createCsvTopComments(ElementsRequestExecutor.URL, ElementsRequestExecutor.TEXT,
              Application.API_VERSION, metadata));
      return null;
    }
    return new RatioGroupByBoundaryResponse(attribution, Application.API_VERSION, metadata,
        groupByResultSet);
  }

  /**
   * Extracts and returns a geometry out of the given contribution. The boolean values specify if it
   * should be clipped/unclipped and if the geometry before/after a contribution should be taken.
   */
  public Geometry getGeometry(OSMContribution contribution, boolean clipGeometries,
      boolean before) {
    Geometry geom = null;
    if (clipGeometries) {
      if (before) {
        geom = contribution.getGeometryBefore();
      } else {
        geom = contribution.getGeometryAfter();
      }
    } else {
      if (before) {
        geom = contribution.getGeometryUnclippedBefore();
      } else {
        geom = contribution.getGeometryUnclippedAfter();
      }
    }
    return geom;
  }

  /** Combines the two given filters with an OR operation. Used in /ratio computation. */
  public String combineFiltersWithOr(String firstFilter, String secondFilter) {
    if (firstFilter.isBlank() || secondFilter.isBlank()) {
      // definition of an empty combined filter if filter1 or filter2 is empty
      return "";
    }
    return "(" + firstFilter + ") or (" + secondFilter + ")";
  }

  /**
   * Creates the csv response for /elements/_/groupBy requests.
   * 
   * @param resultSet GroupByObject array containing GroupByResult objects
   *        containing ElementsResult objects
   * @return Pair containing the column names (left) and the data rows (right)
   */
  private ImmutablePair, List> createCsvResponseForElementsGroupBy(
      GroupByObject[] resultSet) {
    List columnNames = new LinkedList<>();
    columnNames.add("timestamp");
    List rows = new LinkedList<>();
    for (int i = 0; i < resultSet.length; i++) {
      GroupByResult groupByResult = (GroupByResult) resultSet[i];
      Object groupByObject = groupByResult.getGroupByObjectId();
      if (groupByObject instanceof Object[]) {
        Object[] groupByObjectArr = (Object[]) groupByObject;
        columnNames.add(groupByObjectArr[0].toString() + "_" + groupByObjectArr[1].toString());
      } else {
        columnNames.add(groupByObject.toString());
      }
      for (int j = 0; j < groupByResult.getResult().length; j++) {
        ElementsResult elemResult = (ElementsResult) groupByResult.getResult()[j];
        if (i == 0) {
          String[] row = new String[resultSet.length + 1];
          row[0] = elemResult.getTimestamp();
          row[1] = String.valueOf(elemResult.getValue());
          rows.add(row);
        } else {
          rows.get(j)[i + 1] = String.valueOf(elemResult.getValue());
        }
      }
    }
    return new ImmutablePair<>(columnNames, rows);
  }

  /**
   * Creates the csv response for /elements/_/ratio/groupBy requests.
   * 
   * @param resultSet GroupByObject array containing RatioGroupByResult
   *        objects containing RatioResult objects
   * @return Pair containing the column names (left) and the data rows (right)
   */
  private ImmutablePair, List> createCsvResponseForElementsRatioGroupBy(
      GroupByObject[] resultSet) {
    List columnNames = new LinkedList<>();
    columnNames.add("timestamp");
    List rows = new LinkedList<>();
    for (int i = 0; i < resultSet.length; i++) {
      RatioGroupByResult ratioGroupByResult = (RatioGroupByResult) resultSet[i];
      columnNames.add(ratioGroupByResult.getGroupByObjectId() + "_value");
      columnNames.add(ratioGroupByResult.getGroupByObjectId() + "_value2");
      columnNames.add(ratioGroupByResult.getGroupByObjectId() + "_ratio");
      for (int j = 0; j < ratioGroupByResult.getRatioResult().length; j++) {
        RatioResult ratioResult = ratioGroupByResult.getRatioResult()[j];
        if (i == 0) {
          String[] row = new String[resultSet.length * 3 + 1];
          row[0] = ratioResult.getTimestamp();
          row[1] = String.valueOf(ratioResult.getValue());
          row[2] = String.valueOf(ratioResult.getValue2());
          row[3] = String.valueOf(ratioResult.getRatio());
          rows.add(row);
        } else {
          int count = i * 3 + 1;
          rows.get(j)[count] = String.valueOf(ratioResult.getValue());
          rows.get(j)[count + 1] = String.valueOf(ratioResult.getValue2());
          rows.get(j)[count + 2] = String.valueOf(ratioResult.getRatio());
        }
      }
    }
    return new ImmutablePair<>(columnNames, rows);
  }

  /**
   * Creates the csv response for /users/_/groupBy requests.
   * 
   * @param resultSet GroupByObject array containing GroupByResult objects
   *        containing UsersResult objects
   * @return Pair containing the column names (left) and the data rows (right)
   */
  private ImmutablePair, List> createCsvResponseForUsersGroupBy(
      GroupByObject[] resultSet) {
    List columnNames = new LinkedList<>();
    columnNames.add("fromTimestamp");
    columnNames.add("toTimestamp");
    List rows = new LinkedList<>();
    for (int i = 0; i < resultSet.length; i++) {
      GroupByResult groupByResult = (GroupByResult) resultSet[i];
      columnNames.add(groupByResult.getGroupByObjectId().toString());
      for (int j = 0; j < groupByResult.getResult().length; j++) {
        UsersResult usersResult = (UsersResult) groupByResult.getResult()[j];
        if (i == 0) {
          String[] row = new String[resultSet.length + 2];
          row[0] = usersResult.getFromTimestamp();
          row[1] = usersResult.getToTimestamp();
          row[2] = String.valueOf(usersResult.getValue());
          rows.add(row);
        } else {
          int count = i + 2;
          rows.get(j)[count] = String.valueOf(usersResult.getValue());
        }
      }
    }
    return new ImmutablePair<>(columnNames, rows);
  }

  /**
   * Fills the given stream with output data using multiple parallel threads.
   * 
   * @throws RuntimeException if any one thread experiences an exception, or it only wraps
   *         {@link IOException}
   * @throws DatabaseAccessException if the access to keytables or database is not possible
   * @throws ExecutionException thrown by {@link java.util.concurrent.ForkJoinTask#get() get}
   * @throws InterruptedException thrown by {@link java.util.concurrent.ForkJoinTask#get() get}
   * @throws IOException thrown by {@link java.io.OutputStream#flush() flush}
   */
  private void writeStreamResponse(ThreadLocal outputJsonGen,
      Stream stream, ThreadLocal outputBuffers,
      final ServletOutputStream outputStream)
      throws ExecutionException, InterruptedException, IOException {
    ThreadLocal tts;
    HikariDataSource keytablesConnectionPool;
    if (DbConnData.keytablesDbPoolConfig != null) {
      keytablesConnectionPool = new HikariDataSource(DbConnData.keytablesDbPoolConfig);
      tts = ThreadLocal.withInitial(() -> {
        try {
          return new TagTranslator(keytablesConnectionPool.getConnection());
        } catch (OSHDBKeytablesNotFoundException | SQLException e) {
          throw new DatabaseAccessException(ExceptionMessages.DATABASE_ACCESS);
        }
      });
    } else {
      keytablesConnectionPool = null;
      tts = ThreadLocal.withInitial(() -> DbConnData.tagTranslator);
    }
    ReentrantLock lock = new ReentrantLock();
    AtomicBoolean errored = new AtomicBoolean(false);
    ForkJoinPool threadPool = new ForkJoinPool(ProcessingData.getNumberOfDataExtractionThreads());
    try {
      threadPool.submit(() -> stream.parallel().map(data -> {
        // 0. resolve tags
        Map tags = data.getProperties();
        List keysToDelete = new LinkedList<>();
        List tagsToAdd = new LinkedList<>();
        for (Entry tag : tags.entrySet()) {
          String key = tag.getKey();
          if (key.charAt(0) != '@') {
            keysToDelete.add(key);
            tagsToAdd.add(tts.get().getOSMTagOf((OSHDBTag) tag.getValue()));
          }
        }
        tags.keySet().removeAll(keysToDelete);
        for (OSMTag tag : tagsToAdd) {
          tags.put(tag.getKey(), tag.getValue());
        }
        // 1. convert features to geojson
        try {
          outputBuffers.get().reset();
          outputJsonGen.get().writeObject(data);
          return outputBuffers.get().toByteArray();
        } catch (IOException e) {
          throw new RuntimeException(e);
        }
      }).forEach(data -> {
        // 2. write data out to client
        // only 1 thread is allowed to write at once!
        lock.lock();
        if (errored.get()) {
          // when any one thread experienced an exception (e.g. a client disconnects):
          // the "errored" flag is set and all threads abort themselves by throwing an exception
          lock.unlock();
          throw new RuntimeException();
        }
        try {
          // separate features in the result by a comma, except for the very first one
          if (isFirst.get()) {
            isFirst.set(false);
          } else {
            outputStream.print(", ");
          }
          // write the feature
          outputStream.write(data);
        } catch (IOException e) {
          errored.set(true);
          throw new RuntimeException(e);
        } finally {
          lock.unlock();
        }
      })).get();
    } finally {
      threadPool.shutdown();
      if (keytablesConnectionPool != null) {
        keytablesConnectionPool.close();
      }
      outputStream.flush();
    }
  }

  /** Defines character encoding, content type and cache header in given servlet response object. */
  private HttpServletResponse setCsvSettingsInServletResponse(HttpServletResponse servletResponse) {
    servletResponse.setCharacterEncoding("UTF-8");
    servletResponse.setContentType("text/csv");
    if (!RequestUtils.cacheNotAllowed(processingData.getRequestUrl(),
        processingData.getRequestParameters().getTime())) {
      servletResponse.setHeader("Cache-Control", "no-transform, public, max-age=31556926");
    }
    return servletResponse;
  }

  /**
   * Creates a new CSVWriter, writes the given comments and returns the writer object.
   * 
   * @throws IOException thrown by {@link javax.servlet.ServletResponse#getWriter() getWriter}
   */
  private CSVWriter writeComments(HttpServletResponse servletResponse, List comments)
      throws IOException {
    CSVWriter writer =
        new CSVWriter(servletResponse.getWriter(), ';', CSVWriter.DEFAULT_QUOTE_CHARACTER,
            CSVWriter.DEFAULT_ESCAPE_CHARACTER, CSVWriter.DEFAULT_LINE_END);
    writer.writeAll(comments, false);
    return writer;
  }

  /** Adds additional properties like the version or the changeset ID to the feature. */
  private Map addAdditionalProperties(OSMEntity entity,
      Map properties, boolean isContributionsEndpoint,
      EnumSet contributionTypes) {
    properties.put("@version", entity.getVersion());
    properties.put("@osmType", entity.getType());
    properties.put("@changesetId", entity.getChangesetId());
    if (isContributionsEndpoint) {
      if (contributionTypes.contains(ContributionType.CREATION)) {
        properties.put("@creation", "true");
      }
      if (contributionTypes.contains(ContributionType.DELETION)) {
        properties.put("@deletion", "true");
      }
      if (contributionTypes.contains(ContributionType.TAG_CHANGE)) {
        properties.put("@tagChange", "true");
      }
      if (contributionTypes.contains(ContributionType.GEOMETRY_CHANGE)) {
        properties.put("@geometryChange", "true");
      }
    }
    return properties;
  }

  /** Enum type used in /ratio computation. */
  public enum MatchType {
    MATCHES1, MATCHES2, MATCHESBOTH, MATCHESNONE
  }
}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy