org.opentripplanner.graph_builder.GraphBuilderDataSources Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of otp Show documentation
The OpenTripPlanner multimodal journey planning system
There is a newer version: 2.6.0
package org.opentripplanner.graph_builder;

import static org.opentripplanner.datastore.api.FileType.DEM;
import static org.opentripplanner.datastore.api.FileType.GTFS;
import static org.opentripplanner.datastore.api.FileType.NETEX;
import static org.opentripplanner.datastore.api.FileType.OSM;

import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
import java.io.File;
import java.net.URI;
import java.util.EnumSet;
import java.util.Set;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.opentripplanner.datastore.OtpDataStore;
import org.opentripplanner.datastore.api.CompositeDataSource;
import org.opentripplanner.datastore.api.DataSource;
import org.opentripplanner.datastore.api.FileType;
import org.opentripplanner.graph_builder.module.ned.parameter.DemExtractParameters;
import org.opentripplanner.graph_builder.module.ned.parameter.DemExtractParametersBuilder;
import org.opentripplanner.graph_builder.module.osm.parameters.OsmExtractParameters;
import org.opentripplanner.graph_builder.module.osm.parameters.OsmExtractParametersBuilder;
import org.opentripplanner.gtfs.graphbuilder.GtfsFeedParameters;
import org.opentripplanner.gtfs.graphbuilder.GtfsFeedParametersBuilder;
import org.opentripplanner.netex.config.NetexFeedParameters;
import org.opentripplanner.standalone.config.BuildConfig;
import org.opentripplanner.standalone.config.CommandLineParameters;
import org.opentripplanner.standalone.config.api.OtpBaseDirectory;
import org.opentripplanner.util.OtpAppException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * This is a wrapper around an {@link OtpDataStore} adding the ability to filter which data source
 * input files should be used and validate the available input files against the command line
 * parameters set.
 * 
 * After this class is validated the {@link #has(FileType)} method can be used to determine if a the
 * build process should include a file in the build.
 * 
 * By separating this from the builder, this class can be constructed early, causing a validation of
 * the available data-sources against the configuration - and then if not valid - abort the entire
 * OTP startup early, before spending time on loading any data - like the streetGraph.
 */
@Singleton
public class GraphBuilderDataSources {

  private static final Logger LOG = LoggerFactory.getLogger(GraphBuilderDataSources.class);
  private static final String BULLET_POINT = "- ";

  private final OtpDataStore store;
  private final Multimap inputData = ArrayListMultimap.create();
  private final Multimap skipData = ArrayListMultimap.create();
  private final Set includeTypes = EnumSet.complementOf(EnumSet.of(FileType.UNKNOWN));
  private final File cacheDirectory;
  private final DataSource outputGraph;
  private final BuildConfig buildConfig;
  private final File baseDirectory;

  /**
   * Create a wrapper around the data-store and resolve which files to import and export. Validate
   * these files against the given command line arguments and the graph build parameters.
   */
  @Inject
  public GraphBuilderDataSources(
    CommandLineParameters cli,
    BuildConfig bc,
    OtpDataStore store,
    @OtpBaseDirectory File baseDirectory
  ) {
    this.store = store;
    this.buildConfig = bc;
    this.cacheDirectory = cli.cacheDirectory;
    this.outputGraph = getOutputGraph(cli);
    this.baseDirectory = baseDirectory;

    // Select which files to import
    include(cli.doBuildStreet(), OSM);
    include(cli.doBuildStreet(), DEM);
    include(cli.doBuildTransit(), GTFS);
    include(cli.doBuildTransit(), NETEX);

    selectFilesToImport();

    // Log all files and expected action to take
    logSkippedAndSelectedFiles();

    // init input vil automatically validate it.
    // Check that the CLI is consistent with the input available
    validateCliMatchesInputData(cli);
  }

  public DataSource getOutputGraph() {
    return outputGraph;
  }

  /**
   * @return {@code true} if and only if the data source exist, proper command line parameters is
   * set and not disabled by the loaded configuration files.
   */
  public boolean has(FileType type) {
    return inputData.containsKey(type);
  }

  public Iterable get(FileType type) {
    return inputData.get(type);
  }

  public Iterable> getOsmConfiguredDatasource() {
    return inputData
      .get(OSM)
      .stream()
      .map(it -> new ConfiguredDataSource<>(it, getOsmExtractConfig(it)))
      .toList();
  }

  private OsmExtractParameters getOsmExtractConfig(DataSource dataSource) {
    return buildConfig.osm.parameters
      .stream()
      .filter(osmExtractConfig -> uriMatch(osmExtractConfig.source(), dataSource.uri()))
      .findFirst()
      .orElse(new OsmExtractParametersBuilder().withSource(dataSource.uri()).build());
  }

  public Iterable> getDemConfiguredDatasource() {
    return inputData
      .get(DEM)
      .stream()
      .map(it -> new ConfiguredDataSource<>(it, getDemExtractConfig(it)))
      .toList();
  }

  private DemExtractParameters getDemExtractConfig(DataSource dataSource) {
    return buildConfig.dem
      .demExtracts()
      .stream()
      .filter(demExtractConfig -> uriMatch(demExtractConfig.source(), dataSource.uri()))
      .findFirst()
      .orElse(new DemExtractParametersBuilder().withSource(dataSource.uri()).build());
  }

  public Iterable> getGtfsConfiguredDatasource() {
    return inputData
      .get(GTFS)
      .stream()
      .map(it -> new ConfiguredDataSource<>(it, getGtfsFeedConfig(it)))
      .toList();
  }

  private GtfsFeedParameters getGtfsFeedConfig(DataSource dataSource) {
    return buildConfig.transitFeeds
      .gtfsFeeds()
      .stream()
      .filter(gtfsFeedConfig -> uriMatch(gtfsFeedConfig.source(), dataSource.uri()))
      .findFirst()
      .orElse(new GtfsFeedParametersBuilder().withSource(dataSource.uri()).build());
  }

  public Iterable> getNetexConfiguredDatasource() {
    return inputData
      .get(NETEX)
      .stream()
      .map(it -> new ConfiguredDataSource<>(it, getNetexConfig(it)))
      .toList();
  }

  public NetexFeedParameters getNetexConfig(DataSource dataSource) {
    return buildConfig.transitFeeds
      .netexFeeds()
      .stream()
      .filter(netexFeedConfig -> uriMatch(netexFeedConfig.source(), dataSource.uri()))
      .findFirst()
      .orElse(buildConfig.netexDefaults.copyOf().withSource(dataSource.uri()).build());
  }

  /**
   * Match the URI provided in the configuration with the URI of a datasource,
   * either by comparing directly the two URIs or by first prepending the OTP base directory
   * to the URI provided in the configuration.
   * This covers the case where the source parameter provided in the configuration is relative to
   * the base directory.
   */
  private boolean uriMatch(URI configURI, URI datasourceURI) {
    return (
      configURI.equals(datasourceURI) ||
      (
        !configURI.isAbsolute() &&
        baseDirectory.toPath().resolve(configURI.toString()).toUri().equals(datasourceURI)
      )
    );
  }

  public CompositeDataSource getBuildReportDir() {
    return store.getBuildReportDir();
  }

  public File getCacheDirectory() {
    return cacheDirectory;
  }

  /* private methods */

  private boolean hasOneOf(FileType... types) {
    for (FileType type : types) {
      if (has(type)) {
        return true;
      }
    }
    return false;
  }

  private void logSkippedAndSelectedFiles() {
    LOG.info("Data source location(s): {}", String.join(", ", store.getRepositoryDescriptions()));

    // Sort data input files by type
    LOG.info("Existing files expected to be read or written:");
    for (FileType type : FileType.values()) {
      for (DataSource source : inputData.get(type)) {
        LOG.info(BULLET_POINT + "{}", source.detailedInfo());
      }
    }

    if (!skipData.values().isEmpty()) {
      LOG.info("Files excluded due to command line switches or unknown type:");
      for (FileType type : FileType.values()) {
        for (DataSource source : skipData.get(type)) {
          LOG.info(BULLET_POINT + "{}", source.detailedInfo());
        }
      }
    }
  }

  private void validateCliMatchesInputData(CommandLineParameters cli) {
    if (cli.build) {
      if (!hasOneOf(OSM, GTFS, NETEX)) {
        throw new OtpAppException("Unable to build graph, no transit data available.");
      }
    } else if (cli.buildStreet) {
      if (!has(OSM)) {
        throw new OtpAppException("Unable to build street graph, no OSM data available.");
      }
    } else if (cli.load) {
      if (!store.getGraph().exists()) {
        throw new OtpAppException(
          "Unable to load graph, no graph file found: %s",
          store.getGraph().path()
        );
      }
    } else if (cli.loadStreet) {
      if (!store.getStreetGraph().exists()) {
        throw new OtpAppException(
          "Unable to load street graph, no street graph file found: %s",
          store.getStreetGraph().path()
        );
      }
    }
  }

  private DataSource getOutputGraph(CommandLineParameters cli) {
    if (cli.doSaveGraph()) {
      return store.getGraph();
    } else if (cli.doSaveStreetGraph()) {
      return store.getStreetGraph();
    }
    return null;
  }

  private void include(boolean include, FileType type) {
    // Add or remove type - we do not care if the element already exist or not
    if (include) {
      includeTypes.add(type);
    } else {
      includeTypes.remove(type);
    }
  }

  private void selectFilesToImport() {
    for (FileType type : FileType.values()) {
      if (includeTypes.contains(type)) {
        inputData.putAll(type, store.listExistingSourcesFor(type));
      } else {
        skipData.putAll(type, store.listExistingSourcesFor(type));
      }
    }
  }
}