• Home
• com.cerner.bunsen
• bunsen-r4
• 0.4.4
• source code
• ConceptMaps.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

com.cerner.bunsen.r4.codes.ConceptMaps Maven / Gradle / Ivy

package com.cerner.bunsen.r4.codes;

import static org.apache.spark.sql.functions.lit;

import ca.uhn.fhir.context.FhirVersionEnum;
import com.cerner.bunsen.FhirEncoders;
import com.cerner.bunsen.codes.Mapping;
import com.cerner.bunsen.codes.UrlAndVersion;
import com.cerner.bunsen.codes.base.AbstractConceptMaps;
import com.cerner.bunsen.codes.broadcast.BroadcastableConceptMap;
import com.cerner.bunsen.codes.broadcast.BroadcastableMappings;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.MapFunction;
import org.apache.spark.broadcast.Broadcast;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoder;
import org.apache.spark.sql.SparkSession;
import org.hl7.fhir.exceptions.FHIRException;
import org.hl7.fhir.r4.model.ConceptMap;
import org.hl7.fhir.r4.model.ConceptMap.ConceptMapGroupComponent;
import org.hl7.fhir.r4.model.ConceptMap.ConceptMapGroupUnmappedMode;
import org.hl7.fhir.r4.model.ConceptMap.SourceElementComponent;
import org.hl7.fhir.r4.model.ConceptMap.TargetElementComponent;
import org.hl7.fhir.r4.model.UriType;


/**
 * An immutable collection of FHIR ConceptMaps. This class is used to import concept
 * map content, explore it, and persist it to a database.
 */
public class ConceptMaps extends AbstractConceptMaps {

  private static final Encoder CONCEPT_MAP_ENCODER = FhirEncoders.forR4()
      .getOrCreate()
      .of(ConceptMap.class);

  /**
   * Returns the encoder for mappings.
   *
   * @return an encoder for mappings.
   */
  public static Encoder getMappingEncoder() {

    return MAPPING_ENCODER;
  }

  /**
   * Returns the encoder for concept maps.
   *
   * @return an encoder for concept maps.
   */
  public static Encoder getConceptMapEncoder() {

    return CONCEPT_MAP_ENCODER;
  }


  protected ConceptMaps(SparkSession spark,
      Dataset members,
      Dataset conceptMaps,
      Dataset mappings) {

    super(spark, FhirVersionEnum.R4, members, conceptMaps, mappings, CONCEPT_MAP_ENCODER);
  }

  protected ConceptMaps newInstance(SparkSession spark,
      Dataset members,
      Dataset conceptMaps,
      Dataset mappings) {

    return new ConceptMaps(spark, members, conceptMaps, mappings);
  }

  /**
   * Returns the collection of concept maps from the default database and tables.
   *
   * @param spark the spark session
   * @return a ConceptMaps instance.
   */
  public static ConceptMaps getDefault(SparkSession spark) {

    return getFromDatabase(spark, MAPPING_DATABASE);
  }

  /**
   * Returns the collection of concept maps from the tables in the given database.
   *
   * @param spark the spark session
   * @param databaseName name of the database containing the conceptmaps and mappings tables.
   * @return a ConceptMaps instance.
   */
  public static ConceptMaps getFromDatabase(SparkSession spark, String databaseName) {

    Dataset mappings = spark.sql(
        "SELECT * FROM " + databaseName + "." + MAPPING_TABLE).as(MAPPING_ENCODER);

    Dataset conceptMaps = spark
        .sql("SELECT * FROM " + databaseName + "." + CONCEPT_MAP_TABLE)
        .as(CONCEPT_MAP_ENCODER);

    return new ConceptMaps(spark,
        spark.emptyDataset(URL_AND_VERSION_ENCODER),
        conceptMaps,
        mappings);
  }

  /**
   * Returns an empty ConceptMaps instance.
   *
   * @param spark the spark session
   * @return an empty ConceptMaps instance.
   */
  public static ConceptMaps getEmpty(SparkSession spark) {

    Dataset emptyConceptMaps = spark.emptyDataset(CONCEPT_MAP_ENCODER)
        .withColumn("timestamp", lit(null).cast("timestamp"))
        .as(CONCEPT_MAP_ENCODER);

    return new ConceptMaps(spark,
        spark.emptyDataset(URL_AND_VERSION_ENCODER),
        emptyConceptMaps,
        spark.emptyDataset(MAPPING_ENCODER));
  }

  @Override
  protected void addToConceptMap(ConceptMap map, Dataset mappings) {

    // Sort the items so they are grouped together optimally, and so
    // we consistently produce the same ordering, therefore making
    // inspection and comparison of the concept maps easier.
    List sortedMappings = mappings.sort("sourceSystem",
        "targetSystem",
        "sourceValue",
        "targetValue")
        .collectAsList();

    ConceptMapGroupComponent currentGroup = null;
    SourceElementComponent element = null;

    // Workaround for the decoder producing an immutable array by
    // replacing it with a mutable one.
    map.setGroup(new ArrayList<>(map.getGroup()));
    for (Mapping mapping: sortedMappings) {

      // Add a new group if we don't match the previous one.
      if (currentGroup == null
          || !mapping.getSourceSystem().equals(currentGroup.getSource())
          || !mapping.getTargetSystem().equals(currentGroup.getTarget())) {

        currentGroup = null;

        // Find a matching group.
        for (ConceptMapGroupComponent candidate: map.getGroup()) {

          if (mapping.getSourceSystem().equals(candidate.getSource())
              && mapping.getTargetSystem().equals(candidate.getTarget())) {

            currentGroup = candidate;

            // Workaround for the decoder producing an immutable array by
            // replacing it with a mutable one.
            currentGroup.setElement(new ArrayList<>(currentGroup.getElement()));
            break;
          }
        }

        // No matching group found, so add it.
        if (currentGroup == null) {
          currentGroup = map.addGroup();

          currentGroup.setSource(mapping.getSourceSystem());
          currentGroup.setTarget(mapping.getTargetSystem());

          // Ensure a new element is created for the newly created group.
          element = null;
        }
      }

      // There is an element for each distinct source value in the map,
      // so add one if it does not match the previous.
      if (element == null
          || !mapping.getSourceValue().equals(element.getCode())) {

        element = currentGroup.addElement();
        element.setCode(mapping.getSourceValue());
      }

      element.addTarget().setCode(mapping.getTargetValue());
    }
  }

  /**
   * Given a concept map, returns a list of mapping records it contains.
   *
   * @param map a concept map
   * @return a list of Mapping records.
   */
  public static List expandMappings(ConceptMap map) {

    List mappings = new ArrayList<>();

    expandMappingsIterator(map).forEachRemaining(mappings::add);

    return mappings;
  }

  private static Iterator expandMappingsIterator(ConceptMap map) {

    List mappings = new ArrayList<>();

    for (ConceptMapGroupComponent group: map.getGroup()) {

      for (SourceElementComponent element: group.getElement()) {

        for (TargetElementComponent target: element.getTarget()) {

          Mapping mapping = new Mapping();

          mapping.setConceptMapUri(map.getUrl());
          mapping.setConceptMapVersion(map.getVersion());

          try {
            String sourceValue = map.getSource() instanceof UriType
                ? map.getSourceUriType().getValue()
                : map.getSourceUriType().getValue();

            mapping.setSourceValueSet(sourceValue);

            String targetValue = map.getTarget() instanceof UriType
                ? map.getTargetUriType().getValue()
                : map.getTargetUriType().getValue();

            mapping.setTargetValueSet(targetValue);

          } catch (FHIRException fhirException) {

            // This should not happen because we check the types,
            // but rethrow to avoid any possibility of swallowing
            // an exception.
            throw new RuntimeException(fhirException);
          }

          mapping.setSourceSystem(group.getSource());
          mapping.setSourceValue(element.getCode());

          mapping.setTargetSystem(group.getTarget());
          mapping.setTargetValue(target.getCode());

          if (target.getEquivalence() != null) {
            mapping.setEquivalence(target.getEquivalence().toCode());
          }

          mappings.add(mapping);
        }
      }
    }

    return mappings.iterator();
  }

  @Override
  public ConceptMaps withConceptMaps(Dataset conceptMaps) {

    Dataset newMembers = getUrlAndVersions(conceptMaps);

    if (hasDuplicateUrlAndVersions(newMembers) || conceptMaps.count() != newMembers.count()) {

      throw new IllegalArgumentException(
          "Cannot add concept maps having duplicate conceptMapUri and conceptMapVersion");
    }

    // Remove the concept contents for persistence. This is most easily done in the ConceptMap
    // object by setting the group to an empty list.
    Dataset withoutConcepts = conceptMaps
        .map((MapFunction) conceptMap -> {

          // Remove the elements rather than the groups to preserved the
          // "unmapped" structure in a group that can refer to other
          // concept maps.
          ConceptMap withoutElements = conceptMap.copy();

          List updatedGroups = new ArrayList<>();

          for (ConceptMapGroupComponent group: withoutElements.getGroup()) {

            group.setElement(new ArrayList<>());
            updatedGroups.add(group);
          }

          withoutElements.setGroup(updatedGroups);

          return withoutElements;
        }, CONCEPT_MAP_ENCODER);

    Dataset newMappings = conceptMaps.flatMap(ConceptMaps::expandMappingsIterator,
        MAPPING_ENCODER);

    return withConceptMaps(withoutConcepts, newMappings);
  }

  @Override
  public Broadcast broadcast(Map conceptMapUriToVersion) {

    List mapsList = getMaps().collectAsList();

    Map allMaps = mapsList.stream()
        .collect(Collectors.toMap(ConceptMap::getUrl, Function.identity()));

    Map mapsToLoad = mapsList
        .stream()
        .filter(conceptMap ->
            conceptMap.getVersion().equals(conceptMapUriToVersion.get(conceptMap.getUrl())))
        .collect(Collectors.toMap(ConceptMap::getUrl, Function.identity()));

    // Expand the concept maps to load and sort them so dependencies are before
    // their dependents in the list.
    List sortedMapsToLoad = sortMapsToLoad(conceptMapUriToVersion.keySet(), allMaps);

    // Since this is used to map from one system to another, we use only targets
    // that don't introduce inaccurate meanings. (For instance, we can't map
    // general condition code to a more specific type, since that is not
    // representative of the source data.)
    Dataset mappings = getMappings(conceptMapUriToVersion)
        .filter("equivalence in ('equivalent', 'equals', 'wider', 'subsumes')");

    // Group mappings by their concept map URI
    Map> groupedMappings =  mappings
        .collectAsList()
        .stream()
        .collect(Collectors.groupingBy(Mapping::getConceptMapUri));

    Map broadcastableMaps = new HashMap<>();

    for (String conceptMapUri: sortedMapsToLoad) {

      ConceptMap map = allMaps.get(conceptMapUri);

      Set children = getMapChildren(map);

      List childMaps = children.stream()
          .map(child -> broadcastableMaps.get(child))
          .collect(Collectors.toList());

      BroadcastableConceptMap broadcastableConceptMap = new BroadcastableConceptMap(conceptMapUri,
          groupedMappings.getOrDefault(conceptMapUri, Collections.emptyList()),
          childMaps);

      broadcastableMaps.put(conceptMapUri, broadcastableConceptMap);
    }

    JavaSparkContext ctx = new JavaSparkContext(getMaps()
        .sparkSession()
        .sparkContext());

    return ctx.broadcast(new BroadcastableMappings(broadcastableMaps));
  }

  /**
   * Returns a list of concept maps in the order they need to be loaded.
   */
  private static List sortMapsToLoad(Set mapsToLoad,
      Map allMaps) {

    Deque pendingMaps = new ArrayDeque<>(mapsToLoad);

    List loadOrder = new ArrayList<>();

    Set loadedMaps = new HashSet<>();

    while (!pendingMaps.isEmpty()) {

      String nextMap = pendingMaps.peek();

      // If the map has already been loaded,
      // remove it and continue.
      if (loadedMaps.contains(nextMap)) {

        pendingMaps.pop();
        continue;
      }

      ConceptMap mapToLoad = allMaps.get(nextMap);

      if (mapToLoad == null) {
        throw new IllegalStateException("Concept map " + nextMap + " "
            + " is referenced but not in the collection of concept maps.");
      }

      // Get the set of children we need to load before the pending map.
      Set childrenToLoad = getMapChildren(mapToLoad);

      childrenToLoad.removeAll(loadedMaps);

      // If the pending map has no children to load, we can
      // add it to our load order.
      if (childrenToLoad.isEmpty()) {

        loadedMaps.add(nextMap);
        loadOrder.add(nextMap);
        pendingMaps.pop();

      } else {

        // The pending map has children, so we need to load them first.
        for (String child: childrenToLoad) {
          pendingMaps.push(child);
        }
      }
    }

    return loadOrder;
  }

  /**
   * Returns the children of a given concept map to which the parent
   * may delegate mapped items.
   */
  private static Set getMapChildren(ConceptMap map) {

    return map.getGroup()
        .stream()
        .filter(group -> group.getUnmapped() != null
            && group.getUnmapped().getMode() == ConceptMapGroupUnmappedMode.OTHERMAP)
        .map(group -> group.getUnmapped().getUrl())
        .collect(Collectors.toSet());
  }
}