org.openmetadata.service.search.indexes.SearchIndex Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of openmetadata-service Show documentation
There is a newer version: 1.6.2
package org.openmetadata.service.search.indexes;

import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
import static org.openmetadata.service.Entity.FIELD_NAME;
import static org.openmetadata.service.jdbi3.LineageRepository.buildRelationshipDetailsMap;
import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM;
import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME;
import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS;
import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.LineageDetails;
import org.openmetadata.schema.type.Relationship;
import org.openmetadata.service.Entity;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.search.SearchIndexUtils;
import org.openmetadata.service.search.models.SearchSuggest;
import org.openmetadata.service.util.FullyQualifiedName;
import org.openmetadata.service.util.JsonUtils;

public interface SearchIndex {
  Set DEFAULT_EXCLUDED_FIELDS =
      Set.of("changeDescription", "lineage.pipeline.changeDescription", "connection");

  default Map buildSearchIndexDoc() {
    // Build Index Doc
    Map esDoc = this.buildSearchIndexDocInternal(JsonUtils.getMap(getEntity()));

    // Non Indexable Fields
    removeNonIndexableFields(esDoc);

    return esDoc;
  }

  default void removeNonIndexableFields(Map esDoc) {
    // Remove non indexable fields
    SearchIndexUtils.removeNonIndexableFields(esDoc, DEFAULT_EXCLUDED_FIELDS);

    // Remove Entity Specific Field
    SearchIndexUtils.removeNonIndexableFields(esDoc, getExcludedFields());
  }

  Object getEntity();

  default Set getExcludedFields() {
    return Collections.emptySet();
  }

  Map buildSearchIndexDocInternal(Map esDoc);

  default List getSuggest() {
    return null;
  }

  default Map getCommonAttributesMap(EntityInterface entity, String entityType) {
    Map map = new HashMap<>();
    List suggest = getSuggest();
    map.put("entityType", entityType);
    map.put("owners", getEntitiesWithDisplayName(entity.getOwners()));
    map.put("domain", getEntityWithDisplayName(entity.getDomain()));
    map.put("followers", SearchIndexUtils.parseFollowers(entity.getFollowers()));
    map.put(
        "totalVotes",
        nullOrEmpty(entity.getVotes())
            ? 0
            : entity.getVotes().getUpVotes() - entity.getVotes().getDownVotes());
    map.put("descriptionStatus", getDescriptionStatus(entity));
    map.put("suggest", suggest);
    map.put(
        "fqnParts",
        getFQNParts(
            entity.getFullyQualifiedName(),
            suggest.stream().map(SearchSuggest::getInput).toList()));
    return map;
  }

  default Set getFQNParts(String fqn, List fqnSplits) {
    Set fqnParts = new HashSet<>();
    fqnParts.add(fqn);
    String parent = FullyQualifiedName.getParentFQN(fqn);
    while (parent != null) {
      fqnParts.add(parent);
      parent = FullyQualifiedName.getParentFQN(parent);
    }
    fqnParts.addAll(fqnSplits);
    return fqnParts;
  }

  default List getEntitiesWithDisplayName(List entities) {
    if (nullOrEmpty(entities)) {
      return Collections.emptyList();
    }
    List clone = new ArrayList<>();
    for (EntityReference entity : entities) {
      EntityReference cloneEntity = JsonUtils.deepCopy(entity, EntityReference.class);
      cloneEntity.setDisplayName(
          nullOrEmpty(cloneEntity.getDisplayName())
              ? cloneEntity.getName()
              : cloneEntity.getDisplayName());
      cloneEntity.setFullyQualifiedName(cloneEntity.getFullyQualifiedName().replace("\"", "\\'"));
      clone.add(cloneEntity);
    }
    return clone;
  }

  default EntityReference getEntityWithDisplayName(EntityReference entity) {
    if (entity == null) {
      return null;
    }
    EntityReference cloneEntity = JsonUtils.deepCopy(entity, EntityReference.class);
    cloneEntity.setDisplayName(
        nullOrEmpty(cloneEntity.getDisplayName())
            ? cloneEntity.getName()
            : cloneEntity.getDisplayName());
    return cloneEntity;
  }

  default String getDescriptionStatus(EntityInterface entity) {
    return nullOrEmpty(entity.getDescription()) ? "INCOMPLETE" : "COMPLETE";
  }

  static List> getLineageData(EntityReference entity) {
    List> data = new ArrayList<>();
    CollectionDAO dao = Entity.getCollectionDAO();
    List toRelationshipsRecords =
        dao.relationshipDAO()
            .findTo(entity.getId(), entity.getType(), Relationship.UPSTREAM.ordinal());
    for (CollectionDAO.EntityRelationshipRecord entityRelationshipRecord : toRelationshipsRecords) {
      EntityReference ref =
          Entity.getEntityReferenceById(
              entityRelationshipRecord.getType(), entityRelationshipRecord.getId(), Include.ALL);
      LineageDetails lineageDetails =
          JsonUtils.readValue(entityRelationshipRecord.getJson(), LineageDetails.class);
      data.add(buildRelationshipDetailsMap(entity, ref, lineageDetails));
    }
    List fromRelationshipsRecords =
        dao.relationshipDAO()
            .findFrom(entity.getId(), entity.getType(), Relationship.UPSTREAM.ordinal());
    for (CollectionDAO.EntityRelationshipRecord entityRelationshipRecord :
        fromRelationshipsRecords) {
      EntityReference ref =
          Entity.getEntityReferenceById(
              entityRelationshipRecord.getType(), entityRelationshipRecord.getId(), Include.ALL);
      LineageDetails lineageDetails =
          JsonUtils.readValue(entityRelationshipRecord.getJson(), LineageDetails.class);
      data.add(buildRelationshipDetailsMap(ref, entity, lineageDetails));
    }
    return data;
  }

  static Map getDefaultFields() {
    Map fields = new HashMap<>();
    fields.put(FIELD_DISPLAY_NAME, 10.0f);
    fields.put(FIELD_DISPLAY_NAME_NGRAM, 1.0f);
    fields.put(FIELD_NAME, 10.0f);
    fields.put(FIELD_NAME_NGRAM, 1.0f);
    fields.put(DISPLAY_NAME_KEYWORD, 8.0f);
    fields.put(NAME_KEYWORD, 8.0f);
    fields.put(FIELD_DESCRIPTION, 2.0f);
    fields.put(FULLY_QUALIFIED_NAME, 5.0f);
    fields.put(FULLY_QUALIFIED_NAME_PARTS, 5.0f);
    return fields;
  }

  static Map getAllFields() {
    Map fields = getDefaultFields();
    fields.putAll(TableIndex.getFields());
    fields.putAll(DashboardIndex.getFields());
    fields.putAll(DashboardDataModelIndex.getFields());
    fields.putAll(PipelineIndex.getFields());
    fields.putAll(TopicIndex.getFields());
    fields.putAll(MlModelIndex.getFields());
    fields.putAll(ContainerIndex.getFields());
    fields.putAll(SearchEntityIndex.getFields());
    fields.putAll(GlossaryTermIndex.getFields());
    fields.putAll(TagIndex.getFields());
    fields.putAll(DataProductIndex.getFields());
    fields.putAll(APIEndpointIndex.getFields());
    return fields;
  }
}