com.yahoo.schema.derived.SchemaInfo Maven / Gradle / Ivy
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.schema.derived;
import com.yahoo.document.ArrayDataType;
import com.yahoo.document.DataType;
import com.yahoo.document.MapDataType;
import com.yahoo.document.PrimitiveDataType;
import com.yahoo.document.ReferenceDataType;
import com.yahoo.document.StructuredDataType;
import com.yahoo.document.TensorDataType;
import com.yahoo.document.WeightedSetDataType;
import com.yahoo.document.annotation.AnnotationReferenceDataType;
import com.yahoo.documentmodel.NewDocumentReferenceDataType;
import com.yahoo.schema.document.Attribute;
import com.yahoo.schema.document.FieldSet;
import com.yahoo.schema.document.ImmutableSDField;
import com.yahoo.search.config.SchemaInfoConfig;
import com.yahoo.schema.Index;
import com.yahoo.schema.RankProfile;
import com.yahoo.schema.RankProfileRegistry;
import com.yahoo.schema.Schema;
import com.yahoo.searchlib.rankingexpression.Reference;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
/**
* Information about a schema.
*
* @author bratseth
*/
public final class SchemaInfo extends Derived {
private final Schema schema;
// Info about profiles needed in memory after build.
// The rank profile registry itself is not kept around due to its size.
private final Map rankProfiles;
private final Summaries summaries;
private final IndexMode indexMode;
public enum IndexMode {INDEX, STREAMING, STORE_ONLY}
public SchemaInfo(Schema schema, String indexMode, RankProfileRegistry rankProfileRegistry, Summaries summaries) {
this(schema, indexMode(indexMode), rankProfileRegistry, summaries);
}
public SchemaInfo(Schema schema, IndexMode indexMode, RankProfileRegistry rankProfileRegistry, Summaries summaries) {
this.schema = schema;
this.rankProfiles = Collections.unmodifiableMap(toRankProfiles(rankProfileRegistry.rankProfilesOf(schema)));
this.summaries = summaries;
this.indexMode = indexMode;
}
private static IndexMode indexMode(String mode) {
if (mode == null) return IndexMode.INDEX;
return switch (mode) {
case "index" -> IndexMode.INDEX;
case "streaming" -> IndexMode.STREAMING;
case "store-only" -> IndexMode.STORE_ONLY;
default -> IndexMode.STORE_ONLY;
};
}
public IndexMode getIndexMode() { return indexMode; }
public String name() { return schema.getName(); }
@Override
public String getDerivedName() { return "schema-info"; }
public Schema fullSchema() { return schema; }
public Map rankProfiles() { return rankProfiles; }
private Map toRankProfiles(Collection rankProfiles) {
Map rankProfileInfos = new LinkedHashMap<>();
rankProfiles.forEach(profile -> rankProfileInfos.put(profile.name(), new RankProfileInfo(profile)));
return rankProfileInfos;
}
public void getConfig(SchemaInfoConfig.Builder builder) {
// Append
var schemaBuilder = new SchemaInfoConfig.Schema.Builder();
schemaBuilder.name(schema.getName());
addFieldsConfig(schemaBuilder);
addFieldSetConfig(schemaBuilder);
addSummaryConfig(schemaBuilder);
addRankProfilesConfig(schemaBuilder);
builder.schema(schemaBuilder);
}
public void export(String toDirectory) throws IOException {
var builder = new SchemaInfoConfig.Builder();
getConfig(builder);
export(toDirectory, builder.build());
}
private void addFieldsConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) {
for (var field : schema.allFieldsList()) {
addFieldConfig(field, schemaBuilder);
for (var index : field.getIndices().values()) {
if ( ! index.getName().equals(field.getName())) // additional index
addFieldConfig(index, field.getDataType(), schemaBuilder);
}
for (var attribute : field.getAttributes().values()) {
if ( ! attribute.getName().equals(field.getName())) // additional attribute
addFieldConfig(attribute, field.getDataType(), schemaBuilder);
}
}
}
private void addFieldConfig(ImmutableSDField field, SchemaInfoConfig.Schema.Builder schemaBuilder) {
var fieldBuilder = new SchemaInfoConfig.Schema.Field.Builder();
fieldBuilder.name(field.getName());
fieldBuilder.type(toTypeSpec(field.getDataType()));
for (var alias : field.getAliasToName().entrySet()) {
if (alias.getValue().equals(field.getName()))
fieldBuilder.alias(alias.getKey());
}
fieldBuilder.attribute(field.doesAttributing());
fieldBuilder.index(field.doesIndexing());
schemaBuilder.field(fieldBuilder);
}
// TODO: Make fields and indexes 1-1 so that this can be removed
private void addFieldConfig(Index index, DataType type, SchemaInfoConfig.Schema.Builder schemaBuilder) {
var fieldBuilder = new SchemaInfoConfig.Schema.Field.Builder();
fieldBuilder.name(index.getName());
fieldBuilder.type(toTypeSpec(type));
for (Iterator i = index.aliasIterator(); i.hasNext(); )
fieldBuilder.alias(i.next());
fieldBuilder.attribute(false);
fieldBuilder.index(true);
schemaBuilder.field(fieldBuilder);
}
// TODO: Make fields and attributes 1-1 so that this can be removed
private void addFieldConfig(Attribute attribute, DataType type, SchemaInfoConfig.Schema.Builder schemaBuilder) {
var fieldBuilder = new SchemaInfoConfig.Schema.Field.Builder();
fieldBuilder.name(attribute.getName());
fieldBuilder.type(toTypeSpec(type));
for (var alias : attribute.getAliases())
fieldBuilder.alias(alias);
fieldBuilder.attribute(true);
fieldBuilder.index(false);
schemaBuilder.field(fieldBuilder);
}
private void addFieldSetConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) {
for (var fieldSet : schema.fieldSets().builtInFieldSets().values())
addFieldSetConfig(fieldSet, schemaBuilder);
for (var fieldSet : schema.fieldSets().userFieldSets().values())
addFieldSetConfig(fieldSet, schemaBuilder);
}
private void addFieldSetConfig(FieldSet fieldSet, SchemaInfoConfig.Schema.Builder schemaBuilder) {
var fieldSetBuilder = new SchemaInfoConfig.Schema.Fieldset.Builder();
fieldSetBuilder.name(fieldSet.getName());
for (String fieldName : fieldSet.getFieldNames())
fieldSetBuilder.field(fieldName);
schemaBuilder.fieldset(fieldSetBuilder);
}
private void addSummaryConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) {
for (var summary : summaries.asList()) {
var summaryBuilder = new SchemaInfoConfig.Schema.Summaryclass.Builder();
summaryBuilder.name(summary.getName());
for (var field : summary.fields().values()) {
var fieldsBuilder = new SchemaInfoConfig.Schema.Summaryclass.Fields.Builder();
fieldsBuilder.name(field.getName())
.type(field.getType().getName())
.dynamic(SummaryClass.commandRequiringQuery(field.getCommand()));
summaryBuilder.fields(fieldsBuilder);
}
schemaBuilder.summaryclass(summaryBuilder);
}
}
private void addRankProfilesConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) {
for (RankProfileInfo rankProfile : rankProfiles().values()) {
var rankProfileConfig = new SchemaInfoConfig.Schema.Rankprofile.Builder()
.name(rankProfile.name())
.hasSummaryFeatures(rankProfile.hasSummaryFeatures())
.hasRankFeatures(rankProfile.hasRankFeatures())
.significance(new SchemaInfoConfig.Schema.Rankprofile.Significance.Builder()
.useModel(rankProfile.useSignificanceModel()));
for (var input : rankProfile.inputs().entrySet()) {
var inputConfig = new SchemaInfoConfig.Schema.Rankprofile.Input.Builder();
inputConfig.name(input.getKey().toString());
inputConfig.type(input.getValue().type().toString());
rankProfileConfig.input(inputConfig);
}
schemaBuilder.rankprofile(rankProfileConfig);
}
}
/** Returns this type as a spec on the form following "field [name] type " in schemas. */
private String toTypeSpec(DataType dataType) {
if (dataType instanceof PrimitiveDataType)
return dataType.getName();
if (dataType instanceof AnnotationReferenceDataType annotationType)
return "annotationreference<" + annotationType.getAnnotationType().getName() + ">";
if (dataType instanceof ArrayDataType arrayType)
return "array<" + toTypeSpec(arrayType.getNestedType()) + ">";
if (dataType instanceof MapDataType mapType)
return "map<" + toTypeSpec(mapType.getKeyType()) + "," + toTypeSpec(mapType.getValueType()) + ">";
if (dataType instanceof ReferenceDataType referenceType)
return "reference<" + toTypeSpec(referenceType.getTargetType()) + ">";
if (dataType instanceof NewDocumentReferenceDataType referenceType)
return "reference<" + toTypeSpec(referenceType.getTargetType()) + ">";
if (dataType instanceof StructuredDataType structType)
return structType.getName();
if (dataType instanceof TensorDataType tensorType)
return tensorType.getTensorType().toString();
if (dataType instanceof WeightedSetDataType weightedSetDataType)
return "weightedset<" + toTypeSpec(weightedSetDataType.getNestedType()) + ">";
throw new IllegalArgumentException("Unknown data type " + dataType + " class " + dataType.getClass());
}
/** A store of a *small* (in memory) amount of rank profile info. */
public static final class RankProfileInfo {
private final String name;
private final boolean hasSummaryFeatures;
private final boolean hasRankFeatures;
private final boolean useSignificanceModel;
private final Map inputs;
public RankProfileInfo(RankProfile profile) {
this.name = profile.name();
this.hasSummaryFeatures = ! profile.getSummaryFeatures().isEmpty();
this.hasRankFeatures = ! profile.getRankFeatures().isEmpty();
this.inputs = profile.inputs();
useSignificanceModel = profile.useSignificanceModel();
}
public String name() { return name; }
public boolean hasSummaryFeatures() { return hasSummaryFeatures; }
public boolean hasRankFeatures() { return hasRankFeatures; }
public boolean useSignificanceModel() { return useSignificanceModel; }
public Map inputs() { return inputs; }
}
}