com.yahoo.schema.processing.IndexingValidation Maven / Gradle / Ivy

// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.schema.processing;

import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.document.ArrayDataType;
import com.yahoo.document.DataType;
import com.yahoo.document.MapDataType;
import com.yahoo.document.WeightedSetDataType;
import com.yahoo.schema.RankProfileRegistry;
import com.yahoo.schema.Schema;
import com.yahoo.schema.document.Attribute;
import com.yahoo.schema.document.GeoPos;
import com.yahoo.schema.document.SDField;
import com.yahoo.vespa.documentmodel.SummaryField;
import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression;
import com.yahoo.vespa.indexinglanguage.expressions.Expression;
import com.yahoo.vespa.indexinglanguage.expressions.FieldTypeAdapter;
import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression;
import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression;
import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression;
import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression;
import com.yahoo.vespa.indexinglanguage.expressions.VerificationContext;
import com.yahoo.vespa.indexinglanguage.expressions.VerificationException;
import com.yahoo.vespa.model.container.search.QueryProfiles;
import com.yahoo.yolean.Exceptions;

import java.util.HashSet;
import java.util.Set;

/**
 * @author Simon Thoresen Hult
 */
public class IndexingValidation extends Processor {

    IndexingValidation(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
        super(schema, deployLogger, rankProfileRegistry, queryProfiles);
    }

    @Override
    public void process(boolean validate, boolean documentsOnly) {
        if ( ! validate) return;

        VerificationContext context = new VerificationContext(new MyAdapter(schema));
        for (SDField field : schema.allConcreteFields()) {
            ScriptExpression script = field.getIndexingScript();
            try {
                script.verify(context);
                MyConverter converter = new MyConverter();
                for (StatementExpression exp : script) {
                    converter.convert(exp); // TODO: stop doing this explicitly when visiting a script does not branch
                }
            } catch (VerificationException e) {
                fail(schema, field, "For expression '" + e.getExpression() + "': " + Exceptions.toMessageString(e));
            }
        }
    }

    private static class MyConverter extends ExpressionConverter {

        final Set outputs = new HashSet<>();
        final Set prevNames = new HashSet<>();

        @Override
        public ExpressionConverter branch() {
            MyConverter ret = new MyConverter();
            ret.outputs.addAll(outputs);
            ret.prevNames.addAll(prevNames);
            return ret;
        }

        @Override
        protected boolean shouldConvert(Expression exp) {
            if (exp instanceof OutputExpression) {
                String fieldName = ((OutputExpression)exp).getFieldName();
                if (outputs.contains(fieldName) && !prevNames.contains(fieldName)) {
                    throw new VerificationException(exp, "Attempting to assign conflicting values to field '" +
                                                         fieldName + "'.");
                }
                outputs.add(fieldName);
                prevNames.add(fieldName);
            }
            if (exp.createdOutputType() != null) {
                prevNames.clear();
            }
            return false;
        }

        @Override
        protected Expression doConvert(Expression exp) {
            throw new UnsupportedOperationException();
        }
    }

    private static class MyAdapter implements FieldTypeAdapter {

        final Schema schema;

        MyAdapter(Schema schema) {
            this.schema = schema;
        }

        @Override
        public DataType getInputType(Expression exp, String fieldName) {
            SDField field = schema.getDocumentField(fieldName);
            if (field == null) {
                throw new VerificationException(exp, "Input field '" + fieldName + "' not found.");
            }
            return field.getDataType();
        }

        @Override
        public void tryOutputType(Expression exp, String fieldName, DataType valueType) {
            String fieldDesc;
            DataType fieldType;
            if (exp instanceof AttributeExpression) {
                Attribute attribute = schema.getAttribute(fieldName);
                if (attribute == null) {
                    throw new VerificationException(exp, "Attribute '" + fieldName + "' not found.");
                }
                fieldDesc = "attribute";
                fieldType = attribute.getDataType();
            } else if (exp instanceof IndexExpression) {
                SDField field = schema.getConcreteField(fieldName);
                if (field == null) {
                    throw new VerificationException(exp, "Index field '" + fieldName + "' not found.");
                }
                fieldDesc = "index field";
                fieldType = field.getDataType();
            } else if (exp instanceof SummaryExpression) {
                SummaryField field = schema.getSummaryField(fieldName);
                if (field == null) {
                    // Use document field if summary field is not found
                    SDField sdField = schema.getConcreteField(fieldName);
                    if (sdField != null && sdField.doesSummarying()) {
                        fieldDesc = "document field";
                        fieldType = sdField.getDataType();
                    } else {
                        throw new VerificationException(exp, "Summary field '" + fieldName + "' not found.");
                    }
                } else {
                    fieldDesc = "summary field";
                    fieldType = field.getDataType();
                }
            } else {
                throw new UnsupportedOperationException();
            }
            if ( ! fieldType.isAssignableFrom(valueType) &&
                 ! fieldType.isAssignableFrom(createCompatType(valueType))) {
                throw new VerificationException(exp, "Can not assign " + valueType.getName() + " to " + fieldDesc +
                                                     " '" + fieldName + "' which is " + fieldType.getName() + ".");
            }
        }

        private static DataType createCompatType(DataType origType) {
            if (origType instanceof ArrayDataType) {
                return DataType.getArray(createCompatType(((ArrayDataType)origType).getNestedType()));
            } else if (origType instanceof MapDataType) {
                MapDataType mapType = (MapDataType)origType;
                return DataType.getMap(createCompatType(mapType.getKeyType()),
                                       createCompatType(mapType.getValueType()));
            } else if (origType instanceof WeightedSetDataType) {
                return DataType.getWeightedSet(createCompatType(((WeightedSetDataType)origType).getNestedType()));
            } else if (GeoPos.isPos(origType)) {
                return DataType.LONG;
            } else {
                return origType;
            }
        }
    }

}