com.yahoo.vespa.model.filedistribution.UserConfiguredFiles Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of config-model Show documentation
There is a newer version: 8.441.21
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.filedistribution;

import com.yahoo.config.FileReference;
import com.yahoo.config.ModelReference;
import com.yahoo.config.application.api.ApplicationFile;
import com.yahoo.config.application.api.ApplicationPackage;
import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.config.application.api.FileRegistry;
import com.yahoo.config.model.api.ModelContext;
import com.yahoo.config.model.producer.AnyConfigProducer;
import com.yahoo.config.model.producer.UserConfigRepo;
import com.yahoo.path.Path;
import com.yahoo.vespa.config.ConfigDefinition;
import com.yahoo.vespa.config.ConfigDefinitionKey;
import com.yahoo.vespa.config.ConfigPayloadBuilder;
import com.yahoo.yolean.Exceptions;

import java.io.Serializable;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;

import static com.yahoo.vespa.model.container.ApplicationContainerCluster.UserConfiguredUrls;
import static java.util.logging.Level.INFO;
import static java.util.logging.Level.WARNING;

/**
 * Utility methods for registering file distribution of files/paths/urls/models defined by the user.
 *
 * @author gjoranv
 * @author hmusum
 */
public class UserConfiguredFiles implements Serializable {

    private final FileRegistry fileRegistry;
    private final DeployLogger logger;
    private final UserConfiguredUrls userConfiguredUrls;
    private final String unknownConfigDefinition;
    private final ApplicationPackage applicationPackage;

    public UserConfiguredFiles(FileRegistry fileRegistry, DeployLogger logger,
                               ModelContext.FeatureFlags featureFlags,
                               UserConfiguredUrls userConfiguredUrls,
                               ApplicationPackage applicationPackage) {
        this.fileRegistry = fileRegistry;
        this.logger = logger;
        this.userConfiguredUrls = userConfiguredUrls;
        this.unknownConfigDefinition = featureFlags.unknownConfigDefinition();
        this.applicationPackage = applicationPackage;
    }

    /**
     * Registers user configured files for a producer for file distribution.
     */
    public  void register(PRODUCER producer) {
        UserConfigRepo userConfigs = producer.getUserConfigs();
        Map registeredFiles = new HashMap<>();
        for (ConfigDefinitionKey key : userConfigs.configsProduced()) {
            ConfigPayloadBuilder builder = userConfigs.get(key);
            try {
                register(builder, registeredFiles, key);
            } catch (IllegalArgumentException e) {
                throw new IllegalArgumentException("Invalid config in services.xml for '" + key + "': " +
                                                   Exceptions.toMessageString(e));
            }
        }
    }

    private void register(ConfigPayloadBuilder builder, Map registeredFiles, ConfigDefinitionKey key) {
        ConfigDefinition configDefinition = builder.getConfigDefinition();
        if (configDefinition == null) {
            String message = "Unable to find config definition " + key + ". Will not register files for file distribution for this config";
            switch (unknownConfigDefinition) {
                case "warning" -> logger.logApplicationPackage(WARNING, message);
                case "fail" -> throw new IllegalArgumentException("Unable to find config definition for " + key);
            }
            return;
        }

        // Inspect fields at this level
        registerEntries(builder, registeredFiles, configDefinition.getFileDefs(), false);
        registerEntries(builder, registeredFiles, configDefinition.getPathDefs(), false);
        registerEntries(builder, registeredFiles, configDefinition.getOptionalPathDefs(), false);
        registerEntries(builder, registeredFiles, configDefinition.getModelDefs(), true);

        // Inspect arrays
        for (Map.Entry entry : configDefinition.getArrayDefs().entrySet()) {
            if (isNotAnyFileType(entry.getValue().getTypeSpec().getType())) continue;
            ConfigPayloadBuilder.Array array = builder.getArray(entry.getKey());
            registerFileEntries(array.getElements(), registeredFiles, "model".equals(entry.getValue().getTypeSpec().getType()));
        }

        // Inspect maps
        for (Map.Entry entry : configDefinition.getLeafMapDefs().entrySet()) {
            if (isNotAnyFileType(entry.getValue().getTypeSpec().getType())) continue;
            ConfigPayloadBuilder.MapBuilder map = builder.getMap(entry.getKey());
            registerFileEntries(map.getElements(), registeredFiles, "model".equals(entry.getValue().getTypeSpec().getType()));
        }

        // Inspect inner fields
        for (String name : configDefinition.getStructDefs().keySet()) {
            register(builder.getObject(name), registeredFiles, key);
        }
        for (String name : configDefinition.getInnerArrayDefs().keySet()) {
            ConfigPayloadBuilder.Array array = builder.getArray(name);
            for (ConfigPayloadBuilder element : array.getElements()) {
                register(element, registeredFiles, key);
            }
        }
        for (String name : configDefinition.getStructMapDefs().keySet()) {
            ConfigPayloadBuilder.MapBuilder map = builder.getMap(name);
            for (ConfigPayloadBuilder element : map.getElements()) {
                register(element, registeredFiles, key);
            }
        }
    }

    private static boolean isNotAnyFileType(String type) {
        return ! "file".equals(type) && ! "path".equals(type) && ! "model".equals(type);
    }

    private void registerEntries(ConfigPayloadBuilder builder,
                                 Map registeredFiles,
                                 Map entries,
                                 boolean isModelType) {
        for (Map.Entry entry : entries.entrySet()) {
            String name = entry.getKey();
            ConfigPayloadBuilder fileEntry = builder.getObject(name);
            if (isEmptyOptionalPath(entry, fileEntry)) continue;
            if (fileEntry.getValue() == null || fileEntry.getValue().equals("."))
                throw new IllegalArgumentException("Invalid config value '" + fileEntry.getValue() + "' for field '" + name);
            registerFileEntry(fileEntry, registeredFiles, isModelType);
        }
    }

    private static boolean isEmptyOptionalPath(Map.Entry entry, ConfigPayloadBuilder fileEntry) {
        return entry.getValue() instanceof ConfigDefinition.OptionalPathDef && fileEntry.getValue() == null;
    }

    private void registerFileEntries(Collection builders, Map registeredFiles, boolean isModelType) {
        for (ConfigPayloadBuilder builder : builders) {
            registerFileEntry(builder, registeredFiles, isModelType);
        }
    }

    private void registerFileEntry(ConfigPayloadBuilder builder, Map registeredFiles, boolean isModelType) {
        Path path;
        if (isModelType) {
            var modelReference = ModelReference.valueOf(builder.getValue());
            if (modelReference.path().isEmpty()) {
                modelReference.url().ifPresent(url -> userConfiguredUrls.add(url.value()));
                return;
            }
            path = Path.fromString(modelReference.path().get().value());
        }
        else {
            path = Path.fromString(builder.getValue());
        }

        ApplicationFile file = applicationPackage.getFile(path);
        if (file.isDirectory() && (file.listFiles() == null || file.listFiles().isEmpty()))
            logger.logApplicationPackage(INFO, "Directory '" + path.getRelative() + "' is empty");

        FileReference reference = registeredFiles.get(path);
        if (reference == null) {
            reference = fileRegistry.addFile(path.getRelative());
            registeredFiles.put(path, reference);
        }
        if (reference == null)
            throw new IllegalArgumentException("No such file or directory '" + path.getRelative() + "'");

        if (isModelType) {
            var model = ModelReference.valueOf(builder.getValue());
            var modelWithReference = ModelReference.unresolved(model.modelId(), model.url(), Optional.of(reference));
            builder.setValue(modelWithReference.toString());
        }
        else {
            builder.setValue(reference.value());
        }
    }

}