All Downloads are FREE. Search and download functionalities are using the official Maven repository.
Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
com.yahoo.vespa.model.filedistribution.UserConfiguredFiles Maven / Gradle / Ivy
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.filedistribution;
import com.yahoo.config.FileReference;
import com.yahoo.config.ModelReference;
import com.yahoo.config.application.api.ApplicationFile;
import com.yahoo.config.application.api.ApplicationPackage;
import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.config.application.api.FileRegistry;
import com.yahoo.config.model.api.ModelContext;
import com.yahoo.config.model.producer.AnyConfigProducer;
import com.yahoo.config.model.producer.UserConfigRepo;
import com.yahoo.path.Path;
import com.yahoo.vespa.config.ConfigDefinition;
import com.yahoo.vespa.config.ConfigDefinitionKey;
import com.yahoo.vespa.config.ConfigPayloadBuilder;
import com.yahoo.yolean.Exceptions;
import java.io.Serializable;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import static com.yahoo.vespa.model.container.ApplicationContainerCluster.UserConfiguredUrls;
import static java.util.logging.Level.INFO;
import static java.util.logging.Level.WARNING;
/**
* Utility methods for registering file distribution of files/paths/urls/models defined by the user.
*
* @author gjoranv
* @author hmusum
*/
public class UserConfiguredFiles implements Serializable {
private final FileRegistry fileRegistry;
private final DeployLogger logger;
private final UserConfiguredUrls userConfiguredUrls;
private final String unknownConfigDefinition;
private final ApplicationPackage applicationPackage;
public UserConfiguredFiles(FileRegistry fileRegistry, DeployLogger logger,
ModelContext.FeatureFlags featureFlags,
UserConfiguredUrls userConfiguredUrls,
ApplicationPackage applicationPackage) {
this.fileRegistry = fileRegistry;
this.logger = logger;
this.userConfiguredUrls = userConfiguredUrls;
this.unknownConfigDefinition = featureFlags.unknownConfigDefinition();
this.applicationPackage = applicationPackage;
}
/**
* Registers user configured files for a producer for file distribution.
*/
public void register(PRODUCER producer) {
UserConfigRepo userConfigs = producer.getUserConfigs();
Map registeredFiles = new HashMap<>();
for (ConfigDefinitionKey key : userConfigs.configsProduced()) {
ConfigPayloadBuilder builder = userConfigs.get(key);
try {
register(builder, registeredFiles, key);
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException("Invalid config in services.xml for '" + key + "': " +
Exceptions.toMessageString(e));
}
}
}
private void register(ConfigPayloadBuilder builder, Map registeredFiles, ConfigDefinitionKey key) {
ConfigDefinition configDefinition = builder.getConfigDefinition();
if (configDefinition == null) {
String message = "Unable to find config definition " + key + ". Will not register files for file distribution for this config";
switch (unknownConfigDefinition) {
case "warning" -> logger.logApplicationPackage(WARNING, message);
case "fail" -> throw new IllegalArgumentException("Unable to find config definition for " + key);
}
return;
}
// Inspect fields at this level
registerEntries(builder, registeredFiles, configDefinition.getFileDefs(), false);
registerEntries(builder, registeredFiles, configDefinition.getPathDefs(), false);
registerEntries(builder, registeredFiles, configDefinition.getOptionalPathDefs(), false);
registerEntries(builder, registeredFiles, configDefinition.getModelDefs(), true);
// Inspect arrays
for (Map.Entry entry : configDefinition.getArrayDefs().entrySet()) {
if (isNotAnyFileType(entry.getValue().getTypeSpec().getType())) continue;
ConfigPayloadBuilder.Array array = builder.getArray(entry.getKey());
registerFileEntries(array.getElements(), registeredFiles, "model".equals(entry.getValue().getTypeSpec().getType()));
}
// Inspect maps
for (Map.Entry entry : configDefinition.getLeafMapDefs().entrySet()) {
if (isNotAnyFileType(entry.getValue().getTypeSpec().getType())) continue;
ConfigPayloadBuilder.MapBuilder map = builder.getMap(entry.getKey());
registerFileEntries(map.getElements(), registeredFiles, "model".equals(entry.getValue().getTypeSpec().getType()));
}
// Inspect inner fields
for (String name : configDefinition.getStructDefs().keySet()) {
register(builder.getObject(name), registeredFiles, key);
}
for (String name : configDefinition.getInnerArrayDefs().keySet()) {
ConfigPayloadBuilder.Array array = builder.getArray(name);
for (ConfigPayloadBuilder element : array.getElements()) {
register(element, registeredFiles, key);
}
}
for (String name : configDefinition.getStructMapDefs().keySet()) {
ConfigPayloadBuilder.MapBuilder map = builder.getMap(name);
for (ConfigPayloadBuilder element : map.getElements()) {
register(element, registeredFiles, key);
}
}
}
private static boolean isNotAnyFileType(String type) {
return ! "file".equals(type) && ! "path".equals(type) && ! "model".equals(type);
}
private void registerEntries(ConfigPayloadBuilder builder,
Map registeredFiles,
Map entries,
boolean isModelType) {
for (Map.Entry entry : entries.entrySet()) {
String name = entry.getKey();
ConfigPayloadBuilder fileEntry = builder.getObject(name);
if (isEmptyOptionalPath(entry, fileEntry)) continue;
if (fileEntry.getValue() == null || fileEntry.getValue().equals("."))
throw new IllegalArgumentException("Invalid config value '" + fileEntry.getValue() + "' for field '" + name);
registerFileEntry(fileEntry, registeredFiles, isModelType);
}
}
private static boolean isEmptyOptionalPath(Map.Entry entry, ConfigPayloadBuilder fileEntry) {
return entry.getValue() instanceof ConfigDefinition.OptionalPathDef && fileEntry.getValue() == null;
}
private void registerFileEntries(Collection builders, Map registeredFiles, boolean isModelType) {
for (ConfigPayloadBuilder builder : builders) {
registerFileEntry(builder, registeredFiles, isModelType);
}
}
private void registerFileEntry(ConfigPayloadBuilder builder, Map registeredFiles, boolean isModelType) {
Path path;
if (isModelType) {
var modelReference = ModelReference.valueOf(builder.getValue());
if (modelReference.path().isEmpty()) {
modelReference.url().ifPresent(url -> userConfiguredUrls.add(url.value()));
return;
}
path = Path.fromString(modelReference.path().get().value());
}
else {
path = Path.fromString(builder.getValue());
}
ApplicationFile file = applicationPackage.getFile(path);
if (file.isDirectory() && (file.listFiles() == null || file.listFiles().isEmpty()))
logger.logApplicationPackage(INFO, "Directory '" + path.getRelative() + "' is empty");
FileReference reference = registeredFiles.get(path);
if (reference == null) {
reference = fileRegistry.addFile(path.getRelative());
registeredFiles.put(path, reference);
}
if (reference == null)
throw new IllegalArgumentException("No such file or directory '" + path.getRelative() + "'");
if (isModelType) {
var model = ModelReference.valueOf(builder.getValue());
var modelWithReference = ModelReference.unresolved(model.modelId(), model.url(), Optional.of(reference));
builder.setValue(modelWithReference.toString());
}
else {
builder.setValue(reference.value());
}
}
}
