com.yahoo.application.Application Maven / Gradle / Ivy
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.application;
import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter;
import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter;
import ai.vespa.rankingexpression.importer.onnx.OnnxImporter;
import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter;
import ai.vespa.rankingexpression.importer.vespa.VespaImporter;
import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter;
import com.google.common.annotations.Beta;
import com.yahoo.application.container.JDisc;
import com.yahoo.application.container.impl.StandaloneContainerRunner;
import com.yahoo.application.content.ContentCluster;
import com.yahoo.config.*;
import com.yahoo.config.application.api.ApplicationPackage;
import com.yahoo.config.model.NullConfigModelRegistry;
import com.yahoo.config.model.application.provider.FilesApplicationPackage;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.docproc.DocumentProcessor;
import com.yahoo.io.IOUtils;
import com.yahoo.jdisc.handler.RequestHandler;
import com.yahoo.jdisc.service.ClientProvider;
import com.yahoo.jdisc.service.ServerProvider;
import com.yahoo.search.Searcher;
import com.yahoo.search.rendering.Renderer;
import com.yahoo.text.StringUtilities;
import com.yahoo.text.Utf8;
import com.yahoo.vespa.model.VespaModel;
import com.yahoo.yolean.Exceptions;
import org.xml.sax.SAXException;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.lang.reflect.Field;
import java.net.BindException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
/**
* Contains one or more containers built from services.xml.
* Other services present in the services.xml file might be mocked in future versions.
*
* Currently, only a single top level JDisc Container is allowed. Other clusters are ignored.
*
* @author Tony Vaagenes
*/
@Beta
public final class Application implements AutoCloseable {
/**
* This system property is set to "true" upon creation of an Application.
* This is useful for components which are created by dependency injection which needs to modify
* their behavior to function without reliance on any processes outside the JVM.
*/
public static final String vespaLocalProperty = "vespa.local";
private final JDisc container;
private final List contentClusters;
private final Path path;
private final boolean deletePathWhenClosing;
// For internal use only
Application(Path path, Networking networking, boolean deletePathWhenClosing) {
System.setProperty(vespaLocalProperty, "true");
this.path = path;
this.deletePathWhenClosing = deletePathWhenClosing;
contentClusters = ContentCluster.fromPath(path);
container = JDisc.fromPath(path, networking, createVespaModel().configModelRepo());
}
@Beta
public static Application fromBuilder(Builder builder) throws Exception {
return builder.build();
}
/**
* Factory method to create an Application from an XML String. Note that any components that are referenced in
* the XML must be present on the classpath. To deploy OSGi bundles in memory,
* use {@link Application#fromApplicationPackage(Path, Networking)}.
*
* @param xml the XML configuration to use
* @return a new JDisc instance
*/
public static Application fromServicesXml(String xml, Networking networking) {
Path applicationDir = StandaloneContainerRunner.createApplicationPackage(xml);
return new Application(applicationDir, networking, true);
}
/**
* Factory method to create an Application from an application package.
* This method allows deploying OSGi bundles(contained in the components subdirectory).
* All the OSGi bundles will share the same class loader.
*
* @param path the reference to the application package to use
* @return a new JDisc instance
*/
public static Application fromApplicationPackage(Path path, Networking networking) {
return new Application(path, networking, false);
}
/**
* Factory method to create an Application from an application package.
* This method allows deploying OSGi bundles(contained in the components subdirectory).
* All the OSGi bundles will share the same class loader.
*
* @param file the reference to the application package to use
* @return a new JDisc instance
*/
public static Application fromApplicationPackage(File file, Networking networking) {
return fromApplicationPackage(file.toPath(), networking);
}
private VespaModel createVespaModel() {
try {
List modelImporters = List.of(new VespaImporter(),
new TensorFlowImporter(),
new OnnxImporter(),
new LightGBMImporter(),
new XGBoostImporter());
DeployState deployState = new DeployState.Builder()
.applicationPackage(FilesApplicationPackage.fromFile(path.toFile(), true))
.modelImporters(modelImporters)
.deployLogger((level, s) -> { })
.accessLoggingEnabledByDefault(false)
.build();
return new VespaModel(new NullConfigModelRegistry(), deployState);
} catch (IOException | SAXException e) {
throw new IllegalArgumentException("Error creating application from '" + path + "'", e);
}
}
/**
* @param id from the jdisc element in services xml. Default id in services.xml is "jdisc"
*/
public JDisc getJDisc(String id) {
return container;
}
/**
* Shuts down all services.
*/
@Override
public void close() {
container.close();
IOUtils.recursiveDeleteDir(new File(path.toFile(), "models.generated"));
if (deletePathWhenClosing)
IOUtils.recursiveDeleteDir(path.toFile());
}
/**
* A wrapper around ApplicationBuilder that generates a services.xml
*/
@Beta
public static class Builder {
private static final ThreadLocal random = new ThreadLocal<>();
private static final String DEFAULT_CHAIN = "default";
private final Map containers = new LinkedHashMap<>();
private final Path path;
private Networking networking = Networking.disable;
public Builder() throws IOException {
this.path = makeTempDir("app", "standalone").toPath();
}
public Builder container(String id, Container container) {
if (containers.size() > 0) {
throw new RuntimeException("Only a single JDisc container is currently supported.");
}
containers.put(id, container);
return this;
}
/**
* Create a temporary directory using @{link File.createTempFile()}, but creating
* a directory instead of a file.
*
* @param prefix directory prefix
* @param suffix directory suffix
* @return The created directory
* @throws IOException if the temporary directory could not be created
*/
private static File makeTempDir(String prefix, String suffix) throws IOException {
File tmpDir = File.createTempFile(prefix, suffix, getTempDir());
if (!tmpDir.delete()) {
throw new RuntimeException("Could not delete temp directory: " + tmpDir);
}
if (!tmpDir.mkdirs()) {
throw new RuntimeException("Could not create temp directory: " + tmpDir);
}
return tmpDir;
}
/**
* Get the temporary directory
*
* @return The temporary directory File object
*/
private static File getTempDir() {
String rootPath = getResourceFile("/");
String tmpPath = rootPath + "/tmp/";
File tmpDir = new File(tmpPath);
if (!tmpDir.exists() && !tmpDir.mkdirs()) {
if (!tmpDir.exists()) { // possible race condition may cause mkdirs() to fail, check a second time before failing
throw new RuntimeException("Could not create temp dir: " + tmpDir.getAbsolutePath());
}
}
if (!tmpDir.isDirectory()) {
throw new RuntimeException("Temp dir path is not a directory: " + tmpDir.getAbsolutePath());
}
return tmpDir;
}
/**
* Get the file name (path) of a resource or fail if it can not be found
*
* @param resource Name of desired resource
* @return Path of resource
*/
private static String getResourceFile(String resource) {
URL resourceUrl = Application.class.getResource(resource);
if (resourceUrl == null || resourceUrl.getFile() == null || resourceUrl.getFile().isEmpty()) {
throw new RuntimeException("Could not access resource: " + resource);
}
return resourceUrl.getFile();
}
// copy from com.yahoo.application.ApplicationBuilder
private void createFile(Path path, String content) throws IOException {
Files.createDirectories(path.getParent());
Files.write(path, Utf8.toBytes(content));
}
/**
* @return a random number between 2000 and 62000
*/
private static int getRandomPort() {
Random r = random.get();
if (r == null) {
r = new Random(System.currentTimeMillis());
random.set(r);
}
return r.nextInt(60000) + 2000;
}
/**
* @param name name of document type (search definition)
* @param searchDefinition add this search definition to the application
* @throws java.io.IOException e.g.if file not found
*/
public Builder documentType(String name, String searchDefinition) throws IOException {
Path path = nestedResource(ApplicationPackage.SCHEMAS_DIR, name, ApplicationPackage.SD_NAME_SUFFIX);
createFile(path, searchDefinition);
return this;
}
public Builder expressionInclude(String name, String searchDefinition) throws IOException {
Path path = nestedResource(ApplicationPackage.SCHEMAS_DIR, name, ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX);
createFile(path, searchDefinition);
return this;
}
/**
* @param name name of rank expression
* @param rankExpressionContent add this rank expression to the application
* @throws java.io.IOException e.g.if file not found
*/
public Builder rankExpression(String name, String rankExpressionContent) throws IOException {
Path path = nestedResource(ApplicationPackage.SCHEMAS_DIR, name, ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX);
createFile(path, rankExpressionContent);
return this;
}
/**
* @param name name of query profile
* @param queryProfile add this queyr profile to the application
* @return builder
* @throws java.io.IOException e.g.if file not found
*/
public Builder queryProfile(String name, String queryProfile) throws IOException {
Path path = nestedResource(ApplicationPackage.QUERY_PROFILES_DIR, name, ".xml");
createFile(path, queryProfile);
return this;
}
/**
* @param name name of query profile type
* @param queryProfileType add this query profile type to the application
* @return builder
* @throws java.io.IOException e.g.if file not found
*/
public Builder queryProfileType(String name, String queryProfileType) throws IOException {
Path path = nestedResource(ApplicationPackage.QUERY_PROFILE_TYPES_DIR, name, ".xml");
createFile(path, queryProfileType);
return this;
}
// copy from com.yahoo.application.ApplicationBuilder
private Path nestedResource(com.yahoo.path.Path nestedPath, String name, String fileType) {
String nameWithoutSuffix = StringUtilities.stripSuffix(name, fileType);
return path.resolve(nestedPath.getRelative()).resolve(nameWithoutSuffix + fileType);
}
/**
* @param networking enable or disable networking (disabled by default)
* @return builder
*/
public Builder networking(Networking networking) {
this.networking = networking;
return this;
}
// generate the services xml and load the container
private Application build() throws Exception {
Application app = null;
Exception exception = null;
// if we get a bind exception, then retry a few times (may conflict with parallel test runs)
for (int i = 0; i < 5; i++) {
try {
generateXml();
app = new Application(path, networking, true);
break;
} catch (Error e) { // the container thinks this is really serious, in this case is it not in the cause is a BindException
// catch bind error and reset container
Optional bindException = Exceptions.findCause(e, BindException.class);
if (bindException.isPresent()) {
exception = bindException.get();
com.yahoo.container.Container.resetInstance(); // this is needed to be able to recreate the container from config again
} else {
throw new Exception(e.getCause());
}
}
}
if (app == null) {
throw exception;
}
return app;
}
private void generateXml() throws Exception {
try (PrintWriter xml = new PrintWriter(Files.newOutputStream(path.resolve("services.xml")))) {
xml.println("");
for (Map.Entry entry : containers.entrySet()) {
entry.getValue().build(xml, entry.getKey(), (networking == Networking.enable ? getRandomPort() : -1));
}
}
}
public static class Container {
private final Map>> docprocs = new LinkedHashMap<>();
private final Map>> searchers = new LinkedHashMap<>();
private final List> renderers = new ArrayList<>();
private final List> handlers = new ArrayList<>();
private final List> clients = new ArrayList<>();
private final List> servers = new ArrayList<>();
private final List> components = new ArrayList<>();
private final List configs = new ArrayList<>();
private boolean enableSearch = false;
private static class ComponentItem {
private String id;
private Class component;
private List configs = new ArrayList<>();
public ComponentItem(String id, Class component, ConfigInstance... configs) {
this.id = id;
this.component = component;
if (configs != null) {
Collections.addAll(this.configs, configs);
}
}
}
/**
* @param docproc add this docproc to the default document processing chain
* @return builder
*/
public Container documentProcessor(Class docproc) {
return documentProcessor(DEFAULT_CHAIN, docproc);
}
/**
* @param chainName chain name to add docproc
* @param docproc add this docproc to the document processing chain
* @param configs local docproc configs
* @return builder
*/
public Container documentProcessor(String chainName, Class docproc, ConfigInstance... configs) {
return documentProcessor(docproc.getName(), chainName, docproc, configs);
}
/**
* @param id component id
* @param chainName chain name to add docproc
* @param docproc add this docproc to the document processing chain
* @param configs local docproc configs
* @return builder
*/
public Container documentProcessor(String id, String chainName, Class docproc, ConfigInstance... configs) {
List> chain = docprocs.get(chainName);
if (chain == null) {
chain = new ArrayList<>();
docprocs.put(chainName, chain);
}
chain.add(new ComponentItem<>(id, docproc, configs));
return this;
}
/**
* @param enableSearch if true, enable search even without any searchers defined
* @return builder
*/
public Container search(boolean enableSearch) {
this.enableSearch = enableSearch;
return this;
}
/**
* @param searcher add this searcher to the default search chain
* @return builder
*/
public Container searcher(Class searcher) {
return searcher(DEFAULT_CHAIN, searcher);
}
/**
* @param chainName chain name to add searcher
* @param searcher add this searcher to the search chain
* @param configs local searcher configs
* @return builder
*/
public Container searcher(String chainName, Class searcher, ConfigInstance... configs) {
return searcher(searcher.getName(), chainName, searcher, configs);
}
/**
* @param id component id
* @param chainName chain name to add searcher
* @param searcher add this searcher to the search chain
* @param configs local searcher configs
* @return builder
*/
public Container searcher(String id, String chainName, Class searcher, ConfigInstance... configs) {
List> chain = searchers.get(chainName);
if (chain == null) {
chain = new ArrayList<>();
searchers.put(chainName, chain);
}
chain.add(new ComponentItem<>(id, searcher, configs));
return this;
}
/**
* @param id component id, enable template with ?format=id or ?presentation.format=id
* @param renderer add this renderer
* @param configs local renderer configs
* @return builder
*/
public Container renderer(String id, Class renderer, ConfigInstance... configs) {
renderers.add(new ComponentItem<>(id, renderer, configs));
return this;
}
/**
* @param binding binding string
* @param handler the handler class
* @return builder
*/
public Container handler(String binding, Class handler) {
handlers.add(new ComponentItem<>(binding, handler));
return this;
}
/**
* @param binding binding string
* @param client the client class
* @return builder
*/
public Container client(String binding, Class client) {
clients.add(new ComponentItem<>(binding, client));
return this;
}
/**
* @param id server compoent id
* @param server the server class
* @return builder
*/
public Container server(String id, Class server) {
servers.add(new ComponentItem<>(id, server));
return this;
}
/**
* @param component make this component available to the container
* @return builder
*/
public Container component(Class component) {
return component(component.getName(), component, (ConfigInstance) null);
}
/**
* @param component make this component available to the container
* @return builder
*/
public Container component(String id, Class component, ConfigInstance... configs) {
components.add(new ComponentItem<>(id, component, configs));
return this;
}
/**
* @param config add this config to the application
* @return builder
*/
public Container config(final ConfigInstance config) {
configs.add(config);
return this;
}
// generate services.xml based on this builder
private void build(PrintWriter xml, String id, int port) throws Exception {
xml.println("");
if (port > 0) {
xml.println("");
xml.println(" ");
}
for (ComponentItem entry : handlers) {
xml.println("");
xml.println("" + entry.id + " ");
xml.println(" ");
}
for (ComponentItem entry : clients) {
xml.println("");
xml.println("" + entry.id + " ");
xml.println(" ");
}
for (ComponentItem server : servers) {
generateComponent(xml, server, "server");
}
// container scoped configs
for (ConfigInstance config : configs) {
generateConfig(xml, config);
}
for (ComponentItem component : components) {
generateComponent(xml, component, "component");
}
if (!docprocs.isEmpty()) {
xml.println("");
for (Map.Entry>> entry : docprocs.entrySet()) {
xml.println("");
for (ComponentItem docproc : entry.getValue()) {
generateComponent(xml, docproc, "documentprocessor");
}
xml.println(" ");
}
xml.println(" ");
}
if (enableSearch || !searchers.isEmpty() || !renderers.isEmpty()) {
xml.println("");
for (Map.Entry>> entry : searchers.entrySet()) {
xml.println("");
for (ComponentItem searcher : entry.getValue()) {
generateComponent(xml, searcher, "searcher");
}
xml.println(" ");
}
for (ComponentItem renderer : renderers) {
generateComponent(xml, renderer, "renderer");
}
xml.println(" ");
}
xml.println(" ");
}
private void generateComponent(PrintWriter xml, ComponentItem componentItem, String elementName) throws Exception {
xml.print("<" + elementName + " id=\"" + componentItem.id + "\" class=\"" + componentItem.component.getName() + "\"");
if (componentItem.configs.isEmpty() || (!componentItem.configs.isEmpty() && componentItem.configs.get(0) == null)) {
xml.println(" />");
} else {
xml.println(">");
for (ConfigInstance config : componentItem.configs) {
generateConfig(xml, config);
}
xml.println("");
}
}
// uses reflection to generate XML from a config object
private void generateConfig(PrintWriter xml, ConfigInstance config) throws Exception {
Field nameField = config.getClass().getField("CONFIG_DEF_NAME");
String name = (String) nameField.get(config);
Field namespaceField = config.getClass().getField("CONFIG_DEF_NAMESPACE");
String namespace = (String) namespaceField.get(config);
xml.println("");
generateConfigNode(xml, config);
xml.println(" ");
}
private void generateConfigNode(PrintWriter xml, InnerNode node) throws Exception {
// print all leaf nodes as config values
Field[] fields = node.getClass().getDeclaredFields();
for (Field field : fields) {
generateConfigField(xml, node, field);
}
}
private void generateConfigField(PrintWriter xml, InnerNode node, Field field) throws Exception {
field.setAccessible(true);
if (LeafNode.class.isAssignableFrom(field.getType())) {
LeafNode value = (LeafNode) field.get(node);
if (value.value() != null) {
xml.print("<" + field.getName());
String v = value.getValue();
if (v.isEmpty()) {
xml.println(" />");
} else {
xml.println(">" + v + "");
}
}
} else if (InnerNode.class.isAssignableFrom(field.getType())) {
xml.println("<" + field.getName() + ">");
generateConfigNode(xml, (InnerNode) field.get(node));
xml.println("");
} else if (Map.class.isAssignableFrom(field.getType())) {
Map map = (Map) field.get(node);
if (!map.isEmpty()) {
xml.println("<" + field.getName() + ">");
for (Map.Entry entry : map.entrySet()) {
if (entry.getValue() instanceof InnerNode) {
xml.println("- ");
generateConfigNode(xml, (InnerNode) entry.getValue());
xml.println("
");
} else if (entry.getValue() instanceof LeafNode) {
xml.println("- " + ((LeafNode) entry.getValue()).getValue() + "
");
}
}
xml.println("");
}
} else if (InnerNodeVector.class.isAssignableFrom(field.getType())) {
InnerNodeVector vector = (InnerNodeVector) field.get(node);
if (!vector.isEmpty()) {
xml.println("<" + field.getName() + ">");
for (InnerNode innerNode : vector) {
xml.println("- ");
generateConfigNode(xml, innerNode);
xml.println("
");
}
xml.println("");
}
} else if (LeafNodeVector.class.isAssignableFrom(field.getType())) {
LeafNodeVector> vector = (LeafNodeVector>) field.get(node);
if (!vector.isEmpty()) {
xml.println("<" + field.getName() + ">");
for (LeafNode item : vector) {
xml.println("- " + item.getValue() + "
");
}
xml.println("");
}
}
}
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy