nstream.adapter.common.AdapterUtils Maven / Gradle / Ivy
Show all versions of nstream-adapter-common Show documentation
// Copyright 2015-2024 Nstream, inc.
//
// Licensed under the Redis Source Available License 2.0 (RSALv2) Agreement;
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://redis.com/legal/rsalv2-agreement/
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package nstream.adapter.common;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.Optional;
import java.util.Properties;
import java.util.function.BiConsumer;
import java.util.function.Function;
import nstream.adapter.common.ingress.AssemblerFactory;
import nstream.adapter.common.ingress.AssemblyException;
import nstream.adapter.common.ingress.ContentAssembler;
import nstream.adapter.common.ingress.ValueAssembler;
import nstream.adapter.common.ingress.XmlContentAssembler;
import nstream.adapter.common.relay.DslInterpret;
import nstream.adapter.common.schedule.DeferrableException;
import swim.api.agent.AgentContext;
import swim.codec.Utf8;
import swim.json.Json;
import swim.recon.Recon;
import swim.structure.Attr;
import swim.structure.Item;
import swim.structure.Record;
import swim.structure.Slot;
import swim.structure.Text;
import swim.structure.Value;
import swim.util.Log;
/**
* A collection of convenience methods that will commonly be invoked within
* Nstream adapter implementations.
*/
public final class AdapterUtils {
private AdapterUtils() {
}
// ===========================================================================
// File loading utilities
// ===========================================================================
/**
* Returns an {@code InputStream} for reading a system file, falling back to a
* Java resource if required.
*
* @param diskPath the path of the system file to load
* @param clazz the class associated with the fallback resource
* @param resourcePath the path of the fallback resource
*
* @return an {@code InputStream} for reading the file or its fallback;
* {@code null} if neither one could be accessed
*/
public static InputStream openFileAsStream(String diskPath,
Class clazz, String resourcePath) {
if (diskPath == null || diskPath.isEmpty()) {
return openResourceAsStream(resourcePath, clazz);
}
try {
return new FileInputStream(diskPath);
} catch (IOException e) {
return openResourceAsStream(resourcePath, clazz);
}
}
/**
* Returns an {@code InputStream} for reading a system file, falling back to a
* Java resource if required.
*
* @param path the path of either a system file or fallback resource to load
* @param clazz the class associated with the fallback resource
*
* @return an {@code InputStream} for reading the file or its fallback;
* {@code null} if neither one could be accessed
*/
public static InputStream openFileAsStream(String path, Class clazz) {
return openFileAsStream(path, clazz, path);
}
/**
* Returns an {@code InputStream} for reading a resource.
*
* @param path the path of the resource to be read
* @param clazz the class whose classloader (first) or class will load the
* specified resource
*
* @return an {@code InputStream} for reading the resource, or {@code null}
* if it could not be accessed
*/
public static InputStream openResourceAsStream(String path, Class clazz) {
final InputStream result = clazz.getClassLoader().getResourceAsStream(path);
return result == null ? clazz.getResourceAsStream(path) : result;
}
// ===========================================================================
// Ingress utility functions
// ===========================================================================
private static final AssemblerFactory ASSEMBLER_FACTORY = new AssemblerFactory();
static {
ASSEMBLER_FACTORY.registerContentAssembler(new XmlContentAssembler(Value.extant()));
}
/**
* Translates a {@code InputStream} out of an input that was encoded
* or compressed with a known encoding.
* Specifically, calling {@link InputStream#read()} against the returned
* {@code InputStream} must return a byte of UTF-8 encoded, uncompressed data.
* Be aware that the original input may be modified as a side effect.
*
* @param encoded the {@code InputStream} to be decoded
* @param minimalContentEncoding an identifier of the encoding scheme used by
* {@code encoded}
* @return an {@code InputStream} whose {@code read} method(s) are UTF-8
* compatible
* @throws IOException if there was an issue collecting {@code encoded} into
* a UTF-8 compatible {@code InputStream}
*/
// TODO: enum contentEncoding argument
// TODO: throw RuntimeException for bad encodings?
public static InputStream decodeStream(InputStream encoded, String minimalContentEncoding)
throws IOException {
return ASSEMBLER_FACTORY.decodeStream(encoded, minimalContentEncoding);
}
/**
* Parses a textual UTF-8 {@code InputStream} with a known data format into
* Swim's structured data model.
*
* @param stream the {@code InputStream} holding the data of interest
* @param minimalContentType the data format
*
* @return a {@code Value} representing the data in {@code stream}
*
* @throws IOException if there was an issue reading from {@code stream}
*/
public static Value assembleContent(InputStream stream, String minimalContentType)
throws AssemblyException, IOException {
return ASSEMBLER_FACTORY.assemble(stream, minimalContentType);
}
/**
* Parses a {@code String} with a known data format into Swim's structured
* data model.
*
* @param msg the data of interest
* @param minimalContentType the data format
*
* @return a {@code Value} representing the data in {@code msg}
*/
public static Value assembleContent(String msg, String minimalContentType)
throws AssemblyException {
return ASSEMBLER_FACTORY.assemble(msg, minimalContentType);
}
/**
* Parses an {@code Object} that is assumed to hold formatted textual data
* into Swim's structured data model.
*
* @param content an object holding the data of interest
* @param minimalContentType the data format
*
* @return a {@code Value} representing the data in {@code content}
*
* @throws IllegalArgumentException if the type of {@code content} is
* unrecognized for translation
*/
public static Value assembleContent(Object content, String minimalContentType)
throws AssemblyException {
if (minimalContentType == null) {
return Value.fromObject(content); // last resort
}
return ASSEMBLER_FACTORY.assembleContent(content, minimalContentType);
}
/**
* Parses an {@code Object} that is assumed to hold formatted textual data
* into Swim's structured data model.
*
* @param content an object holding the data of interest
* @param assembler the parsing logic to use, falling back to {@code
* swim.structure.Value#fromObject}
*
* @return a {@code Value} representing the data in {@code content}
*
* @throws IllegalArgumentException if the type of {@code content} is
* unrecognized for translation
*/
public static Value assembleContent(Object content, ContentAssembler assembler)
throws AssemblyException {
if (assembler == null) {
return Value.fromObject(content); // last resort
}
return AssemblerFactory.assembleContent(content, assembler);
}
/**
* Parses an {@code Object} that is assumed to hold possibly but not
* necessarily formatted textual data into Swim's structured data model.
*
* @param content an object holding the data of interest
* @param assembler the parsing logic to use, falling back to {@code
* swim.structure.Value#fromObject}
*
* @return a {@code Value} representing the data in {@code content}
*
* @throws IllegalArgumentException if the type of {@code content} is
* unrecognized for translation
*/
@SuppressWarnings("unchecked")
public static Value assemble(Object content, ValueAssembler assembler)
throws AssemblyException {
if (assembler == null) {
return Value.fromObject(content); // last resort
} else if (assembler instanceof ContentAssembler) {
return assembleContent(content, (ContentAssembler) assembler);
} else {
return assembler.assemble((V) content);
}
}
/**
* Executes logic from a specified {@link AgentContext} against some input.
*
* @param schema the DSL-interpretable logic to run
* @param agentContext the {@code AgentContext} from which to run the logic
* @param scope the input value for the logic
*
* @throws DeferrableException if there was any issue interpreting {@code
* scope} against {@code schema}
*/
public static void ingressDslRelay(Value schema, AgentContext agentContext, Value scope)
throws DeferrableException {
if (schema == null || !schema.isDistinct() || !(schema instanceof Record)) {
throw new DeferrableException(schema + " is not DSL-interpretable");
} else {
try {
DslInterpret.interpret(agentContext, (Record) schema, scope);
} catch (Exception e) {
throw new DeferrableException(schema + " is not DSL-interpretable against scope " + scope, e);
}
}
}
// ===========================================================================
// Egress utility functions
// ===========================================================================
public static String messageDismantle(Value value, String minimalContentType) {
if (minimalContentType == null) {
minimalContentType = "";
}
switch (minimalContentType) {
case "xml":
throw new UnsupportedOperationException("Xml not yet implemented");
case "recon":
return Recon.toString(value);
case "json":
return Json.toString(value);
default:
return value.stringValue();
}
}
// ===========================================================================
// @config-related utilities
// ===========================================================================
public static void loadPropsFromUse(Log log, String name, Properties properties, Value useDef) {
loadFromUse(log, name, properties, useDef, (s, p) -> loadPropsFromFile(log, name, p, s));
}
private static void loadPropsFromFile(Log log, String name, Properties properties, String fname) {
final Properties temp = new Properties();
try (InputStream is = openFileAsStream(fname, AdapterUtils.class)) {
temp.load(is);
} catch (Exception e) {
log.warn("Failed to populate " + name + " from file " + fname + ": " + e.getMessage());
try (StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw)) {
e.printStackTrace(pw);
log.warn(sw.toString());
} catch (IOException ioe) {
// swallow
}
}
for (String prop : temp.stringPropertyNames()) {
if (properties.getProperty(prop) != null) {
log.debug("Ignored already-defined property " + prop + " when loading from " + fname);
} else {
properties.setProperty(prop, temp.getProperty(prop));
}
}
}
public static void loadPropsFromDef(Log log, String name, Properties properties, Value defDef) {
if (defDef instanceof Record) {
for (Item i : defDef) {
if (i instanceof Slot) {
if (!(i.key() instanceof Text)) {
log.warn("Ignored non-Text Slot key " + i.key() + " when loading " + name + " from def");
continue;
}
final String key = i.key().stringValue();
if (properties.getProperty(key) != null) {
log.debug("Ignored already-defined property " + key + " when loading " + name + " from def");
continue;
}
if (isConfigFieldMold(i.toValue())) {
evaluateConfigProp(log, name, i.toValue())
.ifPresentOrElse(
s -> properties.setProperty(key, s),
() -> log.warn("Ignored property " + key + " that had no definition at runtime via " + i.toValue() + " when loading " + name + " from def"));
} else if (!i.toValue().isDistinct() || i.toValue() instanceof Record) {
log.warn("Ignored non-string Slot value " + i.toValue() + " under " + key + " when loading " + name + " from def");
} else {
properties.setProperty(key, i.toValue().stringValue());
}
} else {
log.warn("Ignored non-Slot item " + i + " when loading " + name + " from def");
}
}
} else if (defDef.isDistinct()) {
log.warn("Ignored non-Record def " + defDef + " when loading " + name + " from def");
}
}
public static S settingsFromConf(Log log, String name, String tag,
Value conf, Function mold,
Function cast) {
if (!(conf instanceof Record)) {
log.warn("Ignored invalid " + name + " configuration " + conf);
return null;
}
if (conf.head() instanceof Attr && tag.equals(conf.head().key().stringValue(null))) {
final Value use = conf.get("use"),
def = conf.get("def");
if (use.isDistinct() || def.isDistinct()) {
final Value evaluated = AdapterUtils.loadConf(log, tag, tag, conf, mold);
return cast.apply(evaluated);
} else { // legacy
return cast.apply(conf);
}
} else {
log.warn("Ignored invalid " + name + " configuration " + conf);
return null;
}
}
public static Record loadConf(Log log, String name, Value conf,
Function mold) {
final Record result = Record.create();
loadStructureFromUse(log, name, result, conf.get("use"), mold);
loadStructureFromDef(log, name, result, conf.get("def"), mold);
return result;
}
public static Record loadConf(Log log, String name, String tag, Value conf,
Function mold) {
final Record result = Record.create().attr(tag);
loadStructureFromUse(log, name, result, conf.get("use"), mold);
loadStructureFromDef(log, name, result, conf.get("def"), mold);
return result;
}
static void loadStructureFromUse(Log log, String name, Record structure, Value useDef,
Function mold) {
loadFromUse(log, name, structure, useDef, (s, v) -> loadStructureFromFile(log, name, v, s, mold));
}
// TODO: more trace/debug/info-level logging
private static void loadStructureFromFile(Log log, String name, Record structure, String fname,
Function mold) {
try (InputStream is = openFileAsStream(fname, AdapterUtils.class)) {
if (fname.endsWith(".recon")) {
loadStructureFromRecord(log, name, fname, structure, (Record) Utf8.read(is, Recon.structureParser().blockParser()));
} else if (fname.endsWith(".properties")) {
final Properties toAdd = new Properties();
toAdd.load(is);
loadStructureFromProperties(log, name, fname, structure, toAdd, mold);
} else {
log.warn("Ignored file " + fname + " with invalid extension");
}
} catch (Exception e) {
try (StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw)) {
e.printStackTrace(pw);
log.warn(sw.toString());
} catch (IOException ioe) {
// swallow
}
}
}
// TODO: logging
private static void loadStructureFromProperties(Log log, String name, String fname, Record structure, Properties input,
Function moldFromProperties) {
final Record toAdd = moldFromProperties.apply(input);
for (Item i : toAdd) {
if (i instanceof Slot) {
structure.updatedSlot(i.key(), i.toValue());
}
}
}
private static void loadStructureFromRecord(Log log, String name, String fname, Record structure, Record input) {
for (Item i : input) {
if (!(i instanceof Slot)) {
log.warn("Ignored non-Slot Item " + i + " when loading " + name + " from " + fname);
} else if (structure.get(i.key()).isDistinct()) {
log.debug("Ignored already-defined key " + i.key().stringValue(null) + " when loading " + name + " from " + fname);
} else {
structure.slot(i.key(), i.toValue());
}
}
}
private static void loadFromUse(Log log, String name, V accumulator, Value useDef, BiConsumer load) {
if (useDef instanceof Text) {
load.accept(useDef.stringValue(), accumulator);
} else if (useDef instanceof Record) {
if (isConfigFieldMold(useDef)) {
evaluateConfigProp(log, name, useDef)
.ifPresentOrElse(
s -> load.accept(s, accumulator),
() -> log.warn("Failed to coerce " + useDef + " into a filename when evaluating " + name));
} else {
for (Item i : useDef) {
if (i instanceof Text) {
load.accept(i.stringValue(), accumulator);
} else if (isConfigFieldMold(i.toValue())) {
evaluateConfigProp(log, name, i.toValue())
.ifPresentOrElse(
s -> load.accept(s, accumulator),
() -> log.warn("Failed to coerce " + i + " into a filename when evaluating " + name));
}
}
}
}
}
static void loadStructureFromDef(Log log, String name, Record structure, Value defDef,
Function moldFromProperties) {
final Properties properties = new Properties();
if (defDef instanceof Record) {
for (Item i : defDef) {
if (i instanceof Slot) {
if (!(i.key() instanceof Text)) {
log.warn("Ignored non-Text Slot key " + i.key() + " when loading " + name + " from def");
continue;
}
final String key = i.key().stringValue();
if (structure.get(key).isDistinct() || properties.getProperty(key) != null) {
log.debug("Ignored already-defined key " + key + " when loading " + name + " from def");
continue;
}
if (isConfigFieldMold(i.toValue())) {
evaluateConfigMold(log, key, i.toValue(), properties, structure);
} else if (i.toValue().isDistinct()) {
structure.updatedSlot(key, i.toValue());
} else {
log.warn("Ignored non-string Slot value " + i.toValue() + " under " + key + " when loading " + name + " from def");
}
} else {
log.warn("Ignored non-Slot item " + i + " when loading " + name + " from def");
}
}
final Record mold = moldFromProperties.apply(properties);
for (Item i : mold) {
if (i instanceof Slot && !structure.get(i.key()).isDistinct()) {
structure.updatedSlot(i.key(), i.toValue());
}
}
} else if (defDef.isDistinct()) {
log.warn("Ignored non-Record def " + defDef + " when loading " + name);
}
}
private static boolean isConfigFieldMold(Value mold) {
return mold instanceof Record
&& mold.head() instanceof Attr
&& "config".equals(mold.head().key().stringValue(null));
}
private static Optional evaluateConfigProp(Log log, String field, Value configMold) {
final ConfigField cf = new ConfigField(log, field, configMold.get("env").stringValue(null),
configMold.get("prop").stringValue(null), configMold.get("def").stringValue(null));
return cf.evaluate();
}
private static void evaluateConfigMold(Log log, String field, Value configMold,
Properties props, Record structure) {
final ConfigField cf = new ConfigField(log, field, configMold.get("env").stringValue(null),
configMold.get("prop").stringValue(null), configMold.get("def"));
cf.evaluate(props, structure, configMold, field);
}
static class ConfigField {
final Log log;
final String field;
final String env;
final String prop;
final String def;
final Value defVal;
private ConfigField(Log log, String field, String env, String prop, String def, Value defVal) {
this.log = log;
this.field = field;
this.env = env;
this.prop = prop;
this.def = def;
this.defVal = defVal;
}
ConfigField(Log log, String field, String env, String prop, String def) {
this(log, field, env, prop, def, null);
}
ConfigField(Log log, String field, String env, String prop, Value def) {
this(log, field, env, prop, null, def);
}
Optional evaluate() {
return evaluateEnv()
.or(this::evaluateProp)
.or(this::evaluateDef);
}
void evaluate(Properties properties, Record structure, Value mold, String name) {
evaluateEnv()
.or(this::evaluateProp)
.ifPresentOrElse(
s -> properties.setProperty(this.field, s),
() -> evaluateDefVal().ifPresentOrElse(
v -> structure.slot(this.field, v),
() -> this.log.warn("Ignored Field " + this.field + " that had no Value at runtime via "
+ mold + " when loading " + name)));
}
Optional evaluateEnv() {
if (this.env == null || this.env.isEmpty()) {
return Optional.empty();
}
final String result = System.getenv(this.env);
if (result == null) {
this.log.debug("No env configured under " + this.env + ", falling back");
return Optional.empty();
} else {
this.log.info("Used env " + this.env + " in @config interpretation");
return Optional.of(result);
}
}
Optional evaluateProp() {
if (this.prop == null || this.prop.isEmpty()) {
return Optional.empty();
}
final String result = System.getProperty(this.prop);
if (result == null) {
this.log.debug("No prop configured under " + this.prop + ", falling back");
return Optional.empty();
} else {
this.log.info("Used prop " + this.prop + " in @config interpretation of " + this.field);
return Optional.of(result);
}
}
Optional evaluateDef() {
return this.def == null ? Optional.empty() : Optional.of(this.def);
}
Optional evaluateDefVal() {
return this.defVal == null ? Optional.empty() : Optional.of(this.defVal);
}
}
}