io.nosqlbench.engine.api.scenarios.NBCLIScenarioParser Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of engine-api Show documentation
Show all versions of engine-api Show documentation
The engine API for nosqlbench;
Provides the interfaces needed to build internal modules for the
nosqlbench core engine
package io.nosqlbench.engine.api.scenarios;
/*
* Copyright (c) 2022 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.rawyaml.RawStmtsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.Scenarios;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.templating.StrInterpolator;
import io.nosqlbench.nb.api.config.params.Synonyms;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.nb.api.content.NBPathsAPI;
import io.nosqlbench.nb.api.errors.BasicError;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import java.io.IOException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class NBCLIScenarioParser {
public final static String SILENT_LOCKED = "==";
public final static String VERBOSE_LOCKED = "===";
public final static String UNLOCKED = "=";
private final static Logger logger = LogManager.getLogger("SCENARIOS");
private static final String SEARCH_IN = "activities";
public static final String WORKLOAD_SCENARIO_STEP = "WORKLOAD_SCENARIO_STEP";
public static boolean isFoundWorkload(String workload, String... includes) {
Optional> found = NBIO.all()
.prefix("activities")
.prefix(includes)
.name(workload)
.extension(RawStmtsLoader.YAML_EXTENSIONS)
.first();
return found.isPresent();
}
public static void parseScenarioCommand(LinkedList arglist,
Set RESERVED_WORDS,
String... includes) {
String workloadName = arglist.removeFirst();
Optional> found = NBIO.all()
.prefix("activities")
.prefix(includes)
.name(workloadName)
.extension(RawStmtsLoader.YAML_EXTENSIONS)
.first();
//
Content workloadContent = found.orElseThrow();
// Optional workloadPathSearch = NBPaths.findOptionalPath(workloadName, "yaml", false, "activities");
// Path workloadPath = workloadPathSearch.orElseThrow();
// Buffer in scenario names from CLI, only counting non-options non-parameters and non-reserved words
List scenarioNames = new ArrayList<>();
while (arglist.size() > 0
&& !arglist.peekFirst().contains("=")
&& !arglist.peekFirst().startsWith("-")
&& !RESERVED_WORDS.contains(arglist.peekFirst())) {
scenarioNames.add(arglist.removeFirst());
}
if (scenarioNames.size() == 0) {
scenarioNames.add("default");
}
// Parse CLI command into keyed parameters, in order
LinkedHashMap userProvidedParams = new LinkedHashMap<>();
while (arglist.size() > 0
&& arglist.peekFirst().contains("=")
&& !arglist.peekFirst().startsWith("-")) {
String[] arg = arglist.removeFirst().split("=", 2);
arg[0] = Synonyms.canonicalize(arg[0], logger);
if (userProvidedParams.containsKey(arg[0])) {
throw new BasicError("duplicate occurrence of option on command line: " + arg[0]);
}
userProvidedParams.put(arg[0], arg[1]);
}
// This will buffer the new command before adding it to the main arg list
LinkedList buildCmdBuffer = new LinkedList<>();
StrInterpolator userParamsInterp = new StrInterpolator(userProvidedParams);
for (String scenarioName : scenarioNames) {
// Load in named scenario
Content yamlWithNamedScenarios = NBIO.all()
.prefix(SEARCH_IN)
.prefix(includes)
.name(workloadName)
.extension(RawStmtsLoader.YAML_EXTENSIONS)
.first().orElseThrow();
// TODO: The yaml needs to be parsed with arguments from each command independently to support template vars
StmtsDocList scenariosYaml = StatementsLoader.loadContent(logger, yamlWithNamedScenarios, new LinkedHashMap<>(userProvidedParams));
Scenarios scenarios = scenariosYaml.getDocScenarios();
Map namedSteps = scenarios.getNamedScenario(scenarioName);
if (namedSteps == null) {
throw new BasicError("Unable to find named scenario '" + scenarioName + "' in workload '" + workloadName
+ "', but you can pick from one of: " +
scenarios.getScenarioNames().stream().collect(Collectors.joining(", ")));
}
// each named command line step of the named scenario
for (Map.Entry cmdEntry : namedSteps.entrySet()) {
String stepName = cmdEntry.getKey();
String cmd = cmdEntry.getValue();
cmd = userParamsInterp.apply(cmd);
LinkedHashMap parsedStep = parseStep(cmd);
LinkedHashMap usersCopy = new LinkedHashMap<>(userProvidedParams);
LinkedHashMap buildingCmd = new LinkedHashMap<>();
// consume each of the parameters from the steps to produce a composited command
// order is primarily based on the step template, then on user-provided parameters
for (CmdArg cmdarg : parsedStep.values()) {
// allow user provided parameter values to override those in the template,
// if the assignment operator used in the template allows for it
if (usersCopy.containsKey(cmdarg.getName())) {
cmdarg = cmdarg.override(usersCopy.remove(cmdarg.getName()));
}
buildingCmd.put(cmdarg.getName(), cmdarg.toString());
}
usersCopy.forEach((k, v) -> buildingCmd.put(k, k + "=" + v));
// Undefine any keys with a value of 'undef'
List undefKeys = buildingCmd.entrySet()
.stream()
.filter(e -> e.getValue().toLowerCase().endsWith("=undef"))
.map(Map.Entry::getKey)
.collect(Collectors.toList());
undefKeys.forEach(buildingCmd::remove);
if (!buildingCmd.containsKey("workload")) {
buildingCmd.put("workload", "workload=" + workloadName);
}
if (!buildingCmd.containsKey("alias")) {
buildingCmd.put("alias", "alias=" + WORKLOAD_SCENARIO_STEP);
}
String alias = buildingCmd.get("alias");
for (String token : new String[]{"WORKLOAD", "SCENARIO", "STEP"}) {
if (!alias.contains(token)) {
logger.warn("Your alias template '" + alias + "' does not contain " + token + ", which will " +
"cause your metrics to be combined under the same name. It is strongly advised that you " +
"include them in a template like " + WORKLOAD_SCENARIO_STEP + ".");
}
}
String workloadToken = workloadContent.asPath().getFileName().toString();
alias = alias.replaceAll("WORKLOAD", sanitize(workloadToken));
alias = alias.replaceAll("SCENARIO", sanitize(scenarioName));
alias = alias.replaceAll("STEP", sanitize(stepName));
alias = (alias.startsWith("alias=") ? alias : "alias=" + alias);
buildingCmd.put("alias", alias);
logger.debug("rebuilt command: " + String.join(" ", buildingCmd.values()));
buildCmdBuffer.addAll(buildingCmd.values());
}
}
buildCmdBuffer.descendingIterator().forEachRemaining(arglist::addFirst);
}
public static String sanitize(String word) {
String sanitized = word;
sanitized = sanitized.replaceAll("\\..+$", "");
sanitized = sanitized.replaceAll("[^a-zA-Z0-9]+", "");
return sanitized;
}
private static final Pattern WordAndMaybeAssignment = Pattern.compile("(?\\w[-_\\d\\w.]+)((?=+)(?.+))?");
private static LinkedHashMap parseStep(String cmd) {
LinkedHashMap parsedStep = new LinkedHashMap<>();
String[] namedStepPieces = cmd.split(" ");
for (String commandFragment : namedStepPieces) {
Matcher matcher = WordAndMaybeAssignment.matcher(commandFragment);
if (!matcher.matches()) {
throw new BasicError("Unable to recognize scenario cmd spec in '" + commandFragment + "'");
}
String commandName = matcher.group("name");
commandName = Synonyms.canonicalize(commandName, logger);
String assignmentOp = matcher.group("oper");
String assignedValue = matcher.group("val");
parsedStep.put(commandName, new CmdArg(commandName, assignmentOp, assignedValue));
}
return parsedStep;
}
private final static class CmdArg {
private final String name;
private final String operator;
private final String value;
private String scenarioName;
public CmdArg(String name, String operator, String value) {
this.name = name;
this.operator = operator;
this.value = value;
}
public boolean isReassignable() {
return UNLOCKED.equals(operator);
}
public boolean isFinalSilent() {
return SILENT_LOCKED.equals(operator);
}
public boolean isFinalVerbose() {
return VERBOSE_LOCKED.equals(operator);
}
public CmdArg override(String value) {
if (isReassignable()) {
return new CmdArg(this.name, this.operator, value);
} else if (isFinalSilent()) {
return this;
} else if (isFinalVerbose()) {
throw new BasicError("Unable to reassign value for locked param '" + name + operator + value + "'");
} else {
throw new RuntimeException("impossible!");
}
}
@Override
public String toString() {
return name + (operator != null ? "=" : "") + (value != null ? value : "");
}
public String getName() {
return name;
}
}
private static final Pattern templatePattern = Pattern.compile("TEMPLATE\\((.+?)\\)");
private static final Pattern innerTemplatePattern = Pattern.compile("TEMPLATE\\((.+?)$");
private static final Pattern templatePattern2 = Pattern.compile("<<(.+?)>>");
public static List filterForScenarios(List> candidates) {
List yamlPathList = candidates.stream().map(Content::asPath).collect(Collectors.toList());
List workloadDescriptions = new ArrayList<>();
for (Path yamlPath : yamlPathList) {
try {
String referenced = yamlPath.toString();
if (referenced.startsWith("/")) {
if (yamlPath.getFileSystem() == FileSystems.getDefault()) {
Path relative = Paths.get(System.getProperty("user.dir")).toAbsolutePath().relativize(yamlPath);
if (!relative.toString().contains("..")) {
referenced = relative.toString();
}
}
}
Content content = NBIO.all().prefix(SEARCH_IN)
.name(referenced).extension(RawStmtsLoader.YAML_EXTENSIONS)
.one();
StmtsDocList stmts = StatementsLoader.loadContent(logger, content, Map.of());
if (stmts.getStmtDocs().size() == 0) {
logger.warn("Encountered yaml with no docs in '" + referenced + "'");
continue;
}
Map templates = new LinkedHashMap<>();
try {
List lines = Files.readAllLines(yamlPath);
for (String line : lines) {
templates = matchTemplates(line, templates);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
Scenarios scenarios = stmts.getDocScenarios();
List scenarioNames = scenarios.getScenarioNames();
if (scenarioNames != null && scenarioNames.size() > 0) {
// String path = yamlPath.toString();
// path = path.startsWith(FileSystems.getDefault().getSeparator()) ? path.substring(1) : path;
LinkedHashMap sortedTemplates = new LinkedHashMap<>();
ArrayList keyNames = new ArrayList<>(templates.keySet());
Collections.sort(keyNames);
for (String keyName : keyNames) {
sortedTemplates.put(keyName, templates.get(keyName));
}
String description = stmts.getDescription();
workloadDescriptions.add(new WorkloadDesc(referenced, scenarioNames, sortedTemplates, description, ""));
}
} catch (Exception e) {
throw new RuntimeException("Error while scanning path '" + yamlPath.toString() + "':" + e.getMessage(), e);
}
}
Collections.sort(workloadDescriptions);
return workloadDescriptions;
}
public static List getWorkloadsWithScenarioScripts(boolean defaultIncludes, Set includes) {
return getWorkloadsWithScenarioScripts(defaultIncludes, includes.toArray(new String[0]));
}
public static List getWorkloadsWithScenarioScripts(boolean defaultIncludes, String... includes) {
NBPathsAPI.GetPrefix searchin = NBIO.all();
if (defaultIncludes) {
searchin = searchin.prefix(SEARCH_IN);
}
List> activities = searchin
.prefix(includes)
.extension(RawStmtsLoader.YAML_EXTENSIONS)
.list();
return filterForScenarios(activities);
}
public static List getScripts(boolean defaultIncludes, String... includes) {
NBPathsAPI.GetPrefix searchin = NBIO.all();
if (defaultIncludes) {
searchin = searchin.prefix(SEARCH_IN);
}
List scriptPaths = searchin
.prefix("scripts/auto")
.prefix(includes)
.extension("js")
.list().stream().map(Content::asPath).collect(Collectors.toList());
List scriptNames = new ArrayList();
for (Path scriptPath : scriptPaths) {
String name = scriptPath.getFileName().toString();
name = name.substring(0, name.lastIndexOf('.'));
scriptNames.add(name);
}
return scriptNames;
}
public static Map matchTemplates(String line, Map templates) {
Matcher matcher = templatePattern.matcher(line);
while (matcher.find()) {
String match = matcher.group(1);
Matcher innerMatcher = innerTemplatePattern.matcher(match);
String[] matchArray = match.split(",");
// if (matchArray.length!=2) {
// throw new BasicError("TEMPLATE form must have two arguments separated by a comma, like 'TEMPLATE(a,b), not '" + match +"'");
// }
//TODO: support recursive matches
if (innerMatcher.find()) {
String[] innerMatch = innerMatcher.group(1).split("[,:]");
//We want the outer name with the inner default value
templates.put(matchArray[0], innerMatch[1]);
} else {
templates.put(matchArray[0], matchArray[1]);
}
}
matcher = templatePattern2.matcher(line);
while (matcher.find()) {
String match = matcher.group(1);
String[] matchArray = match.split(":");
if (matchArray.length == 1) {
templates.put(matchArray[0], "-none-");
} else {
templates.put(matchArray[0], matchArray[1]);
}
}
return templates;
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy