io.nosqlbench.engine.cli.NBCLIOptions Maven / Gradle / Ivy
package io.nosqlbench.engine.cli;
import ch.qos.logback.classic.Level;
import io.nosqlbench.engine.api.metrics.IndicatorMode;
import io.nosqlbench.engine.api.scenarios.NBCLIScenarioParser;
import io.nosqlbench.engine.api.util.Unit;
import io.nosqlbench.engine.core.script.Scenario;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.security.InvalidParameterException;
import java.util.*;
import java.util.stream.Collectors;
/**
* No CLI parser lib is useful for command structures, it seems. So we have this instead, which is good enough.
* If something better is needed later, this can be replaced.
*/
public class NBCLIOptions {
private final static Logger logger = LoggerFactory.getLogger(NBCLIOptions.class);
// Options which may contextualize other CLI options or commands.
// These must be parsed first
private static final String INCLUDE = "--include";
private static final String METRICS_PREFIX = "--metrics-prefix";
// Discovery
private static final String HELP = "--help";
private static final String LIST_METRICS = "--list-metrics";
private static final String LIST_DRIVERS = "--list-drivers";
private static final String LIST_ACTIVITY_TYPES = "--list-activity-types";
private static final String LIST_WORKLOADS = "--list-workloads";
private static final String LIST_SCENARIOS = "--list-scenarios";
private static final String LIST_INPUT_TYPES = "--list-input-types";
private static final String LIST_OUTPUT_TYPES = "--list-output-types";
private static final String VERSION_COORDS = "--version-coords";
private static final String VERSION = "--version";
private static final String SHOW_SCRIPT = "--show-script";
private static final String COPY = "--copy";
// Execution
private static final String SCRIPT = "script";
private static final String ACTIVITY = "activity";
private static final String SCENARIO = "scenario";
private static final String RUN = "run";
private static final String START = "start";
private static final String FRAGMENT = "fragment";
private static final String STOP = "stop";
private static final String AWAIT = "await";
private static final String WAIT_MILLIS = "waitmillis";
private static final String EXPORT_CYCLE_LOG = "--export-cycle-log";
private static final String IMPORT_CYCLE_LOG = "--import-cycle-log";
private static final String HDR_DIGITS = "--hdr-digits";
// Execution Options
private static final String SESSION_NAME = "--session-name";
private static final String LOGS_DIR = "--logs-dir";
private static final String LOGS_MAX = "--logs-max";
private static final String LOGS_LEVEL = "--logs-level";
private static final String DASH_V_INFO = "-v";
private static final String DASH_VV_DEBUG = "-vv";
private static final String DASH_VVV_TRACE = "-vvv";
private static final String REPORT_INTERVAL = "--report-interval";
private static final String REPORT_GRAPHITE_TO = "--report-graphite-to";
private static final String REPORT_CSV_TO = "--report-csv-to";
private static final String PROGRESS = "--progress";
private static final String WITH_LOGGING_PATTERN = "--with-logging-pattern";
private static final String LOG_HISTOGRAMS = "--log-histograms";
private static final String LOG_HISTOSTATS = "--log-histostats";
private static final String CLASSIC_HISTOGRAMS = "--classic-histograms";
private final static String LOG_LEVEL_OVERRIDE = "--log-level-override";
private final static String ENABLE_CHART = "--enable-chart";
private final static String DOCKER_METRICS = "--docker-metrics";
private static final String GRAALJS_ENGINE = "--graaljs";
private static final String NASHORN_ENGINE = "--nashorn";
private static final String GRAALJS_COMPAT = "--graaljs-compat";
public static final Set RESERVED_WORDS = new HashSet<>() {{
addAll(
Arrays.asList(
SCRIPT, ACTIVITY, SCENARIO, RUN, START,
FRAGMENT, STOP, AWAIT, WAIT_MILLIS, LIST_ACTIVITY_TYPES, HELP
)
);
}};
private static final String DEFAULT_CONSOLE_LOGGING_PATTERN = "%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n";
private final LinkedList cmdList = new LinkedList<>();
private int logsMax = 0;
private boolean wantsVersionShort = false;
private boolean wantsVersionCoords = false;
private boolean wantsActivityHelp = false;
private String wantsActivityHelpFor;
private boolean wantsActivityTypes = false;
private boolean wantsBasicHelp = false;
private String reportGraphiteTo = null;
private String reportCsvTo = null;
private int reportInterval = 10;
private String metricsPrefix = "nosqlbench.";
private String wantsMetricsForActivity;
private String sessionName = "";
private boolean showScript = false;
private Level consoleLevel = Level.WARN;
private final List histoLoggerConfigs = new ArrayList<>();
private final List statsLoggerConfigs = new ArrayList<>();
private final List classicHistoConfigs = new ArrayList<>();
private String progressSpec = "console:1m";
private String logsDirectory = "logs";
private boolean wantsInputTypes = false;
private boolean wantsMarkerTypes = false;
private String[] rleDumpOptions = new String[0];
private String[] cyclelogImportOptions = new String[0];
private String consoleLoggingPattern = DEFAULT_CONSOLE_LOGGING_PATTERN;
private String logsLevel = "INFO";
private Map logLevelsOverrides = new HashMap<>();
private boolean enableChart = false;
private boolean dockerMetrics = false;
private boolean wantsScenariosList = false;
private String wantsToCopyWorkload = null;
private boolean wantsWorkloadsList = false;
private final List wantsToIncludePaths = new ArrayList<>();
private Scenario.Engine engine = Scenario.Engine.Graalvm;
private boolean graaljs_compat = false;
private int hdr_digits = 4;
public NBCLIOptions(String[] args) {
parse(args);
}
private void parse(String[] args) {
LinkedList arglist = new LinkedList<>() {{
addAll(Arrays.asList(args));
}};
if (arglist.peekFirst() == null) {
wantsBasicHelp = true;
return;
}
// Preprocess --include regardless of position
LinkedList nonincludes = new LinkedList<>();
while (arglist.peekFirst() != null) {
String word = arglist.peekFirst();
if (word.startsWith("--") && word.contains("=")) {
String wordToSplit = arglist.removeFirst();
String[] split = wordToSplit.split("=", 2);
arglist.offerFirst(split[1]);
arglist.offerFirst(split[0]);
continue;
}
if (INCLUDE.equals(word)) {
arglist.removeFirst();
String include = readWordOrThrow(arglist, "path to include");
wantsToIncludePaths.add(include);
} else if (METRICS_PREFIX.equals(word)) {
arglist.removeFirst();
metricsPrefix = arglist.removeFirst();
} else {
nonincludes.addLast(arglist.removeFirst());
}
}
arglist = nonincludes;
nonincludes = new LinkedList<>();
PathCanonicalizer canonicalizer = new PathCanonicalizer(wantsIncludes());
while (arglist.peekFirst() != null) {
String word = arglist.peekFirst();
if (word.startsWith("--") && word.contains("=")) {
String wordToSplit = arglist.removeFirst();
String[] split = wordToSplit.split("=", 2);
arglist.offerFirst(split[1]);
arglist.offerFirst(split[0]);
continue;
}
switch (word) {
case GRAALJS_COMPAT:
graaljs_compat = true;
arglist.removeFirst();
break;
case GRAALJS_ENGINE:
engine = Scenario.Engine.Graalvm;
arglist.removeFirst();
break;
case NASHORN_ENGINE:
engine = Scenario.Engine.Nashorn;
arglist.removeFirst();
break;
case SHOW_SCRIPT:
arglist.removeFirst();
showScript = true;
break;
case LIST_METRICS:
arglist.removeFirst();
arglist.addFirst("start");
Cmd cmd = Cmd.parseArg(arglist,canonicalizer);
wantsMetricsForActivity = cmd.getArg("driver");
break;
case SESSION_NAME:
arglist.removeFirst();
sessionName = readWordOrThrow(arglist, "a session name");
break;
case LOGS_DIR:
arglist.removeFirst();
logsDirectory = readWordOrThrow(arglist, "a log directory");
break;
case HDR_DIGITS:
arglist.removeFirst();
hdr_digits = Integer.parseInt(readWordOrThrow(arglist, "significant digits"));
break;
case LOGS_MAX:
arglist.removeFirst();
logsMax = Integer.parseInt(readWordOrThrow(arglist, "max logfiles to keep"));
break;
case LOGS_LEVEL:
arglist.removeFirst();
logsLevel = readWordOrThrow(arglist, "a log level");
break;
case LOG_LEVEL_OVERRIDE:
arglist.removeFirst();
logLevelsOverrides = parseLogLevelOverrides(readWordOrThrow(arglist, "log levels in name:LEVEL,... format"));
break;
case PROGRESS:
arglist.removeFirst();
progressSpec = readWordOrThrow(arglist, "a progress indicator, like 'log:1m' or 'screen:10s', or just 'log' or 'screen'");
break;
case VERSION:
arglist.removeFirst();
wantsVersionShort = true;
break;
case VERSION_COORDS:
arglist.removeFirst();
wantsVersionCoords = true;
break;
case ENABLE_CHART:
arglist.removeFirst();
enableChart = true;
break;
case DOCKER_METRICS:
arglist.removeFirst();
dockerMetrics = true;
break;
case HELP:
case "-h":
case "help":
arglist.removeFirst();
if (arglist.peekFirst() == null) {
wantsBasicHelp = true;
logger.info("getting basic help");
} else {
wantsActivityHelp = true;
wantsActivityHelpFor = readWordOrThrow(arglist, "topic");
}
break;
case EXPORT_CYCLE_LOG:
arglist.removeFirst();
rleDumpOptions = readAllWords(arglist);
break;
case IMPORT_CYCLE_LOG:
arglist.removeFirst();
cyclelogImportOptions = readAllWords(arglist);
break;
case LOG_HISTOGRAMS:
arglist.removeFirst();
String logto = arglist.removeFirst();
histoLoggerConfigs.add(logto);
break;
case LOG_HISTOSTATS:
arglist.removeFirst();
String logStatsTo = arglist.removeFirst();
statsLoggerConfigs.add(logStatsTo);
break;
case CLASSIC_HISTOGRAMS:
arglist.removeFirst();
String classicHistos = arglist.removeFirst();
classicHistoConfigs.add(classicHistos);
break;
case REPORT_INTERVAL:
arglist.removeFirst();
reportInterval = Integer.parseInt(readWordOrThrow(arglist, "report interval"));
break;
case REPORT_CSV_TO:
arglist.removeFirst();
reportCsvTo = arglist.removeFirst();
break;
case REPORT_GRAPHITE_TO:
arglist.removeFirst();
reportGraphiteTo = arglist.removeFirst();
break;
case LIST_DRIVERS:
case LIST_ACTIVITY_TYPES:
arglist.removeFirst();
wantsActivityTypes = true;
break;
case LIST_INPUT_TYPES:
arglist.removeFirst();
wantsInputTypes = true;
break;
case LIST_OUTPUT_TYPES:
arglist.removeFirst();
wantsMarkerTypes = true;
break;
case DASH_V_INFO:
consoleLevel = Level.INFO;
arglist.removeFirst();
break;
case DASH_VV_DEBUG:
consoleLevel = Level.DEBUG;
arglist.removeFirst();
break;
case DASH_VVV_TRACE:
consoleLevel = Level.TRACE;
arglist.removeFirst();
break;
case WITH_LOGGING_PATTERN:
arglist.removeFirst();
consoleLoggingPattern = readWordOrThrow(arglist, "logging pattern");
break;
case LIST_SCENARIOS:
arglist.removeFirst();
wantsScenariosList = true;
break;
case LIST_WORKLOADS:
arglist.removeFirst();
wantsWorkloadsList = true;
break;
case COPY:
arglist.removeFirst();
wantsToCopyWorkload = readWordOrThrow(arglist, "workload to copy");
break;
default:
nonincludes.addLast(arglist.removeFirst());
}
}
arglist = nonincludes;
while (arglist.peekFirst() != null) {
String word = arglist.peekFirst();
if (word.startsWith("--") && word.contains("=")) {
String wordToSplit = arglist.removeFirst();
String[] split = wordToSplit.split("=", 2);
arglist.offerFirst(split[1]);
arglist.offerFirst(split[0]);
continue;
}
Cmd cmd=null;
switch (word) {
case FRAGMENT:
case SCRIPT:
case START:
case RUN:
case AWAIT:
case STOP:
case WAIT_MILLIS:
cmd = Cmd.parseArg(arglist,canonicalizer);
cmdList.add(cmd);
break;
default:
Optional> scriptfile = NBIO.local()
.prefix("scripts/auto")
.name(word)
.extension("js")
.first();
//Script
if (scriptfile.isPresent()) {
arglist.removeFirst();
arglist.addFirst("scripts/auto/" + word);
arglist.addFirst("script");
cmd = Cmd.parseArg(arglist,canonicalizer);
cmdList.add(cmd);
} else if (NBCLIScenarioParser.isFoundWorkload(word, wantsIncludes())) {
NBCLIScenarioParser.parseScenarioCommand(arglist, RESERVED_WORDS, wantsIncludes());
} else {
throw new InvalidParameterException("unrecognized option:" + word);
}
break;
}
}
}
public String[] wantsIncludes() {
return wantsToIncludePaths.toArray(new String[0]);
}
private Map parseLogLevelOverrides(String levelsSpec) {
Map levels = new HashMap<>();
Arrays.stream(levelsSpec.split("[,;]")).forEach(kp -> {
String[] ll = kp.split(":");
if (ll.length != 2) {
throw new RuntimeException("Log level must have name:level format");
}
levels.put(ll[0], Level.toLevel(ll[1]));
});
return levels;
}
public Scenario.Engine getScriptingEngine() {
return engine;
}
public boolean wantsGraaljsCompatMode() {
return graaljs_compat;
}
public List getHistoLoggerConfigs() {
List configs = histoLoggerConfigs.stream().map(LoggerConfig::new).collect(Collectors.toList());
checkLoggerConfigs(configs, LOG_HISTOGRAMS);
return configs;
}
public List getStatsLoggerConfigs() {
List configs = statsLoggerConfigs.stream().map(LoggerConfig::new).collect(Collectors.toList());
checkLoggerConfigs(configs, LOG_HISTOSTATS);
return configs;
}
public List getClassicHistoConfigs() {
List configs = classicHistoConfigs.stream().map(LoggerConfig::new).collect(Collectors.toList());
checkLoggerConfigs(configs, CLASSIC_HISTOGRAMS);
return configs;
}
public List getCommands() {
return cmdList;
}
public boolean wantsShowScript() {
return showScript;
}
public boolean wantsVersionCoords() {
return wantsVersionCoords;
}
public boolean isWantsVersionShort() {
return wantsVersionShort;
}
public boolean wantsActivityTypes() {
return wantsActivityTypes;
}
public boolean wantsTopicalHelp() {
return wantsActivityHelp;
}
public String wantsTopicalHelpFor() {
return wantsActivityHelpFor;
}
public boolean wantsBasicHelp() {
return wantsBasicHelp;
}
public boolean wantsEnableChart() {
return enableChart;
}
public boolean wantsDockerMetrics() {
return dockerMetrics;
}
public int getReportInterval() {
return reportInterval;
}
public String wantsReportGraphiteTo() {
return reportGraphiteTo;
}
public String wantsMetricsPrefix() {
return metricsPrefix;
}
public String wantsMetricsForActivity() {
return wantsMetricsForActivity;
}
public String getSessionName() {
return sessionName;
}
public Level wantsConsoleLogLevel() {
return consoleLevel;
}
private void assertNotParameter(String scriptName) {
if (scriptName.contains("=")) {
throw new InvalidParameterException("script name must precede script arguments");
}
}
private void assertNotReserved(String name) {
if (RESERVED_WORDS.contains(name)) {
throw new InvalidParameterException(name + " is a reserved word and may not be used here.");
}
}
private String readWordOrThrow(LinkedList arglist, String required) {
if (arglist.peekFirst() == null) {
throw new InvalidParameterException(required + " not found");
}
return arglist.removeFirst();
}
private String[] readAllWords(LinkedList arglist) {
String[] args = arglist.toArray(new String[0]);
arglist.clear();
return args;
}
// private Cmd parseScriptCmd(LinkedList arglist) {
// String cmdType = arglist.removeFirst();
// String scriptName = readWordOrThrow(arglist, "script name");
// assertNotReserved(scriptName);
// assertNotParameter(scriptName);
// Map scriptParams = new LinkedHashMap<>();
// while (arglist.size() > 0 && !RESERVED_WORDS.contains(arglist.peekFirst())
// && arglist.peekFirst().contains("=")) {
// String[] split = arglist.removeFirst().split("=", 2);
// scriptParams.put(split[0], split[1]);
// }
// return new Cmd(CmdType.script, scriptName, scriptParams);
// }
public int getHdrDigits() {
return hdr_digits;
}
public String getProgressSpec() {
ProgressSpec spec = parseProgressSpec(this.progressSpec);// sanity check
if (spec.indicatorMode == IndicatorMode.console
&& Level.INFO.isGreaterOrEqual(wantsConsoleLogLevel())) {
logger.warn("Console is already logging info or more, so progress data on console is suppressed.");
spec.indicatorMode = IndicatorMode.logonly;
}
return spec.toString();
}
private void checkLoggerConfigs(List configs, String configName) {
Set files = new HashSet<>();
configs.stream().map(LoggerConfig::getFilename).forEach(s -> {
if (files.contains(s)) {
logger.warn(s + " is included in " + configName + " more than once. It will only be included " +
"in the first matching config. Reorder your options if you need to control this.");
}
files.add(s);
});
}
public String wantsReportCsvTo() {
return reportCsvTo;
}
public String getLogsDirectory() {
return logsDirectory;
}
public int getLogsMax() {
return logsMax;
}
public String getLogsLevel() {
return logsLevel;
}
public boolean wantsInputTypes() {
return this.wantsInputTypes;
}
public boolean wantsMarkerTypes() {
return wantsMarkerTypes;
}
public boolean wantsToDumpCyclelog() {
return rleDumpOptions.length > 0;
}
public boolean wantsToImportCycleLog() {
return cyclelogImportOptions.length > 0;
}
public String[] getCyclelogImportOptions() {
return cyclelogImportOptions;
}
public String[] getCycleLogExporterOptions() {
return rleDumpOptions;
}
public String getConsoleLoggingPattern() {
return consoleLoggingPattern;
}
public Map getLogLevelOverrides() {
return logLevelsOverrides;
}
public void setHistoLoggerConfigs(String pattern, String file, String interval) {
//--log-histograms 'hdrdata.log:.*:2m'
histoLoggerConfigs.add(String.format("%s:%s:%s", file, pattern, interval));
}
public boolean wantsScenariosList() {
return wantsScenariosList;
}
public boolean wantsToCopyResource() {
return wantsToCopyWorkload != null;
}
public String wantsToCopyResourceNamed() {
return wantsToCopyWorkload;
}
public boolean wantsWorkloadsList() {
return wantsWorkloadsList;
}
public static class LoggerConfig {
public String file;
public String pattern = ".*";
public String interval = "30 seconds";
public LoggerConfig(String histoLoggerSpec) {
String[] words = histoLoggerSpec.split(":");
switch (words.length) {
case 3:
interval = words[2].isEmpty() ? interval : words[2];
case 2:
pattern = words[1].isEmpty() ? pattern : words[1];
case 1:
file = words[0];
if (file.isEmpty()) {
throw new RuntimeException("You must not specify an empty file here for logging data.");
}
break;
default:
throw new RuntimeException(
LOG_HISTOGRAMS +
" options must be in either 'regex:filename:interval' or 'regex:filename' or 'filename' format"
);
}
}
public String getFilename() {
return file;
}
}
private static class ProgressSpec {
public String intervalSpec;
public IndicatorMode indicatorMode;
public String toString() {
return indicatorMode.toString() + ":" + intervalSpec;
}
}
private ProgressSpec parseProgressSpec(String interval) {
ProgressSpec progressSpec = new ProgressSpec();
String[] parts = interval.split(":");
switch (parts.length) {
case 2:
Unit.msFor(parts[1]).orElseThrow(
() -> new RuntimeException("Unable to parse progress indicator indicatorSpec '" + parts[1] + "'")
);
progressSpec.intervalSpec = parts[1];
case 1:
progressSpec.indicatorMode = IndicatorMode.valueOf(parts[0]);
break;
default:
throw new RuntimeException("This should never happen.");
}
return progressSpec;
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy