org.elasticsearch.test.ESTestCase Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of framework Show documentation
Show all versions of framework Show documentation
Elasticsearch subproject :test:framework
The newest version!
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.test;
import com.carrotsearch.randomizedtesting.RandomizedTest;
import com.carrotsearch.randomizedtesting.annotations.Listeners;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.Appender;
import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.core.LoggerContext;
import org.apache.logging.log4j.core.appender.AbstractAppender;
import org.apache.logging.log4j.core.config.Configurator;
import org.apache.logging.log4j.core.layout.PatternLayout;
import org.apache.logging.log4j.status.StatusConsoleListener;
import org.apache.logging.log4j.status.StatusData;
import org.apache.logging.log4j.status.StatusLogger;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.tests.util.TestRuleMarkFailure;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.tests.util.TimeUnits;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.TransportVersion;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.RequestBuilder;
import org.elasticsearch.action.support.ActionTestUtils;
import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.action.support.SubscribableListener;
import org.elasticsearch.bootstrap.BootstrapForTesting;
import org.elasticsearch.client.internal.Requests;
import org.elasticsearch.cluster.ClusterModule;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.CheckedSupplier;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.bytes.CompositeBytesReference;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.NamedWriteable;
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.logging.HeaderWarning;
import org.elasticsearch.common.logging.HeaderWarningAppender;
import org.elasticsearch.common.logging.LogConfigurator;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.time.DateUtils;
import org.elasticsearch.common.time.FormatNames;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.common.util.Maps;
import org.elasticsearch.common.util.MockBigArrays;
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.core.Booleans;
import org.elasticsearch.core.CheckedRunnable;
import org.elasticsearch.core.PathUtils;
import org.elasticsearch.core.PathUtilsForTesting;
import org.elasticsearch.core.RefCounted;
import org.elasticsearch.core.RestApiVersion;
import org.elasticsearch.core.Strings;
import org.elasticsearch.core.SuppressForbidden;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.env.TestEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexService.IndexCreationContext;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.IndexVersion;
import org.elasticsearch.index.analysis.AnalysisRegistry;
import org.elasticsearch.index.analysis.CharFilterFactory;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.TokenFilterFactory;
import org.elasticsearch.index.analysis.TokenizerFactory;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.plugins.AnalysisPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.plugins.scanners.StablePluginsRegistry;
import org.elasticsearch.script.MockScriptEngine;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.search.MockSearchService;
import org.elasticsearch.search.SearchService;
import org.elasticsearch.test.junit.listeners.LoggingListener;
import org.elasticsearch.test.junit.listeners.ReproduceInfoPrinter;
import org.elasticsearch.threadpool.ExecutorBuilder;
import org.elasticsearch.threadpool.TestThreadPool;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.LeakTracker;
import org.elasticsearch.transport.netty4.Netty4Plugin;
import org.elasticsearch.xcontent.MediaType;
import org.elasticsearch.xcontent.NamedXContentRegistry;
import org.elasticsearch.xcontent.ToXContent;
import org.elasticsearch.xcontent.XContent;
import org.elasticsearch.xcontent.XContentBuilder;
import org.elasticsearch.xcontent.XContentFactory;
import org.elasticsearch.xcontent.XContentParser;
import org.elasticsearch.xcontent.XContentParser.Token;
import org.elasticsearch.xcontent.XContentParserConfiguration;
import org.elasticsearch.xcontent.XContentType;
import org.hamcrest.Matcher;
import org.hamcrest.MatcherAssert;
import org.hamcrest.Matchers;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.internal.AssumptionViolatedException;
import org.junit.rules.RuleChain;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.lang.annotation.ElementType;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.lang.invoke.MethodHandles;
import java.math.BigInteger;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.nio.file.Path;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
import java.security.Provider;
import java.security.SecureRandom;
import java.time.Instant;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Random;
import java.util.Set;
import java.util.TimeZone;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BooleanSupplier;
import java.util.function.Consumer;
import java.util.function.IntFunction;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.DoubleStream;
import java.util.stream.IntStream;
import java.util.stream.LongStream;
import java.util.stream.Stream;
import static java.util.Collections.emptyMap;
import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.emptyCollectionOf;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.startsWith;
/**
* Base testcase for randomized unit testing with Elasticsearch
*/
@Listeners({ ReproduceInfoPrinter.class, LoggingListener.class })
@ThreadLeakScope(Scope.SUITE)
@ThreadLeakLingering(linger = 5000) // 5 sec lingering
@TimeoutSuite(millis = 20 * TimeUnits.MINUTE)
@ThreadLeakFilters(filters = { GraalVMThreadsFilter.class, NettyGlobalThreadsFilter.class })
@LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose")
// we suppress pretty much all the lucene codecs for now, except asserting
// assertingcodec is the winner for a codec here: it finds bugs and gives clear exceptions.
@SuppressCodecs(
{
"SimpleText",
"Memory",
"CheapBastard",
"Direct",
"Compressing",
"FST50",
"FSTOrd50",
"TestBloomFilteredLucenePostings",
"MockRandom",
"BlockTreeOrds",
"LuceneFixedGap",
"LuceneVarGapFixedInterval",
"LuceneVarGapDocFreqInterval",
"Lucene50" }
)
@LuceneTestCase.SuppressReproduceLine
public abstract class ESTestCase extends LuceneTestCase {
protected static final List JAVA_TIMEZONE_IDS;
protected static final List JAVA_ZONE_IDS;
private static final AtomicInteger portGenerator = new AtomicInteger();
private static final Collection loggedLeaks = new ArrayList<>();
private HeaderWarningAppender headerWarningAppender;
@AfterClass
public static void resetPortCounter() {
portGenerator.set(0);
}
// Allows distinguishing between parallel test processes
public static final String TEST_WORKER_VM_ID;
public static final String TEST_WORKER_SYS_PROPERTY = "org.gradle.test.worker";
public static final String DEFAULT_TEST_WORKER_ID = "--not-gradle--";
public static final String FIPS_SYSPROP = "tests.fips.enabled";
private static final SetOnce WARN_SECURE_RANDOM_FIPS_NOT_DETERMINISTIC = new SetOnce<>();
static {
Random random = initTestSeed();
TEST_WORKER_VM_ID = System.getProperty(TEST_WORKER_SYS_PROPERTY, DEFAULT_TEST_WORKER_ID);
setTestSysProps(random);
// TODO: consolidate logging initialization for tests so it all occurs in logconfigurator
LogConfigurator.loadLog4jPlugins();
LogConfigurator.configureESLogging();
MockLog.init();
final List testAppenders = new ArrayList<>(3);
for (String leakLoggerName : Arrays.asList("io.netty.util.ResourceLeakDetector", LeakTracker.class.getName())) {
Logger leakLogger = LogManager.getLogger(leakLoggerName);
Appender leakAppender = new AbstractAppender(leakLoggerName, null, PatternLayout.newBuilder().withPattern("%m").build()) {
@Override
public void append(LogEvent event) {
String message = event.getMessage().getFormattedMessage();
if (Level.ERROR.equals(event.getLevel()) && message.contains("LEAK:")) {
synchronized (loggedLeaks) {
loggedLeaks.add(message);
}
}
}
};
leakAppender.start();
Loggers.addAppender(leakLogger, leakAppender);
testAppenders.add(leakAppender);
}
Logger promiseUncaughtLogger = LogManager.getLogger("io.netty.util.concurrent.DefaultPromise");
final Appender uncaughtAppender = new AbstractAppender(
promiseUncaughtLogger.getName(),
null,
PatternLayout.newBuilder().withPattern("%m").build()
) {
@Override
public void append(LogEvent event) {
if (Level.WARN.equals(event.getLevel())) {
synchronized (loggedLeaks) {
loggedLeaks.add(event.getMessage().getFormattedMessage());
}
}
}
};
uncaughtAppender.start();
Loggers.addAppender(promiseUncaughtLogger, uncaughtAppender);
testAppenders.add(uncaughtAppender);
// shutdown hook so that when the test JVM exits, logging is shutdown too
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
for (Appender testAppender : testAppenders) {
testAppender.stop();
}
LoggerContext context = (LoggerContext) LogManager.getContext(false);
Configurator.shutdown(context);
}));
BootstrapForTesting.ensureInitialized();
/*
* We need to exclude time zones not supported by joda (like SystemV* timezones)
* because they cannot be converted back to DateTimeZone which we currently
* still need to do internally e.g. in bwc serialization and in the extract() method
* //TODO remove once tests do not send time zone ids back to versions of ES using Joda
*/
Set unsupportedJodaTZIds = Set.of(
"ACT",
"AET",
"AGT",
"ART",
"AST",
"BET",
"BST",
"CAT",
"CNT",
"CST",
"CTT",
"EAT",
"ECT",
"EST",
"HST",
"IET",
"IST",
"JST",
"MIT",
"MST",
"NET",
"NST",
"PLT",
"PNT",
"PRT",
"PST",
"SST",
"VST"
);
Predicate unsupportedZoneIdsPredicate = tz -> tz.startsWith("System/") || tz.equals("Eire");
Predicate unsupportedTZIdsPredicate = unsupportedJodaTZIds::contains;
JAVA_TIMEZONE_IDS = Arrays.stream(TimeZone.getAvailableIDs())
.filter(unsupportedTZIdsPredicate.negate())
.filter(unsupportedZoneIdsPredicate.negate())
.sorted()
.toList();
JAVA_ZONE_IDS = ZoneId.getAvailableZoneIds().stream().filter(unsupportedZoneIdsPredicate.negate()).sorted().toList();
}
static Random initTestSeed() {
String inputSeed = System.getProperty("tests.seed");
long seed;
if (inputSeed == null) {
// when running tests in intellij, we don't have a seed. Setup the seed early here, before getting to RandomizedRunner,
// so that we can use it in ESTestCase static init
seed = System.nanoTime();
setTestSeed(Long.toHexString(seed));
} else {
String[] seedParts = inputSeed.split("[\\:]");
seed = Long.parseUnsignedLong(seedParts[0], 16);
}
if (Booleans.parseBoolean(System.getProperty("tests.hackImmutableCollections", "false"))) {
forceImmutableCollectionsSeed(seed);
}
return new Random(seed);
}
@SuppressForbidden(reason = "set tests.seed for intellij")
static void setTestSeed(String seed) {
System.setProperty("tests.seed", seed);
}
private static void forceImmutableCollectionsSeed(long seed) {
try {
MethodHandles.Lookup lookup = MethodHandles.lookup();
Class collectionsClass = Class.forName("java.util.ImmutableCollections");
var salt32l = lookup.findStaticVarHandle(collectionsClass, "SALT32L", long.class);
var reverse = lookup.findStaticVarHandle(collectionsClass, "REVERSE", boolean.class);
salt32l.set(seed & 0xFFFF_FFFFL);
reverse.set((seed & 1) == 0);
} catch (Exception e) {
throw new AssertionError(e);
}
}
@SuppressForbidden(reason = "force log4j and netty sysprops")
private static void setTestSysProps(Random random) {
System.setProperty("log4j.shutdownHookEnabled", "false");
System.setProperty("log4j2.disable.jmx", "true");
// Enable Netty leak detection and monitor logger for logged leak errors
System.setProperty("io.netty.leakDetection.level", "paranoid");
if (System.getProperty("es.use_unpooled_allocator") == null) {
// unless explicitly forced to unpooled, always test with the pooled allocator to get the best possible coverage from Netty's
// leak detection which does not cover simple unpooled heap buffers
System.setProperty("es.use_unpooled_allocator", "false");
}
// We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
// other if we allow them to set the number of available processors as it's set-once in Netty.
System.setProperty("es.set.netty.runtime.available.processors", "false");
// sometimes use the java.time date formatters
if (random.nextBoolean()) {
System.setProperty("es.datetime.java_time_parsers", "true");
}
}
protected final Logger logger = LogManager.getLogger(getClass());
private ThreadContext threadContext;
// -----------------------------------------------------------------
// Suite and test case setup/cleanup.
// -----------------------------------------------------------------
@Rule
public RuleChain failureAndSuccessEvents = RuleChain.outerRule(new TestRuleAdapter() {
@Override
protected void afterIfSuccessful() throws Throwable {
ESTestCase.this.afterIfSuccessful();
}
@Override
protected void afterAlways(List errors) throws Throwable {
if (errors != null && errors.isEmpty() == false) {
boolean allAssumption = true;
for (Throwable error : errors) {
if (false == error instanceof AssumptionViolatedException) {
allAssumption = false;
break;
}
}
if (false == allAssumption) {
ESTestCase.this.afterIfFailed(errors);
}
}
super.afterAlways(errors);
}
});
/**
* Generates a new transport address using {@link TransportAddress#META_ADDRESS} with an incrementing port number.
* The port number starts at 0 and is reset after each test suite run.
*/
public static TransportAddress buildNewFakeTransportAddress() {
return new TransportAddress(TransportAddress.META_ADDRESS, portGenerator.incrementAndGet());
}
/**
* Called when a test fails, supplying the errors it generated. Not called when the test fails because assumptions are violated.
*/
protected void afterIfFailed(List errors) {}
/** called after a test is finished, but only if successful */
protected void afterIfSuccessful() throws Exception {}
/**
* Marks a test suite or a test method that should run without security manager enabled.
*/
@Retention(RetentionPolicy.RUNTIME)
@Target({ ElementType.TYPE })
@Inherited
public @interface WithoutSecurityManager {
}
private static Closeable securityManagerRestorer;
// disable security manager if test is annotated to run without it
@BeforeClass
public static void maybeStashClassSecurityManager() {
if (getTestClass().isAnnotationPresent(WithoutSecurityManager.class)) {
securityManagerRestorer = BootstrapForTesting.disableTestSecurityManager();
}
}
@AfterClass
public static void maybeRestoreClassSecurityManager() throws IOException {
if (securityManagerRestorer != null) {
securityManagerRestorer.close();
securityManagerRestorer = null;
}
}
// setup mock filesystems for this test run. we change PathUtils
// so that all accesses are plumbed thru any mock wrappers
@BeforeClass
public static void setFileSystem() throws Exception {
PathUtilsForTesting.setup();
}
@AfterClass
public static void restoreFileSystem() throws Exception {
PathUtilsForTesting.teardown();
}
// randomize content type for request builders
@BeforeClass
public static void setContentType() throws Exception {
Requests.INDEX_CONTENT_TYPE = randomFrom(XContentType.values());
}
@AfterClass
public static void restoreContentType() {
Requests.INDEX_CONTENT_TYPE = XContentType.JSON;
}
@BeforeClass
public static void ensureSupportedLocale() {
if (isUnusableLocale()) {
Logger logger = LogManager.getLogger(ESTestCase.class);
logger.warn(
"Attempting to run tests in an unusable locale in a FIPS JVM. Certificate expiration validation will fail, "
+ "switching to English. See: https://github.com/bcgit/bc-java/issues/405"
);
Locale.setDefault(Locale.ENGLISH);
}
}
@Before
public void setHeaderWarningAppender() {
this.headerWarningAppender = HeaderWarningAppender.createAppender("header_warning", null);
this.headerWarningAppender.start();
Loggers.addAppender(LogManager.getLogger("org.elasticsearch.deprecation"), this.headerWarningAppender);
}
@After
public void removeHeaderWarningAppender() {
if (this.headerWarningAppender != null) {
Loggers.removeAppender(LogManager.getLogger("org.elasticsearch.deprecation"), this.headerWarningAppender);
this.headerWarningAppender = null;
}
}
@Before
public final void before() {
LeakTracker.setContextHint(getTestName());
logger.info("{}before test", getTestParamsForLogging());
assertNull("Thread context initialized twice", threadContext);
if (enableWarningsCheck()) {
this.threadContext = new ThreadContext(Settings.EMPTY);
HeaderWarning.setThreadContext(threadContext);
}
}
/**
* Whether or not we check after each test whether it has left warnings behind. That happens if any deprecated feature or syntax
* was used by the test and the test didn't assert on it using {@link #assertWarnings(String...)}.
*/
protected boolean enableWarningsCheck() {
return true;
}
@After
public final void after() throws Exception {
checkStaticState();
// We check threadContext != null rather than enableWarningsCheck()
// because after methods are still called in the event that before
// methods failed, in which case threadContext might not have been
// initialized
if (threadContext != null) {
ensureNoWarnings();
HeaderWarning.removeThreadContext(threadContext);
threadContext = null;
}
ensureAllSearchContextsReleased();
ensureCheckIndexPassed();
logger.info("{}after test", getTestParamsForLogging());
LeakTracker.setContextHint("");
}
private String getTestParamsForLogging() {
String name = getTestName();
int start = name.indexOf('{');
if (start < 0) return "";
int end = name.lastIndexOf('}');
if (end < 0) return "";
return "[" + name.substring(start + 1, end) + "] ";
}
public void ensureNoWarnings() {
// Check that there are no unaccounted warning headers. These should be checked with {@link #assertWarnings(String...)} in the
// appropriate test
try {
final List warnings = threadContext.getResponseHeaders().get("Warning");
if (warnings != null) {
// unit tests do not run with the bundled JDK, if there are warnings we need to filter the no-jdk deprecation warning
final List filteredWarnings = warnings.stream()
.filter(k -> filteredWarnings().stream().noneMatch(s -> k.contains(s)))
.collect(Collectors.toList());
assertThat("unexpected warning headers", filteredWarnings, empty());
} else {
assertNull("unexpected warning headers", warnings);
}
} finally {
resetDeprecationLogger();
}
}
protected List filteredWarnings() {
List filtered = new ArrayList<>();
filtered.add(
"Configuring multiple [path.data] paths is deprecated. Use RAID or other system level features for utilizing"
+ " multiple disks. This feature will be removed in a future release."
);
filtered.add("Configuring [path.data] with a list is deprecated. Instead specify as a string value");
filtered.add("setting [path.shared_data] is deprecated and will be removed in a future release");
return filtered;
}
/**
* Convenience method to assert warnings for settings deprecations and general deprecation warnings.
* @param settings the settings that are expected to be deprecated
* @param warnings other expected general deprecation warnings
*/
protected final void assertSettingDeprecationsAndWarnings(final Setting[] settings, final DeprecationWarning... warnings) {
assertWarnings(true, Stream.concat(Arrays.stream(settings).map(setting -> {
String warningMessage = String.format(
Locale.ROOT,
"[%s] setting was deprecated in Elasticsearch and will be removed in a future release.",
setting.getKey()
);
return new DeprecationWarning(
setting.getProperties().contains(Setting.Property.Deprecated) ? DeprecationLogger.CRITICAL : Level.WARN,
warningMessage
);
}), Arrays.stream(warnings)).toArray(DeprecationWarning[]::new));
}
/**
* Convenience method to assert warnings for settings deprecations and general deprecation warnings. All warnings passed to this method
* are assumed to be at WARNING level.
* @param expectedWarnings expected general deprecation warning messages.
*/
protected final void assertWarnings(String... expectedWarnings) {
assertWarnings(
true,
Arrays.stream(expectedWarnings)
.map(expectedWarning -> new DeprecationWarning(Level.WARN, expectedWarning))
.toArray(DeprecationWarning[]::new)
);
}
/**
* Convenience method to assert warnings for settings deprecations and general deprecation warnings. All warnings passed to this method
* are assumed to be at CRITICAL level.
* @param expectedWarnings expected general deprecation warning messages.
*/
protected final void assertCriticalWarnings(String... expectedWarnings) {
assertWarnings(
true,
Arrays.stream(expectedWarnings)
.map(expectedWarning -> new DeprecationWarning(DeprecationLogger.CRITICAL, expectedWarning))
.toArray(DeprecationWarning[]::new)
);
}
protected final void assertWarnings(boolean stripXContentPosition, DeprecationWarning... expectedWarnings) {
if (enableWarningsCheck() == false) {
throw new IllegalStateException("unable to check warning headers if the test is not set to do so");
}
try {
final List actualWarningStrings = threadContext.getResponseHeaders().get("Warning");
if (expectedWarnings == null || expectedWarnings.length == 0) {
assertNull("expected 0 warnings, actual: " + actualWarningStrings, actualWarningStrings);
} else {
assertNotNull("no warnings, expected: " + Arrays.asList(expectedWarnings), actualWarningStrings);
final Set actualDeprecationWarnings = actualWarningStrings.stream().map(warningString -> {
String warningText = HeaderWarning.extractWarningValueFromWarningHeader(warningString, stripXContentPosition);
final Level level;
if (warningString.startsWith(Integer.toString(DeprecationLogger.CRITICAL.intLevel()))) {
level = DeprecationLogger.CRITICAL;
} else if (warningString.startsWith(Integer.toString(Level.WARN.intLevel()))) {
level = Level.WARN;
} else {
throw new IllegalArgumentException("Unknown level in deprecation message " + warningString);
}
return new DeprecationWarning(level, warningText);
}).collect(Collectors.toSet());
for (DeprecationWarning expectedWarning : expectedWarnings) {
DeprecationWarning escapedExpectedWarning = new DeprecationWarning(
expectedWarning.level,
HeaderWarning.escapeAndEncode(expectedWarning.message)
);
assertThat(actualDeprecationWarnings, hasItem(escapedExpectedWarning));
}
assertEquals(
"Expected "
+ expectedWarnings.length
+ " warnings but found "
+ actualWarningStrings.size()
+ "\nExpected: "
+ Arrays.asList(expectedWarnings)
+ "\nActual: "
+ actualWarningStrings,
expectedWarnings.length,
actualWarningStrings.size()
);
}
} finally {
resetDeprecationLogger();
}
}
/**
* Reset the deprecation logger by clearing the current thread context.
*/
private void resetDeprecationLogger() {
// "clear" context by stashing current values and dropping the returned StoredContext
threadContext.stashContext();
}
private static final List statusData = new ArrayList<>();
static {
// ensure that the status logger is set to the warn level so we do not miss any warnings with our Log4j usage
StatusLogger.getLogger().setLevel(Level.WARN);
// Log4j will write out status messages indicating problems with the Log4j usage to the status logger; we hook into this logger and
// assert that no such messages were written out as these would indicate a problem with our logging configuration
StatusLogger.getLogger().registerListener(new StatusConsoleListener(Level.WARN) {
@Override
public void log(StatusData data) {
synchronized (statusData) {
statusData.add(data);
}
}
});
}
// Tolerate the absence or otherwise denial of these specific lookup classes.
// At some future time, we should require the JDNI warning.
private static final List LOG_4J_MSG_PREFIXES = List.of(
"JNDI lookup class is not available because this JRE does not support JNDI. "
+ "JNDI string lookups will not be available, continuing configuration.",
"JMX runtime input lookup class is not available because this JRE does not support JMX. "
+ "JMX lookups will not be available, continuing configuration. "
);
// separate method so that this can be checked again after suite scoped cluster is shut down
protected static void checkStaticState() throws Exception {
MockBigArrays.ensureAllArraysAreReleased();
// ensure no one changed the status logger level on us
assertThat(StatusLogger.getLogger().getLevel(), equalTo(Level.WARN));
synchronized (statusData) {
try {
// ensure that there are no status logger messages which would indicate a problem with our Log4j usage; we map the
// StatusData instances to Strings as otherwise their toString output is useless
assertThat(
statusData.stream().map(status -> status.getMessage().getFormattedMessage()).collect(Collectors.toList()),
anyOf(
emptyCollectionOf(String.class),
contains(startsWith(LOG_4J_MSG_PREFIXES.get(0)), startsWith(LOG_4J_MSG_PREFIXES.get(1))),
contains(startsWith(LOG_4J_MSG_PREFIXES.get(1)))
)
);
} finally {
// we clear the list so that status data from other tests do not interfere with tests within the same JVM
statusData.clear();
}
}
synchronized (loggedLeaks) {
try {
assertThat(loggedLeaks, empty());
} finally {
loggedLeaks.clear();
}
}
}
/**
* Assert that at least one leak was detected, also clear the list of detected leaks
* so the test won't fail for leaks detected up until this point.
*/
protected static void assertLeakDetected() {
synchronized (loggedLeaks) {
assertFalse("No leaks have been detected", loggedLeaks.isEmpty());
loggedLeaks.clear();
}
}
// this must be a separate method from other ensure checks above so suite scoped integ tests can call...TODO: fix that
public final void ensureAllSearchContextsReleased() throws Exception {
assertBusy(() -> MockSearchService.assertNoInFlightContext());
}
// mockdirectorywrappers currently set this boolean if checkindex fails
// TODO: can we do this cleaner???
/** MockFSDirectoryService sets this: */
public static final List checkIndexFailures = new CopyOnWriteArrayList<>();
@Before
public final void resetCheckIndexStatus() throws Exception {
checkIndexFailures.clear();
}
public final void ensureCheckIndexPassed() {
if (checkIndexFailures.isEmpty() == false) {
final AssertionError e = new AssertionError("at least one shard failed CheckIndex");
for (Exception failure : checkIndexFailures) {
e.addSuppressed(failure);
}
throw e;
}
}
// -----------------------------------------------------------------
// Test facilities and facades for subclasses.
// -----------------------------------------------------------------
// TODO: decide on one set of naming for between/scaledBetween and remove others
// TODO: replace frequently() with usually()
/**
* Returns a "scaled" random number between min and max (inclusive).
*
* @see RandomizedTest#scaledRandomIntBetween(int, int)
*/
public static int scaledRandomIntBetween(int min, int max) {
return RandomizedTest.scaledRandomIntBetween(min, max);
}
/**
* A random integer from min to max (inclusive).
*
* @see #scaledRandomIntBetween(int, int)
*/
public static int randomIntBetween(int min, int max) {
return RandomNumbers.randomIntBetween(random(), min, max);
}
/**
* A random long number between min (inclusive) and max (inclusive).
*/
public static long randomLongBetween(long min, long max) {
return RandomNumbers.randomLongBetween(random(), min, max);
}
/**
* @return a random instant between a min and a max value with a random nanosecond precision
*/
public static Instant randomInstantBetween(Instant minInstant, Instant maxInstant) {
return Instant.ofEpochSecond(
randomLongBetween(minInstant.getEpochSecond(), maxInstant.getEpochSecond()),
randomLongBetween(0, 999999999)
);
}
/**
* The maximum value that can be represented as an unsigned long.
*/
public static final BigInteger UNSIGNED_LONG_MAX = BigInteger.ONE.shiftLeft(Long.SIZE).subtract(BigInteger.ONE);
/**
* A unsigned long in a {@link BigInteger} between min (inclusive) and max (inclusive).
*/
public static BigInteger randomUnsignedLongBetween(BigInteger min, BigInteger max) {
if (min.compareTo(BigInteger.ZERO) < 0) {
throw new IllegalArgumentException("Must be between [0] and [" + UNSIGNED_LONG_MAX + "]");
}
if (0 < max.compareTo(UNSIGNED_LONG_MAX)) {
throw new IllegalArgumentException("Must be between [0] and [" + UNSIGNED_LONG_MAX + "]");
}
// Shift the min and max down into the long range
long minShifted = min.add(BigInteger.valueOf(Long.MIN_VALUE)).longValueExact();
long maxShifted = max.add(BigInteger.valueOf(Long.MIN_VALUE)).longValueExact();
// Grab a random number in that range
long randomShifted = randomLongBetween(minShifted, maxShifted);
// Shift back up into long range
return BigInteger.valueOf(randomShifted).subtract(BigInteger.valueOf(Long.MIN_VALUE));
}
/**
* Returns a "scaled" number of iterations for loops which can have a variable
* iteration count. This method is effectively
* an alias to {@link #scaledRandomIntBetween(int, int)}.
*/
public static int iterations(int min, int max) {
return scaledRandomIntBetween(min, max);
}
/**
* An alias for {@link #randomIntBetween(int, int)}.
*
* @see #scaledRandomIntBetween(int, int)
*/
public static int between(int min, int max) {
return randomIntBetween(min, max);
}
/**
* The exact opposite of {@link #rarely()}.
*/
public static boolean frequently() {
return rarely() == false;
}
public static boolean randomBoolean() {
return random().nextBoolean();
}
public static Boolean randomOptionalBoolean() {
return randomBoolean() ? Boolean.TRUE : randomFrom(Boolean.FALSE, null);
}
public static byte randomByte() {
return (byte) random().nextInt();
}
public static byte randomNonNegativeByte() {
byte randomByte = randomByte();
return (byte) (randomByte == Byte.MIN_VALUE ? 0 : Math.abs(randomByte));
}
/**
* Helper method to create a byte array of a given length populated with random byte values
*
* @see #randomByte()
*/
public static byte[] randomByteArrayOfLength(int size) {
byte[] bytes = new byte[size];
for (int i = 0; i < size; i++) {
bytes[i] = randomByte();
}
return bytes;
}
public static byte randomByteBetween(byte minInclusive, byte maxInclusive) {
return (byte) randomIntBetween(minInclusive, maxInclusive);
}
public static void randomBytesBetween(byte[] bytes, byte minInclusive, byte maxInclusive) {
for (int i = 0, len = bytes.length; i < len;) {
bytes[i++] = randomByteBetween(minInclusive, maxInclusive);
}
}
public static BytesReference randomBytesReference(int length) {
final var slices = new ArrayList();
var remaining = length;
while (remaining > 0) {
final var sliceLen = between(1, remaining);
slices.add(new BytesArray(randomByteArrayOfLength(sliceLen)));
remaining -= sliceLen;
}
return CompositeBytesReference.of(slices.toArray(BytesReference[]::new));
}
public static short randomShort() {
return (short) random().nextInt();
}
public static int randomInt() {
return random().nextInt();
}
public static IntStream randomInts() {
return random().ints();
}
public static IntStream randomInts(long streamSize) {
return random().ints(streamSize);
}
/**
* @return a long between 0 and Long.MAX_VALUE (inclusive) chosen uniformly at random.
*/
public static long randomNonNegativeLong() {
return randomLong() & Long.MAX_VALUE;
}
/**
* @return an int between 0 and Integer.MAX_VALUE (inclusive) chosen uniformly at random.
*/
public static int randomNonNegativeInt() {
return randomInt() & Integer.MAX_VALUE;
}
public static float randomFloat() {
return random().nextFloat();
}
/**
* Returns a float value in the interval [start, end) if lowerInclusive is
* set to true, (start, end) otherwise.
*
* @param start lower bound of interval to draw uniformly distributed random numbers from
* @param end upper bound
* @param lowerInclusive whether or not to include lower end of the interval
*/
public static float randomFloatBetween(float start, float end, boolean lowerInclusive) {
float result;
if (start == -Float.MAX_VALUE || end == Float.MAX_VALUE) {
// formula below does not work with very large floats
result = Float.intBitsToFloat(randomInt());
while (result < start || result > end || Double.isNaN(result)) {
result = Float.intBitsToFloat(randomInt());
}
} else {
result = randomFloat();
if (lowerInclusive == false) {
while (result <= 0.0f) {
result = randomFloat();
}
}
result = result * end + (1.0f - result) * start;
}
return result;
}
public static double randomDouble() {
return random().nextDouble();
}
public static DoubleStream randomDoubles() {
return random().doubles();
}
public static DoubleStream randomDoubles(long streamSize) {
return random().doubles(streamSize);
}
/**
* Returns a double value in the interval [start, end) if lowerInclusive is
* set to true, (start, end) otherwise.
*
* @param start lower bound of interval to draw uniformly distributed random numbers from
* @param end upper bound
* @param lowerInclusive whether or not to include lower end of the interval
*/
public static double randomDoubleBetween(double start, double end, boolean lowerInclusive) {
double result = 0.0;
if (start == -Double.MAX_VALUE || end == Double.MAX_VALUE) {
// formula below does not work with very large doubles
result = Double.longBitsToDouble(randomLong());
while (result < start || result > end || Double.isNaN(result)) {
result = Double.longBitsToDouble(randomLong());
}
} else {
result = randomDouble();
if (lowerInclusive == false) {
while (result <= 0.0) {
result = randomDouble();
}
}
result = result * end + (1.0 - result) * start;
}
return result;
}
public static long randomLong() {
return random().nextLong();
}
public static LongStream randomLongs() {
return random().longs();
}
public static LongStream randomLongs(long streamSize) {
return random().longs(streamSize);
}
/**
* Returns a random BigInteger uniformly distributed over the range 0 to (2^64 - 1) inclusive
* Currently BigIntegers are only used for unsigned_long field type, where the max value is 2^64 - 1.
* Modify this random generator if a wider range for BigIntegers is necessary.
* @return a random bigInteger in the range [0 ; 2^64 - 1]
*/
public static BigInteger randomBigInteger() {
return new BigInteger(64, random());
}
/** A random integer from 0..max (inclusive). */
public static int randomInt(int max) {
return RandomizedTest.randomInt(max);
}
/** A random byte size value. */
public static ByteSizeValue randomByteSizeValue() {
return ByteSizeValue.ofBytes(randomLongBetween(0L, Long.MAX_VALUE >> 16));
}
/** Pick a random object from the given array. The array must not be empty. */
@SafeVarargs
@SuppressWarnings("varargs")
public static T randomFrom(T... array) {
return randomFrom(random(), array);
}
/** Pick a random object from the given array. The array must not be empty. */
@SafeVarargs
@SuppressWarnings("varargs")
public static T randomFrom(Random random, T... array) {
return RandomPicks.randomFrom(random, array);
}
/** Pick a random object from the given array of suppliers. The array must not be empty. */
@SafeVarargs
@SuppressWarnings("varargs")
public static T randomFrom(Random random, Supplier... array) {
Supplier supplier = RandomPicks.randomFrom(random, array);
return supplier.get();
}
/** Pick a random object from the given list. */
public static T randomFrom(List list) {
return RandomPicks.randomFrom(random(), list);
}
/** Pick a random object from the given collection. */
public static T randomFrom(Collection collection) {
return randomFrom(random(), collection);
}
/** Pick a random object from the given collection. */
public static T randomFrom(Random random, Collection collection) {
return RandomPicks.randomFrom(random, collection);
}
public static String randomAlphaOfLengthBetween(int minCodeUnits, int maxCodeUnits) {
return RandomizedTest.randomAsciiOfLengthBetween(minCodeUnits, maxCodeUnits);
}
public static String randomAlphaOfLength(int codeUnits) {
return RandomizedTest.randomAsciiOfLength(codeUnits);
}
public static SecureString randomSecureStringOfLength(int codeUnits) {
var randomAlpha = randomAlphaOfLength(codeUnits);
return new SecureString(randomAlpha.toCharArray());
}
public static String randomNullOrAlphaOfLength(int codeUnits) {
return randomBoolean() ? null : randomAlphaOfLength(codeUnits);
}
/**
* Creates a valid random identifier such as node id or index name
*/
public static String randomIdentifier() {
return randomAlphaOfLengthBetween(8, 12).toLowerCase(Locale.ROOT);
}
public static String randomUUID() {
return UUIDs.randomBase64UUID(random());
}
public static String randomUnicodeOfLengthBetween(int minCodeUnits, int maxCodeUnits) {
return RandomizedTest.randomUnicodeOfLengthBetween(minCodeUnits, maxCodeUnits);
}
public static String randomUnicodeOfLength(int codeUnits) {
return RandomizedTest.randomUnicodeOfLength(codeUnits);
}
public static String randomUnicodeOfCodepointLengthBetween(int minCodePoints, int maxCodePoints) {
return RandomizedTest.randomUnicodeOfCodepointLengthBetween(minCodePoints, maxCodePoints);
}
public static String randomUnicodeOfCodepointLength(int codePoints) {
return RandomizedTest.randomUnicodeOfCodepointLength(codePoints);
}
public static String randomRealisticUnicodeOfLengthBetween(int minCodeUnits, int maxCodeUnits) {
return RandomizedTest.randomRealisticUnicodeOfLengthBetween(minCodeUnits, maxCodeUnits);
}
public static String randomRealisticUnicodeOfLength(int codeUnits) {
return RandomizedTest.randomRealisticUnicodeOfLength(codeUnits);
}
public static String randomRealisticUnicodeOfCodepointLengthBetween(int minCodePoints, int maxCodePoints) {
return RandomizedTest.randomRealisticUnicodeOfCodepointLengthBetween(minCodePoints, maxCodePoints);
}
public static String randomRealisticUnicodeOfCodepointLength(int codePoints) {
return RandomizedTest.randomRealisticUnicodeOfCodepointLength(codePoints);
}
/**
* @param maxArraySize The maximum number of elements in the random array
* @param stringSize The length of each String in the array
* @param allowNull Whether the returned array may be null
* @param allowEmpty Whether the returned array may be empty (have zero elements)
*/
public static String[] generateRandomStringArray(int maxArraySize, int stringSize, boolean allowNull, boolean allowEmpty) {
if (allowNull && random().nextBoolean()) {
return null;
}
int arraySize = randomIntBetween(allowEmpty ? 0 : 1, maxArraySize);
String[] array = new String[arraySize];
for (int i = 0; i < arraySize; i++) {
array[i] = RandomStrings.randomAsciiOfLength(random(), stringSize);
}
return array;
}
public static String[] generateRandomStringArray(int maxArraySize, int stringSize, boolean allowNull) {
return generateRandomStringArray(maxArraySize, stringSize, allowNull, true);
}
public static T[] randomArray(int maxArraySize, IntFunction arrayConstructor, Supplier valueConstructor) {
return randomArray(0, maxArraySize, arrayConstructor, valueConstructor);
}
public static T[] randomArray(int minArraySize, int maxArraySize, IntFunction arrayConstructor, Supplier valueConstructor) {
final int size = randomIntBetween(minArraySize, maxArraySize);
final T[] array = arrayConstructor.apply(size);
for (int i = 0; i < array.length; i++) {
array[i] = valueConstructor.get();
}
return array;
}
public static List randomList(int maxListSize, Supplier valueConstructor) {
return randomList(0, maxListSize, valueConstructor);
}
public static List randomList(int minListSize, int maxListSize, Supplier valueConstructor) {
final int size = randomIntBetween(minListSize, maxListSize);
List list = new ArrayList<>();
for (int i = 0; i < size; i++) {
list.add(valueConstructor.get());
}
return list;
}
public static Map randomMap(int minMapSize, int maxMapSize, Supplier> entryConstructor) {
final int size = randomIntBetween(minMapSize, maxMapSize);
Map list = Maps.newMapWithExpectedSize(size);
for (int i = 0; i < size; i++) {
Tuple entry = entryConstructor.get();
list.put(entry.v1(), entry.v2());
}
return list;
}
public static Set randomSet(int minSetSize, int maxSetSize, Supplier valueConstructor) {
return new HashSet<>(randomList(minSetSize, maxSetSize, valueConstructor));
}
public static TimeValue randomTimeValue(int lower, int upper, TimeUnit... units) {
return new TimeValue(between(lower, upper), randomFrom(units));
}
public static TimeValue randomTimeValue(int lower, int upper) {
return randomTimeValue(lower, upper, TimeUnit.values());
}
public static TimeValue randomTimeValue() {
return randomTimeValue(0, 1000);
}
public static TimeValue randomPositiveTimeValue() {
return randomTimeValue(1, 1000);
}
/**
* generate a random epoch millis in a range 1 to 9999-12-31T23:59:59.999
*/
public static long randomMillisUpToYear9999() {
return randomLongBetween(1, DateUtils.MAX_MILLIS_BEFORE_9999);
}
/**
* generate a random TimeZone from the ones available in java.util
*/
public static TimeZone randomTimeZone() {
return TimeZone.getTimeZone(randomFrom(JAVA_TIMEZONE_IDS));
}
/**
* generate a random TimeZone from the ones available in java.time
*/
public static ZoneId randomZone() {
return ZoneId.of(randomFrom(JAVA_ZONE_IDS));
}
/**
* Generate a random valid date formatter pattern.
*/
public static String randomDateFormatterPattern() {
return randomFrom(FormatNames.values()).getName();
}
/**
* Randomly choose between {@link EsExecutors#DIRECT_EXECUTOR_SERVICE} (which does not fork), {@link ThreadPool#generic}, and one of the
* other named threadpool executors.
*/
public static Executor randomExecutor(ThreadPool threadPool, String... otherExecutorNames) {
final var choice = between(0, otherExecutorNames.length + 1);
if (choice < otherExecutorNames.length) {
return threadPool.executor(otherExecutorNames[choice]);
} else if (choice == otherExecutorNames.length) {
return threadPool.generic();
} else {
return EsExecutors.DIRECT_EXECUTOR_SERVICE;
}
}
/**
* helper to randomly perform on consumer with value
*/
public static void maybeSet(Consumer consumer, T value) {
if (randomBoolean()) {
consumer.accept(value);
}
}
/**
* helper to get a random value in a certain range that's different from the input
*/
public static T randomValueOtherThan(T input, Supplier randomSupplier) {
return randomValueOtherThanMany(v -> Objects.equals(input, v), randomSupplier);
}
/**
* helper to get a random value in a certain range that's different from the input
*/
public static T randomValueOtherThanMany(Predicate input, Supplier randomSupplier) {
T randomValue = null;
do {
randomValue = randomSupplier.get();
} while (input.test(randomValue));
return randomValue;
}
/**
* Runs the code block for 10 seconds waiting for no assertion to trip.
*/
public static void assertBusy(CheckedRunnable codeBlock) throws Exception {
assertBusy(codeBlock, 10, TimeUnit.SECONDS);
}
/**
* Runs the code block for the provided interval, waiting for no assertions to trip.
*/
public static void assertBusy(CheckedRunnable codeBlock, long maxWaitTime, TimeUnit unit) throws Exception {
long maxTimeInMillis = TimeUnit.MILLISECONDS.convert(maxWaitTime, unit);
// In case you've forgotten your high-school studies, log10(x) / log10(y) == log y(x)
long iterations = Math.max(Math.round(Math.log10(maxTimeInMillis) / Math.log10(2)), 1);
long timeInMillis = 1;
long sum = 0;
List failures = new ArrayList<>();
for (int i = 0; i < iterations; i++) {
try {
codeBlock.run();
return;
} catch (AssertionError e) {
failures.add(e);
}
sum += timeInMillis;
Thread.sleep(timeInMillis);
timeInMillis *= 2;
}
timeInMillis = maxTimeInMillis - sum;
Thread.sleep(Math.max(timeInMillis, 0));
try {
codeBlock.run();
} catch (AssertionError e) {
for (AssertionError failure : failures) {
e.addSuppressed(failure);
}
throw e;
}
}
/**
* Periodically execute the supplied function until it returns true, or a timeout
* is reached. This version uses a timeout of 10 seconds. If at all possible,
* use {@link ESTestCase#assertBusy(CheckedRunnable)} instead.
*
* @param breakSupplier determines whether to return immediately or continue waiting.
* @return the last value returned by breakSupplier
* @throws InterruptedException if any sleep calls were interrupted.
*/
public static boolean waitUntil(BooleanSupplier breakSupplier) throws InterruptedException {
return waitUntil(breakSupplier, 10, TimeUnit.SECONDS);
}
// After 1s, we stop growing the sleep interval exponentially and just sleep 1s until maxWaitTime
private static final long AWAIT_BUSY_THRESHOLD = 1000L;
/**
* Periodically execute the supplied function until it returns true, or until the
* specified maximum wait time has elapsed. If at all possible, use
* {@link ESTestCase#assertBusy(CheckedRunnable)} instead.
*
* @param breakSupplier determines whether to return immediately or continue waiting.
* @param maxWaitTime the maximum amount of time to wait
* @param unit the unit of tie for maxWaitTime
* @return the last value returned by breakSupplier
* @throws InterruptedException if any sleep calls were interrupted.
*/
public static boolean waitUntil(BooleanSupplier breakSupplier, long maxWaitTime, TimeUnit unit) throws InterruptedException {
long maxTimeInMillis = TimeUnit.MILLISECONDS.convert(maxWaitTime, unit);
long timeInMillis = 1;
long sum = 0;
while (sum + timeInMillis < maxTimeInMillis) {
if (breakSupplier.getAsBoolean()) {
return true;
}
Thread.sleep(timeInMillis);
sum += timeInMillis;
timeInMillis = Math.min(AWAIT_BUSY_THRESHOLD, timeInMillis * 2);
}
timeInMillis = maxTimeInMillis - sum;
Thread.sleep(Math.max(timeInMillis, 0));
return breakSupplier.getAsBoolean();
}
protected TestThreadPool createThreadPool(ExecutorBuilder... executorBuilders) {
return new TestThreadPool(getTestName(), executorBuilders);
}
public static boolean terminate(ExecutorService... services) {
boolean terminated = true;
for (ExecutorService service : services) {
if (service != null) {
terminated &= ThreadPool.terminate(service, 10, TimeUnit.SECONDS);
}
}
return terminated;
}
public static boolean terminate(ThreadPool threadPool) {
return ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
}
/**
* Returns a {@link java.nio.file.Path} pointing to the class path relative resource given
* as the first argument. In contrast to
* getClass().getResource(...).getFile() this method will not
* return URL encoded paths if the parent path contains spaces or other
* non-standard characters.
*/
@Override
public Path getDataPath(String relativePath) {
// we override LTC behavior here: wrap even resources with mockfilesystems,
// because some code is buggy when it comes to multiple nio.2 filesystems
// (e.g. FileSystemUtils, and likely some tests)
try {
return PathUtils.get(getClass().getResource(relativePath).toURI()).toAbsolutePath().normalize();
} catch (Exception e) {
throw new RuntimeException("resource not found: " + relativePath, e);
}
}
/** Returns a random number of temporary paths. */
public String[] tmpPaths() {
final int numPaths = TestUtil.nextInt(random(), 1, 3);
final String[] absPaths = new String[numPaths];
for (int i = 0; i < numPaths; i++) {
absPaths[i] = createTempDir().toAbsolutePath().toString();
}
return absPaths;
}
public NodeEnvironment newNodeEnvironment() throws IOException {
return newNodeEnvironment(Settings.EMPTY);
}
public Settings buildEnvSettings(Settings settings) {
return Settings.builder()
.put(settings)
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath())
.putList(Environment.PATH_DATA_SETTING.getKey(), tmpPaths())
.build();
}
public NodeEnvironment newNodeEnvironment(Settings settings) throws IOException {
Settings build = buildEnvSettings(settings);
return new NodeEnvironment(build, TestEnvironment.newEnvironment(build));
}
public Environment newEnvironment() {
Settings build = buildEnvSettings(Settings.EMPTY);
return TestEnvironment.newEnvironment(build);
}
public Environment newEnvironment(Settings settings) {
Settings build = buildEnvSettings(settings);
return TestEnvironment.newEnvironment(build);
}
/** Return consistent index settings for the provided index version. */
public static Settings.Builder settings(IndexVersion version) {
return Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version);
}
/** Return consistent index settings for the provided index version, shard- and replica-count. */
public static Settings.Builder indexSettings(IndexVersion indexVersionCreated, int shards, int replicas) {
return settings(indexVersionCreated).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shards)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replicas);
}
/** Return consistent index settings for the provided shard- and replica-count. */
public static Settings.Builder indexSettings(int shards, int replicas) {
return Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shards)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replicas);
}
/**
* Returns size random values
*/
@SafeVarargs
@SuppressWarnings("varargs")
public static List randomSubsetOf(int size, T... values) {
List list = arrayAsArrayList(values);
return randomSubsetOf(size, list);
}
/**
* Returns a random subset of values (including a potential empty list, or the full original list)
*/
public static List randomSubsetOf(Collection collection) {
return randomSubsetOf(randomInt(collection.size()), collection);
}
public static List randomNonEmptySubsetOf(Collection collection) {
if (collection.isEmpty()) {
throw new IllegalArgumentException("Can't pick non-empty subset of an empty collection");
}
return randomSubsetOf(randomIntBetween(1, collection.size()), collection);
}
/**
* Returns size random values
*/
public static List randomSubsetOf(int size, Collection collection) {
if (size > collection.size()) {
throw new IllegalArgumentException(
"Can't pick " + size + " random objects from a collection of " + collection.size() + " objects"
);
}
List tempList = new ArrayList<>(collection);
Collections.shuffle(tempList, random());
return tempList.subList(0, size);
}
public static List shuffledList(List list) {
return randomSubsetOf(list.size(), list);
}
/**
* Builds a set of unique items. Usually you'll get the requested count but you might get less than that number if the supplier returns
* lots of repeats. Make sure that the items properly implement equals and hashcode.
*/
public static Set randomUnique(Supplier supplier, int targetCount) {
Set things = new HashSet<>();
int maxTries = targetCount * 10;
for (int t = 0; t < maxTries; t++) {
if (things.size() == targetCount) {
return things;
}
things.add(supplier.get());
}
// Oh well, we didn't get enough unique things. It'll be ok.
return things;
}
public static String randomGeohash(int minPrecision, int maxPrecision) {
return geohashGenerator.ofStringLength(random(), minPrecision, maxPrecision);
}
public static String getTestTransportType() {
return Netty4Plugin.NETTY_TRANSPORT_NAME;
}
public static Class getTestTransportPlugin() {
return Netty4Plugin.class;
}
private static final GeohashGenerator geohashGenerator = new GeohashGenerator();
public String randomCompatibleMediaType(RestApiVersion version) {
XContentType type = randomFrom(XContentType.VND_JSON, XContentType.VND_SMILE, XContentType.VND_CBOR, XContentType.VND_YAML);
return compatibleMediaType(type, version);
}
public String compatibleMediaType(XContentType type, RestApiVersion version) {
return type.toParsedMediaType()
.responseContentTypeHeader(Map.of(MediaType.COMPATIBLE_WITH_PARAMETER_NAME, String.valueOf(version.major)));
}
public XContentType randomVendorType() {
return randomFrom(XContentType.VND_JSON, XContentType.VND_SMILE, XContentType.VND_CBOR, XContentType.VND_YAML);
}
public static class GeohashGenerator extends CodepointSetGenerator {
private static final char[] ASCII_SET = "0123456789bcdefghjkmnpqrstuvwxyz".toCharArray();
public GeohashGenerator() {
super(ASCII_SET);
}
}
/**
* Returns the bytes that represent the XContent output of the provided {@link ToXContent} object, using the provided
* {@link XContentType}. Wraps the output into a new anonymous object according to the value returned
* by the {@link ToXContent#isFragment()} method returns. Shuffles the keys to make sure that parsing never relies on keys ordering.
*/
protected final BytesReference toShuffledXContent(
ToXContent toXContent,
XContentType xContentType,
ToXContent.Params params,
boolean humanReadable,
String... exceptFieldNames
) throws IOException {
BytesReference bytes = XContentHelper.toXContent(toXContent, xContentType, params, humanReadable);
try (XContentParser parser = createParser(xContentType.xContent(), bytes)) {
try (XContentBuilder builder = shuffleXContent(parser, rarely(), exceptFieldNames)) {
return BytesReference.bytes(builder);
}
}
}
/**
* Randomly shuffles the fields inside objects in the {@link XContentBuilder} passed in.
* Recursively goes through inner objects and also shuffles them. Exceptions for this
* recursive shuffling behavior can be made by passing in the names of fields which
* internally should stay untouched.
*/
protected final XContentBuilder shuffleXContent(XContentBuilder builder, String... exceptFieldNames) throws IOException {
try (XContentParser parser = createParser(builder)) {
return shuffleXContent(parser, builder.isPrettyPrint(), exceptFieldNames);
}
}
/**
* Randomly shuffles the fields inside objects parsed using the {@link XContentParser} passed in.
* Recursively goes through inner objects and also shuffles them. Exceptions for this
* recursive shuffling behavior can be made by passing in the names of fields which
* internally should stay untouched.
*/
public static XContentBuilder shuffleXContent(XContentParser parser, boolean prettyPrint, String... exceptFieldNames)
throws IOException {
XContentBuilder xContentBuilder = XContentFactory.contentBuilder(parser.contentType());
if (prettyPrint) {
xContentBuilder.prettyPrint();
}
Token token = parser.currentToken() == null ? parser.nextToken() : parser.currentToken();
if (token == Token.START_ARRAY) {
List