org.elasticsearch.common.settings.KeyStoreWrapper Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of elasticsearch Show documentation
Show all versions of elasticsearch Show documentation
Elasticsearch - Open Source, Distributed, RESTful Search Engine
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
package org.elasticsearch.common.settings;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.index.IndexFormatTooNewException;
import org.apache.lucene.index.IndexFormatTooOldException;
import org.apache.lucene.store.BufferedChecksumIndexInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.SimpleFSDirectory;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.cli.ExitCodes;
import org.elasticsearch.cli.UserException;
import org.elasticsearch.common.Randomness;
import org.elasticsearch.common.hash.MessageDigests;
import javax.crypto.AEADBadTagException;
import javax.crypto.Cipher;
import javax.crypto.CipherInputStream;
import javax.crypto.CipherOutputStream;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.GCMParameterSpec;
import javax.crypto.spec.PBEKeySpec;
import javax.crypto.spec.SecretKeySpec;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.AccessDeniedException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.PosixFileAttributeView;
import java.nio.file.attribute.PosixFilePermissions;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
import java.security.SecureRandom;
import java.util.Arrays;
import java.util.Base64;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
/**
* A disk based container for sensitive settings in Elasticsearch.
*
* Loading a keystore has 2 phases. First, call {@link #load(Path)}. Then call
* {@link #decrypt(char[])} with the keystore password, or an empty char array if
* {@link #hasPassword()} is {@code false}. Loading and decrypting should happen
* in a single thread. Once decrypted, settings may be read in multiple threads.
*/
public class KeyStoreWrapper implements SecureSettings {
/** An identifier for the type of data that may be stored in a keystore entry. */
private enum EntryType {
STRING,
FILE
}
/** An entry in the keystore. The bytes are opaque and interpreted based on the entry type. */
private static class Entry {
final byte[] bytes;
final byte[] sha256Digest;
Entry(byte[] bytes) {
this.bytes = bytes;
this.sha256Digest = MessageDigests.sha256().digest(bytes);
}
}
/**
* A regex for the valid characters that a setting name in the keystore may use.
*/
private static final Pattern ALLOWED_SETTING_NAME = Pattern.compile("[A-Za-z0-9_\\-.]+");
public static final Setting SEED_SETTING = SecureSetting.secureString("keystore.seed", null);
/** Characters that may be used in the bootstrap seed setting added to all keystores. */
private static final char[] SEED_CHARS = ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" +
"~!@#$%^&*-_=+?").toCharArray();
/** The name of the keystore file to read and write. */
private static final String KEYSTORE_FILENAME = "elasticsearch.keystore";
/** The version of the metadata written before the keystore data. */
static final int FORMAT_VERSION = 4;
/** The oldest metadata format version that can be read. */
private static final int MIN_FORMAT_VERSION = 1;
/** The algorithm used to derive the cipher key from a password. */
private static final String KDF_ALGO = "PBKDF2WithHmacSHA512";
/** The number of iterations to derive the cipher key. */
private static final int KDF_ITERS = 10000;
/**
* The number of bits for the cipher key.
*
* Note: The Oracle JDK 8 ships with a limited JCE policy that restricts key length for AES to 128 bits.
* This can be increased to 256 bits once minimum java 9 is the minimum java version.
* See http://www.oracle.com/technetwork/java/javase/terms/readme/jdk9-readme-3852447.html#jce
* */
private static final int CIPHER_KEY_BITS = 128;
/** The number of bits for the GCM tag. */
private static final int GCM_TAG_BITS = 128;
/** The cipher used to encrypt the keystore data. */
private static final String CIPHER_ALGO = "AES";
/** The mode used with the cipher algorithm. */
private static final String CIPHER_MODE = "GCM";
/** The padding used with the cipher algorithm. */
private static final String CIPHER_PADDING = "NoPadding";
// format version changelog:
// 1: initial version, ES 5.3
// 2: file setting, ES 5.4
// 3: FIPS compliant algos, ES 6.3
// 4: remove distinction between string/files, ES 6.8/7.1
/** The metadata format version used to read the current keystore wrapper. */
private final int formatVersion;
/** True iff the keystore has a password needed to read. */
private final boolean hasPassword;
/** The raw bytes of the encrypted keystore. */
private final byte[] dataBytes;
/** The decrypted secret data. See {@link #decrypt(char[])}. */
private final SetOnce> entries = new SetOnce<>();
private volatile boolean closed;
private KeyStoreWrapper(int formatVersion, boolean hasPassword, byte[] dataBytes) {
this.formatVersion = formatVersion;
this.hasPassword = hasPassword;
this.dataBytes = dataBytes;
}
/**
* Get the metadata format version for the keystore
**/
public int getFormatVersion() {
return formatVersion;
}
/** Returns a path representing the ES keystore in the given config dir. */
public static Path keystorePath(Path configDir) {
return configDir.resolve(KEYSTORE_FILENAME);
}
/** Constructs a new keystore with the given password. */
public static KeyStoreWrapper create() {
KeyStoreWrapper wrapper = new KeyStoreWrapper(FORMAT_VERSION, false, null);
wrapper.entries.set(new HashMap<>());
addBootstrapSeed(wrapper);
return wrapper;
}
/** Add the bootstrap seed setting, which may be used as a unique, secure, random value by the node */
public static void addBootstrapSeed(KeyStoreWrapper wrapper) {
assert wrapper.getSettingNames().contains(SEED_SETTING.getKey()) == false;
SecureRandom random = Randomness.createSecure();
int passwordLength = 20; // Generate 20 character passwords
char[] characters = new char[passwordLength];
for (int i = 0; i < passwordLength; ++i) {
characters[i] = SEED_CHARS[random.nextInt(SEED_CHARS.length)];
}
wrapper.setString(SEED_SETTING.getKey(), characters);
Arrays.fill(characters, (char)0);
}
/**
* Loads information about the Elasticsearch keystore from the provided config directory.
*
* {@link #decrypt(char[])} must be called before reading or writing any entries.
* Returns {@code null} if no keystore exists.
*/
public static KeyStoreWrapper load(Path configDir) throws IOException {
Path keystoreFile = keystorePath(configDir);
if (Files.exists(keystoreFile) == false) {
return null;
}
SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
try (IndexInput indexInput = directory.openInput(KEYSTORE_FILENAME, IOContext.READONCE)) {
ChecksumIndexInput input = new BufferedChecksumIndexInput(indexInput);
final int formatVersion;
try {
formatVersion = CodecUtil.checkHeader(input, KEYSTORE_FILENAME, MIN_FORMAT_VERSION, FORMAT_VERSION);
} catch (IndexFormatTooOldException e) {
throw new IllegalStateException("The Elasticsearch keystore [" + keystoreFile + "] format is too old. " +
"You should delete and recreate it in order to upgrade.", e);
} catch (IndexFormatTooNewException e) {
throw new IllegalStateException("The Elasticsearch keystore [" + keystoreFile + "] format is too new. " +
"Are you trying to downgrade? You should delete and recreate it in order to downgrade.", e);
}
byte hasPasswordByte = input.readByte();
boolean hasPassword = hasPasswordByte == 1;
if (hasPassword == false && hasPasswordByte != 0) {
throw new IllegalStateException("hasPassword boolean is corrupt: "
+ String.format(Locale.ROOT, "%02x", hasPasswordByte));
}
if (formatVersion <= 2) {
String type = input.readString();
if (type.equals("PKCS12") == false) {
throw new IllegalStateException("Corrupted legacy keystore string encryption algorithm");
}
final String stringKeyAlgo = input.readString();
if (stringKeyAlgo.equals("PBE") == false) {
throw new IllegalStateException("Corrupted legacy keystore string encryption algorithm");
}
if (formatVersion == 2) {
final String fileKeyAlgo = input.readString();
if (fileKeyAlgo.equals("PBE") == false) {
throw new IllegalStateException("Corrupted legacy keystore file encryption algorithm");
}
}
}
final byte[] dataBytes;
if (formatVersion == 2) {
// For v2 we had a map of strings containing the types for each setting. In v3 this map is now
// part of the encrypted bytes. Unfortunately we cannot seek backwards with checksum input, so
// we cannot just read the map and find out how long it is. So instead we read the map and
// store it back using java's builtin DataOutput in a byte array, along with the actual keystore bytes
Map settingTypes = input.readMapOfStrings();
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
try (DataOutputStream output = new DataOutputStream(bytes)) {
output.writeInt(settingTypes.size());
for (Map.Entry entry : settingTypes.entrySet()) {
output.writeUTF(entry.getKey());
output.writeUTF(entry.getValue());
}
int keystoreLen = input.readInt();
byte[] keystoreBytes = new byte[keystoreLen];
input.readBytes(keystoreBytes, 0, keystoreLen);
output.write(keystoreBytes);
}
dataBytes = bytes.toByteArray();
} else {
int dataBytesLen = input.readInt();
dataBytes = new byte[dataBytesLen];
input.readBytes(dataBytes, 0, dataBytesLen);
}
CodecUtil.checkFooter(input);
return new KeyStoreWrapper(formatVersion, hasPassword, dataBytes);
}
}
/** Upgrades the format of the keystore, if necessary. */
public static void upgrade(KeyStoreWrapper wrapper, Path configDir, char[] password) throws Exception {
if (wrapper.getFormatVersion() == FORMAT_VERSION && wrapper.getSettingNames().contains(SEED_SETTING.getKey())) {
return;
}
// add keystore.seed if necessary
if (wrapper.getSettingNames().contains(SEED_SETTING.getKey()) == false) {
addBootstrapSeed(wrapper);
}
wrapper.save(configDir, password);
}
@Override
public boolean isLoaded() {
return entries.get() != null;
}
/** Return true iff calling {@link #decrypt(char[])} requires a non-empty password. */
public boolean hasPassword() {
return hasPassword;
}
private Cipher createCipher(int opmode, char[] password, byte[] salt, byte[] iv) throws GeneralSecurityException {
PBEKeySpec keySpec = new PBEKeySpec(password, salt, KDF_ITERS, CIPHER_KEY_BITS);
SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(KDF_ALGO);
SecretKey secretKey;
try {
secretKey = keyFactory.generateSecret(keySpec);
} catch (Error e) {
// Security Providers might throw a subclass of Error in FIPS 140 mode, if some prerequisite like
// salt, iv, or password length is not met. We catch this because we don't want the JVM to exit.
throw new GeneralSecurityException("Error generating an encryption key from the provided password", e);
}
SecretKeySpec secret = new SecretKeySpec(secretKey.getEncoded(), CIPHER_ALGO);
GCMParameterSpec spec = new GCMParameterSpec(GCM_TAG_BITS, iv);
Cipher cipher = Cipher.getInstance(CIPHER_ALGO + "/" + CIPHER_MODE + "/" + CIPHER_PADDING);
cipher.init(opmode, secret, spec);
cipher.updateAAD(salt);
return cipher;
}
/**
* Decrypts the underlying keystore data.
*
* This may only be called once.
*/
public void decrypt(char[] password) throws GeneralSecurityException, IOException {
if (entries.get() != null) {
throw new IllegalStateException("Keystore has already been decrypted");
}
if (formatVersion <= 2) {
decryptLegacyEntries();
if (password.length != 0) {
throw new IllegalArgumentException("Keystore format does not accept non-empty passwords");
}
return;
}
final byte[] salt;
final byte[] iv;
final byte[] encryptedBytes;
try (ByteArrayInputStream bytesStream = new ByteArrayInputStream(dataBytes);
DataInputStream input = new DataInputStream(bytesStream)) {
int saltLen = input.readInt();
salt = new byte[saltLen];
input.readFully(salt);
int ivLen = input.readInt();
iv = new byte[ivLen];
input.readFully(iv);
int encryptedLen = input.readInt();
encryptedBytes = new byte[encryptedLen];
input.readFully(encryptedBytes);
if (input.read() != -1) {
throw new SecurityException("Keystore has been corrupted or tampered with");
}
} catch (EOFException e) {
throw new SecurityException("Keystore has been corrupted or tampered with", e);
}
Cipher cipher = createCipher(Cipher.DECRYPT_MODE, password, salt, iv);
try (ByteArrayInputStream bytesStream = new ByteArrayInputStream(encryptedBytes);
CipherInputStream cipherStream = new CipherInputStream(bytesStream, cipher);
DataInputStream input = new DataInputStream(cipherStream)) {
entries.set(new HashMap<>());
int numEntries = input.readInt();
while (numEntries-- > 0) {
String setting = input.readUTF();
if (formatVersion == 3) {
// legacy, the keystore format would previously store the entry type
input.readUTF();
}
int entrySize = input.readInt();
byte[] entryBytes = new byte[entrySize];
input.readFully(entryBytes);
entries.get().put(setting, new Entry(entryBytes));
}
if (input.read() != -1) {
throw new SecurityException("Keystore has been corrupted or tampered with");
}
} catch (IOException e) {
if (e.getCause() instanceof AEADBadTagException) {
throw new SecurityException("Provided keystore password was incorrect", e);
}
throw new SecurityException("Keystore has been corrupted or tampered with", e);
}
}
/** Encrypt the keystore entries and return the encrypted data. */
private byte[] encrypt(char[] password, byte[] salt, byte[] iv) throws GeneralSecurityException, IOException {
assert isLoaded();
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
Cipher cipher = createCipher(Cipher.ENCRYPT_MODE, password, salt, iv);
try (CipherOutputStream cipherStream = new CipherOutputStream(bytes, cipher);
DataOutputStream output = new DataOutputStream(cipherStream)) {
output.writeInt(entries.get().size());
for (Map.Entry mapEntry : entries.get().entrySet()) {
output.writeUTF(mapEntry.getKey());
byte[] entryBytes = mapEntry.getValue().bytes;
output.writeInt(entryBytes.length);
output.write(entryBytes);
}
}
return bytes.toByteArray();
}
private void decryptLegacyEntries() throws GeneralSecurityException, IOException {
// v1 and v2 keystores never had passwords actually used, so we always use an empty password
KeyStore keystore = KeyStore.getInstance("PKCS12");
Map settingTypes = new HashMap<>();
ByteArrayInputStream inputBytes = new ByteArrayInputStream(dataBytes);
try (DataInputStream input = new DataInputStream(inputBytes)) {
// first read the setting types map
if (formatVersion == 2) {
int numSettings = input.readInt();
for (int i = 0; i < numSettings; ++i) {
String key = input.readUTF();
String value = input.readUTF();
settingTypes.put(key, EntryType.valueOf(value));
}
}
// then read the actual keystore
keystore.load(input, "".toCharArray());
}
// verify the settings metadata matches the keystore entries
Enumeration aliases = keystore.aliases();
if (formatVersion == 1) {
while (aliases.hasMoreElements()) {
settingTypes.put(aliases.nextElement(), EntryType.STRING);
}
} else {
// verify integrity: keys in keystore match what the metadata thinks exist
Set expectedSettings = new HashSet<>(settingTypes.keySet());
while (aliases.hasMoreElements()) {
String settingName = aliases.nextElement();
if (expectedSettings.remove(settingName) == false) {
throw new SecurityException("Keystore has been corrupted or tampered with");
}
}
if (expectedSettings.isEmpty() == false) {
throw new SecurityException("Keystore has been corrupted or tampered with");
}
}
// fill in the entries now that we know all the types to expect
this.entries.set(new HashMap<>());
SecretKeyFactory keyFactory = SecretKeyFactory.getInstance("PBE");
KeyStore.PasswordProtection password = new KeyStore.PasswordProtection("".toCharArray());
for (Map.Entry settingEntry : settingTypes.entrySet()) {
String setting = settingEntry.getKey();
EntryType settingType = settingEntry.getValue();
KeyStore.SecretKeyEntry keystoreEntry = (KeyStore.SecretKeyEntry) keystore.getEntry(setting, password);
PBEKeySpec keySpec = (PBEKeySpec) keyFactory.getKeySpec(keystoreEntry.getSecretKey(), PBEKeySpec.class);
char[] chars = keySpec.getPassword();
keySpec.clearPassword();
final byte[] bytes;
if (settingType == EntryType.STRING) {
ByteBuffer byteBuffer = StandardCharsets.UTF_8.encode(CharBuffer.wrap(chars));
bytes = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.position(), byteBuffer.limit());
Arrays.fill(byteBuffer.array(), (byte)0);
} else {
assert settingType == EntryType.FILE;
// The PBE keyspec gives us chars, we convert to bytes
byte[] tmpBytes = new byte[chars.length];
for (int i = 0; i < tmpBytes.length; ++i) {
tmpBytes[i] = (byte)chars[i]; // PBE only stores the lower 8 bits, so this narrowing is ok
}
bytes = Base64.getDecoder().decode(tmpBytes);
Arrays.fill(tmpBytes, (byte)0);
}
Arrays.fill(chars, '\0');
entries.get().put(setting, new Entry(bytes));
}
}
/** Write the keystore to the given config directory. */
public synchronized void save(Path configDir, char[] password) throws Exception {
ensureOpen();
SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
// write to tmp file first, then overwrite
String tmpFile = KEYSTORE_FILENAME + ".tmp";
try (IndexOutput output = directory.createOutput(tmpFile, IOContext.DEFAULT)) {
CodecUtil.writeHeader(output, KEYSTORE_FILENAME, FORMAT_VERSION);
output.writeByte(password.length == 0 ? (byte)0 : (byte)1);
// new cipher params
SecureRandom random = Randomness.createSecure();
// use 64 bytes salt, which surpasses that recommended by OWASP
// see https://www.owasp.org/index.php/Password_Storage_Cheat_Sheet
byte[] salt = new byte[64];
random.nextBytes(salt);
// use 96 bits (12 bytes) for IV as recommended by NIST
// see http://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38d.pdf section 5.2.1.1
byte[] iv = new byte[12];
random.nextBytes(iv);
// encrypted data
byte[] encryptedBytes = encrypt(password, salt, iv);
// size of data block
output.writeInt(4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length);
output.writeInt(salt.length);
output.writeBytes(salt, salt.length);
output.writeInt(iv.length);
output.writeBytes(iv, iv.length);
output.writeInt(encryptedBytes.length);
output.writeBytes(encryptedBytes, encryptedBytes.length);
CodecUtil.writeFooter(output);
} catch (final AccessDeniedException e) {
final String message = String.format(
Locale.ROOT,
"unable to create temporary keystore at [%s], write permissions required for [%s] or run [elasticsearch-keystore upgrade]",
configDir.resolve(tmpFile),
configDir);
throw new UserException(ExitCodes.CONFIG, message, e);
}
Path keystoreFile = keystorePath(configDir);
Files.move(configDir.resolve(tmpFile), keystoreFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
PosixFileAttributeView attrs = Files.getFileAttributeView(keystoreFile, PosixFileAttributeView.class);
if (attrs != null) {
// don't rely on umask: ensure the keystore has minimal permissions
attrs.setPermissions(PosixFilePermissions.fromString("rw-rw----"));
}
}
/**
* It is possible to retrieve the setting names even if the keystore is closed.
* This allows {@link SecureSetting} to correctly determine that a entry exists even though it cannot be read. Thus attempting to
* read a secure setting after the keystore is closed will generate a "keystore is closed" exception rather than using the fallback
* setting.
*/
@Override
public Set getSettingNames() {
assert entries.get() != null : "Keystore is not loaded";
return entries.get().keySet();
}
// TODO: make settings accessible only to code that registered the setting
@Override
public synchronized SecureString getString(String setting) {
ensureOpen();
Entry entry = entries.get().get(setting);
ByteBuffer byteBuffer = ByteBuffer.wrap(entry.bytes);
CharBuffer charBuffer = StandardCharsets.UTF_8.decode(byteBuffer);
return new SecureString(Arrays.copyOfRange(charBuffer.array(), charBuffer.position(), charBuffer.limit()));
}
@Override
public synchronized InputStream getFile(String setting) {
ensureOpen();
Entry entry = entries.get().get(setting);
return new ByteArrayInputStream(entry.bytes);
}
/**
* Returns the SHA256 digest for the setting's value, even after {@code #close()} has been called. The setting must exist. The digest is
* used to check for value changes without actually storing the value.
*/
@Override
public byte[] getSHA256Digest(String setting) {
assert entries.get() != null : "Keystore is not loaded";
Entry entry = entries.get().get(setting);
return entry.sha256Digest;
}
/**
* Ensure the given setting name is allowed.
*
* @throws IllegalArgumentException if the setting name is not valid
*/
public static void validateSettingName(String setting) {
if (ALLOWED_SETTING_NAME.matcher(setting).matches() == false) {
throw new IllegalArgumentException("Setting name [" + setting + "] does not match the allowed setting name pattern ["
+ ALLOWED_SETTING_NAME.pattern() + "]");
}
}
/**
* Set a string setting.
*/
synchronized void setString(String setting, char[] value) {
ensureOpen();
validateSettingName(setting);
ByteBuffer byteBuffer = StandardCharsets.UTF_8.encode(CharBuffer.wrap(value));
byte[] bytes = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.position(), byteBuffer.limit());
Entry oldEntry = entries.get().put(setting, new Entry(bytes));
if (oldEntry != null) {
Arrays.fill(oldEntry.bytes, (byte)0);
}
}
/**
* Set a file setting.
*/
synchronized void setFile(String setting, byte[] bytes) {
ensureOpen();
validateSettingName(setting);
Entry oldEntry = entries.get().put(setting, new Entry(Arrays.copyOf(bytes, bytes.length)));
if (oldEntry != null) {
Arrays.fill(oldEntry.bytes, (byte)0);
}
}
/**
* Remove the given setting from the keystore.
*/
void remove(String setting) {
ensureOpen();
Entry oldEntry = entries.get().remove(setting);
if (oldEntry != null) {
Arrays.fill(oldEntry.bytes, (byte)0);
}
}
private void ensureOpen() {
if (closed) {
throw new IllegalStateException("Keystore is closed");
}
assert isLoaded() : "Keystore is not loaded";
}
@Override
public synchronized void close() {
this.closed = true;
if (null != entries.get() && entries.get().isEmpty() == false) {
for (Entry entry : entries.get().values()) {
Arrays.fill(entry.bytes, (byte) 0);
}
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy