org.apache.cassandra.service.FileSystemOwnershipCheck Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of cassandra-all Show documentation
Show all versions of cassandra-all Show documentation
The Apache Cassandra Project develops a highly scalable second-generation distributed database, bringing together Dynamo's fully distributed design and Bigtable's ColumnFamily-based data model.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service;
import java.io.BufferedReader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.StartupChecksOptions;
import org.apache.cassandra.exceptions.StartupException;
import org.apache.cassandra.io.util.File;
import static org.apache.cassandra.service.StartupChecks.StartupCheckType.check_filesystem_ownership;
/**
* Ownership markers on disk are compatible with the java property file format.
* (https://docs.oracle.com/javase/8/docs/api/java/util/Properties.html#load-java.io.Reader-)
*
* This simple formatting is intended to enable them to be created either
* manually or by automated tooling using minimal standard tools (editor/shell
* builtins/etc).
* The only mandatory property is version, which must be parseable as an int
* and upon which the futher set of required properties will depend.
*
* In version 1, two further property values are required:
* - volume_count
* to be parsed as an int representing the number of mounted volumes where
* a marker file is expected.
* - ownership_token
* must contain a non-empty token string that can be compared to one
* derived from system properties. For version 1, this is simply the cluster name.
*
* For this check to succeed as of version 1 then:
* - There must be a single properties file found in the fs tree for each
* target directory.
* - Every file found must contain the mandatory version property with the
* literal value '1'.
* - The value of the ownership_token property in each file must match the
* cluster name
* - The value of the volume_count property must be an int which must matches
* the number of distinct marker files found when traversing the filesystem.
*
* In overridden implementations, you will need to override {@link #constructTokenFromProperties(Map)}
* and add the related *_PROPERTY values you will want the system to check on startup to confirm ownership.
*/
public class FileSystemOwnershipCheck implements StartupCheck
{
private static final Logger logger = LoggerFactory.getLogger(FileSystemOwnershipCheck.class);
public static final String FILE_SYSTEM_CHECK_OWNERSHIP_TOKEN = "CassandraOwnershipToken";
public static final String DEFAULT_FS_OWNERSHIP_FILENAME = ".cassandra_fs_ownership";
// Ownership file properties
static final String VERSION = "version";
static final String VOLUME_COUNT = "volume_count";
static final String TOKEN = "ownership_token";
// Error strings
static final String ERROR_PREFIX = "FS ownership check failed; ";
static final String MISSING_PROPERTY = "property '%s' required for fs ownership check not supplied";
static final String NO_OWNERSHIP_FILE = "no file found in tree for %s";
static final String MULTIPLE_OWNERSHIP_FILES = "multiple files found in tree for %s";
static final String INCONSISTENT_FILES_FOUND = "inconsistent ownership files found on disk: %s";
static final String INVALID_FILE_COUNT = "number of ownership files found doesn't match expected";
static final String MISMATCHING_TOKEN = "token found on disk does not match supplied";
static final String UNSUPPORTED_VERSION = "unsupported version '%s' in ownership file";
static final String INVALID_PROPERTY_VALUE = "invalid or missing value for property '%s'";
static final String READ_EXCEPTION = "error when checking for fs ownership file";
private final Supplier> dirs;
FileSystemOwnershipCheck()
{
this(() -> Iterables.concat(Arrays.asList(DatabaseDescriptor.getAllDataFileLocations()),
Arrays.asList(DatabaseDescriptor.getCommitLogLocation(),
DatabaseDescriptor.getSavedCachesLocation(),
DatabaseDescriptor.getHintsDirectory().absolutePath())));
}
@VisibleForTesting
FileSystemOwnershipCheck(Supplier> dirs)
{
this.dirs = dirs;
}
@Override
public StartupChecks.StartupCheckType getStartupCheckType()
{
return check_filesystem_ownership;
}
@Override
public void execute(StartupChecksOptions options) throws StartupException
{
if (!isEnabled(options))
{
logger.info("Filesystem ownership check is not enabled.");
return;
}
Map config = options.getConfig(getStartupCheckType());
String expectedToken = constructTokenFromProperties(config);
String tokenFilename = getFsOwnershipFilename(config);
Map foundPerTargetDir = new HashMap<>();
Map foundProperties = new HashMap<>();
// Step 1: Traverse the filesystem from each target dir upward, looking for marker files
for (String dataDir : dirs.get())
{
logger.info("Checking for fs ownership details in file hierarchy for {}", dataDir);
int foundFiles = 0;
Path dir = File.getPath(dataDir).normalize();
do
{
File tokenFile = resolve(dir, tokenFilename);
if (tokenFile.exists())
{
foundFiles++;
if (!foundProperties.containsKey(tokenFile.toPath().toAbsolutePath()))
{
try (BufferedReader reader = Files.newBufferedReader(tokenFile.toPath()))
{
Properties props = new Properties();
props.load(reader);
foundProperties.put(tokenFile.toPath().toAbsolutePath(), props);
}
catch (Exception e)
{
logger.error("Error reading fs ownership file from disk", e);
throw exception(READ_EXCEPTION);
}
}
}
dir = dir.getParent();
} while (dir != null);
foundPerTargetDir.put(dataDir, foundFiles);
}
// If a marker file couldn't be found for every target directory, error.
if (foundPerTargetDir.containsValue(0))
{
throw exception(String.format(NO_OWNERSHIP_FILE, foundPerTargetDir.entrySet()
.stream()
.filter(e -> e.getValue() == 0)
.map(Map.Entry::getKey)
.collect(Collectors.joining("', '", "'", "'"))));
}
// If more than one marker file was found in the tree for any target directory, error
Set multipleTokens = foundPerTargetDir.entrySet()
.stream()
.filter(e -> e.getValue() > 1)
.map(Map.Entry::getKey)
.collect(Collectors.toSet());
if (!multipleTokens.isEmpty())
throw exception(String.format(MULTIPLE_OWNERSHIP_FILES, String.join(",", multipleTokens)));
// Step 2: assert that the content of each file is identical
assert !foundProperties.isEmpty();
Multimap byHash = HashMultimap.create();
foundProperties.forEach((key, value) -> byHash.put(value.hashCode(), key));
if (byHash.keySet().size() > 1)
{
// Group identical files to highlight where the mismatches are, worst case is
// they're all unique, but even then the number of individual files should be low
throw exception(String.format(INCONSISTENT_FILES_FOUND,
byHash.keySet()
.stream()
.map(hash -> byHash.get(hash)
.stream()
.map(Path::toString)
.sorted()
.collect(Collectors.joining("', '", "['", "']")))
.sorted()
.collect(Collectors.joining(", "))));
}
// Step 3: validate the content of the properties from disk
// Currently, only version 1 is supported which requires:
// volume_count that matches the number of unique files we found
// ownership_token that matches the one constructed from system props
Properties fromDisk = foundProperties.entrySet().iterator().next().getValue();
int version = getIntProperty(fromDisk, VERSION);
if (version != 1)
throw exception(String.format(UNSUPPORTED_VERSION, version));
int volumeCount = getIntProperty(fromDisk, VOLUME_COUNT);
if (volumeCount != foundProperties.size())
throw exception(INVALID_FILE_COUNT);
String token = getRequiredProperty(fromDisk, TOKEN);
if (!expectedToken.equals(token))
throw exception(MISMATCHING_TOKEN);
logger.info("Successfully verified fs ownership");
}
/** In version 1, we check and return the ownership token. Extend this for custom ownership hierarchies. */
protected String constructTokenFromProperties(Map config) throws StartupException
{
String cluster = getOwnershipToken(config);
if (null == cluster || cluster.isEmpty())
throw exception(String.format(MISSING_PROPERTY, FILE_SYSTEM_CHECK_OWNERSHIP_TOKEN));
return cluster;
}
private int getIntProperty(Properties props, String key) throws StartupException
{
String val = getRequiredProperty(props, key);
try
{
return Integer.parseInt(val);
}
catch (NumberFormatException e)
{
throw exception(String.format(INVALID_PROPERTY_VALUE, key));
}
}
private String getRequiredProperty(Properties props, String key) throws StartupException
{
String s = props.getProperty(key);
if (null == s || s.isEmpty())
throw exception(String.format(INVALID_PROPERTY_VALUE, key));
return s;
}
private File resolve(Path dir, String filename) throws StartupException
{
try
{
return new File(dir.resolve(filename));
}
catch (Exception e)
{
logger.error("Encountered error resolving path ownership file {} relative to dir {}", filename, dir);
throw exception(READ_EXCEPTION);
}
}
private StartupException exception(String message)
{
return new StartupException(StartupException.ERR_WRONG_DISK_STATE, ERROR_PREFIX + message);
}
public boolean isEnabled(StartupChecksOptions options)
{
boolean enabledFromYaml = options.isEnabled(getStartupCheckType());
return CassandraRelevantProperties.FILE_SYSTEM_CHECK_ENABLE.getBoolean(enabledFromYaml);
}
public String getFsOwnershipFilename(Map config)
{
if (CassandraRelevantProperties.FILE_SYSTEM_CHECK_OWNERSHIP_FILENAME.isPresent())
{
logger.warn(String.format("Cassandra system property flag %s is deprecated and you should " +
"use startup check configuration in cassandra.yaml",
CassandraRelevantProperties.FILE_SYSTEM_CHECK_OWNERSHIP_FILENAME.getKey()));
return CassandraRelevantProperties.FILE_SYSTEM_CHECK_OWNERSHIP_FILENAME.getString();
}
else
{
Object fsOwnershipFilename = config.get("ownership_filename");
return fsOwnershipFilename == null
? CassandraRelevantProperties.FILE_SYSTEM_CHECK_OWNERSHIP_FILENAME.getDefaultValue()
: (String) fsOwnershipFilename;
}
}
public String getOwnershipToken(Map config)
{
if (CassandraRelevantProperties.FILE_SYSTEM_CHECK_OWNERSHIP_TOKEN.isPresent())
{
logger.warn(String.format("Cassandra system property flag %s is deprecated and you should " +
"use startup check configuration in cassandra.yaml",
CassandraRelevantProperties.FILE_SYSTEM_CHECK_OWNERSHIP_TOKEN.getKey()));
return CassandraRelevantProperties.FILE_SYSTEM_CHECK_OWNERSHIP_TOKEN.getString();
}
else
{
Object ownershipToken = config.get("ownership_token");
return ownershipToken == null
? CassandraRelevantProperties.FILE_SYSTEM_CHECK_OWNERSHIP_TOKEN.getDefaultValue()
: (String) ownershipToken;
}
}
}