org.apache.cassandra.service.DataResurrectionCheck Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of cassandra-all Show documentation
Show all versions of cassandra-all Show documentation
The Apache Cassandra Project develops a highly scalable second-generation distributed database, bringing together Dynamo's fully distributed design and Bigtable's ColumnFamily-based data model.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service;
import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.StartupChecksOptions;
import org.apache.cassandra.exceptions.StartupException;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.SchemaKeyspace;
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.JsonUtils;
import org.apache.cassandra.utils.Pair;
import static java.lang.String.format;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.stream.Collectors.joining;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
import static org.apache.cassandra.exceptions.StartupException.ERR_WRONG_DISK_STATE;
import static org.apache.cassandra.exceptions.StartupException.ERR_WRONG_MACHINE_STATE;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
public class DataResurrectionCheck implements StartupCheck
{
private static final Logger LOGGER = LoggerFactory.getLogger(DataResurrectionCheck.class);
public static final String HEARTBEAT_FILE_CONFIG_PROPERTY = "heartbeat_file";
public static final String EXCLUDED_KEYSPACES_CONFIG_PROPERTY = "excluded_keyspaces";
public static final String EXCLUDED_TABLES_CONFIG_PROPERTY = "excluded_tables";
public static final String DEFAULT_HEARTBEAT_FILE = "cassandra-heartbeat";
@JsonIgnoreProperties(ignoreUnknown = true)
public static class Heartbeat
{
@JsonProperty("last_heartbeat")
public final Instant lastHeartbeat;
/** needed for jackson serialization */
@SuppressWarnings("unused")
private Heartbeat() {
this.lastHeartbeat = null;
}
public Heartbeat(Instant lastHeartbeat)
{
this.lastHeartbeat = lastHeartbeat;
}
public void serializeToJsonFile(File outputFile) throws IOException
{
JsonUtils.serializeToJsonFile(this, outputFile);
}
public static Heartbeat deserializeFromJsonFile(File file) throws IOException
{
return JsonUtils.deserializeFromJsonFile(Heartbeat.class, file);
}
@Override
public boolean equals(Object o)
{
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Heartbeat manifest = (Heartbeat) o;
return Objects.equals(lastHeartbeat, manifest.lastHeartbeat);
}
@Override
public int hashCode()
{
return Objects.hash(lastHeartbeat);
}
}
@VisibleForTesting
static class TableGCPeriod
{
String table;
int gcPeriod;
TableGCPeriod(String table, int gcPeriod)
{
this.table = table;
this.gcPeriod = gcPeriod;
}
}
static File getHeartbeatFile(Map config)
{
String heartbeatFileConfigValue = (String) config.get(HEARTBEAT_FILE_CONFIG_PROPERTY);
File heartbeatFile;
if (heartbeatFileConfigValue != null)
{
heartbeatFile = new File(heartbeatFileConfigValue);
}
else
{
String[] dataFileLocations = DatabaseDescriptor.getLocalSystemKeyspacesDataFileLocations();
assert dataFileLocations.length != 0;
heartbeatFile = new File(dataFileLocations[0], DEFAULT_HEARTBEAT_FILE);
}
LOGGER.trace("Resolved heartbeat file for data resurrection check: " + heartbeatFile);
return heartbeatFile;
}
@Override
public StartupChecks.StartupCheckType getStartupCheckType()
{
return StartupChecks.StartupCheckType.check_data_resurrection;
}
@Override
public void execute(StartupChecksOptions options) throws StartupException
{
if (options.isDisabled(getStartupCheckType()))
return;
Map config = options.getConfig(StartupChecks.StartupCheckType.check_data_resurrection);
File heartbeatFile = getHeartbeatFile(config);
if (!heartbeatFile.exists())
{
LOGGER.debug("Heartbeat file {} not found! Skipping heartbeat startup check.", heartbeatFile.absolutePath());
return;
}
Heartbeat heartbeat;
try
{
heartbeat = Heartbeat.deserializeFromJsonFile(heartbeatFile);
}
catch (IOException ex)
{
throw new StartupException(ERR_WRONG_DISK_STATE, "Failed to deserialize heartbeat file " + heartbeatFile);
}
if (heartbeat.lastHeartbeat == null)
return;
long heartbeatMillis = heartbeat.lastHeartbeat.toEpochMilli();
List> violations = new ArrayList<>();
Set excludedKeyspaces = getExcludedKeyspaces(config);
Set> excludedTables = getExcludedTables(config);
long currentTimeMillis = currentTimeMillis();
for (String keyspace : getKeyspaces())
{
if (excludedKeyspaces.contains(keyspace))
continue;
for (TableGCPeriod userTable : getTablesGcPeriods(keyspace))
{
if (excludedTables.contains(Pair.create(keyspace, userTable.table)))
continue;
long gcGraceMillis = ((long) userTable.gcPeriod) * 1000;
if (heartbeatMillis + gcGraceMillis < currentTimeMillis)
violations.add(Pair.create(keyspace, userTable.table));
}
}
if (!violations.isEmpty())
{
String invalidTables = violations.stream()
.map(p -> format("%s.%s", p.left, p.right))
.collect(joining(","));
String exceptionMessage = format("There are tables for which gc_grace_seconds is older " +
"than the lastly known time Cassandra node was up based " +
"on its heartbeat %s with timestamp %s. Cassandra node will not start " +
"as it would likely introduce data consistency " +
"issues (zombies etc). Please resolve these issues manually, " +
"then remove the heartbeat and start the node again. Invalid tables: %s",
heartbeatFile, heartbeat.lastHeartbeat, invalidTables);
throw new StartupException(ERR_WRONG_MACHINE_STATE, exceptionMessage);
}
}
@Override
public void postAction(StartupChecksOptions options)
{
// Schedule heartbeating after all checks have passed, not as part of the check,
// as it might happen that other checks after it might fail, but we would be heartbeating already.
if (options.isEnabled(StartupChecks.StartupCheckType.check_data_resurrection))
{
Map config = options.getConfig(StartupChecks.StartupCheckType.check_data_resurrection);
File heartbeatFile = DataResurrectionCheck.getHeartbeatFile(config);
ScheduledExecutors.scheduledTasks.scheduleAtFixedRate(() ->
{
Heartbeat heartbeat = new Heartbeat(Instant.ofEpochMilli(Clock.Global.currentTimeMillis()));
try
{
heartbeatFile.parent().createDirectoriesIfNotExists();
DataResurrectionCheck.LOGGER.trace("writing heartbeat to file " + heartbeatFile);
heartbeat.serializeToJsonFile(heartbeatFile);
}
catch (IOException ex)
{
DataResurrectionCheck.LOGGER.error("Unable to serialize heartbeat to " + heartbeatFile, ex);
}
}, 0, CassandraRelevantProperties.CHECK_DATA_RESURRECTION_HEARTBEAT_PERIOD.getInt(), MILLISECONDS);
}
}
@VisibleForTesting
public Set getExcludedKeyspaces(Map config)
{
String excludedKeyspacesConfigValue = (String) config.get(EXCLUDED_KEYSPACES_CONFIG_PROPERTY);
if (excludedKeyspacesConfigValue == null)
return Collections.emptySet();
else
return Arrays.stream(excludedKeyspacesConfigValue.trim().split(","))
.map(String::trim)
.collect(toSet());
}
@VisibleForTesting
public Set> getExcludedTables(Map config)
{
String excludedKeyspacesConfigValue = (String) config.get(EXCLUDED_TABLES_CONFIG_PROPERTY);
if (excludedKeyspacesConfigValue == null)
return Collections.emptySet();
Set> pairs = new HashSet<>();
for (String keyspaceTable : excludedKeyspacesConfigValue.trim().split(","))
{
String[] pair = keyspaceTable.trim().split("\\.");
if (pair.length != 2)
continue;
pairs.add(Pair.create(pair[0].trim(), pair[1].trim()));
}
return pairs;
}
@VisibleForTesting
List getKeyspaces()
{
return SchemaKeyspace.fetchNonSystemKeyspaces()
.stream()
.map(keyspaceMetadata -> keyspaceMetadata.name)
.collect(toList());
}
@VisibleForTesting
List getTablesGcPeriods(String userKeyspace)
{
Optional keyspaceMetadata = SchemaKeyspace.fetchNonSystemKeyspaces().get(userKeyspace);
if (!keyspaceMetadata.isPresent())
return Collections.emptyList();
KeyspaceMetadata ksmd = keyspaceMetadata.get();
return ksmd.tables.stream()
.filter(tmd -> tmd.params.gcGraceSeconds > 0)
.map(tmd -> new TableGCPeriod(tmd.name, tmd.params.gcGraceSeconds)).collect(toList());
}
}