org.graylog2.periodical.IndexerClusterCheckerThread Maven / Gradle / Ivy
/**
* This file is part of Graylog.
*
* Graylog is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Graylog is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Graylog. If not, see fileDescriptorStats = cluster.getFileDescriptorStats();
for (NodeFileDescriptorStats nodeFileDescriptorStats : fileDescriptorStats) {
final String name = nodeFileDescriptorStats.name();
final String ip = nodeFileDescriptorStats.ip();
final String host = nodeFileDescriptorStats.host();
final long maxFileDescriptors = nodeFileDescriptorStats.fileDescriptorMax().orElse(-1L);
if (maxFileDescriptors != -1L && maxFileDescriptors < MINIMUM_OPEN_FILES_LIMIT) {
// Write notification.
final String ipOrHostName = firstNonNull(host, ip);
final Notification notification = notificationService.buildNow()
.addType(Notification.Type.ES_OPEN_FILES)
.addSeverity(Notification.Severity.URGENT)
.addDetail("hostname", ipOrHostName)
.addDetail("max_file_descriptors", maxFileDescriptors);
if (notificationService.publishIfFirst(notification)) {
LOG.warn("Indexer node <{}> ({}) open file limit is too low: [{}]. Set it to at least {}.",
name,
ipOrHostName,
maxFileDescriptors,
MINIMUM_OPEN_FILES_LIMIT);
}
allHigher = false;
}
}
if (allHigher) {
Notification notification = notificationService.build().addType(Notification.Type.ES_OPEN_FILES);
notificationService.fixed(notification);
}
}
@VisibleForTesting()
void checkDiskUsage() {
final Map> notificationTypePerNodeIdentifier = new HashMap<>();
try {
ClusterAllocationDiskSettings settings = cluster.getClusterAllocationDiskSettings();
if (settings.ThresholdEnabled()) {
final Set diskUsageStats = cluster.getDiskUsageStats();
for (NodeDiskUsageStats nodeDiskUsageStats : diskUsageStats) {
Notification.Type currentNodeNotificationType = null;
WatermarkSettings watermarkSettings = settings.watermarkSettings();
if (watermarkSettings instanceof PercentageWatermarkSettings) {
currentNodeNotificationType = getDiskUsageNotificationTypeByPercentage((PercentageWatermarkSettings) watermarkSettings, nodeDiskUsageStats);
} else if (watermarkSettings instanceof AbsoluteValueWatermarkSettings) {
currentNodeNotificationType = getDiskUsageNotificationTypeByAbsoluteValues((AbsoluteValueWatermarkSettings) watermarkSettings, nodeDiskUsageStats);
}
if (currentNodeNotificationType != null) {
String nodeIdentifier = firstNonNull(nodeDiskUsageStats.host(), nodeDiskUsageStats.ip());
if (notificationTypePerNodeIdentifier.containsKey(currentNodeNotificationType)) {
List nodesIdentifier = notificationTypePerNodeIdentifier.get(currentNodeNotificationType);
nodesIdentifier.add(nodeIdentifier);
} else {
notificationTypePerNodeIdentifier.put(currentNodeNotificationType, Arrays.asList(nodeIdentifier));
}
}
}
if (notificationTypePerNodeIdentifier.isEmpty()) {
fixAllDiskUsageNotifications();
} else {
publishDiskUsageNotifications(notificationTypePerNodeIdentifier);
}
}
} catch (Exception e) {
LOG.error("Error while trying to check Elasticsearch disk usage.Details: " + e.getMessage());
}
}
private Notification.Type getDiskUsageNotificationTypeByPercentage(PercentageWatermarkSettings settings, NodeDiskUsageStats nodeDiskUsageStats) {
if (settings.floodStage() != null && nodeDiskUsageStats.diskUsedPercent() >= settings.floodStage()) {
return Notification.Type.ES_NODE_DISK_WATERMARK_FLOOD_STAGE;
} else if (nodeDiskUsageStats.diskUsedPercent() >= settings.high()) {
return Notification.Type.ES_NODE_DISK_WATERMARK_HIGH;
} else if (nodeDiskUsageStats.diskUsedPercent() >= settings.low()) {
return Notification.Type.ES_NODE_DISK_WATERMARK_LOW;
}
return null;
}
private Notification.Type getDiskUsageNotificationTypeByAbsoluteValues(AbsoluteValueWatermarkSettings settings, NodeDiskUsageStats nodeDiskUsageStats) {
if (settings.floodStage() != null && nodeDiskUsageStats.diskAvailable().getBytes() <= settings.floodStage().getBytes()) {
return Notification.Type.ES_NODE_DISK_WATERMARK_FLOOD_STAGE;
} else if (nodeDiskUsageStats.diskAvailable().getBytes() <= settings.high().getBytes()) {
return Notification.Type.ES_NODE_DISK_WATERMARK_HIGH;
} else if (nodeDiskUsageStats.diskAvailable().getBytes() <= settings.low().getBytes()) {
return Notification.Type.ES_NODE_DISK_WATERMARK_LOW;
}
return null;
}
private void fixAllDiskUsageNotifications() {
notificationService.fixed(Notification.Type.ES_NODE_DISK_WATERMARK_FLOOD_STAGE);
notificationService.fixed(Notification.Type.ES_NODE_DISK_WATERMARK_HIGH);
notificationService.fixed(Notification.Type.ES_NODE_DISK_WATERMARK_LOW);
}
private void publishDiskUsageNotifications(Map> notificationTypePerNodeIdentifier) {
for (Map.Entry> entry : notificationTypePerNodeIdentifier.entrySet()) {
if (!notificationExists(entry.getKey())) {
Notification notification = notificationService.buildNow()
.addType(entry.getKey())
.addSeverity(Notification.Severity.URGENT)
.addDetail("nodes", String.join(", ", entry.getValue()));
notificationService.publishIfFirst(notification);
for (String node: entry.getValue()) {
LOG.warn("Elasticsearch node [{}] triggered [{}] due to low free disk space",
node,
entry.getKey());
}
}
}
}
private boolean notificationExists(Notification.Type type) {
return !notificationService.isFirst(type);
}
@Override
protected Logger getLogger() {
return LOG;
}
@Override
public boolean runsForever() {
return false;
}
@Override
public boolean stopOnGracefulShutdown() {
return true;
}
@Override
public boolean masterOnly() {
return true;
}
@Override
public boolean startOnThisNode() {
return true;
}
@Override
public boolean isDaemon() {
return true;
}
@Override
public int getInitialDelaySeconds() {
return 0;
}
@Override
public int getPeriodSeconds() {
return 30;
}
}