org.apache.sling.discovery.oak.SynchronizedClocksHealthCheck Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.sling.discovery.oak;
import java.lang.management.ManagementFactory;
import java.util.Collection;
import java.util.Set;
import javax.management.MBeanServer;
import javax.management.ObjectName;
import org.apache.sling.discovery.base.connectors.announcement.Announcement;
import org.apache.sling.discovery.base.connectors.announcement.AnnouncementRegistry;
import org.apache.felix.hc.api.HealthCheck;
import org.apache.felix.hc.api.Result;
import org.apache.felix.hc.api.FormattingResultLog;
import org.apache.sling.settings.SlingSettingsService;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Reference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* HealthCheck that builds on-top of DocumentNodeStore's
* determineServerTimeDifferenceMillis method which checks how much the local
* time differs from the DocumentStore's time. It then applies low- and
* high-water marks to that time difference:
*
* - if the value is higher than the high-water mark (5sec by default), then
* it issues a critical
* - if the value is lower than the high-water but higher than the low-water
* mark (1sec by default), then it issues only a warn
* - if the value is lower than the low-water mark, then it issues only an
* info
*
*/
@Component(
immediate = true,
service = HealthCheck.class,
property = {
HealthCheck.NAME + "=Synchronized Clocks",
HealthCheck.MBEAN_NAME + "=slingDiscoveryOakSynchronizedClocks"
})
public class SynchronizedClocksHealthCheck implements HealthCheck {
protected final Logger logger = LoggerFactory.getLogger(getClass());
private static final String DOCUMENT_NODE_STORE_MBEAN = "org.apache.jackrabbit.oak:name=*,type=DocumentNodeStore";
private static final String TIME_DIFF_METHOD_NAME = "determineServerTimeDifferenceMillis";
private static final long INTRA_CLUSTER_HIGH_WATER_MARK = 5000;
private static final long INTRA_CLUSTER_LOW_WATER_MARK = 1000;
private static final long INTER_CLUSTER_HIGH_WATER_MARK = 10000;
private static final long INTER_CLUSTER_LOW_WATER_MARK = 5000;
@Reference
private AnnouncementRegistry announcementRegistry;
@Reference
private SlingSettingsService settingsService;
@Override
public Result execute() {
final FormattingResultLog resultLog = new FormattingResultLog();
resultLog.debug("Checking cluster internal clocks");
try {
final MBeanServer jmxServer = ManagementFactory.getPlatformMBeanServer();
ObjectName n = new ObjectName(DOCUMENT_NODE_STORE_MBEAN);
Set names = jmxServer.queryNames(n, null);
if (names.size() == 0) {
resultLog.info("Intra-cluster test n/a (No DocumentNodeStore MBean found)");
} else {
ObjectName firstName = names.iterator().next();
final Object value = jmxServer.invoke(firstName, TIME_DIFF_METHOD_NAME, new Object[0], new String[0]);
logger.debug("{} returns {}", new Object[] { firstName, TIME_DIFF_METHOD_NAME, value });
resultLog.debug("{} returns {}", firstName, TIME_DIFF_METHOD_NAME, value);
if (value != null && (value instanceof Long)) {
Long diffMillis = (Long) value;
if (Math.abs(diffMillis) >= INTRA_CLUSTER_HIGH_WATER_MARK) {
logger.warn(
"execute: clocks in local cluster out of sync by {}ms "
+ "which is equal or higher than the high-water mark of {}ms.",
diffMillis, INTRA_CLUSTER_HIGH_WATER_MARK);
resultLog.critical(
"Clocks heavily out of sync in local cluster: "
+ "time difference of this VM with DocumentStore server: "
+ "{}ms is equal or larger than high-water mark of {}ms",
diffMillis, INTRA_CLUSTER_HIGH_WATER_MARK);
} else if (Math.abs(diffMillis) >= INTRA_CLUSTER_LOW_WATER_MARK) {
logger.warn(
"execute: clocks in local cluster out of sync by {}ms"
+ "ms which is equal or higher than the low-water mark of {}ms.",
diffMillis, INTRA_CLUSTER_LOW_WATER_MARK);
resultLog.warn(
"Clocks noticeably out of sync in local cluster: "
+ "time difference of this VM with DocumentStore server: "
+ "{}ms is equal or larger than low-water mark of {}ms",
diffMillis, INTRA_CLUSTER_LOW_WATER_MARK);
} else {
logger.debug("execute: clocks in local cluster in sync. diff is {}ms"
+ "ms which is within low-water mark of {}ms.", diffMillis, INTRA_CLUSTER_LOW_WATER_MARK);
resultLog.info("Clocks in sync in local cluster: time difference of this VM with DocumentStore server: "
+ "{}ms is within low-water mark of {}ms", diffMillis, INTRA_CLUSTER_LOW_WATER_MARK);
}
}
}
} catch (final Exception e) {
logger.warn("execute: {}, JMX method {} invocation failed: {}",
new Object[] { DOCUMENT_NODE_STORE_MBEAN, TIME_DIFF_METHOD_NAME, e });
resultLog.healthCheckError("{}, JMX method {} invocation failed: {}", DOCUMENT_NODE_STORE_MBEAN, TIME_DIFF_METHOD_NAME,
e);
}
final String slingId = settingsService == null ? "n/a" : settingsService.getSlingId();
if (announcementRegistry == null) {
logger.warn("execute: no announcementRegistry ({}) set", announcementRegistry);
resultLog.warn("Cannot determine topology clocks since no announcementRegistry ({}) set", announcementRegistry);
} else {
final Collection localAnnouncements = announcementRegistry.listLocalAnnouncements();
if (localAnnouncements.isEmpty()) {
logger.debug("execute: no topology connectors connected to local instance.");
resultLog.info("No topology connectors connected to local instance.");
}
for (Announcement ann : localAnnouncements) {
final String peerSlingId = ann.getOwnerId();
final long originallyCreatedAt = ann.getOriginallyCreatedAt();
final long receivedAt = ann.getReceivedAt();
long diffMillis = Math.abs(originallyCreatedAt - receivedAt);
if (Math.abs(diffMillis) >= INTER_CLUSTER_HIGH_WATER_MARK) {
logger.warn(
"execute: clocks between local instance (slingId: {}) and remote instance (slingId: {}) out of sync by {}ms"
+ "ms which is equal or higher than the high-water mark of {}ms.",
new Object[] { slingId, peerSlingId, diffMillis, INTER_CLUSTER_HIGH_WATER_MARK });
resultLog.critical(
"Clocks heavily out of sync between local instance (slingId: {}) and remote instance (slingId: {}): "
+ "by {}ms which is equal or larger than high-water mark of {}ms",
new Object[] { slingId, peerSlingId, diffMillis, INTER_CLUSTER_HIGH_WATER_MARK });
} else if (Math.abs(diffMillis) >= INTER_CLUSTER_LOW_WATER_MARK) {
logger.warn(
"execute: clocks out of sync between local instance (slingId: {}) and remote instance (slingId: {}) by {}ms "
+ "ms which is equal or higher than the low-water mark of {}ms.",
new Object[] { slingId, peerSlingId, diffMillis, INTER_CLUSTER_HIGH_WATER_MARK });
resultLog.warn(
"Clocks noticeably out of sync between local instance (slingId: {}) and remote instance (slingId: {}): "
+ "by {}ms which is equal or larger than low-water mark of {}ms",
new Object[] { slingId, peerSlingId, diffMillis, INTER_CLUSTER_HIGH_WATER_MARK });
} else {
logger.debug(
"execute: clocks in sync between local instance (slingId: {}) and remote instance (slingId: {}). "
+ "diff is {}ms which is within low-water mark of {}ms.",
new Object[] { slingId, peerSlingId, diffMillis, INTER_CLUSTER_HIGH_WATER_MARK });
resultLog.info(
"Clocks in sync between local instance (slingId: {}) and remote instance (slingId: {}): "
+ "diff is {}ms which is within low-water mark of {}ms",
new Object[] { slingId, peerSlingId, diffMillis, INTER_CLUSTER_HIGH_WATER_MARK });
}
}
}
return new Result(resultLog);
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy