org.voltcore.utils.LatencyWatchdog Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of voltdb Show documentation
VoltDB server libraries
There is a newer version: 10.1.1
/* This file is part of VoltDB.
 * Copyright (C) 2008-2018 VoltDB Inc.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with VoltDB.  If not, see .
 */

package org.voltcore.utils;

import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;

import org.voltcore.logging.Level;
import org.voltcore.logging.VoltLogger;


public class LatencyWatchdog extends Thread {

    private static final VoltLogger LOG = new VoltLogger("HOST");

    private static ThreadLocal sLatencyVal = new ThreadLocal() {
        @Override
        public AtomicLong initialValue() {
            final AtomicLong retval = new AtomicLong();
            sLatencyMap.put(Thread.currentThread(), retval);
            return retval;
        }
    };

    private static ConcurrentHashMap sLatencyMap = new ConcurrentHashMap();

    private static final long WATCHDOG_THRESHOLD = Long.getLong("WATCHDOG_THRESHOLD", 100);  /* millisecond, same below */

    private static final long WAKEUP_INTERVAL = Long.getLong("WAKEUP_INTERVAL", 25);

    private static final long MIN_LOG_INTERVAL = Long.getLong("MIN_LOG_INTERVAL", 10 * 1000);

    static LatencyWatchdog sWatchdog;

    public static final boolean sEnable = Boolean.getBoolean("ENABLE_LATENCY_WATCHDOG");  /* Compiler will eliminate code surrounded by its scope when turn it off */

    static {
        if (sEnable) {
            sWatchdog = new LatencyWatchdog();
            sWatchdog.start();
        }
    }

    /**
     * Update latency watchdog time stamp for current thread.
     * Keep this method small so inlining and elimination can work their magic
     */
    public static void pet() {
        if (!sEnable)
            return;
        petImpl();
    }

    private static void petImpl() {
        sLatencyVal.get().lazySet(System.currentTimeMillis());
    }

    /**
     * The watchdog thread will be invoked every WAKEUP_INTERVAL time, to check if any thread that be monitored
     * has not updated its time stamp more than WATCHDOG_THRESHOLD millisecond. Same stack trace messages are
     * rate limited by MIN_LOG_INTERVAL.
     */
    @Override
    public void run() {
        Thread.currentThread().setName("Latency Watchdog");
        LOG.info(String.format("Latency Watchdog enabled -- threshold:%d(ms) " +
                               "wakeup_interval:%d(ms) min_log_interval:%d(ms)\n",
                               WATCHDOG_THRESHOLD, WAKEUP_INTERVAL, MIN_LOG_INTERVAL));
        while (true) {
            for (Entry entry : sLatencyMap.entrySet()) {
                Thread t = entry.getKey();
                long timestamp = entry.getValue().get();
                long now = System.currentTimeMillis();
                if ((now - timestamp > WATCHDOG_THRESHOLD) && t.getState() != Thread.State.TERMINATED) {
                    StringBuilder sb = new StringBuilder();
                    String format = t.getName() + " has been delayed for more than " + WATCHDOG_THRESHOLD + " milliseconds\n %s";
                    for (StackTraceElement ste : t.getStackTrace()) {
                        sb.append(ste);
                        sb.append("\n");
                    }
                    RateLimitedLogger.tryLogForMessage(now, MIN_LOG_INTERVAL, TimeUnit.MILLISECONDS, LOG, Level.INFO, format, sb.toString());
                }
            }
            try {
                Thread.sleep(WAKEUP_INTERVAL);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
}