All Downloads are FREE. Search and download functionalities are using the official Maven repository.

io.undertow.server.handlers.StuckThreadDetectionHandler Maven / Gradle / Ivy

There is a newer version: 62
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package io.undertow.server.handlers;

import io.undertow.UndertowLogger;
import io.undertow.server.HandlerWrapper;
import io.undertow.server.HttpHandler;
import io.undertow.server.HttpServerExchange;
import io.undertow.server.handlers.builder.HandlerBuilder;
import io.undertow.util.WorkerUtils;
import org.xnio.XnioExecutor;
import org.xnio.XnioIoThread;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * This valve allows to detect requests that take a long time to process, which might
 * indicate that the thread that is processing it is stuck.
 * Based on code proposed by TomLu in Bugzilla entry #50306
 *
 * @author slaurent
 *
 */
public class StuckThreadDetectionHandler implements HttpHandler {

    public static final int DEFAULT_THRESHOLD = 600;
    /**
     * Keeps count of the number of stuck threads detected
     */
    private final AtomicInteger stuckCount = new AtomicInteger(0);

    /**
     * In seconds. Default 600 (10 minutes).
     */
    private final int threshold;

    /**
     * The only references we keep to actual running Thread objects are in
     * this Map (which is automatically cleaned in invoke()s finally clause).
     * That way, Threads can be GC'ed, eventhough the Valve still thinks they
     * are stuck (caused by a long monitor interval)
     */
    private final ConcurrentHashMap activeThreads =
            new ConcurrentHashMap();
    /**
     *
     */
    private final Queue completedStuckThreadsQueue =
            new ConcurrentLinkedQueue<>();

    private final HttpHandler next;


    private final Runnable stuckThreadTask = new Runnable() {
        @Override
        public void run() {
            long thresholdInMillis = threshold * 1000L;

            // Check monitored threads, being careful that the request might have
            // completed by the time we examine it
            for (MonitoredThread monitoredThread : activeThreads.values()) {
                long activeTime = monitoredThread.getActiveTimeInMillis();

                if (activeTime >= thresholdInMillis && monitoredThread.markAsStuckIfStillRunning()) {
                    int numStuckThreads = stuckCount.incrementAndGet();
                    notifyStuckThreadDetected(monitoredThread, activeTime, numStuckThreads);
                }
            }
            // Check if any threads previously reported as stuck, have finished.
            for (CompletedStuckThread completedStuckThread = completedStuckThreadsQueue.poll();
                 completedStuckThread != null; completedStuckThread = completedStuckThreadsQueue.poll()) {

                int numStuckThreads = stuckCount.decrementAndGet();
                notifyStuckThreadCompleted(completedStuckThread, numStuckThreads);
            }
            synchronized (StuckThreadDetectionHandler.this) {
                if(activeThreads.isEmpty()) {
                    timerKey = null;
                } else {
                    timerKey = WorkerUtils.executeAfter(((XnioIoThread)Thread.currentThread()), stuckThreadTask, 1, TimeUnit.SECONDS);
                }
            }
        }
    };

    private volatile XnioExecutor.Key timerKey;

    public StuckThreadDetectionHandler(HttpHandler next) {
        this(DEFAULT_THRESHOLD, next);
    }

    public StuckThreadDetectionHandler(int threshold, HttpHandler next) {
        this.threshold = threshold;
        this.next = next;
    }

    /**
     * @return The current threshold in seconds
     */
    public int getThreshold() {
        return threshold;
    }



    private void notifyStuckThreadDetected(MonitoredThread monitoredThread,
        long activeTime, int numStuckThreads) {
        Throwable th = new Throwable();
        th.setStackTrace(monitoredThread.getThread().getStackTrace());
        UndertowLogger.REQUEST_LOGGER.stuckThreadDetected
                (monitoredThread.getThread().getName(), monitoredThread.getThread().getId(),
                        activeTime, monitoredThread.getStartTime(), monitoredThread.getRequestUri(), threshold, numStuckThreads, th);
    }

    private void notifyStuckThreadCompleted(CompletedStuckThread thread,
            int numStuckThreads) {
        UndertowLogger.REQUEST_LOGGER.stuckThreadCompleted
                (thread.getName(), thread.getId(), thread.getTotalActiveTime(), numStuckThreads);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void handleRequest(HttpServerExchange exchange) throws Exception {

        // Save the thread/runnable
        // Keeping a reference to the thread object here does not prevent
        // GC'ing, as the reference is removed from the Map in the finally clause

        Long key = Thread.currentThread().getId();
        MonitoredThread monitoredThread = new MonitoredThread(Thread.currentThread(), exchange.getRequestURI() + exchange.getQueryString());
        activeThreads.put(key, monitoredThread);
        if(timerKey == null) {
            synchronized (this) {
                if(timerKey == null) {
                    timerKey = exchange.getIoThread().executeAfter(stuckThreadTask, 1, TimeUnit.SECONDS);
                }
            }
        }

        try {
            next.handleRequest(exchange);
        } finally {
            activeThreads.remove(key);
            if (monitoredThread.markAsDone() == MonitoredThreadState.STUCK) {
                completedStuckThreadsQueue.add(
                        new CompletedStuckThread(monitoredThread.getThread(),
                            monitoredThread.getActiveTimeInMillis()));
            }
        }
    }

    public long[] getStuckThreadIds() {
        List idList = new ArrayList<>();
        for (MonitoredThread monitoredThread : activeThreads.values()) {
            if (monitoredThread.isMarkedAsStuck()) {
                idList.add(Long.valueOf(monitoredThread.getThread().getId()));
            }
        }

        long[] result = new long[idList.size()];
        for (int i = 0; i < result.length; i++) {
            result[i] = idList.get(i).longValue();
        }
        return result;
    }

    private static class MonitoredThread {

        /**
         * Reference to the thread to get a stack trace from background task
         */
        private final Thread thread;
        private final String requestUri;
        private final long start;
        private final AtomicInteger state = new AtomicInteger(
            MonitoredThreadState.RUNNING.ordinal());

        MonitoredThread(Thread thread, String requestUri) {
            this.thread = thread;
            this.requestUri = requestUri;
            this.start = System.currentTimeMillis();
        }

        public Thread getThread() {
            return this.thread;
        }

        public String getRequestUri() {
            return requestUri;
        }

        public long getActiveTimeInMillis() {
            return System.currentTimeMillis() - start;
        }

        public Date getStartTime() {
            return new Date(start);
        }

        public boolean markAsStuckIfStillRunning() {
            return this.state.compareAndSet(MonitoredThreadState.RUNNING.ordinal(),
                MonitoredThreadState.STUCK.ordinal());
        }

        public MonitoredThreadState markAsDone() {
            int val = this.state.getAndSet(MonitoredThreadState.DONE.ordinal());
            return MonitoredThreadState.values()[val];
        }

        boolean isMarkedAsStuck() {
            return this.state.get() == MonitoredThreadState.STUCK.ordinal();
        }
    }

    private static class CompletedStuckThread {

        private final String threadName;
        private final long threadId;
        private final long totalActiveTime;

        CompletedStuckThread(Thread thread, long totalActiveTime) {
            this.threadName = thread.getName();
            this.threadId = thread.getId();
            this.totalActiveTime = totalActiveTime;
        }

        public String getName() {
            return this.threadName;
        }

        public long getId() {
            return this.threadId;
        }

        public long getTotalActiveTime() {
            return this.totalActiveTime;
        }
    }

    private enum MonitoredThreadState {
        RUNNING, STUCK, DONE;
    }

    public static final class Wrapper implements HandlerWrapper {

        private final int threshhold;

        public Wrapper(int threshhold) {
            this.threshhold = threshhold;
        }

        public Wrapper() {
            this.threshhold = DEFAULT_THRESHOLD;
        }

        @Override
        public HttpHandler wrap(HttpHandler handler) {
            return new StuckThreadDetectionHandler(threshhold, handler);
        }
    }

    public static class Builder implements HandlerBuilder {

        @Override
        public String name() {
            return "stuck-thread-detector";
        }

        @Override
        public Map> parameters() {
            return Collections.>singletonMap("threshhold", Integer.class);
        }

        @Override
        public Set requiredParameters() {
            return Collections.emptySet();
        }

        @Override
        public String defaultParameter() {
            return "threshhold";
        }

        @Override
        public HandlerWrapper build(Map config) {
            Integer threshhold = (Integer) config.get("threshhold");
            if(threshhold == null) {
                return new Wrapper();
            } else {
                return new Wrapper(threshhold);
            }
        }
    }
}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy