com.yahoo.vespa.hosted.provision.restapi.LocksResponse Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of node-repository Show documentation
Keeps track of node assignment in a multi-application setup.
There is a newer version: 8.458.13
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.restapi;

import com.yahoo.container.jdisc.HttpResponse;
import com.yahoo.net.HostName;
import com.yahoo.slime.Cursor;
import com.yahoo.slime.JsonFormat;
import com.yahoo.slime.Slime;
import com.yahoo.vespa.curator.stats.LatencyMetrics;
import com.yahoo.vespa.curator.stats.LockMetrics;
import com.yahoo.vespa.curator.stats.LockStats;
import com.yahoo.vespa.curator.stats.LockAttempt;
import com.yahoo.vespa.curator.stats.RecordedLockAttempts;
import com.yahoo.vespa.curator.stats.ThreadLockStats;

import java.io.IOException;
import java.io.OutputStream;
import java.time.Duration;
import java.time.Instant;
import java.util.Comparator;
import java.util.List;
import java.util.Optional;
import java.util.TreeMap;
import java.util.stream.Collectors;

/**
 * Returns information related to ZooKeeper locks.
 *
 * @author hakon
 */
public class LocksResponse extends HttpResponse {

    private final Slime slime = new Slime();

    public LocksResponse() {
        this(HostName.getLocalhost(),
             new TreeMap<>(LockStats.getGlobal().getLockMetricsByPath()),
             LockStats.getGlobal().getThreadLockStats(),
             LockStats.getGlobal().getLockAttemptSamples());
    }

    /** For testing */
    LocksResponse(String hostname,
                  TreeMap lockMetricsByPath,
                  List threadLockStatsList,
                  List historicSamples) {
        super(200);

        Cursor root = slime.setObject();

        root.setString("hostname", hostname);
        root.setString("time", Instant.now().toString());

        Cursor threadsCursor = root.setArray("threads");
        for (var threadLockStats : threadLockStatsList) {
            Optional ongoingLockAttempt = threadLockStats.getTopMostOngoingLockAttempt();
            Optional ongoingRecording = threadLockStats.getOngoingRecording();
            if (ongoingLockAttempt.isEmpty() && ongoingRecording.isEmpty()) {
                continue;
            }

            Cursor threadCursor = threadsCursor.addObject();
            threadCursor.setString("thread-name", threadLockStats.getThreadName());

            ongoingLockAttempt.ifPresent(lockAttempt -> {
                setLockAttempt(threadCursor.setObject("active-lock"), lockAttempt, false, false);
                threadCursor.setString("stack-trace", threadLockStats.getStackTrace());
            });

            ongoingRecording.ifPresent(recording -> setRecording(threadCursor.setObject("ongoing-recording"), recording));
        }

        Cursor historicSamplesCursor = root.setArray("slow-locks");
        historicSamples.stream()
                .sorted(Comparator.comparing(LockAttempt::getDuration).reversed())
                .forEach(lockAttempt -> setLockAttempt(historicSamplesCursor.addObject(), lockAttempt, true, true));

        List historicRecordings = LockStats.getGlobal().getHistoricRecordings().stream()
                .sorted(Comparator.comparing(RecordedLockAttempts::duration).reversed())
                .toList();
        if (!historicRecordings.isEmpty()) {
            Cursor recordingsCursor = root.setArray("recordings");
            historicRecordings.forEach(recording -> setRecording(recordingsCursor.addObject(), recording));
        }

        Cursor lockPathsCursor = root.setArray("lock-paths");
        lockMetricsByPath.forEach((lockPath, lockMetrics) -> {
            Cursor lockPathCursor = lockPathsCursor.addObject();
            lockPathCursor.setString("path", lockPath);
            setNonZeroLong(lockPathCursor, "acquireCount", lockMetrics.getCumulativeAcquireCount());
            setNonZeroLong(lockPathCursor, "acquireFailedCount", lockMetrics.getCumulativeAcquireFailedCount());
            setNonZeroLong(lockPathCursor, "acquireTimedOutCount", lockMetrics.getCumulativeAcquireTimedOutCount());
            setNonZeroLong(lockPathCursor, "lockedCount", lockMetrics.getCumulativeAcquireSucceededCount());
            setNonZeroLong(lockPathCursor, "releaseCount", lockMetrics.getCumulativeReleaseCount());
            setNonZeroLong(lockPathCursor, "releaseFailedCount", lockMetrics.getCumulativeReleaseFailedCount());
            setNonZeroLong(lockPathCursor, "reentryCount", lockMetrics.getCumulativeReentryCount());
            setNonZeroLong(lockPathCursor, "deadlock", lockMetrics.getCumulativeDeadlockCount());
            setNonZeroLong(lockPathCursor, "nakedRelease", lockMetrics.getCumulativeNakedReleaseCount());
            setNonZeroLong(lockPathCursor, "acquireWithoutRelease", lockMetrics.getCumulativeAcquireWithoutReleaseCount());
            setNonZeroLong(lockPathCursor, "foreignRelease", lockMetrics.getCumulativeForeignReleaseCount());

            setLatency(lockPathCursor, "acquire", lockMetrics.getAcquireLatencyMetrics());
            setLatency(lockPathCursor, "locked", lockMetrics.getLockedLatencyMetrics());
        });
    }

    private static void setNonZeroLong(Cursor cursor, String fieldName, long value) {
        if (value != 0) {
            cursor.setLong(fieldName, value);
        }
    }

    private static void setLatency(Cursor cursor, String name, LatencyMetrics latencyMetrics) {
        setNonZeroDouble(cursor, name + "Latency", latencyMetrics.latencySeconds());
        setNonZeroDouble(cursor, name + "MaxActiveLatency", latencyMetrics.maxActiveLatencySeconds());
        setNonZeroDouble(cursor, name + "Hz", latencyMetrics.endHz());
        setNonZeroDouble(cursor, name + "Load", latencyMetrics.load());
        if (latencyMetrics.loadByThread().isEmpty()) return;
        Cursor loadByThreadCursor = cursor.setArray(name + "LoadByThread");
        latencyMetrics.loadByThread().forEach((threadName, load) -> {
            Cursor loadForThreadCursor = loadByThreadCursor.addObject();
            loadForThreadCursor.setString("name", threadName);
            loadForThreadCursor.setDouble("load", load);
        });
    }

    private static void setNonZeroDouble(Cursor cursor, String fieldName, double value) {
        if (Double.compare(value, 0.0) != 0) {
            cursor.setDouble(fieldName, value);
        }
    }

    @Override
    public void render(OutputStream stream) throws IOException {
        new JsonFormat(true).encode(stream, slime);
    }

    @Override
    public String getContentType() {
        return "application/json";
    }

    private void setRecording(Cursor cursor, RecordedLockAttempts recording) {
        cursor.setString("record-id", recording.recordId());
        cursor.setString("start-time", toString(recording.startInstant()));
        cursor.setString("duration", recording.duration().toString());
        Cursor locksCursor = cursor.setArray("locks");
        recording.lockAttempts().forEach(lockAttempt -> setLockAttempt(locksCursor.addObject(), lockAttempt, false, false));
    }

    private void setLockAttempt(Cursor lockAttemptCursor, LockAttempt lockAttempt, boolean includeThreadName,
                                boolean includeStackTrace) {
        if (includeThreadName) {
            lockAttemptCursor.setString("thread-name", lockAttempt.getThreadName());
        }
        lockAttemptCursor.setString("lock-path", lockAttempt.getLockPath());
        lockAttemptCursor.setString("invoke-acquire-time", toString(lockAttempt.getTimeAcquiredWasInvoked()));
        lockAttemptCursor.setString("acquire-timeout", lockAttempt.getAcquireTimeout().toString());
        lockAttempt.getTimeLockWasAcquired().ifPresent(instant -> lockAttemptCursor.setString("lock-acquired-time", toString(instant)));
        lockAttemptCursor.setString("lock-state", lockAttempt.getLockState().name());
        lockAttempt.getTimeTerminalStateWasReached().ifPresent(instant -> lockAttemptCursor.setString("terminal-state-time", toString(instant)));
        lockAttemptCursor.setString("acquire-duration", toString(lockAttempt.getDurationOfAcquire()));
        lockAttemptCursor.setString("locked-duration", toString(lockAttempt.getDurationWithLock()));
        lockAttemptCursor.setString("total-duration", toString(lockAttempt.getDuration()));
        if (includeStackTrace) {
            lockAttempt.getStackTrace().ifPresent(stackTrace -> lockAttemptCursor.setString("stack-trace", stackTrace));
        }

        List nestedLockAttempts = lockAttempt.getNestedLockAttempts();
        if (!nestedLockAttempts.isEmpty()) {
            Cursor nestedLockAttemptsCursor = lockAttemptCursor.setArray("nested-locks");
            nestedLockAttempts.forEach(nestedLockAttempt ->
                    setLockAttempt(nestedLockAttemptsCursor.addObject(), nestedLockAttempt, false, includeStackTrace));
        }
    }

    private static String toString(Duration duration) {
        return Duration.ofMillis(duration.toMillis()).toString();
    }

    private static String toString(Instant time) {
        return Instant.ofEpochMilli(time.toEpochMilli()).toString();
    }
}