com.yahoo.vespa.hosted.provision.provisioning.ProvisioningThrottler Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of node-repository Show documentation
Keeps track of node assignment in a multi-application setup.
The newest version!
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.provisioning;

import ai.vespa.metrics.ConfigServerMetrics;
import com.yahoo.jdisc.Metric;
import com.yahoo.vespa.flags.IntFlag;
import com.yahoo.vespa.flags.PermanentFlags;
import com.yahoo.vespa.hosted.provision.NodeList;
import com.yahoo.vespa.hosted.provision.NodeRepository;
import com.yahoo.vespa.hosted.provision.node.Agent;
import com.yahoo.vespa.hosted.provision.node.History;

import java.time.Duration;
import java.time.Instant;
import java.util.Objects;
import java.util.logging.Logger;

/**
 * Throttles provisioning of new hosts in dynamically provisioned zones.
 *
 * @author mpolden
 */
public class ProvisioningThrottler {

    /** Metric that indicates whether throttling is active where 1 means active and 0 means inactive */
    private static final String throttlingActiveMetric = ConfigServerMetrics.THROTTLED_HOST_PROVISIONING.baseName();

    private static final Logger LOG = Logger.getLogger(ProvisioningThrottler.class.getName());
    private static final Duration WINDOW = Duration.ofDays(1);

    private final NodeRepository nodeRepository;
    private final Metric metric;
    private final IntFlag maxHostsPerHourFlag;

    public ProvisioningThrottler(NodeRepository nodeRepository, Metric metric) {
        this.nodeRepository = Objects.requireNonNull(nodeRepository);
        this.metric = Objects.requireNonNull(metric);
        this.maxHostsPerHourFlag = PermanentFlags.MAX_HOSTS_PER_HOUR.bindTo(nodeRepository.flagSource());
    }

    /** Returns whether provisioning should be throttled at given instant */
    public boolean throttle(NodeList allNodes, Agent agent) {
        Instant startOfWindow = nodeRepository.clock().instant().minus(WINDOW);
        int provisionedRecently = allNodes.matching(host -> host.history().hasEventAfter(History.Event.Type.provisioned,
                                                                                         startOfWindow))
                                          .size();
        boolean throttle = throttle(provisionedRecently, maxHostsPerHourFlag.value(), agent);
        metric.set(throttlingActiveMetric, throttle ? 1 : 0, null);
        return throttle;
    }

    static boolean throttle(int recentHosts, int maxPerHour, Agent agent) {
        double rate = recentHosts / (double) WINDOW.toMillis();
        boolean throttle = (rate * Duration.ofHours(1).toMillis()) > maxPerHour;
        if (throttle) {
            LOG.warning(String.format("Throttling provisioning of new hosts by %s: %d hosts have been provisioned " +
                                      "in the past %s, which exceeds maximum rate of %d hosts/hour", agent,
                                      recentHosts, WINDOW, maxPerHour));
        }
        return throttle;
    }

}