com.yahoo.vespa.config.server.application.FileDistributionStatus Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of configserver Show documentation
There is a newer version: 8.441.21
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server.application;

import com.yahoo.component.AbstractComponent;
import com.yahoo.concurrent.DaemonThreadFactory;
import com.yahoo.config.model.api.PortInfo;
import com.yahoo.config.model.api.ServiceInfo;
import com.yahoo.jrt.Request;
import com.yahoo.jrt.Spec;
import com.yahoo.jrt.Supervisor;
import com.yahoo.jrt.Target;
import com.yahoo.jrt.Transport;
import java.util.logging.Level;
import com.yahoo.slime.Cursor;
import com.yahoo.vespa.config.server.http.JSONResponse;

import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.logging.Logger;

/**
 * File distribution status for each host in the application
 *
 * @author hmusum
 */
public class FileDistributionStatus extends AbstractComponent {

    private static final Logger log = Logger.getLogger(FileDistributionStatus.class.getName());

    enum Status {UNKNOWN, FINISHED, IN_PROGRESS}

    private final ExecutorService rpcExecutor = Executors.newCachedThreadPool(new DaemonThreadFactory("filedistribution status"));
    private final Supervisor supervisor = new Supervisor(new Transport("filedistribution-status"));

    public StatusAllHosts status(Application application, Duration timeout) {
        List hostStatuses = new ArrayList<>();
        List > results = new ArrayList<>();
        application.getModel().getHosts()
                .forEach(host -> host.getServices()
                        .stream()
                        .filter(service -> "configproxy".equals(service.getServiceType()))
                        .forEach(service -> {
                            results.add(rpcExecutor.submit(() -> getHostStatus(service.getHostName(), getRpcPort(service), timeout)));
                        }));
        // wait for all
        results.forEach(future -> {
            try {
                hostStatuses.add(future.get());
            } catch (InterruptedException | ExecutionException e) {
                log.log(Level.WARNING, "Failed getting file distribution status", e);
            }
        });
        return createStatusForAllHosts(hostStatuses);
    }

    HostStatus getHostStatus(String hostname, int port, Duration timeout) {
        Target target = supervisor.connect(new Spec(hostname, port));
        Request request = new Request("filedistribution.getActiveFileReferencesStatus");
        target.invokeSync(request, timeout);
        HostStatus hostStatus = createHostStatusFromResponse(hostname, request);
        target.close();
        return hostStatus;
    }

    private HostStatus createHostStatusFromResponse(String hostname, Request request) {
        if (request.isError()) {
            return new HostStatus(hostname,
                                  Status.UNKNOWN,
                                  Map.of(),
                                  "error: " + request.errorMessage() + "(" + request.errorCode() + ")");
        } else {
            Map fileReferenceStatuses = new HashMap<>();
            String[] fileReferences = request.returnValues().get(0).asStringArray();
            double[] downloadStatus = request.returnValues().get(1).asDoubleArray();

            boolean allDownloaded = true;
            for (int i = 0; i < fileReferences.length; i++) {
                fileReferenceStatuses.put(fileReferences[i], downloadStatus[i]);
                if (downloadStatus[i] < 1.0) {
                    allDownloaded = false;
                }
            }

            return new HostStatus(hostname, allDownloaded ? Status.FINISHED : Status.IN_PROGRESS, fileReferenceStatuses, "");
        }
    }

    private StatusAllHosts createStatusForAllHosts(List hostStatuses) {
        int countUnknown = 0;
        int countInProgress = 0;
        int countFinished = 0;
        for (HostStatus hostStatus : hostStatuses) {
            switch (hostStatus.status) {
                case IN_PROGRESS -> countInProgress++;
                case FINISHED -> countFinished++;
                case UNKNOWN -> countUnknown++;
            }
        }

        if (countInProgress == 0 && countUnknown == 0)
            return new StatusAllHosts(Status.FINISHED, hostStatuses);
        else if (countInProgress == 0 && countFinished == 0)
            return new StatusAllHosts(Status.UNKNOWN, hostStatuses);
        else
            return new StatusAllHosts(Status.IN_PROGRESS, hostStatuses);
    }

    private static Integer getRpcPort(ServiceInfo service) {
        return service.getPorts().stream()
                .filter(port -> port.getTags().contains("rpc"))
                .map(PortInfo::getPort)
                .findFirst()
                .orElseThrow(() -> new RuntimeException("Could not find rpc port for " + service.getServiceType() + " on " + service.getHostName()));
    }

    private static class StatusAllHosts extends JSONResponse {

        private StatusAllHosts(Status status, List hostStatuses) {
            super(200);
            Cursor hostsArray = object.setArray("hosts");
            for (HostStatus hostStatus : hostStatuses) {
                Cursor host = hostsArray.addObject();
                host.setString("hostname", hostStatus.hostname);
                host.setString("status", hostStatus.status.name());
                hostStatus.errorMessage.ifPresent(message -> host.setString("message", message));
                Cursor fileReferences = host.setArray("fileReferences");
                hostStatus.fileReferenceStatuses.forEach((key, value) -> fileReferences.addObject().setDouble(key, value));
            }

            object.setString("status", status.name());
        }
    }

    @Override
    public void deconstruct() {
        rpcExecutor.shutdownNow();
    }

    static class HostStatus {

        private final String hostname;
        private final Status status;
        private final Map fileReferenceStatuses;
        private final Optional errorMessage;

        HostStatus(String hostname, Status status, Map fileReferenceStatuses) {
            this.hostname = hostname;
            this.status = status;
            this.fileReferenceStatuses = fileReferenceStatuses;
            this.errorMessage = Optional.empty();
        }

        HostStatus(String hostname, Status status, Map fileReferenceStatuses, String errorMessage) {
            this.hostname = hostname;
            this.status = status;
            this.fileReferenceStatuses = fileReferenceStatuses;
            this.errorMessage = Optional.of(errorMessage);
        }

        public String hostname() {
            return hostname;
        }

        @Override
        public String toString() {
            return hostname + ": " + status + ", " + fileReferenceStatuses + " " + errorMessage.orElse("");
        }
    }

}