• Home
• io.hyperfoil
• hyperfoil-clustering
• 0.27.1
• source code
• ControllerVerticle.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

io.hyperfoil.clustering.ControllerVerticle Maven / Gradle / Ivy

package io.hyperfoil.clustering;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiConsumer;
import java.util.stream.Collectors;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.infinispan.commons.api.BasicCacheContainer;

import com.fasterxml.jackson.core.JsonEncoding;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.ObjectMapper;

import io.hyperfoil.api.BenchmarkExecutionException;
import io.hyperfoil.api.config.Agent;
import io.hyperfoil.api.config.Benchmark;
import io.hyperfoil.api.config.BenchmarkSource;
import io.hyperfoil.api.config.Model;
import io.hyperfoil.api.config.Phase;
import io.hyperfoil.api.config.RunHook;
import io.hyperfoil.api.config.SessionLimitPolicy;
import io.hyperfoil.api.deployment.DeployedAgent;
import io.hyperfoil.api.deployment.Deployer;
import io.hyperfoil.api.session.GlobalData;
import io.hyperfoil.api.session.PhaseInstance;
import io.hyperfoil.clustering.messages.AgentControlMessage;
import io.hyperfoil.clustering.messages.AgentHello;
import io.hyperfoil.clustering.messages.AgentReadyMessage;
import io.hyperfoil.clustering.messages.AgentStatusMessage;
import io.hyperfoil.clustering.messages.AuxiliaryHello;
import io.hyperfoil.clustering.messages.ConnectionStatsMessage;
import io.hyperfoil.clustering.messages.DelayStatsCompletionMessage;
import io.hyperfoil.clustering.messages.ErrorMessage;
import io.hyperfoil.clustering.messages.PhaseChangeMessage;
import io.hyperfoil.clustering.messages.PhaseControlMessage;
import io.hyperfoil.clustering.messages.PhaseStatsCompleteMessage;
import io.hyperfoil.clustering.messages.RequestStatsMessage;
import io.hyperfoil.clustering.messages.SessionStatsMessage;
import io.hyperfoil.clustering.messages.StatsMessage;
import io.hyperfoil.clustering.util.PersistenceUtil;
import io.hyperfoil.controller.CsvWriter;
import io.hyperfoil.controller.JsonLoader;
import io.hyperfoil.controller.JsonWriter;
import io.hyperfoil.controller.StatisticsStore;
import io.hyperfoil.core.hooks.ExecRunHook;
import io.hyperfoil.core.parser.BenchmarkParser;
import io.hyperfoil.core.parser.ParserException;
import io.hyperfoil.core.util.CountDown;
import io.hyperfoil.core.util.LowHigh;
import io.hyperfoil.internal.Controller;
import io.hyperfoil.internal.Properties;
import io.vertx.core.AbstractVerticle;
import io.vertx.core.AsyncResult;
import io.vertx.core.DeploymentOptions;
import io.vertx.core.Future;
import io.vertx.core.Handler;
import io.vertx.core.Promise;
import io.vertx.core.eventbus.EventBus;
import io.vertx.core.eventbus.Message;
import io.vertx.core.eventbus.ReplyException;
import io.vertx.core.impl.VertxInternal;
import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;
import io.vertx.core.spi.cluster.ClusterManager;
import io.vertx.core.spi.cluster.NodeListener;
import io.vertx.ext.cluster.infinispan.InfinispanClusterManager;

public class ControllerVerticle extends AbstractVerticle implements NodeListener {
   private static final Logger log = LogManager.getLogger(ControllerVerticle.class);
   private static final int MAX_IN_MEMORY_RUNS = Properties.getInt(Properties.MAX_IN_MEMORY_RUNS, 20);
   static final String DEFAULT_STATS_JSON = "all.json";

   private EventBus eb;
   private ControllerServer server;
   private Deployer deployer;
   private final AtomicInteger runIds = new AtomicInteger();
   private final Map benchmarks = new HashMap<>();
   private final Map templates = new HashMap<>();
   private long timerId = -1;

   Map runs = new HashMap<>();

   @Override
   public void start(Promise future) {
      log.info("Starting in directory {}...", Controller.ROOT_DIR);
      CountDown startCountDown = new CountDown(future, 2);
      server = new ControllerServer(this, startCountDown);
      vertx.exceptionHandler(throwable -> log.error("Uncaught error: ", throwable));
      if (Files.exists(Controller.RUN_DIR)) {
         try {
            Files.list(Controller.RUN_DIR).forEach(this::updateRuns);
         } catch (IOException e) {
            log.error("Could not list run dir contents", e);
         } catch (Exception e) {
            log.error("Cannot load previous runs from {}", Controller.RUN_DIR, e);
         }
      }
      //noinspection ResultOfMethodCallIgnored
      Controller.HOOKS_DIR.resolve("pre").toFile().mkdirs();
      //noinspection ResultOfMethodCallIgnored
      Controller.HOOKS_DIR.resolve("post").toFile().mkdirs();

      eb = vertx.eventBus();

      eb.consumer(Feeds.DISCOVERY, message -> {
         if (message.body() instanceof AgentHello) {
            handleAgentHello(message, (AgentHello) message.body());
         } else if (message.body() instanceof AuxiliaryHello) {
            AuxiliaryHello hello = (AuxiliaryHello) message.body();
            log.info("Noticed auxiliary {} (node {}, {})", hello.name(), hello.nodeId(), hello.deploymentId());
            String nodeId = ((VertxInternal) vertx).getClusterManager().getNodeId();
            message.reply(nodeId);
         } else {
            log.error("Unknown message on discovery feed! {}", message.body());
         }
      });

      eb.consumer(Feeds.RESPONSE, message -> {
         AgentStatusMessage msg = (AgentStatusMessage) message.body();
         Run run = runs.get(msg.runId());
         if (run == null) {
            log.error("No run {}", msg.runId());
            return;
         }
         AgentInfo agent = run.agents.stream().filter(a -> a.deploymentId.equals(msg.senderId())).findAny().orElse(null);
         if (agent == null) {
            log.error("No agent {} in run {}", msg.senderId(), run.id);
            return;
         }
         if (msg instanceof PhaseChangeMessage) {
            handlePhaseChange(run, agent, (PhaseChangeMessage) msg);
         } else if (msg instanceof ErrorMessage) {
            ErrorMessage errorMessage = (ErrorMessage) msg;
            run.errors.add(new Run.Error(agent, errorMessage.error()));
            if (errorMessage.isFatal()) {
               agent.status = AgentInfo.Status.FAILED;
               stopSimulation(run);
            }
         } else if (msg instanceof AgentReadyMessage) {
            if (!run.validation) {
               agent.status = AgentInfo.Status.READY;
               if (run.agents.stream().allMatch(a -> a.status == AgentInfo.Status.READY)) {
                  startSimulation(run);
               }
            } else { //stop simulation from running if a validation run
               agent.status = AgentInfo.Status.STOPPED;
               stopSimulation(run);
            }
         } else {
            log.error("Unexpected type of message: {}", msg);
         }
      });

      eb.consumer(Feeds.STATS, message -> {
         if (!(message.body() instanceof StatsMessage)) {
            log.error("Unknown message type: {}", message.body());
            return;
         }
         StatsMessage statsMessage = (StatsMessage) message.body();
         Run run = runs.get(statsMessage.runId);
         if (run != null) {
            String agentName = run.agents.stream()
                  .filter(ai -> ai.deploymentId.equals(statsMessage.address))
                  .map(ai -> ai.name).findFirst().orElse("");
            if (statsMessage instanceof RequestStatsMessage) {
               RequestStatsMessage rsm = (RequestStatsMessage) statsMessage;
               String phase = run.phase(rsm.phaseId);
               if (rsm.statistics != null) {
                  log.debug("Run {}: Received stats from {}({}): {}/{}/{}:{} ({} requests)",
                        rsm.runId, agentName, rsm.address, phase, rsm.stepId, rsm.metric,
                        rsm.statistics.sequenceId, rsm.statistics.requestCount);
                  boolean added = run.statisticsStore().record(agentName, rsm.phaseId, rsm.stepId, rsm.metric, rsm.statistics);
                  if (!added) {
                     // warning already logged
                     String errorMessage = String.format(
                           "Received statistics for %s/%d/%s:%d with %d requests but the statistics are already completed; these statistics won't be reported.",
                           phase, rsm.stepId, rsm.metric, rsm.statistics.sequenceId, rsm.statistics.requestCount);
                     run.errors.add(new Run.Error(null, new BenchmarkExecutionException(errorMessage)));
                  }
               }
            } else if (statsMessage instanceof PhaseStatsCompleteMessage) {
               PhaseStatsCompleteMessage pscm = (PhaseStatsCompleteMessage) statsMessage;
               log.debug("Run {}: Received stats completion for phase {} from {}", run.id, pscm.phase, pscm.address);
               AgentInfo agent = run.agents.stream().filter(a -> a.deploymentId.equals(pscm.address)).findFirst().orElse(null);
               if (agent == null) {
                  log.error("Run {}: Cannot find agent {}", run.id, pscm.address);
               } else {
                  PhaseInstance.Status prevStatus = agent.phases.put(pscm.phase, PhaseInstance.Status.STATS_COMPLETE);
                  if (prevStatus == PhaseInstance.Status.STATS_COMPLETE) {
                     // TODO: the stats might be completed both regularly and when the agent receives STOP
                     log.info("Run {}: stats for phase {} are already completed, ignoring.", run.id, pscm.phase);
                  } else if (run.agents.stream().map(a -> a.phases.get(pscm.phase))
                        .allMatch(s -> s == PhaseInstance.Status.STATS_COMPLETE)) {
                     ControllerPhase controllerPhase = run.phases.get(pscm.phase);
                     if (controllerPhase != null) {
                        tryCompletePhase(run, pscm.phase, controllerPhase);
                     } else if (!run.validation) {
                        // if run.validation is true, then startSimulation is not executed and the phases are not
                        // added in the list, therefore it is expected that the phase is not found
                        // log the error if and only if the phase is not found, and we are not running just validation
                        log.error("Run {}: Cannot find phase {}!", run.id, pscm.phase);
                     }
                  }
               }
            } else if (statsMessage instanceof SessionStatsMessage) {
               SessionStatsMessage sessionStatsMessage = (SessionStatsMessage) statsMessage;
               log.trace("Run {}: Received session pool stats from {}", sessionStatsMessage.runId, sessionStatsMessage.address);
               for (Map.Entry entry : sessionStatsMessage.sessionStats.entrySet()) {
                  run.statisticsStore().recordSessionStats(agentName, sessionStatsMessage.timestamp,
                        entry.getKey(), entry.getValue().low, entry.getValue().high);
               }
            } else if (statsMessage instanceof ConnectionStatsMessage) {
               ConnectionStatsMessage connectionStatsMessage = (ConnectionStatsMessage) statsMessage;
               log.trace("Run {}: Received connection stats from {}", connectionStatsMessage.runId,
                     connectionStatsMessage.address);
               run.statisticsStore().recordConnectionStats(agentName, connectionStatsMessage.timestamp,
                     connectionStatsMessage.stats);
            } else if (statsMessage instanceof DelayStatsCompletionMessage) {
               DelayStatsCompletionMessage delayStatsCompletionMessage = (DelayStatsCompletionMessage) statsMessage;
               String phase = run.phase(delayStatsCompletionMessage.phaseId);
               log.trace("Run {}: Received request for extension from {} for phase {} by {} ms",
                     delayStatsCompletionMessage.runId, delayStatsCompletionMessage.address,
                     phase, delayStatsCompletionMessage.delay);
               ControllerPhase controllerPhase = run.phases.get(phase);
               controllerPhase.delayStatsCompletionUntil(System.currentTimeMillis() + delayStatsCompletionMessage.delay);
            }
         } else {
            log.error("Unknown run {}", statsMessage.runId);
         }
         message.reply("OK");
      });

      if (vertx.isClustered()) {
         for (Deployer.Factory deployerFactory : ServiceLoader.load(Deployer.Factory.class)) {
            log.debug("Found deployer {}", deployerFactory.name());
            if (Controller.DEPLOYER.equals(deployerFactory.name())) {
               deployer = deployerFactory.create();
               break;
            }
         }
         if (deployer == null) {
            throw new IllegalStateException(
                  "Hyperfoil is running in clustered mode but it couldn't load deployer '" + Controller.DEPLOYER + "'");
         }

         if (vertx instanceof VertxInternal) {
            ClusterManager clusterManager = ((VertxInternal) vertx).getClusterManager();
            clusterManager.nodeListener(this);
         }
      }

      if (!Controller.BENCHMARK_DIR.toFile().exists() && !Controller.BENCHMARK_DIR.toFile().mkdirs()) {
         log.error("Failed to create benchmark directory: {}", Controller.BENCHMARK_DIR);
      }
      startCountDown.increment();
      loadBenchmarks(startCountDown);
      startCountDown.countDown();
   }

   private void tryCompletePhase(Run run, String phase, ControllerPhase controllerPhase) {
      long delay = controllerPhase.delayStatsCompletionUntil() == null ? -1
            : controllerPhase.delayStatsCompletionUntil() - System.currentTimeMillis();
      if (delay <= 0) {
         log.info("Run {}: completing stats for phase {}", run.id, phase);
         run.statisticsStore().completePhase(phase);
         if (!run.statisticsStore().validateSlas()) {
            log.info("SLA validation failed for {}", phase);
            controllerPhase.setFailed();
            if (run.benchmark.failurePolicy() == Benchmark.FailurePolicy.CANCEL) {
               failNotStartedPhases(run, controllerPhase);
            }
         }
      } else {
         log.info("Run {}: all agents completed stats for phase {} but delaying for {} ms", run.id, phase, delay);
         vertx.setTimer(delay, ignored -> tryCompletePhase(run, phase, controllerPhase));
      }
   }

   private void handleAgentHello(Message