com.spotify.helios.agent.AgentParser Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of helios-services Show documentation
There is a newer version: 0.9.9
/*
 * Copyright (c) 2014 Spotify AB.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package com.spotify.helios.agent;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Splitter;
import com.google.common.net.InetAddresses;

import com.spotify.docker.client.DockerHost;
import com.spotify.helios.servicescommon.ServiceParser;

import net.sourceforge.argparse4j.inf.Argument;
import net.sourceforge.argparse4j.inf.ArgumentParser;
import net.sourceforge.argparse4j.inf.ArgumentParserException;
import net.sourceforge.argparse4j.inf.Namespace;

import java.net.InetSocketAddress;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.function.Predicate;

import static com.google.common.io.BaseEncoding.base16;
import static com.spotify.helios.cli.Utils.argToStringMap;
import static net.sourceforge.argparse4j.impl.Arguments.append;
import static net.sourceforge.argparse4j.impl.Arguments.storeTrue;

/**
 * Parses and processes command-line arguments to produce the {@link AgentConfig}.
 */
public class AgentParser extends ServiceParser {

  private static final String ZK_AGENT_PASSWORD_ENVVAR = "HELIOS_ZK_AGENT_PASSWORD";

  private final AgentConfig agentConfig;

  private Argument noHttpArg;
  private Argument httpArg;
  private Argument adminArg;
  private Argument dockerArg;
  private Argument dockerCertPathArg;
  private Argument envArg;
  private Argument syslogRedirectToArg;
  private Argument portRangeArg;
  private Argument agentIdArg;
  private Argument dnsArg;
  private Argument bindArg;
  private Argument addHostArg;
  private Argument labelsArg;
  private Argument zkRegistrationTtlMinutesArg;
  private Argument zkAclMasterDigest;
  private Argument zkAclAgentPassword;

  public AgentParser(final String... args) throws ArgumentParserException {
    super("helios-agent", "Spotify Helios Agent", args);

    final Namespace options = getNamespace();
    final DockerHost dockerHost = DockerHost.from(
        options.getString(dockerArg.getDest()),
        options.getString(dockerCertPathArg.getDest()));

    final Map envVars = argToStringMap(options, envArg);
    final Map labels;
    try {
      labels = argToStringMap(options, labelsArg);
    } catch (IllegalArgumentException e) {
      throw new IllegalArgumentException(e.getMessage() +
                                         "\nLabels need to be in the format key=value.");
    }

    final InetSocketAddress httpAddress = parseSocketAddress(options.getString(httpArg.getDest()));
    final InetSocketAddress adminAddress = parseSocketAddress(
        options.getString(adminArg.getDest()));

    final String portRangeString = options.getString(portRangeArg.getDest());
    final List parts = Splitter.on(':').splitToList(portRangeString);
    if (parts.size() != 2) {
      throw new IllegalArgumentException("Bad port range: " + portRangeString);
    }
    final int start;
    final int end;
    try {
      start = Integer.valueOf(parts.get(0));
      end = Integer.valueOf(parts.get(1));
    } catch (NumberFormatException e) {
      throw new IllegalArgumentException("Bad port range: " + portRangeString);
    }
    if (end <= start) {
      throw new IllegalArgumentException("Bad port range: " + portRangeString);
    }


    String agentPassword = System.getenv(ZK_AGENT_PASSWORD_ENVVAR);
    if (agentPassword == null) {
      agentPassword = options.getString(zkAclAgentPassword.getDest());
    }

    this.agentConfig = new AgentConfig()
        .setName(getName())
        .setZooKeeperConnectionString(getZooKeeperConnectString())
        .setZooKeeperSessionTimeoutMillis(getZooKeeperSessionTimeoutMillis())
        .setZooKeeperConnectionTimeoutMillis(getZooKeeperConnectionTimeoutMillis())
        .setZooKeeperClusterId(getZooKeeperClusterId())
        .setZooKeeperRegistrationTtlMinutes(options.getInt(zkRegistrationTtlMinutesArg.getDest()))
        .setZooKeeperEnableAcls(getZooKeeperEnableAcls())
        .setZookeeperAclMasterUser(getZooKeeperAclMasterUser())
        .setZooKeeperAclMasterDigest(options.getString(zkAclMasterDigest.getDest()))
        .setZookeeperAclAgentUser(getZooKeeperAclAgentUser())
        .setZooKeeperAclAgentPassword(agentPassword)
        .setDomain(getDomain())
        .setEnvVars(envVars)
        .setDockerHost(dockerHost)
        .setInhibitMetrics(getInhibitMetrics())
        .setRedirectToSyslog(options.getString(syslogRedirectToArg.getDest()))
        .setStateDirectory(getStateDirectory())
        .setStatsdHostPort(getStatsdHostPort())
        .setRiemannHostPort(getRiemannHostPort())
        .setPortRange(start, end)
        .setSentryDsn(getSentryDsn())
        .setServiceRegistryAddress(getServiceRegistryAddress())
        .setServiceRegistrarPlugin(getServiceRegistrarPlugin())
        .setAdminAddress(adminAddress)
        .setHttpEndpoint(httpAddress)
        .setNoHttp(options.getBoolean(noHttpArg.getDest()))
        .setKafkaBrokers(getKafkaBrokers())
        .setLabels(labels)
        .setFfwdConfig(ffwdConfig(options));

    final String explicitId = options.getString(agentIdArg.getDest());
    if (explicitId != null) {
      agentConfig.setId(explicitId);
    } else {
      final byte[] idBytes = new byte[20];
      new SecureRandom().nextBytes(idBytes);
      agentConfig.setId(base16().encode(idBytes));
    }

    agentConfig.setDns(validateArgument(
        options.getList(dnsArg.getDest()),
        InetAddresses::isInetAddress,
        arg -> "Invalid IP address " + arg));

    agentConfig.setBinds(validateArgument(
        options.getList(bindArg.getDest()),
        BindVolumeContainerDecorator::isValidBind,
        arg -> "Invalid bind " + arg));

    agentConfig.setExtraHosts(validateArgument(
        options.getList(addHostArg.getDest()),
        AddExtraHostContainerDecorator::isValidArg,
        arg -> "Invalid ExtraHost " + arg));
  }

  /**
   * Verifies that all entries in the Collection satisfy the predicate. If any do not, throw an
   * IllegalArgumentException with the specified message for the first invalid entry.
   */
  @VisibleForTesting
  protected static  List validateArgument(List list, Predicate predicate,
                                              Function msgFn) {

    final Optional firstInvalid = list.stream()
        .filter(predicate.negate())
        .findAny();

    if (firstInvalid.isPresent()) {
      throw new IllegalArgumentException(firstInvalid.map(msgFn).get());
    }
    return list;
  }

  @Override
  protected void addArgs(final ArgumentParser parser) {
    noHttpArg = parser.addArgument("--no-http")
        .action(storeTrue())
        .setDefault(false)
        .help("disable http server");

    httpArg = parser.addArgument("--http")
        .setDefault("http://0.0.0.0:5803")
        .help("http endpoint");

    adminArg = parser.addArgument("--admin")
        .setDefault("http://0.0.0.0:5804")
        .help("admin http port");

    agentIdArg = parser.addArgument("--id")
        .help("Agent unique ID. Generated and persisted on first run if not specified.");

    dockerArg = parser.addArgument("--docker")
        .setDefault(DockerHost.fromEnv().host())
        .help("docker endpoint");

    dockerCertPathArg = parser.addArgument("--docker-cert-path")
        .setDefault(DockerHost.fromEnv().dockerCertPath())
        .help("directory containing client.pem and client.key for connecting to Docker over HTTPS");

    envArg = parser.addArgument("--env")
        .action(append())
        .setDefault(new ArrayList<>())
        .nargs("+")
        .help("Specify environment variables that will pass down to all containers");

    syslogRedirectToArg = parser.addArgument("--syslog-redirect-to")
        .help("redirect container's stdout/stderr to syslog running at host:port");

    portRangeArg = parser.addArgument("--port-range")
        .setDefault("20000:32768")
        .help("Port allocation range, start:end (end exclusive).");

    dnsArg = parser.addArgument("--dns")
        .action(append())
        .setDefault(new ArrayList<>())
        .help("Dns servers to use.");

    bindArg = parser.addArgument("--bind")
        .action(append())
        .setDefault(new ArrayList<>())
        .help("volumes to bind to all containers");

    addHostArg = parser.addArgument("--add-host")
        .action(append())
        .setDefault(new ArrayList<>())
        .help("extra hosts to add to /etc/hosts of created containers, in form `host:ip`. "
              + "See docker documentation for --add-host for more info.");

    labelsArg = parser.addArgument("--labels")
        .action(append())
        .setDefault(new ArrayList())
        .nargs("+")
        .help("labels to apply to this agent. Labels need to be in the format key=value.");

    zkRegistrationTtlMinutesArg = parser.addArgument("--zk-registration-ttl")
        .type(Integer.class)
        .setDefault(10)
        .help("Number of minutes that this agent must be DOWN (i.e. not periodically check-in with "
              + "ZooKeeper) before another agent with the same hostname but lacking the "
              + "registration ID of this one can automatically deregister this one and register "
              + "itself. This is useful when the agent loses its registration ID and you don't "
              + "want to waste time debugging why the master lists your agent as constantly DOWN.");

    zkAclMasterDigest = parser.addArgument("--zk-acl-master-digest")
        .type(String.class);

    zkAclAgentPassword = parser.addArgument("--zk-acl-agent-password")
        .type(String.class)
        .help("ZooKeeper agent password (for ZooKeeper ACLs). If the "
              + ZK_AGENT_PASSWORD_ENVVAR
              + " environment variable is present this argument is ignored.");
  }

  public AgentConfig getAgentConfig() {
    return agentConfig;
  }

}