org.apache.geode.distributed.internal.ServerLocator Maven / Gradle / Ivy

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional information regarding
 * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License. You may obtain a
 * copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */
package org.apache.geode.distributed.internal;

import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.geode.internal.DataSerializableFixedID;
import org.apache.logging.log4j.Logger;

import org.apache.geode.CancelCriterion;
import org.apache.geode.InternalGemFireException;
import org.apache.geode.cache.GemFireCache;
import org.apache.geode.cache.client.internal.locator.ClientConnectionRequest;
import org.apache.geode.cache.client.internal.locator.ClientConnectionResponse;
import org.apache.geode.cache.client.internal.locator.ClientReplacementRequest;
import org.apache.geode.cache.client.internal.locator.GetAllServersRequest;
import org.apache.geode.cache.client.internal.locator.GetAllServersResponse;
import org.apache.geode.cache.client.internal.locator.LocatorListRequest;
import org.apache.geode.cache.client.internal.locator.LocatorListResponse;
import org.apache.geode.cache.client.internal.locator.LocatorStatusResponse;
import org.apache.geode.cache.client.internal.locator.QueueConnectionRequest;
import org.apache.geode.cache.client.internal.locator.QueueConnectionResponse;
import org.apache.geode.cache.client.internal.locator.ServerLocationRequest;
import org.apache.geode.cache.server.ServerLoad;
import org.apache.geode.distributed.DistributedMember;
import org.apache.geode.distributed.DistributedSystem;
import org.apache.geode.distributed.internal.DistributionAdvisor.Profile;
import org.apache.geode.distributed.internal.tcpserver.TcpHandler;
import org.apache.geode.distributed.internal.tcpserver.TcpServer;
import org.apache.geode.i18n.LogWriterI18n;
import org.apache.geode.internal.net.SocketCreator;
import org.apache.geode.internal.cache.CacheServerAdvisor.CacheServerProfile;
import org.apache.geode.internal.cache.ControllerAdvisor;
import org.apache.geode.internal.cache.ControllerAdvisor.ControllerProfile;
import org.apache.geode.internal.cache.FindDurableQueueProcessor;
import org.apache.geode.internal.cache.GridAdvisor.GridProfile;
import org.apache.geode.internal.i18n.LocalizedStrings;
import org.apache.geode.internal.logging.LogService;

/**
 * 
 * @since GemFire 5.7
 */
public class ServerLocator implements TcpHandler, DistributionAdvisee {
  private static final Logger logger = LogService.getLogger();

  private final int port;
  private final String hostNameForClients;
  private InternalDistributedSystem ds;
  private ControllerAdvisor advisor;
  private final int serialNumber = createSerialNumber();
  private final LocatorStats stats;
  private LocatorLoadSnapshot loadSnapshot = new LocatorLoadSnapshot();
  private Map ownerMap =
      new HashMap();
  private volatile List cachedLocators;
  private final Object cachedLocatorsLock = new Object();

  private final static AtomicInteger profileSN = new AtomicInteger();

  private static final long SERVER_LOAD_LOG_INTERVAL = (60 * 60 * 1000); // log server load once an
                                                                         // hour

  private final String logFile;
  private final String hostName;
  private final String memberName;

  private ProductUseLog productUseLog;

  private volatile long lastLogTime;

  ServerLocator() throws IOException {
    this.port = 10334;
    this.hostName = SocketCreator.getLocalHost().getCanonicalHostName();
    this.hostNameForClients = this.hostName;
    this.logFile = null;
    this.memberName = null;
    this.ds = null;
    this.advisor = null;
    this.stats = null;
  }

  public ServerLocator(int port, InetAddress bindAddress, String hostNameForClients, File logFile,
      ProductUseLog productUseLogWriter, String memberName, InternalDistributedSystem ds,
      LocatorStats stats) throws IOException {
    this.port = port;

    if (bindAddress == null) {
      this.hostName = SocketCreator.getLocalHost().getCanonicalHostName();
    } else {
      this.hostName = bindAddress.getHostAddress();
    }

    if (hostNameForClients != null && !hostNameForClients.equals("")) {
      this.hostNameForClients = hostNameForClients;
    } else {
      this.hostNameForClients = this.hostName;
    }

    this.logFile = logFile != null ? logFile.getCanonicalPath() : null;
    this.memberName = memberName;
    this.productUseLog = productUseLogWriter;

    this.ds = ds;
    this.advisor = ControllerAdvisor.createControllerAdvisor(this); // escapes constructor but
                                                                    // allows field to be final
    this.stats = stats;
  }

  public String getHostName() {
    return this.hostNameForClients;
  }

  public int getPort() {
    return this.port;
  }

  public CancelCriterion getCancelCriterion() {
    return this.ds.getCancelCriterion();
  }

  public void init(TcpServer tcpServer) {
    // if the ds is reconnecting we don't want to start server
    // location services until the DS finishes connecting
    if (this.ds != null && !this.ds.isReconnecting()) {
      this.advisor.handshake();
    }
  }

  /**
   * A ServerLocator may be in place but non-functional during auto-reconnect because peer location
   * services have been initialized while the servers reconnect but server location services aren't
   * reconnected until a cache is available
   */
  protected boolean readyToProcessRequests() {
    return this.ds != null;
  }

  public Object processRequest(Object request) {
    if (!readyToProcessRequests()) {
      return null;
    }

    if (logger.isDebugEnabled()) {
      logger.debug("ServerLocator: Received request {}", request);
    }

    if (!(request instanceof ServerLocationRequest)) {
      throw new InternalGemFireException(
          "Expected ServerLocationRequest, got " + request.getClass());
    }

    Object response;
    int id = ((DataSerializableFixedID) request).getDSFID();
    switch (id) {
      case DataSerializableFixedID.LOCATOR_STATUS_REQUEST:
        response = new LocatorStatusResponse().initialize(this.port, this.hostName, this.logFile,
            this.memberName);
        break;
      case DataSerializableFixedID.LOCATOR_LIST_REQUEST:
        response = getLocatorListResponse((LocatorListRequest) request);
        break;
      case DataSerializableFixedID.CLIENT_REPLACEMENT_REQUEST:
        response = pickReplacementServer((ClientReplacementRequest) request);
        break;
      case DataSerializableFixedID.GET_ALL_SERVERS_REQUEST:
        response = pickAllServers((GetAllServersRequest) request);
        break;
      case DataSerializableFixedID.CLIENT_CONNECTION_REQUEST:
        response = pickServer((ClientConnectionRequest) request);
        break;
      case DataSerializableFixedID.QUEUE_CONNECTION_REQUEST:
        response = pickQueueServers((QueueConnectionRequest) request);
        break;
      default:
        throw new InternalGemFireException("Unknown ServerLocationRequest: " + request.getClass());
    }

    if (logger.isDebugEnabled()) {
      logger.debug("ServerLocator: Sending response {}", response);
    }

    return response;
  }

  private ClientConnectionResponse pickServer(ClientConnectionRequest clientRequest) {
    ServerLocation location = loadSnapshot.getServerForConnection(clientRequest.getServerGroup(),
        clientRequest.getExcludedServers());
    return new ClientConnectionResponse(location);
  }

  private ClientConnectionResponse pickReplacementServer(ClientReplacementRequest clientRequest) {
    ServerLocation location =
        loadSnapshot.getReplacementServerForConnection(clientRequest.getCurrentServer(),
            clientRequest.getServerGroup(), clientRequest.getExcludedServers());
    return new ClientConnectionResponse(location);
  }


  private GetAllServersResponse pickAllServers(GetAllServersRequest clientRequest) {
    ArrayList servers = loadSnapshot.getServers(clientRequest.getServerGroup());
    return new GetAllServersResponse(servers);
  }

  private Object getLocatorListResponse(LocatorListRequest request) {
    List controllers = getLocators();
    boolean balanced = loadSnapshot.hasBalancedConnections(request.getServerGroup());
    return new LocatorListResponse(controllers, balanced);
  }

  private Object pickQueueServers(QueueConnectionRequest clientRequest) {
    Set excludedServers = new HashSet(clientRequest.getExcludedServers());

    /* If this is a request to find durable queues, lets go find them */

    ArrayList servers = new ArrayList();
    boolean durableQueueFound = false;
    if (clientRequest.isFindDurable() && clientRequest.getProxyId().isDurable()) {
      servers = FindDurableQueueProcessor.sendAndFind(this, clientRequest.getProxyId(),
          getDistributionManager());
      /* add the found durables to exclude list so they aren't candidates for more queues */
      excludedServers.addAll(servers);
      durableQueueFound = servers.size() > 0;
    }

    List candidates;
    if (clientRequest.getRedundantCopies() == -1) {
      /* We need all the servers we can get */
      candidates = loadSnapshot.getServersForQueue(clientRequest.getProxyId(),
          clientRequest.getServerGroup(), excludedServers, -1);
    } else if (clientRequest.getRedundantCopies() > servers.size()) {
      /* We need more servers. */
      int count = clientRequest.getRedundantCopies() - servers.size();
      candidates = loadSnapshot.getServersForQueue(clientRequest.getProxyId(),
          clientRequest.getServerGroup(), excludedServers, count);
    } else {
      /* Otherwise, we don't need any more servers */
      candidates = Collections.EMPTY_LIST;
    }

    if (candidates.size() > 1) {
      Collections.shuffle(candidates);
    }
    servers.addAll(candidates);

    return new QueueConnectionResponse(durableQueueFound, servers);
  }

  public void shutDown() {
    this.advisor.close();
    this.loadSnapshot.shutDown();
  }

  public void restarting(DistributedSystem ds, GemFireCache cache,
      ClusterConfigurationService sharedConfig) {
    if (ds != null) {
      this.loadSnapshot = new LocatorLoadSnapshot();
      this.ds = (InternalDistributedSystem) ds;
      this.advisor = ControllerAdvisor.createControllerAdvisor(this); // escapes constructor but
                                                                      // allows field to be final
      if (ds.isConnected()) {
        this.advisor.handshake(); // GEODE-1393: need to get server information during restart
      }
    }
  }

  // DistributionAdvisee methods
  public DM getDistributionManager() {
    return getSystem().getDistributionManager();
  }

  public DistributionAdvisor getDistributionAdvisor() {
    return this.advisor;
  }

  public Profile getProfile() {
    return getDistributionAdvisor().createProfile();
  }

  public DistributionAdvisee getParentAdvisee() {
    return null;
  }

  public InternalDistributedSystem getSystem() {
    return this.ds;
  }

  public String getName() {
    return "ServerLocator";
  }

  public int getSerialNumber() {
    return this.serialNumber;
  }

  public String getFullPath() {
    return getName();
  }

  public InternalDistributedSystem getDs() {
    return ds;
  }

  private static int createSerialNumber() {
    return profileSN.incrementAndGet();
  }

  public void fillInProfile(Profile profile) {
    assert profile instanceof ControllerProfile;
    ControllerProfile cp = (ControllerProfile) profile;
    cp.setHost(this.hostNameForClients);
    cp.setPort(this.port);
    cp.serialNumber = getSerialNumber();
    cp.finishInit();
  }


  public void setLocatorCount(int count) {
    this.stats.setLocatorCount(count);
  }

  public void setServerCount(int count) {
    this.stats.setServerCount(count);
  }

  public void endRequest(Object request, long startTime) {
    stats.endLocatorRequest(startTime);
  }

  public void endResponse(Object request, long startTime) {
    stats.endLocatorResponse(startTime);
  }



  private List getLocators() {
    if (cachedLocators != null) {
      return cachedLocators;
    } else {
      synchronized (cachedLocatorsLock) {
        List profiles = advisor.fetchControllers();
        List result = new ArrayList<>(profiles.size() + 1);
        for (ControllerProfile profile : profiles) {
          result.add(buildServerLocation(profile));
        }
        result.add(new ServerLocation(hostNameForClients, port));
        cachedLocators = result;
        return result;
      }
    }
  }

  protected static ServerLocation buildServerLocation(GridProfile p) {
    return new ServerLocation(p.getHost(), p.getPort());
  }

  /**
   * @param profile
   */
  public void profileCreated(Profile profile) {
    if (profile instanceof CacheServerProfile) {
      CacheServerProfile bp = (CacheServerProfile) profile;
      ServerLocation location = buildServerLocation(bp);
      String[] groups = bp.getGroups();
      loadSnapshot.addServer(location, groups, bp.getInitialLoad(), bp.getLoadPollInterval());
      if (logger.isDebugEnabled()) {
        logger.debug("ServerLocator: Received load from a new server {}, {}", location,
            bp.getInitialLoad());
      }
      synchronized (ownerMap) {
        ownerMap.put(location, profile.getDistributedMember());
      }
    } else {
      cachedLocators = null;
    }
  }

  /**
   * @param profile
   */
  public void profileRemoved(Profile profile) {
    if (profile instanceof CacheServerProfile) {
      CacheServerProfile bp = (CacheServerProfile) profile;
      // InternalDistributedMember id = bp.getDistributedMember();
      ServerLocation location = buildServerLocation(bp);
      loadSnapshot.removeServer(location);
      if (logger.isDebugEnabled()) {
        logger.debug("ServerLocator: server departed {}", location);
      }
      synchronized (ownerMap) {
        ownerMap.remove(location);
      }
    } else {
      cachedLocators = null;
    }
  }

  public void profileUpdated(Profile profile) {
    cachedLocators = null;
    getLogWriterI18n()
        .warning(LocalizedStrings.ServerLocator_SERVERLOCATOR_UNEXPECTED_PROFILE_UPDATE);
  }

  public void updateLoad(ServerLocation location, ServerLoad load, List clientIds) {
    if (getLogWriterI18n().fineEnabled()) {
      getLogWriterI18n()
          .fine("ServerLocator: Received a load update from " + location + ", " + load);
    }
    loadSnapshot.updateLoad(location, load, clientIds);
    this.stats.incServerLoadUpdates();
    logServers();
  }

  private void logServers() {
    if (productUseLog != null) {
      Map loadMap = getLoadMap();
      if (loadMap.size() == 0) {
        return;
      }

      long now = System.currentTimeMillis();
      long lastLogTime = this.lastLogTime;
      if (now < lastLogTime + SERVER_LOAD_LOG_INTERVAL) {
        return;
      }
      this.lastLogTime = now;

      int queues = 0;
      int connections = 0;
      for (ServerLoad l : loadMap.values()) {
        queues += l.getSubscriptionConnectionLoad();
        connections = (int) Math.ceil(l.getConnectionLoad() / l.getLoadPerConnection());
      }

      Set servers;
      synchronized (ownerMap) {
        servers = new HashSet<>(ownerMap.values());
      }

      StringBuilder sb = new StringBuilder(1000);
      sb.append("server count: ").append(servers.size()).append(" connected client count: ")
          .append(connections).append(" client subscription queue count: ").append(queues)
          .append(System.lineSeparator()).append("current servers : ");

      String[] ids = new String[servers.size()];
      int i = 0;
      for (DistributedMember id : servers) {
        ids[i++] = id.toString();
      }
      Arrays.sort(ids);
      for (i = 0; i < ids.length; i++) {
        sb.append(ids[i]).append(' ');
      }
      productUseLog.log(sb.toString());
    }
  }

  /**
   * Test hook to get the load on all of the servers. Returns a map of ServerLocation-> Load object
   * with the current load on that server
   */

  public Map getLoadMap() {
    return loadSnapshot.getLoadMap();
  }

  LogWriterI18n getLogWriterI18n() {
    return ds.getLogWriter().convertToLogWriterI18n();
  }

}