io.scalecube.cluster.membership.MembershipProtocolImpl Maven / Gradle / Ivy

Go to download
package io.scalecube.cluster.membership;

import static io.scalecube.cluster.membership.MemberStatus.ALIVE;
import static io.scalecube.cluster.membership.MemberStatus.DEAD;
import static io.scalecube.cluster.membership.MemberStatus.LEAVING;
import static io.scalecube.reactor.RetryNonSerializedEmitFailureHandler.RETRY_NON_SERIALIZED;

import io.scalecube.cluster.ClusterConfig;
import io.scalecube.cluster.ClusterMath;
import io.scalecube.cluster.Member;
import io.scalecube.cluster.TransportWrapper;
import io.scalecube.cluster.fdetector.FailureDetector;
import io.scalecube.cluster.fdetector.FailureDetectorConfig;
import io.scalecube.cluster.fdetector.FailureDetectorEvent;
import io.scalecube.cluster.gossip.GossipProtocol;
import io.scalecube.cluster.metadata.MetadataStore;
import io.scalecube.cluster.transport.api.Message;
import io.scalecube.cluster.transport.api.Transport;
import io.scalecube.net.Address;
import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import reactor.core.Disposable;
import reactor.core.Disposables;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.publisher.MonoSink;
import reactor.core.publisher.Sinks;
import reactor.core.scheduler.Scheduler;

@SuppressWarnings({"FieldCanBeLocal", "unused"})
public final class MembershipProtocolImpl implements MembershipProtocol {

  private static final Logger LOGGER = LoggerFactory.getLogger(MembershipProtocol.class);

  private enum MembershipUpdateReason {
    FAILURE_DETECTOR_EVENT,
    MEMBERSHIP_GOSSIP,
    SYNC,
    INITIAL_SYNC,
    SUSPICION_TIMEOUT
  }

  // Qualifiers

  public static final String SYNC = "sc/membership/sync";
  public static final String SYNC_ACK = "sc/membership/syncAck";
  public static final String MEMBERSHIP_GOSSIP = "sc/membership/gossip";

  private final Member localMember;

  // Injected

  private final Transport transport;
  private final MembershipConfig membershipConfig;
  private final FailureDetectorConfig failureDetectorConfig;
  private final List seedMembers;
  private final FailureDetector failureDetector;
  private final GossipProtocol gossipProtocol;
  private final MetadataStore metadataStore;
  private final TransportWrapper transportWrapper;

  // State

  private final Map membershipTable = new HashMap<>();
  private final Map members = new HashMap<>();
  private final Set aliveEmittedSet = new HashSet<>();

  // Subject

  private final Sinks.Many sink = Sinks.many().multicast().directBestEffort();

  // Disposables
  private final Disposable.Composite actionsDisposables = Disposables.composite();
  private final Disposable.Swap disposable = Disposables.swap();

  // Scheduled
  private final Scheduler scheduler;
  private final Map suspicionTimeoutTasks = new HashMap<>();

  /**
   * Creates new instantiates of cluster membership protocol with given transport and config.
   *
   * @param localMember local cluster member
   * @param transport cluster transport
   * @param failureDetector failure detector
   * @param gossipProtocol gossip protocol
   * @param metadataStore metadata store
   * @param config cluster config parameters
   * @param scheduler scheduler
   */
  public MembershipProtocolImpl(
      Member localMember,
      Transport transport,
      FailureDetector failureDetector,
      GossipProtocol gossipProtocol,
      MetadataStore metadataStore,
      ClusterConfig config,
      Scheduler scheduler) {
    this.transport = Objects.requireNonNull(transport);
    this.failureDetector = Objects.requireNonNull(failureDetector);
    this.gossipProtocol = Objects.requireNonNull(gossipProtocol);
    this.metadataStore = Objects.requireNonNull(metadataStore);
    this.localMember = Objects.requireNonNull(localMember);
    this.scheduler = Objects.requireNonNull(scheduler);
    this.membershipConfig = Objects.requireNonNull(config).membershipConfig();
    this.failureDetectorConfig = Objects.requireNonNull(config).failureDetectorConfig();
    transportWrapper = new TransportWrapper(transport);

    // Prepare seeds
    seedMembers = cleanUpSeedMembers(membershipConfig.seedMembers());

    // Init membership table with local member record
    membershipTable.put(localMember.id(), new MembershipRecord(localMember, ALIVE, 0));

    // fill in the table of members with local member
    members.put(localMember.id(), localMember);

    actionsDisposables.addAll(
        Arrays.asList(
            transport
                .listen() // Listen to incoming SYNC and SYNC ACK requests from other members
                .publishOn(scheduler)
                .subscribe(
                    this::onMessage,
                    ex -> LOGGER.error("[{}][onMessage][error] cause:", localMember, ex)),
            failureDetector
                .listen() // Listen to events from failure detector
                .publishOn(scheduler)
                .subscribe(
                    this::onFailureDetectorEvent,
                    ex ->
                        LOGGER.error(
                            "[{}][onFailureDetectorEvent][error] cause:", localMember, ex)),
            gossipProtocol
                .listen() // Listen to membership gossips
                .publishOn(scheduler)
                .subscribe(
                    this::onMembershipGossip,
                    ex ->
                        LOGGER.error("[{}][onMembershipGossip][error] cause:", localMember, ex))));
  }

  // Remove duplicates and local address(es)
  private List cleanUpSeedMembers(Collection seedMembers) {
    InetAddress localIpAddress = Address.getLocalIpAddress();

    String hostAddress = localIpAddress.getHostAddress();
    String hostName = localIpAddress.getHostName();

    Address transportAddr = transport.address();
    Address transportAddrByHostAddress = Address.create(hostAddress, transportAddr.port());
    Address transportAddrByHostName = Address.create(hostName, transportAddr.port());

    return new LinkedHashSet<>(seedMembers)
        .stream()
            .filter(addr -> checkAddressesNotEqual(addr, localMember, hostAddress, hostName))
            .filter(addr -> checkAddressesNotEqual(addr, transportAddr))
            .filter(addr -> checkAddressesNotEqual(addr, transportAddrByHostAddress))
            .filter(addr -> checkAddressesNotEqual(addr, transportAddrByHostName))
            .collect(Collectors.toList());
  }

  private boolean checkAddressesNotEqual(
      Address smAddress, Member localMember, String hostAddress, String hostName) {
    return localMember.addresses().stream()
        .allMatch(
            memberAddress ->
                checkAddressesNotEqual(smAddress, memberAddress)
                    && checkAddressesNotEqual(
                        smAddress, Address.create(hostAddress, memberAddress.port()))
                    && checkAddressesNotEqual(
                        smAddress, Address.create(hostName, memberAddress.port())));
  }

  private boolean checkAddressesNotEqual(Address address0, Address address1) {
    if (!address0.equals(address1)) {
      return true;
    } else {
      LOGGER.warn("[{}] Filtering out seed address: {}", localMember, address0);
      return false;
    }
  }

  @Override
  public Flux listen() {
    return sink.asFlux().onBackpressureBuffer();
  }

  /**
   * Updates local member incarnation number.
   *
   * @return mono handle
   */
  public Mono updateIncarnation() {
    return Mono.defer(
        () -> {
          // Update membership table
          MembershipRecord curRecord = membershipTable.get(localMember.id());
          MembershipRecord newRecord =
              new MembershipRecord(localMember, ALIVE, curRecord.incarnation() + 1);
          membershipTable.put(localMember.id(), newRecord);

          // Spread new membership record over the cluster
          return spreadMembershipGossip(newRecord);
        });
  }

  /**
   * Spreads leave notification to other cluster members.
   *
   * @return mono handle
   */
  public Mono leaveCluster() {
    return Mono.defer(
        () -> {
          MembershipRecord curRecord = membershipTable.get(localMember.id());
          MembershipRecord newRecord =
              new MembershipRecord(localMember, LEAVING, curRecord.incarnation() + 1);
          membershipTable.put(localMember.id(), newRecord);
          return spreadMembershipGossip(newRecord);
        });
  }

  @Override
  public Mono start() {
    // Make initial sync with all seed members
    return Mono.create(this::start0).then();
  }

  private void start0(MonoSink