org.apache.brooklyn.entity.nosql.cassandra.CassandraDatacenterImpl Maven / Gradle / Ivy
Show all versions of brooklyn-software-nosql Show documentation
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.brooklyn.entity.nosql.cassandra;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.annotation.Nullable;
import org.apache.brooklyn.api.entity.Entity;
import org.apache.brooklyn.api.entity.EntitySpec;
import org.apache.brooklyn.api.location.Location;
import org.apache.brooklyn.api.policy.PolicySpec;
import org.apache.brooklyn.api.sensor.AttributeSensor;
import org.apache.brooklyn.api.sensor.SensorEvent;
import org.apache.brooklyn.api.sensor.SensorEventListener;
import org.apache.brooklyn.core.effector.EffectorBody;
import org.apache.brooklyn.core.entity.Attributes;
import org.apache.brooklyn.core.entity.Entities;
import org.apache.brooklyn.core.entity.EntityPredicates;
import org.apache.brooklyn.core.entity.lifecycle.Lifecycle;
import org.apache.brooklyn.core.entity.lifecycle.ServiceStateLogic.ServiceNotUpLogic;
import org.apache.brooklyn.core.location.Machines;
import org.apache.brooklyn.enricher.stock.Enrichers;
import org.apache.brooklyn.entity.group.AbstractMembershipTrackingPolicy;
import org.apache.brooklyn.entity.group.DynamicClusterImpl;
import org.apache.brooklyn.entity.group.DynamicGroup;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.brooklyn.util.collections.MutableList;
import org.apache.brooklyn.util.collections.MutableMap;
import org.apache.brooklyn.util.collections.MutableSet;
import org.apache.brooklyn.util.core.ResourceUtils;
import org.apache.brooklyn.util.core.config.ConfigBag;
import org.apache.brooklyn.util.text.Strings;
import org.apache.brooklyn.util.time.Time;
import com.google.common.base.Objects;
import com.google.common.base.Optional;
import com.google.common.base.Supplier;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import com.google.common.net.HostAndPort;
/**
* Implementation of {@link CassandraDatacenter}.
*
* Several subtleties to note:
* - a node may take some time after it is running and serving JMX to actually be contactable on its thrift port
* (so we wait for thrift port to be contactable)
* - sometimes new nodes take a while to peer, and/or take a while to get a consistent schema
* (each up to 1m; often very close to the 1m)
*/
public class CassandraDatacenterImpl extends DynamicClusterImpl implements CassandraDatacenter {
/*
* TODO Seed management is hard!
* - The ServiceRestarter is not doing customize(), so is not refreshing the seeds in cassandra.yaml.
* If we have two nodes that were seeds for each other and they both restart at the same time, we'll have a split brain.
*/
private static final Logger log = LoggerFactory.getLogger(CassandraDatacenterImpl.class);
// Mutex for synchronizing during re-size operations
private final Object mutex = new Object[0];
private final Supplier> defaultSeedSupplier = new Supplier>() {
// Mutex for (re)calculating our seeds
// TODO is this very dangerous?! Calling out to SeedTracker, which calls out to alien getAttribute()/getConfig(). But I think that's ok.
// TODO might not need mutex? previous race was being caused by something else, other than concurrent calls!
private final Object seedMutex = new Object();
@Override
public Set get() {
synchronized (seedMutex) {
boolean hasPublishedSeeds = Boolean.TRUE.equals(getAttribute(HAS_PUBLISHED_SEEDS));
int quorumSize = getSeedQuorumSize();
Set potentialSeeds = gatherPotentialSeeds();
Set potentialRunningSeeds = gatherPotentialRunningSeeds();
boolean stillWaitingForQuorum = (!hasPublishedSeeds) && (potentialSeeds.size() < quorumSize);
if (stillWaitingForQuorum) {
if (log.isDebugEnabled()) log.debug("Not refreshed seeds of cluster {}, because still waiting for quorum (need {}; have {} potentials)", new Object[] {CassandraDatacenterImpl.class, quorumSize, potentialSeeds.size()});
return ImmutableSet.of();
} else if (hasPublishedSeeds) {
Set currentSeeds = getAttribute(CURRENT_SEEDS);
if (getAttribute(SERVICE_STATE_ACTUAL) == Lifecycle.STARTING) {
if (Sets.intersection(currentSeeds, potentialSeeds).isEmpty()) {
log.warn("Cluster {} lost all its seeds while starting! Subsequent failure likely, but changing seeds during startup would risk split-brain: seeds={}", new Object[] {CassandraDatacenterImpl.this, currentSeeds});
}
return currentSeeds;
} else if (potentialRunningSeeds.isEmpty()) {
// TODO Could be race where nodes have only just returned from start() and are about to
// transition to serviceUp; so don't just abandon all our seeds!
log.warn("Cluster {} has no running seeds (yet?); leaving seeds as-is; but risks split-brain if these seeds come back up!", new Object[] {CassandraDatacenterImpl.this});
return currentSeeds;
} else {
Set result = trim(quorumSize, potentialRunningSeeds);
log.debug("Cluster {} updating seeds: chosen={}; potentialRunning={}", new Object[] {CassandraDatacenterImpl.this, result, potentialRunningSeeds});
return result;
}
} else {
Set result = trim(quorumSize, potentialSeeds);
if (log.isDebugEnabled()) log.debug("Cluster {} has reached seed quorum: seeds={}", new Object[] {CassandraDatacenterImpl.this, result});
return result;
}
}
}
private Set trim(int num, Set contenders) {
// Prefer existing seeds wherever possible; otherwise accept any other contenders
Set currentSeeds = (getAttribute(CURRENT_SEEDS) != null) ? getAttribute(CURRENT_SEEDS) : ImmutableSet.of();
Set result = Sets.newLinkedHashSet();
result.addAll(Sets.intersection(currentSeeds, contenders));
result.addAll(contenders);
return ImmutableSet.copyOf(Iterables.limit(result, num));
}
};
protected SeedTracker seedTracker = new SeedTracker();
protected TokenGenerator tokenGenerator = null;
public CassandraDatacenterImpl() {
}
@Override
public void init() {
super.init();
/*
* subscribe to hostname, and keep an accurate set of current seeds in a sensor;
* then at nodes we set the initial seeds to be the current seeds when ready (non-empty)
*/
subscriptions().subscribeToMembers(this, Attributes.HOSTNAME, new SensorEventListener() {
@Override
public void onEvent(SensorEvent event) {
seedTracker.onHostnameChanged(event.getSource(), event.getValue());
}
});
subscriptions().subscribe(this, DynamicGroup.MEMBER_REMOVED, new SensorEventListener() {
@Override public void onEvent(SensorEvent event) {
seedTracker.onMemberRemoved(event.getValue());
}
});
subscriptions().subscribeToMembers(this, Attributes.SERVICE_UP, new SensorEventListener() {
@Override
public void onEvent(SensorEvent event) {
seedTracker.onServiceUpChanged(event.getSource(), event.getValue());
}
});
subscriptions().subscribeToMembers(this, Attributes.SERVICE_STATE_ACTUAL, new SensorEventListener() {
@Override
public void onEvent(SensorEvent event) {
// trigger a recomputation also when lifecycle state changes,
// because it might not have ruled a seed as inviable when service up went true
// because service state was not yet running
seedTracker.onServiceUpChanged(event.getSource(), Lifecycle.RUNNING==event.getValue());
}
});
// Track the datacenters for this cluster
subscriptions().subscribeToMembers(this, CassandraNode.DATACENTER_NAME, new SensorEventListener() {
@Override
public void onEvent(SensorEvent event) {
Entity member = event.getSource();
String dcName = event.getValue();
if (dcName != null) {
Multimap datacenterUsage = getAttribute(DATACENTER_USAGE);
Multimap mutableDatacenterUsage = (datacenterUsage == null) ? LinkedHashMultimap.create() : LinkedHashMultimap.create(datacenterUsage);
Optional oldDcName = getKeyOfVal(mutableDatacenterUsage, member);
if (!(oldDcName.isPresent() && dcName.equals(oldDcName.get()))) {
mutableDatacenterUsage.values().remove(member);
mutableDatacenterUsage.put(dcName, member);
sensors().set(DATACENTER_USAGE, mutableDatacenterUsage);
sensors().set(DATACENTERS, Sets.newLinkedHashSet(mutableDatacenterUsage.keySet()));
}
}
}
private Optional getKeyOfVal(Multimap map, V val) {
for (Map.Entry entry : map.entries()) {
if (Objects.equal(val, entry.getValue())) {
return Optional.of(entry.getKey());
}
}
return Optional.absent();
}
});
subscriptions().subscribe(this, DynamicGroup.MEMBER_REMOVED, new SensorEventListener() {
@Override public void onEvent(SensorEvent event) {
Entity entity = event.getSource();
Multimap datacenterUsage = getAttribute(DATACENTER_USAGE);
if (datacenterUsage != null && datacenterUsage.containsValue(entity)) {
Multimap mutableDatacenterUsage = LinkedHashMultimap.create(datacenterUsage);
mutableDatacenterUsage.values().remove(entity);
sensors().set(DATACENTER_USAGE, mutableDatacenterUsage);
sensors().set(DATACENTERS, Sets.newLinkedHashSet(mutableDatacenterUsage.keySet()));
}
}
});
getMutableEntityType().addEffector(EXECUTE_SCRIPT, new EffectorBody() {
@Override
public String call(ConfigBag parameters) {
return executeScript((String)parameters.getStringKey("commands"));
}
});
}
protected Supplier> getSeedSupplier() {
Supplier> seedSupplier = getConfig(SEED_SUPPLIER);
return (seedSupplier == null) ? defaultSeedSupplier : seedSupplier;
}
protected boolean useVnodes() {
return Boolean.TRUE.equals(getConfig(USE_VNODES));
}
protected synchronized TokenGenerator getTokenGenerator() {
if (tokenGenerator!=null)
return tokenGenerator;
try {
tokenGenerator = getConfig(TOKEN_GENERATOR_CLASS).newInstance();
BigInteger shift = getConfig(TOKEN_SHIFT);
if (shift==null)
shift = BigDecimal.valueOf(Math.random()).multiply(
new BigDecimal(tokenGenerator.range())).toBigInteger();
tokenGenerator.setOrigin(shift);
return tokenGenerator;
} catch (Exception e) {
throw Throwables.propagate(e);
}
}
protected int getSeedQuorumSize() {
Integer quorumSize = getConfig(INITIAL_QUORUM_SIZE);
if (quorumSize!=null && quorumSize>0)
return quorumSize;
// default 2 is recommended, unless initial size is smaller
return Math.min(Math.max(getConfig(INITIAL_SIZE), 1), DEFAULT_SEED_QUORUM);
}
@Override
public Set gatherPotentialSeeds() {
return seedTracker.gatherPotentialSeeds();
}
@Override
public Set gatherPotentialRunningSeeds() {
return seedTracker.gatherPotentialRunningSeeds();
}
/**
* Sets the default {@link #MEMBER_SPEC} to describe the Cassandra nodes.
*/
@Override
protected EntitySpec> getMemberSpec() {
return getConfig(MEMBER_SPEC, EntitySpec.create(CassandraNode.class));
}
@Override
public String getClusterName() {
return getAttribute(CLUSTER_NAME);
}
@Override
public Collection grow(int delta) {
if (useVnodes()) {
// nothing to do for token generator
} else {
if (getCurrentSize() == 0) {
getTokenGenerator().growingCluster(delta);
}
}
return super.grow(delta);
}
@Override
protected Entity createNode(@Nullable Location loc, Map,?> flags) {
Map