package brooklyn.policy.loadbalancing;
import static brooklyn.util.GroovyJavaMethods.elvis;
import static brooklyn.util.GroovyJavaMethods.truth;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import brooklyn.config.ConfigKey;
import brooklyn.entity.Entity;
import brooklyn.entity.basic.EntityLocal;
import brooklyn.entity.basic.EntityInternal;
import brooklyn.event.AttributeSensor;
import brooklyn.event.Sensor;
import brooklyn.event.SensorEvent;
import brooklyn.event.SensorEventListener;
import brooklyn.policy.autoscaling.AutoScalerPolicy;
import brooklyn.policy.basic.AbstractPolicy;
import brooklyn.policy.loadbalancing.BalanceableWorkerPool.ContainerItemPair;
import brooklyn.util.MutableMap;
import brooklyn.util.flags.SetFromFlag;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
* Policy that is attached to a pool of "containers", each of which can host one or more migratable "items".
* The policy monitors the workrates of the items and effects migrations in an attempt to ensure that the containers
* are all sufficiently utilized without any of them being overloaded.
*
*
The particular sensor that defines the items' workrates is specified when the policy is constructed. High- and
* low-thresholds are defined as configuration keys on each of the container entities in the pool:
* for an item sensor named {@code foo.bar.sensorName}, the corresponding container config keys would be named
* {@code foo.bar.sensorName.threshold.low} and {@code foo.bar.sensorName.threshold.high}.
*
*
In addition to balancing items among the available containers, this policy causes the pool Entity to emit
* {@code POOL_COLD} and {@code POOL_HOT} events when it is determined that there is a surplus or shortfall
* of container resource in the pool respectively. These events may be consumed by a separate policy that is capable
* of resizing the container pool.
*/
public class LoadBalancingPolicy extends AbstractPolicy {
private static final Logger LOG = LoggerFactory.getLogger(LoadBalancingPolicy.class);
@SetFromFlag(defaultVal="100")
private long minPeriodBetweenExecs;
private final AttributeSensor metric;
private final String lowThresholdConfigKeyName;
private final String highThresholdConfigKeyName;
private final BalanceablePoolModel model;
private final BalancingStrategy strategy;
private BalanceableWorkerPool poolEntity;
private volatile ScheduledExecutorService executor;
private final AtomicBoolean executorQueued = new AtomicBoolean(false);
private volatile long executorTime = 0;
private int lastEmittedDesiredPoolSize = 0;
private String lastEmittedPoolTemperature = null; // "cold" or "hot"
private final SensorEventListener eventHandler = new SensorEventListener() {
public void onEvent(SensorEvent event) {
if (LOG.isTraceEnabled()) LOG.trace("{} received event {}", LoadBalancingPolicy.this, event);
Entity source = event.getSource();
Object value = event.getValue();
Sensor sensor = event.getSensor();
if (sensor.equals(metric)) {
onItemMetricUpdate((ItemType)source, ((Number) value).doubleValue(), true);
} else if (sensor.equals(BalanceableWorkerPool.CONTAINER_ADDED)) {
onContainerAdded((NodeType) value, true);
} else if (sensor.equals(BalanceableWorkerPool.CONTAINER_REMOVED)) {
onContainerRemoved((NodeType) value, true);
} else if (sensor.equals(BalanceableWorkerPool.ITEM_ADDED)) {
ContainerItemPair pair = (ContainerItemPair) value;
onItemAdded((ItemType)pair.item, (NodeType)pair.container, true);
} else if (sensor.equals(BalanceableWorkerPool.ITEM_REMOVED)) {
ContainerItemPair pair = (ContainerItemPair) value;
onItemRemoved((ItemType)pair.item, (NodeType)pair.container, true);
} else if (sensor.equals(BalanceableWorkerPool.ITEM_MOVED)) {
ContainerItemPair pair = (ContainerItemPair) value;
onItemMoved((ItemType)pair.item, (NodeType)pair.container, true);
}
}
};
public LoadBalancingPolicy(AttributeSensor metric,
BalanceablePoolModel model) {
this(MutableMap.of(), metric, model);
}
public LoadBalancingPolicy(Map props, AttributeSensor metric,
BalanceablePoolModel model) {
super(props);
this.metric = metric;
this.lowThresholdConfigKeyName = metric.getName()+".threshold.low";
this.highThresholdConfigKeyName = metric.getName()+".threshold.high";
this.model = model;
this.strategy = new BalancingStrategy(getName(), model); // TODO: extract interface, inject impl
// TODO Should re-use the execution manager's thread pool, somehow
executor = Executors.newSingleThreadScheduledExecutor(newThreadFactory());
}
@Override
public void setEntity(EntityLocal entity) {
Preconditions.checkArgument(entity instanceof BalanceableWorkerPool, "Provided entity must be a BalanceableWorkerPool");
super.setEntity(entity);
this.poolEntity = (BalanceableWorkerPool) entity;
// Detect when containers are added to or removed from the pool.
subscribe(poolEntity, BalanceableWorkerPool.CONTAINER_ADDED, eventHandler);
subscribe(poolEntity, BalanceableWorkerPool.CONTAINER_REMOVED, eventHandler);
subscribe(poolEntity, BalanceableWorkerPool.ITEM_ADDED, eventHandler);
subscribe(poolEntity, BalanceableWorkerPool.ITEM_REMOVED, eventHandler);
subscribe(poolEntity, BalanceableWorkerPool.ITEM_MOVED, eventHandler);
// Take heed of any extant containers.
for (Entity container : poolEntity.getContainerGroup().getMembers()) {
onContainerAdded((NodeType)container, false);
}
for (Entity item : poolEntity.getItemGroup().getMembers()) {
onItemAdded((ItemType)item, (NodeType)item.getAttribute(Movable.CONTAINER), false);
}
scheduleRebalance();
}
@Override
public void suspend() {
// TODO unsubscribe from everything? And resubscribe on resume?
super.suspend();
if (executor != null) executor.shutdownNow();;
executorQueued.set(false);
}
@Override
public void resume() {
super.resume();
executor = Executors.newSingleThreadScheduledExecutor(newThreadFactory());
executorTime = 0;
executorQueued.set(false);
}
private ThreadFactory newThreadFactory() {
return new ThreadFactoryBuilder()
.setNameFormat("brooklyn-followthesunpolicy-%d")
.build();
}
private void scheduleRebalance() {
if (isRunning() && executorQueued.compareAndSet(false, true)) {
long now = System.currentTimeMillis();
long delay = Math.max(0, (executorTime + minPeriodBetweenExecs) - now);
executor.schedule(new Runnable() {
public void run() {
try {
executorTime = System.currentTimeMillis();
executorQueued.set(false);
strategy.rebalance();
if (LOG.isDebugEnabled()) LOG.debug("{} post-rebalance: poolSize={}; workrate={}; lowThreshold={}; " +
"highThreshold={}", new Object[] {this, model.getPoolSize(), model.getCurrentPoolWorkrate(),
model.getPoolLowThreshold(), model.getPoolHighThreshold()});
if (model.isCold()) {
Map eventVal = ImmutableMap.of(
AutoScalerPolicy.POOL_CURRENT_SIZE_KEY, model.getPoolSize(),
AutoScalerPolicy.POOL_CURRENT_WORKRATE_KEY, model.getCurrentPoolWorkrate(),
AutoScalerPolicy.POOL_LOW_THRESHOLD_KEY, model.getPoolLowThreshold(),
AutoScalerPolicy.POOL_HIGH_THRESHOLD_KEY, model.getPoolHighThreshold());
poolEntity.emit(AutoScalerPolicy.DEFAULT_POOL_COLD_SENSOR, eventVal);
if (LOG.isInfoEnabled()) {
int desiredPoolSize = (int) Math.ceil(model.getCurrentPoolWorkrate() / (model.getPoolLowThreshold()/model.getPoolSize()));
if (desiredPoolSize != lastEmittedDesiredPoolSize || lastEmittedPoolTemperature != "cold") {
LOG.info("{} emitted COLD (suggesting {}): {}", new Object[] {this, desiredPoolSize, eventVal});
lastEmittedDesiredPoolSize = desiredPoolSize;
lastEmittedPoolTemperature = "cold";
}
}
} else if (model.isHot()) {
Map eventVal = ImmutableMap.of(
AutoScalerPolicy.POOL_CURRENT_SIZE_KEY, model.getPoolSize(),
AutoScalerPolicy.POOL_CURRENT_WORKRATE_KEY, model.getCurrentPoolWorkrate(),
AutoScalerPolicy.POOL_LOW_THRESHOLD_KEY, model.getPoolLowThreshold(),
AutoScalerPolicy.POOL_HIGH_THRESHOLD_KEY, model.getPoolHighThreshold());
poolEntity.emit(AutoScalerPolicy.DEFAULT_POOL_HOT_SENSOR, eventVal);
if (LOG.isInfoEnabled()) {
int desiredPoolSize = (int) Math.ceil(model.getCurrentPoolWorkrate() / (model.getPoolHighThreshold()/model.getPoolSize()));
if (desiredPoolSize != lastEmittedDesiredPoolSize || lastEmittedPoolTemperature != "hot") {
LOG.info("{} emitted HOT (suggesting {}): {}", new Object[] {this, desiredPoolSize, eventVal});
lastEmittedDesiredPoolSize = desiredPoolSize;
lastEmittedPoolTemperature = "hot";
}
}
}
} catch (Exception e) {
if (isRunning()) {
LOG.error("Error rebalancing", e);
} else {
LOG.debug("Error rebalancing, but no longer running", e);
}
}
}},
delay,
TimeUnit.MILLISECONDS);
}
}
// TODO Can get duplicate onContainerAdded events.
// I presume it's because we subscribe and then iterate over the extant containers.
// Solution would be for subscription to give you events for existing / current value(s).
// Also current impl messes up single-threaded updates model: the setEntity is a different thread than for subscription events.
private void onContainerAdded(NodeType newContainer, boolean rebalanceNow) {
Preconditions.checkArgument(newContainer instanceof BalanceableContainer, "Added container must be a BalanceableContainer");
if (LOG.isTraceEnabled()) LOG.trace("{} recording addition of container {}", this, newContainer);
// Low and high thresholds for the metric we're interested in are assumed to be present
// in the container's configuration.
Number lowThreshold = (Number) findConfigValue(newContainer, lowThresholdConfigKeyName);
Number highThreshold = (Number) findConfigValue(newContainer, highThresholdConfigKeyName);
if (lowThreshold == null || highThreshold == null) {
LOG.warn(
"Balanceable container '"+newContainer+"' does not define low- and high- threshold configuration keys: '"+
lowThresholdConfigKeyName+"' and '"+highThresholdConfigKeyName+"', skipping");
return;
}
model.onContainerAdded(newContainer, lowThreshold.doubleValue(), highThreshold.doubleValue());
// Note: no need to scan the container for items; they will appear via the ITEM_ADDED events.
// Also, must abide by any item-filters etc defined in the pool.
if (rebalanceNow) scheduleRebalance();
}
private static Object findConfigValue(Entity entity, String configKeyName) {
Map, Object> config = ((EntityInternal)entity).getAllConfig();
for (Entry, Object> entry : config.entrySet()) {
if (configKeyName.equals(entry.getKey().getName()))
return entry.getValue();
}
return null;
}
// TODO Receiving duplicates of onContainerRemoved (e.g. when running LoadBalancingInmemorySoakTest)
private void onContainerRemoved(NodeType oldContainer, boolean rebalanceNow) {
if (LOG.isTraceEnabled()) LOG.trace("{} recording removal of container {}", this, oldContainer);
model.onContainerRemoved(oldContainer);
if (rebalanceNow) scheduleRebalance();
}
private void onItemAdded(ItemType item, NodeType parentContainer, boolean rebalanceNow) {
Preconditions.checkArgument(item instanceof Movable, "Added item "+item+" must implement Movable");
if (LOG.isTraceEnabled()) LOG.trace("{} recording addition of item {} in container {}", new Object[] {this, item, parentContainer});
subscribe(item, metric, eventHandler);
// Update the model, including the current metric value (if any).
boolean immovable = elvis(item.getConfig(Movable.IMMOVABLE), false);
Number currentValue = item.getAttribute(metric);
model.onItemAdded(item, parentContainer, immovable);
if (currentValue != null)
model.onItemWorkrateUpdated(item, currentValue.doubleValue());
if (rebalanceNow) scheduleRebalance();
}
private void onItemRemoved(ItemType item, NodeType parentContainer, boolean rebalanceNow) {
if (LOG.isTraceEnabled()) LOG.trace("{} recording removal of item {}", this, item);
unsubscribe(item);
model.onItemRemoved(item);
if (rebalanceNow) scheduleRebalance();
}
private void onItemMoved(ItemType item, NodeType parentContainer, boolean rebalanceNow) {
if (LOG.isTraceEnabled()) LOG.trace("{} recording moving of item {} to {}", new Object[] {this, item, parentContainer});
model.onItemMoved(item, parentContainer);
if (rebalanceNow) scheduleRebalance();
}
private void onItemMetricUpdate(ItemType item, double newValue, boolean rebalanceNow) {
if (LOG.isTraceEnabled()) LOG.trace("{} recording metric update for item {}, new value {}", new Object[] {this, item, newValue});
model.onItemWorkrateUpdated(item, newValue);
if (rebalanceNow) scheduleRebalance();
}
@Override
public String toString() {
return getClass().getSimpleName() + (truth(name) ? "("+name+")" : "");
}
}
