org.apache.accumulo.server.master.balancer.HostRegexTableLoadBalancer Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of accumulo-server-base Show documentation
Show all versions of accumulo-server-base Show documentation
A common base library for Apache Accumulo servers.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.accumulo.server.master.balancer;
import static java.util.concurrent.TimeUnit.HOURS;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.accumulo.core.client.admin.TableOperations;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.AccumuloConfiguration.Deriver;
import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.master.thrift.TableInfo;
import org.apache.accumulo.core.master.thrift.TabletServerStatus;
import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.master.state.TabletMigration;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
/**
* This balancer creates groups of tablet servers using user-provided regular expressions over the
* tablet server hostnames. Then it delegates to the table balancer to balance the tablets within
* the resulting group of tablet servers. All tablet servers that do not match a regex are grouped
* into a default group.
* Regex properties for this balancer are specified as:
* table.custom.balancer.host.regex.<tablename>=<regex>
* Periodically (default 5m) this balancer will check to see if a tablet server is hosting tablets
* that it should not be according to the regex configuration. If this occurs then the offending
* tablets will be reassigned. This would cover the case where the configuration is changed and the
* manager is restarted while the tablet servers are up. To change the out of bounds check time
* period, set the following property:
* table.custom.balancer.host.regex.oob.period=5m
* Regex matching can be based on either the host name (default) or host ip address. To set this
* balancer to match the regular expressions to the tablet server IP address, then set the following
* property:
* table.custom.balancer.host.regex.is.ip=true
* It's possible that this balancer may create a lot of migrations. To limit the number of
* migrations that are created during a balance call, set the following property (default 250):
* table.custom.balancer.host.regex.concurrent.migrations This balancer can continue
* balancing even if there are outstanding migrations. To limit the number of outstanding migrations
* in which this balancer will continue balancing, set the following property (default 0):
* table.custom.balancer.host.regex.max.outstanding.migrations
*
* @deprecated since 2.1.0. Use
* {@link org.apache.accumulo.core.spi.balancer.HostRegexTableLoadBalancer} instead
*/
@Deprecated(since = "2.1.0")
public class HostRegexTableLoadBalancer extends TableLoadBalancer {
private static final SecureRandom random = new SecureRandom();
private static final String PROP_PREFIX = Property.TABLE_ARBITRARY_PROP_PREFIX.getKey();
private static final Logger LOG = LoggerFactory.getLogger(HostRegexTableLoadBalancer.class);
public static final String HOST_BALANCER_PREFIX = PROP_PREFIX + "balancer.host.regex.";
public static final String HOST_BALANCER_OOB_CHECK_KEY =
PROP_PREFIX + "balancer.host.regex.oob.period";
private static final String HOST_BALANCER_OOB_DEFAULT = "5m";
public static final String HOST_BALANCER_REGEX_USING_IPS_KEY =
PROP_PREFIX + "balancer.host.regex.is.ip";
public static final String HOST_BALANCER_REGEX_MAX_MIGRATIONS_KEY =
PROP_PREFIX + "balancer.host.regex.concurrent.migrations";
private static final int HOST_BALANCER_REGEX_MAX_MIGRATIONS_DEFAULT = 250;
protected static final String DEFAULT_POOL = "HostTableLoadBalancer.ALL";
private static final int DEFAULT_OUTSTANDING_MIGRATIONS = 0;
public static final String HOST_BALANCER_OUTSTANDING_MIGRATIONS_KEY =
PROP_PREFIX + "balancer.host.regex.max.outstanding.migrations";
private static Map getRegexes(AccumuloConfiguration aconf) {
Map regexes = new HashMap<>();
Map customProps =
aconf.getAllPropertiesWithPrefix(Property.TABLE_ARBITRARY_PROP_PREFIX);
if (customProps != null && !customProps.isEmpty()) {
for (Entry customProp : customProps.entrySet()) {
if (customProp.getKey().startsWith(HOST_BALANCER_PREFIX)) {
if (customProp.getKey().equals(HOST_BALANCER_OOB_CHECK_KEY)
|| customProp.getKey().equals(HOST_BALANCER_REGEX_USING_IPS_KEY)
|| customProp.getKey().equals(HOST_BALANCER_REGEX_MAX_MIGRATIONS_KEY)
|| customProp.getKey().equals(HOST_BALANCER_OUTSTANDING_MIGRATIONS_KEY)) {
continue;
}
String tableName = customProp.getKey().substring(HOST_BALANCER_PREFIX.length());
String regex = customProp.getValue();
regexes.put(tableName, regex);
}
}
}
return Map.copyOf(regexes);
}
/**
* Host Regex Table Load Balance Config
*/
static class HrtlbConf {
protected long oobCheckMillis =
ConfigurationTypeHelper.getTimeInMillis(HOST_BALANCER_OOB_DEFAULT);
private int maxTServerMigrations = HOST_BALANCER_REGEX_MAX_MIGRATIONS_DEFAULT;
private int maxOutstandingMigrations = DEFAULT_OUTSTANDING_MIGRATIONS;
private boolean isIpBasedRegex = false;
private Map regexes;
private Map poolNameToRegexPattern = null;
HrtlbConf(AccumuloConfiguration aconf) {
System.out.println("building hrtlb conf");
String oobProperty = aconf.get(HOST_BALANCER_OOB_CHECK_KEY);
if (oobProperty != null) {
oobCheckMillis = ConfigurationTypeHelper.getTimeInMillis(oobProperty);
}
String ipBased = aconf.get(HOST_BALANCER_REGEX_USING_IPS_KEY);
if (ipBased != null) {
isIpBasedRegex = Boolean.parseBoolean(ipBased);
}
String migrations = aconf.get(HOST_BALANCER_REGEX_MAX_MIGRATIONS_KEY);
if (migrations != null) {
maxTServerMigrations = Integer.parseInt(migrations);
}
String outstanding = aconf.get(HOST_BALANCER_OUTSTANDING_MIGRATIONS_KEY);
if (outstanding != null) {
maxOutstandingMigrations = Integer.parseInt(outstanding);
}
this.regexes = getRegexes(aconf);
Map poolNameToRegexPatternBuilder = new HashMap<>();
regexes.forEach((k, v) -> {
poolNameToRegexPatternBuilder.put(k, Pattern.compile(v));
});
poolNameToRegexPattern = Map.copyOf(poolNameToRegexPatternBuilder);
}
}
private static final Set EMPTY_MIGRATIONS = Collections.emptySet();
private volatile long lastOOBCheck = System.currentTimeMillis();
private Map> pools = new HashMap<>();
private final Map migrationsFromLastPass = new HashMap<>();
private final Map tableToTimeSinceNoMigrations = new HashMap<>();
private Deriver hrtlbConf;
private LoadingCache>> tablesRegExCache;
/**
* Group the set of current tservers by pool name. Tservers that don't match a regex are put into
* a default pool. This could be expensive in the terms of the amount of time to recompute the
* groups, so HOST_BALANCER_POOL_RECHECK_KEY should be specified in the terms of minutes, not
* seconds or less.
*
* @param current map of current tservers
* @return current servers grouped by pool name, if not a match it is put into a default pool.
*/
protected synchronized Map>
splitCurrentByRegex(SortedMap current) {
LOG.debug("Performing pool recheck - regrouping tablet servers based on regular expressions");
Map> newPools = new HashMap<>();
for (Entry e : current.entrySet()) {
List poolNames = getPoolNamesForHost(e.getKey().getHost());
for (String pool : poolNames) {
SortedMap np = newPools.get(pool);
if (np == null) {
np = new TreeMap<>(current.comparator());
newPools.put(pool, np);
}
np.put(e.getKey(), e.getValue());
}
}
if (newPools.get(DEFAULT_POOL) == null) {
LOG.warn("Default pool is empty; assigning all tablet servers to the default pool");
SortedMap dp = new TreeMap<>(current.comparator());
dp.putAll(current);
newPools.put(DEFAULT_POOL, dp);
}
pools = newPools;
LOG.trace("Pool to TabletServer mapping:");
if (LOG.isTraceEnabled()) {
for (Entry> e : pools.entrySet()) {
LOG.trace("\tpool: {} -> tservers: {}", e.getKey(), e.getValue().keySet());
}
}
return pools;
}
/**
* Matches host against the regexes and returns the matching pool names
*
* @param host tablet server host
* @return pool names, will return default pool if host matches more no regex
*/
protected List getPoolNamesForHost(String host) {
String test = host;
if (!hrtlbConf.derive().isIpBasedRegex) {
try {
test = getNameFromIp(host);
} catch (UnknownHostException e1) {
LOG.error("Unable to determine host name for IP: " + host + ", setting to default pool",
e1);
return Collections.singletonList(DEFAULT_POOL);
}
}
List pools = new ArrayList<>();
for (Entry e : hrtlbConf.derive().poolNameToRegexPattern.entrySet()) {
if (e.getValue().matcher(test).matches()) {
pools.add(e.getKey());
}
}
if (pools.isEmpty()) {
pools.add(DEFAULT_POOL);
}
return pools;
}
protected String getNameFromIp(String hostIp) throws UnknownHostException {
return InetAddress.getByName(hostIp).getHostName();
}
private void checkTableConfig(TableId tableId) {
Map tableRegexes = tablesRegExCache.getUnchecked(tableId).derive();
if (!hrtlbConf.derive().regexes.equals(tableRegexes)) {
LoggerFactory.getLogger(HostRegexTableLoadBalancer.class).warn(
"Table id {} has different config than system. The per table config is ignored.",
tableId);
}
}
private Map createdTableNameMap(Map tableIdMap) {
HashMap tableNameMap = new HashMap<>();
tableIdMap.forEach((tableName, tableId) -> {
tableNameMap.put(TableId.of(tableId), tableName);
});
return tableNameMap;
}
/**
* Matches table name against pool names, returns matching pool name or DEFAULT_POOL.
*
* @param tableName name of table
* @return tablet server pool name (table name or DEFAULT_POOL)
*/
protected String getPoolNameForTable(String tableName) {
if (tableName == null) {
return DEFAULT_POOL;
}
return hrtlbConf.derive().poolNameToRegexPattern.containsKey(tableName) ? tableName
: DEFAULT_POOL;
}
@Override
public String toString() {
HrtlbConf myConf = hrtlbConf.derive();
ToStringBuilder buf = new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE);
buf.append("\nTablet Out Of Bounds Check Interval", myConf.oobCheckMillis);
buf.append("\nMax Tablet Server Migrations", myConf.maxTServerMigrations);
buf.append("\nRegular Expressions use IPs", myConf.isIpBasedRegex);
buf.append("\nPools", myConf.poolNameToRegexPattern);
return buf.toString();
}
public Map getPoolNameToRegexPattern() {
return hrtlbConf.derive().poolNameToRegexPattern;
}
public int getMaxMigrations() {
return hrtlbConf.derive().maxTServerMigrations;
}
public int getMaxOutstandingMigrations() {
return hrtlbConf.derive().maxOutstandingMigrations;
}
public long getOobCheckMillis() {
return hrtlbConf.derive().oobCheckMillis;
}
public boolean isIpBasedRegex() {
return hrtlbConf.derive().isIpBasedRegex;
}
@Override
public void init(ServerContext context) {
super.init(context);
this.hrtlbConf = context.getConfiguration().newDeriver(HrtlbConf::new);
tablesRegExCache =
CacheBuilder.newBuilder().expireAfterAccess(1, HOURS).build(new CacheLoader<>() {
@Override
public Deriver> load(TableId key) throws Exception {
return context.getTableConfiguration(key)
.newDeriver(HostRegexTableLoadBalancer::getRegexes);
}
});
LOG.info("{}", this);
}
@Override
public void getAssignments(SortedMap current,
Map unassigned, Map assignments) {
Map> pools = splitCurrentByRegex(current);
// group the unassigned into tables
Map> groupedUnassigned = new HashMap<>();
unassigned.forEach((ke, lastTserver) -> groupedUnassigned
.computeIfAbsent(ke.tableId(), k -> new HashMap<>()).put(ke, lastTserver));
Map tableIdToTableName = createdTableNameMap(getTableOperations().tableIdMap());
// Send a view of the current servers to the tables tablet balancer
for (Entry> e : groupedUnassigned.entrySet()) {
Map newAssignments = new HashMap<>();
String tableName = tableIdToTableName.get(e.getKey());
String poolName = getPoolNameForTable(tableName);
SortedMap currentView = pools.get(poolName);
if (currentView == null || currentView.isEmpty()) {
LOG.warn("No tablet servers online for table {}, assigning within default pool", tableName);
currentView = pools.get(DEFAULT_POOL);
if (currentView == null) {
LOG.error(
"No tablet servers exist in the default pool, unable to assign tablets for table {}",
tableName);
continue;
}
}
LOG.debug("Sending {} tablets to balancer for table {} for assignment within tservers {}",
e.getValue().size(), tableName, currentView.keySet());
getBalancerForTable(e.getKey()).getAssignments(currentView, e.getValue(), newAssignments);
assignments.putAll(newAssignments);
}
}
@Override
public long balance(SortedMap current,
Set migrations, List migrationsOut) {
long minBalanceTime = 20_000;
// Iterate over the tables and balance each of them
TableOperations t = getTableOperations();
if (t == null) {
return minBalanceTime;
}
Map tableIdMap = t.tableIdMap();
Map tableIdToTableName = createdTableNameMap(tableIdMap);
tableIdToTableName.keySet().forEach(this::checkTableConfig);
long now = System.currentTimeMillis();
HrtlbConf myConf = hrtlbConf.derive();
Map> currentGrouped =
splitCurrentByRegex(current);
if ((now - this.lastOOBCheck) > myConf.oobCheckMillis) {
try {
// Check to see if a tablet is assigned outside the bounds of the pool. If so, migrate it.
for (String table : tableIdMap.keySet()) {
LOG.debug("Checking for out of bounds tablets for table {}", table);
String tablePoolName = getPoolNameForTable(table);
for (Entry e : current.entrySet()) {
// pool names are the same as table names, except in the DEFAULT case.
// If this table is assigned to a pool for this host, then move on.
List hostPools = getPoolNamesForHost(e.getKey().getHost());
if (hostPools.contains(tablePoolName)) {
continue;
}
String tid = tableIdMap.get(table);
if (tid == null) {
LOG.warn("Unable to check for out of bounds tablets for table {},"
+ " it may have been deleted or renamed.", table);
continue;
}
try {
List outOfBoundsTablets =
getOnlineTabletsForTable(e.getKey(), TableId.of(tid));
if (outOfBoundsTablets == null) {
continue;
}
for (TabletStats ts : outOfBoundsTablets) {
KeyExtent ke = KeyExtent.fromThrift(ts.getExtent());
if (migrations.contains(ke)) {
LOG.debug("Migration for out of bounds tablet {} has already been requested", ke);
continue;
}
String poolName = getPoolNameForTable(table);
SortedMap currentView =
currentGrouped.get(poolName);
if (currentView != null) {
int skip = random.nextInt(currentView.size());
Iterator iter = currentView.keySet().iterator();
for (int i = 0; i < skip; i++) {
iter.next();
}
TServerInstance nextTS = iter.next();
LOG.info("Tablet {} is currently outside the bounds of the"
+ " regex, migrating from {} to {}", ke, e.getKey(), nextTS);
migrationsOut.add(new TabletMigration(ke, e.getKey(), nextTS));
if (migrationsOut.size() >= myConf.maxTServerMigrations) {
break;
}
} else {
LOG.warn("No tablet servers online for pool {}, unable to"
+ " migrate out of bounds tablets", poolName);
}
}
} catch (TException e1) {
LOG.error("Error in OOB check getting tablets for table {} from server {} {}", tid,
e.getKey().getHost(), e);
}
}
}
} finally {
// this could have taken a while...get a new time
this.lastOOBCheck = System.currentTimeMillis();
}
}
if (!migrationsOut.isEmpty()) {
LOG.warn("Not balancing tables due to moving {} out of bounds tablets", migrationsOut.size());
LOG.info("Migrating out of bounds tablets: {}", migrationsOut);
return minBalanceTime;
}
if (migrations != null && !migrations.isEmpty()) {
if (migrations.size() >= myConf.maxOutstandingMigrations) {
LOG.warn("Not balancing tables due to {} outstanding migrations", migrations.size());
if (LOG.isTraceEnabled()) {
LOG.trace("Sample up to 10 outstanding migrations: {}", limitTen(migrations));
}
return minBalanceTime;
}
LOG.debug("Current outstanding migrations of {} being applied", migrations.size());
if (LOG.isTraceEnabled()) {
LOG.trace("Sample up to 10 outstanding migrations: {}", limitTen(migrations));
}
migrationsFromLastPass.keySet().retainAll(migrations);
SortedMap currentCopy = new TreeMap<>(current);
Multimap serverTableIdCopied = HashMultimap.create();
for (TabletMigration migration : migrationsFromLastPass.values()) {
TableInfo fromInfo = getTableInfo(currentCopy, serverTableIdCopied,
migration.tablet.tableId().toString(), migration.oldServer);
if (fromInfo != null) {
fromInfo.setOnlineTablets(fromInfo.getOnlineTablets() - 1);
}
TableInfo toInfo = getTableInfo(currentCopy, serverTableIdCopied,
migration.tablet.tableId().toString(), migration.newServer);
if (toInfo != null) {
toInfo.setOnlineTablets(toInfo.getOnlineTablets() + 1);
}
}
migrations = EMPTY_MIGRATIONS;
} else {
migrationsFromLastPass.clear();
}
for (String s : tableIdMap.values()) {
TableId tableId = TableId.of(s);
String tableName = tableIdToTableName.get(tableId);
String regexTableName = getPoolNameForTable(tableName);
SortedMap currentView =
currentGrouped.get(regexTableName);
if (currentView == null) {
LOG.warn("Skipping balance for table {} as no tablet servers are online.", tableName);
continue;
}
ArrayList newMigrations = new ArrayList<>();
getBalancerForTable(tableId).balance(currentView, migrations, newMigrations);
if (newMigrations.isEmpty()) {
tableToTimeSinceNoMigrations.remove(s);
} else if (tableToTimeSinceNoMigrations.containsKey(s)) {
if ((now - tableToTimeSinceNoMigrations.get(s)) > HOURS.toMillis(1)) {
LOG.warn("We have been consistently producing migrations for {}: {}", tableName,
limitTen(newMigrations));
}
} else {
tableToTimeSinceNoMigrations.put(s, now);
}
migrationsOut.addAll(newMigrations);
if (migrationsOut.size() >= myConf.maxTServerMigrations) {
break;
}
}
for (TabletMigration migration : migrationsOut) {
migrationsFromLastPass.put(migration.tablet, migration);
}
LOG.info("Migrating tablets for balance: {}", migrationsOut);
return minBalanceTime;
}
/**
* Get a mutable table info for the specified table and server
*/
private TableInfo getTableInfo(SortedMap currentCopy,
Multimap serverTableIdCopied, String tableId,
TServerInstance server) {
TableInfo newInfo = null;
if (currentCopy.containsKey(server)) {
Map newTableMap = currentCopy.get(server).getTableMap();
if (newTableMap != null) {
newInfo = newTableMap.get(tableId);
if (newInfo != null) {
Collection tableIdCopied = serverTableIdCopied.get(server);
if (tableIdCopied.isEmpty()) {
newTableMap = new HashMap<>(newTableMap);
currentCopy.get(server).setTableMap(newTableMap);
}
if (!tableIdCopied.contains(tableId)) {
newInfo = new TableInfo(newInfo);
newTableMap.put(tableId, newInfo);
tableIdCopied.add(tableId);
}
}
}
}
return newInfo;
}
// helper to prepare log messages
private static String limitTen(Collection iterable) {
return iterable.stream().limit(10).map(String::valueOf)
.collect(Collectors.joining(", ", "[", "]"));
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy