org.apache.accumulo.server.util.Admin Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of accumulo-server-base Show documentation
Show all versions of accumulo-server-base Show documentation
A common base library for Apache Accumulo servers.
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.accumulo.server.util;
import static java.util.Objects.requireNonNull;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.NamespaceNotFoundException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.InstanceOperations;
import org.apache.accumulo.core.client.impl.ClientContext;
import org.apache.accumulo.core.client.impl.ClientExec;
import org.apache.accumulo.core.client.impl.Credentials;
import org.apache.accumulo.core.client.impl.MasterClient;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.master.thrift.MasterClientService;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.NamespacePermission;
import org.apache.accumulo.core.security.SystemPermission;
import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.core.trace.Tracer;
import org.apache.accumulo.core.util.AddressUtil;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooCache;
import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
import org.apache.accumulo.fate.zookeeper.ZooLock;
import org.apache.accumulo.server.AccumuloServerContext;
import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.server.conf.ServerConfigurationFactory;
import org.apache.accumulo.server.security.SecurityUtil;
import org.apache.accumulo.start.spi.KeywordExecutable;
import org.apache.hadoop.conf.Configuration;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.google.auto.service.AutoService;
import com.google.common.collect.Lists;
import com.google.common.net.HostAndPort;
@AutoService(KeywordExecutable.class)
public class Admin implements KeywordExecutable {
private static final Logger log = LoggerFactory.getLogger(Admin.class);
static class AdminOpts extends ClientOpts {
@Parameter(names = {"-f", "--force"}, description = "force the given server to stop by removing its lock")
boolean force = false;
}
@Parameters(commandDescription = "stop the tablet server on the given hosts")
static class StopCommand {
@Parameter(description = " { ... }")
List args = new ArrayList();
}
@Parameters(commandDescription = "Ping tablet servers. If no arguments, pings all.")
static class PingCommand {
@Parameter(description = "{ ... }")
List args = new ArrayList();
}
@Parameters(commandDescription = "print tablets that are offline in online tables")
static class CheckTabletsCommand {
@Parameter(names = "--fixFiles", description = "Remove dangling file pointers")
boolean fixFiles = false;
@Parameter(names = {"-t", "--table"}, description = "Table to check, if not set checks all tables")
String table = null;
}
@Parameters(commandDescription = "stop the master")
static class StopMasterCommand {}
@Parameters(commandDescription = "stop all the servers")
static class StopAllCommand {}
@Parameters(commandDescription = "list Accumulo instances in zookeeper")
static class ListInstancesCommand {
@Parameter(names = "--print-errors", description = "display errors while listing instances")
boolean printErrors = false;
@Parameter(names = "--print-all", description = "print information for all instances, not just those with names")
boolean printAll = false;
}
@Parameters(commandDescription = "Accumulo volume utility")
static class VolumesCommand {
@Parameter(names = {"-l", "--list"}, description = "list volumes currently in use")
boolean printErrors = false;
}
@Parameters(commandDescription = "print out non-default configuration settings")
static class DumpConfigCommand {
@Parameter(names = {"-a", "--all"}, description = "print the system and all table configurations")
boolean allConfiguration = false;
@Parameter(names = {"-d", "--directory"}, description = "directory to place config files")
String directory = null;
@Parameter(names = {"-s", "--system"}, description = "print the system configuration")
boolean systemConfiguration = false;
@Parameter(names = {"-n", "--namespaces"}, description = "print the namespace configuration")
boolean namespaceConfiguration = false;
@Parameter(names = {"-t", "--tables"}, description = "print per-table configuration")
List tables = new ArrayList();
@Parameter(names = {"-u", "--users"}, description = "print users and their authorizations and permissions")
boolean users = false;
}
@Parameters(commandDescription = "redistribute tablet directories across the current volume list")
static class RandomizeVolumesCommand {
@Parameter(names = {"-t"}, description = "table to update", required = true)
String table = null;
}
public static void main(String[] args) {
new Admin().execute(args);
}
@Override
public String keyword() {
return "admin";
}
@Override
public void execute(final String[] args) {
boolean everything;
AdminOpts opts = new AdminOpts();
JCommander cl = new JCommander(opts);
cl.setProgramName(Admin.class.getName());
CheckTabletsCommand checkTabletsCommand = new CheckTabletsCommand();
cl.addCommand("checkTablets", checkTabletsCommand);
ListInstancesCommand listIntancesOpts = new ListInstancesCommand();
cl.addCommand("listInstances", listIntancesOpts);
PingCommand pingCommand = new PingCommand();
cl.addCommand("ping", pingCommand);
DumpConfigCommand dumpConfigCommand = new DumpConfigCommand();
cl.addCommand("dumpConfig", dumpConfigCommand);
VolumesCommand volumesCommand = new VolumesCommand();
cl.addCommand("volumes", volumesCommand);
StopCommand stopOpts = new StopCommand();
cl.addCommand("stop", stopOpts);
StopAllCommand stopAllOpts = new StopAllCommand();
cl.addCommand("stopAll", stopAllOpts);
StopMasterCommand stopMasterOpts = new StopMasterCommand();
cl.addCommand("stopMaster", stopMasterOpts);
RandomizeVolumesCommand randomizeVolumesOpts = new RandomizeVolumesCommand();
cl.addCommand("randomizeVolumes", randomizeVolumesOpts);
cl.parse(args);
if (opts.help || cl.getParsedCommand() == null) {
cl.usage();
return;
}
AccumuloConfiguration siteConf = SiteConfiguration.getInstance();
// Login as the server on secure HDFS
if (siteConf.getBoolean(Property.INSTANCE_RPC_SASL_ENABLED)) {
SecurityUtil.serverLogin(siteConf);
}
Instance instance = opts.getInstance();
ServerConfigurationFactory confFactory = new ServerConfigurationFactory(instance);
try {
ClientContext context;
if (opts.getToken() == null) {
context = new AccumuloServerContext(confFactory);
} else {
final Credentials userCreds = new Credentials(opts.getPrincipal(), opts.getToken());
context = new AccumuloServerContext(confFactory) {
@Override
public synchronized Credentials getCredentials() {
return userCreds;
}
};
}
int rc = 0;
if (cl.getParsedCommand().equals("listInstances")) {
ListInstances.listInstances(instance.getZooKeepers(), listIntancesOpts.printAll, listIntancesOpts.printErrors);
} else if (cl.getParsedCommand().equals("ping")) {
if (ping(context, pingCommand.args) != 0)
rc = 4;
} else if (cl.getParsedCommand().equals("checkTablets")) {
System.out.println("\n*** Looking for offline tablets ***\n");
if (FindOfflineTablets.findOffline(context, checkTabletsCommand.table) != 0)
rc = 5;
System.out.println("\n*** Looking for missing files ***\n");
if (checkTabletsCommand.table == null) {
if (RemoveEntriesForMissingFiles.checkAllTables(context, checkTabletsCommand.fixFiles) != 0)
rc = 6;
} else {
if (RemoveEntriesForMissingFiles.checkTable(context, checkTabletsCommand.table, checkTabletsCommand.fixFiles) != 0)
rc = 6;
}
} else if (cl.getParsedCommand().equals("stop")) {
stopTabletServer(context, stopOpts.args, opts.force);
} else if (cl.getParsedCommand().equals("dumpConfig")) {
printConfig(context, dumpConfigCommand);
} else if (cl.getParsedCommand().equals("volumes")) {
ListVolumesUsed.listVolumes(context);
} else if (cl.getParsedCommand().equals("randomizeVolumes")) {
rc = RandomizeVolumes.randomize(context.getConnector(), randomizeVolumesOpts.table);
} else {
everything = cl.getParsedCommand().equals("stopAll");
if (everything)
flushAll(context);
stopServer(context, everything);
}
if (rc != 0)
System.exit(rc);
} catch (AccumuloException e) {
log.error("{}", e.getMessage(), e);
System.exit(1);
} catch (AccumuloSecurityException e) {
log.error("{}", e.getMessage(), e);
System.exit(2);
} catch (Exception e) {
log.error("{}", e.getMessage(), e);
System.exit(3);
}
}
private static int ping(ClientContext context, List args) throws AccumuloException, AccumuloSecurityException {
InstanceOperations io = context.getConnector().instanceOperations();
if (args.size() == 0) {
args = io.getTabletServers();
}
int unreachable = 0;
for (String tserver : args) {
try {
io.ping(tserver);
System.out.println(tserver + " OK");
} catch (AccumuloException ae) {
System.out.println(tserver + " FAILED (" + ae.getMessage() + ")");
unreachable++;
}
}
System.out.printf("\n%d of %d tablet servers unreachable\n\n", unreachable, args.size());
return unreachable;
}
/**
* flushing during shutdown is a performance optimization, its not required. The method will make an attempt to initiate flushes of all tables and give up if
* it takes too long.
*
*/
private static void flushAll(final ClientContext context) throws AccumuloException, AccumuloSecurityException {
final AtomicInteger flushesStarted = new AtomicInteger(0);
Runnable flushTask = new Runnable() {
@Override
public void run() {
try {
Connector conn = context.getConnector();
Set tables = conn.tableOperations().tableIdMap().keySet();
for (String table : tables) {
if (table.equals(MetadataTable.NAME))
continue;
try {
conn.tableOperations().flush(table, null, null, false);
flushesStarted.incrementAndGet();
} catch (TableNotFoundException e) {
// ignore
}
}
} catch (Exception e) {
log.warn("Failed to intiate flush {}", e.getMessage());
}
}
};
Thread flusher = new Thread(flushTask);
flusher.setDaemon(true);
flusher.start();
long start = System.currentTimeMillis();
try {
flusher.join(3000);
} catch (InterruptedException e) {
// ignore
}
while (flusher.isAlive() && System.currentTimeMillis() - start < 15000) {
int flushCount = flushesStarted.get();
try {
flusher.join(1000);
} catch (InterruptedException e) {
// ignore
}
if (flushCount == flushesStarted.get()) {
// no progress was made while waiting for join... maybe its stuck, stop waiting on it
break;
}
}
}
private static void stopServer(final ClientContext context, final boolean tabletServersToo) throws AccumuloException, AccumuloSecurityException {
MasterClient.execute(context, new ClientExec() {
@Override
public void execute(MasterClientService.Client client) throws Exception {
client.shutdown(Tracer.traceInfo(), context.rpcCreds(), tabletServersToo);
}
});
}
private static void stopTabletServer(final ClientContext context, List servers, final boolean force) throws AccumuloException,
AccumuloSecurityException {
if (context.getInstance().getMasterLocations().size() == 0) {
log.info("No masters running. Not attempting safe unload of tserver.");
return;
}
final Instance instance = context.getInstance();
final String zTServerRoot = getTServersZkPath(instance);
final ZooCache zc = new ZooCacheFactory().getZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
for (String server : servers) {
HostAndPort address = AddressUtil.parseAddress(server, context.getConfiguration().getPort(Property.TSERV_CLIENTPORT));
final String finalServer = qualifyWithZooKeeperSessionId(zTServerRoot, zc, address.toString());
log.info("Stopping server " + finalServer);
MasterClient.execute(context, new ClientExec() {
@Override
public void execute(MasterClientService.Client client) throws Exception {
client.shutdownTabletServer(Tracer.traceInfo(), context.rpcCreds(), finalServer, force);
}
});
}
}
/**
* Get the parent ZNode for tservers for the given instance
*
* @param instance
* The Instance
* @return The tservers znode for the instance
*/
static String getTServersZkPath(Instance instance) {
requireNonNull(instance);
final String instanceRoot = ZooUtil.getRoot(instance);
return instanceRoot + Constants.ZTSERVERS;
}
/**
* Look up the TabletServers in ZooKeeper and try to find a sessionID for this server reference
*
* @param hostAndPort
* The host and port for a TabletServer
* @return The host and port with the session ID in square-brackets appended, or the original value.
*/
static String qualifyWithZooKeeperSessionId(String zTServerRoot, ZooCache zooCache, String hostAndPort) {
try {
long sessionId = ZooLock.getSessionId(zooCache, zTServerRoot + "/" + hostAndPort);
if (0 == sessionId) {
return hostAndPort;
}
return hostAndPort + "[" + Long.toHexString(sessionId) + "]";
} catch (InterruptedException | KeeperException e) {
log.warn("Failed to communicate with ZooKeeper to find session ID for TabletServer.");
return hostAndPort;
}
}
private static final String ACCUMULO_SITE_BACKUP_FILE = "accumulo-site.xml.bak";
private static final String NS_FILE_SUFFIX = "_ns.cfg";
private static final String USER_FILE_SUFFIX = "_user.cfg";
private static final MessageFormat configFormat = new MessageFormat("config -t {0} -s {1}\n");
private static final MessageFormat createNsFormat = new MessageFormat("createnamespace {0}\n");
private static final MessageFormat createTableFormat = new MessageFormat("createtable {0}\n");
private static final MessageFormat createUserFormat = new MessageFormat("createuser {0}\n");
private static final MessageFormat nsConfigFormat = new MessageFormat("config -ns {0} -s {1}\n");
private static final MessageFormat sysPermFormat = new MessageFormat("grant System.{0} -s -u {1}\n");
private static final MessageFormat nsPermFormat = new MessageFormat("grant Namespace.{0} -ns {1} -u {2}\n");
private static final MessageFormat tablePermFormat = new MessageFormat("grant Table.{0} -t {1} -u {2}\n");
private static final MessageFormat userAuthsFormat = new MessageFormat("setauths -u {0} -s {1}\n");
private DefaultConfiguration defaultConfig;
private Map siteConfig, systemConfig;
private List localUsers;
public void printConfig(ClientContext context, DumpConfigCommand opts) throws Exception {
File outputDirectory = null;
if (opts.directory != null) {
outputDirectory = new File(opts.directory);
if (!outputDirectory.isDirectory()) {
throw new IllegalArgumentException(opts.directory + " does not exist on the local filesystem.");
}
if (!outputDirectory.canWrite()) {
throw new IllegalArgumentException(opts.directory + " is not writable");
}
}
Connector connector = context.getConnector();
defaultConfig = AccumuloConfiguration.getDefaultConfiguration();
siteConfig = connector.instanceOperations().getSiteConfiguration();
systemConfig = connector.instanceOperations().getSystemConfiguration();
if (opts.allConfiguration || opts.users) {
localUsers = Lists.newArrayList(connector.securityOperations().listLocalUsers());
Collections.sort(localUsers);
}
if (opts.allConfiguration) {
// print accumulo site
printSystemConfiguration(connector, outputDirectory);
// print namespaces
for (String namespace : connector.namespaceOperations().list()) {
printNameSpaceConfiguration(connector, namespace, outputDirectory);
}
// print tables
SortedSet tableNames = connector.tableOperations().list();
for (String tableName : tableNames) {
printTableConfiguration(connector, tableName, outputDirectory);
}
// print users
for (String user : localUsers) {
printUserConfiguration(connector, user, outputDirectory);
}
} else {
if (opts.systemConfiguration) {
printSystemConfiguration(connector, outputDirectory);
}
if (opts.namespaceConfiguration) {
for (String namespace : connector.namespaceOperations().list()) {
printNameSpaceConfiguration(connector, namespace, outputDirectory);
}
}
if (opts.tables.size() > 0) {
for (String tableName : opts.tables) {
printTableConfiguration(connector, tableName, outputDirectory);
}
}
if (opts.users) {
for (String user : localUsers) {
printUserConfiguration(connector, user, outputDirectory);
}
}
}
}
private String getDefaultConfigValue(String key) {
if (null == key)
return null;
String defaultValue = null;
try {
Property p = Property.getPropertyByKey(key);
if (null == p)
return defaultValue;
defaultValue = defaultConfig.get(p);
} catch (IllegalArgumentException e) {
// ignore
}
return defaultValue;
}
private void printNameSpaceConfiguration(Connector connector, String namespace, File outputDirectory) throws IOException, AccumuloException,
AccumuloSecurityException, NamespaceNotFoundException {
File namespaceScript = new File(outputDirectory, namespace + NS_FILE_SUFFIX);
FileWriter nsWriter = new FileWriter(namespaceScript);
nsWriter.write(createNsFormat.format(new String[] {namespace}));
TreeMap props = new TreeMap();
for (Entry p : connector.namespaceOperations().getProperties(namespace)) {
props.put(p.getKey(), p.getValue());
}
for (Entry entry : props.entrySet()) {
String defaultValue = getDefaultConfigValue(entry.getKey());
if (defaultValue == null || !defaultValue.equals(entry.getValue())) {
if (!entry.getValue().equals(siteConfig.get(entry.getKey())) && !entry.getValue().equals(systemConfig.get(entry.getKey()))) {
nsWriter.write(nsConfigFormat.format(new String[] {namespace, entry.getKey() + "=" + entry.getValue()}));
}
}
}
nsWriter.close();
}
private static void printUserConfiguration(Connector connector, String user, File outputDirectory) throws IOException, AccumuloException,
AccumuloSecurityException {
File userScript = new File(outputDirectory, user + USER_FILE_SUFFIX);
FileWriter userWriter = new FileWriter(userScript);
userWriter.write(createUserFormat.format(new String[] {user}));
Authorizations auths = connector.securityOperations().getUserAuthorizations(user);
userWriter.write(userAuthsFormat.format(new String[] {user, auths.toString()}));
for (SystemPermission sp : SystemPermission.values()) {
if (connector.securityOperations().hasSystemPermission(user, sp)) {
userWriter.write(sysPermFormat.format(new String[] {sp.name(), user}));
}
}
for (String namespace : connector.namespaceOperations().list()) {
for (NamespacePermission np : NamespacePermission.values()) {
if (connector.securityOperations().hasNamespacePermission(user, namespace, np)) {
userWriter.write(nsPermFormat.format(new String[] {np.name(), namespace, user}));
}
}
}
for (String tableName : connector.tableOperations().list()) {
for (TablePermission perm : TablePermission.values()) {
if (connector.securityOperations().hasTablePermission(user, tableName, perm)) {
userWriter.write(tablePermFormat.format(new String[] {perm.name(), tableName, user}));
}
}
}
userWriter.close();
}
private void printSystemConfiguration(Connector connector, File outputDirectory) throws IOException, AccumuloException, AccumuloSecurityException {
Configuration conf = new Configuration(false);
TreeMap site = new TreeMap(siteConfig);
for (Entry prop : site.entrySet()) {
String defaultValue = getDefaultConfigValue(prop.getKey());
if (!prop.getValue().equals(defaultValue) && !systemConfig.containsKey(prop.getKey())) {
conf.set(prop.getKey(), prop.getValue());
}
}
TreeMap system = new TreeMap(systemConfig);
for (Entry prop : system.entrySet()) {
String defaultValue = getDefaultConfigValue(prop.getKey());
if (!prop.getValue().equals(defaultValue)) {
conf.set(prop.getKey(), prop.getValue());
}
}
File siteBackup = new File(outputDirectory, ACCUMULO_SITE_BACKUP_FILE);
FileOutputStream fos = new FileOutputStream(siteBackup);
try {
conf.writeXml(fos);
} finally {
fos.close();
}
}
private void printTableConfiguration(Connector connector, String tableName, File outputDirectory) throws AccumuloException, TableNotFoundException,
IOException, AccumuloSecurityException {
File tableBackup = new File(outputDirectory, tableName + ".cfg");
FileWriter writer = new FileWriter(tableBackup);
writer.write(createTableFormat.format(new String[] {tableName}));
TreeMap props = new TreeMap();
for (Entry p : connector.tableOperations().getProperties(tableName)) {
props.put(p.getKey(), p.getValue());
}
for (Entry prop : props.entrySet()) {
if (prop.getKey().startsWith(Property.TABLE_PREFIX.getKey())) {
String defaultValue = getDefaultConfigValue(prop.getKey());
if (defaultValue == null || !defaultValue.equals(prop.getValue())) {
if (!prop.getValue().equals(siteConfig.get(prop.getKey())) && !prop.getValue().equals(systemConfig.get(prop.getKey()))) {
writer.write(configFormat.format(new String[] {tableName, prop.getKey() + "=" + prop.getValue()}));
}
}
}
}
writer.close();
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy