org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of hbase-server Show documentation
Show all versions of hbase-server Show documentation
Server functionality for HBase
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
import java.math.BigInteger;
import java.security.PrivilegedAction;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Consumer;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.AsyncConnection;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.regionserver.HRegion.BulkLoadListener;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.UserProvider;
import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier;
import org.apache.hadoop.hbase.security.token.ClientTokenUtil;
import org.apache.hadoop.hbase.security.token.FsDelegationToken;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Methods;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CleanupBulkLoadRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.PrepareBulkLoadRequest;
/**
* Bulk loads in secure mode. This service addresses two issues:
*
* - Moving files in a secure filesystem wherein the HBase Client and HBase Server are different
* filesystem users.
* - Does moving in a secure manner. Assuming that the filesystem is POSIX compliant.
*
* The algorithm is as follows:
*
* - Create an hbase owned staging directory which is world traversable (711):
* {@code /hbase/staging}
* - A user writes out data to his secure output directory: {@code /user/foo/data}
* - A call is made to hbase to create a secret staging directory which globally rwx (777):
* {@code /user/staging/averylongandrandomdirectoryname}
* - The user moves the data into the random staging directory, then calls bulkLoadHFiles()
*
* Like delegation tokens the strength of the security lies in the length and randomness of the
* secret directory.
*/
@InterfaceAudience.Private
public class SecureBulkLoadManager {
public static final long VERSION = 0L;
// 320/5 = 64 characters
private static final int RANDOM_WIDTH = 320;
private static final int RANDOM_RADIX = 32;
private static final Logger LOG = LoggerFactory.getLogger(SecureBulkLoadManager.class);
private final static FsPermission PERM_ALL_ACCESS = FsPermission.valueOf("-rwxrwxrwx");
private final static FsPermission PERM_HIDDEN = FsPermission.valueOf("-rwx--x--x");
private SecureRandom random;
private FileSystem fs;
private Configuration conf;
// two levels so it doesn't get deleted accidentally
// no sticky bit in Hadoop 1.0
private Path baseStagingDir;
private UserProvider userProvider;
private ConcurrentHashMap ugiReferenceCounter;
private AsyncConnection conn;
SecureBulkLoadManager(Configuration conf, AsyncConnection conn) {
this.conf = conf;
this.conn = conn;
}
public void start() throws IOException {
random = new SecureRandom();
userProvider = UserProvider.instantiate(conf);
ugiReferenceCounter = new ConcurrentHashMap<>();
fs = FileSystem.get(conf);
baseStagingDir = new Path(CommonFSUtils.getRootDir(conf), HConstants.BULKLOAD_STAGING_DIR_NAME);
if (conf.get("hbase.bulkload.staging.dir") != null) {
LOG.warn("hbase.bulkload.staging.dir " + " is deprecated. Bulkload staging directory is "
+ baseStagingDir);
}
if (!fs.exists(baseStagingDir)) {
fs.mkdirs(baseStagingDir, PERM_HIDDEN);
if (!PERM_HIDDEN.equals(PERM_HIDDEN.applyUMask(FsPermission.getUMask(conf)))) {
LOG.info("Modifying permissions to " + PERM_HIDDEN);
fs.setPermission(baseStagingDir, PERM_HIDDEN);
}
}
}
public void stop() throws IOException {
}
public String prepareBulkLoad(final HRegion region, final PrepareBulkLoadRequest request)
throws IOException {
User user = getActiveUser();
region.getCoprocessorHost().prePrepareBulkLoad(user);
String bulkToken =
createStagingDir(baseStagingDir, user, region.getTableDescriptor().getTableName()).toString();
return bulkToken;
}
public void cleanupBulkLoad(final HRegion region, final CleanupBulkLoadRequest request)
throws IOException {
region.getCoprocessorHost().preCleanupBulkLoad(getActiveUser());
Path path = new Path(request.getBulkToken());
if (!fs.delete(path, true)) {
if (fs.exists(path)) {
throw new IOException("Failed to clean up " + path);
}
}
LOG.trace("Cleaned up {} successfully.", path);
}
private Consumer fsCreatedListener;
void setFsCreatedListener(Consumer fsCreatedListener) {
this.fsCreatedListener = fsCreatedListener;
}
private void incrementUgiReference(UserGroupInformation ugi) {
// if we haven't seen this ugi before, make a new counter
ugiReferenceCounter.compute(ugi, (key, value) -> {
if (value == null) {
value = new MutableInt(1);
} else {
value.increment();
}
return value;
});
}
private void decrementUgiReference(UserGroupInformation ugi) {
// if the count drops below 1 we remove the entry by returning null
ugiReferenceCounter.computeIfPresent(ugi, (key, value) -> {
if (value.intValue() > 1) {
value.decrement();
} else {
value = null;
}
return value;
});
}
private boolean isUserReferenced(UserGroupInformation ugi) {
// if the ugi is in the map, based on invariants above
// the count must be above zero
return ugiReferenceCounter.containsKey(ugi);
}
public Map> secureBulkLoadHFiles(final HRegion region,
final BulkLoadHFileRequest request) throws IOException {
return secureBulkLoadHFiles(region, request, null);
}
public Map> secureBulkLoadHFiles(final HRegion region,
final BulkLoadHFileRequest request, List clusterIds) throws IOException {
final List> familyPaths = new ArrayList<>(request.getFamilyPathCount());
for (ClientProtos.BulkLoadHFileRequest.FamilyPath el : request.getFamilyPathList()) {
familyPaths.add(new Pair<>(el.getFamily().toByteArray(), el.getPath()));
}
Token userToken = null;
if (userProvider.isHadoopSecurityEnabled()) {
userToken = new Token<>(request.getFsToken().getIdentifier().toByteArray(),
request.getFsToken().getPassword().toByteArray(), new Text(request.getFsToken().getKind()),
new Text(request.getFsToken().getService()));
}
final String bulkToken = request.getBulkToken();
User user = getActiveUser();
final UserGroupInformation ugi = user.getUGI();
if (userProvider.isHadoopSecurityEnabled()) {
try {
Token tok = ClientTokenUtil.obtainToken(conn).get();
if (tok != null) {
boolean b = ugi.addToken(tok);
LOG.debug("token added " + tok + " for user " + ugi + " return=" + b);
}
} catch (Exception ioe) {
LOG.warn("unable to add token", ioe);
}
}
if (userToken != null) {
ugi.addToken(userToken);
} else if (userProvider.isHadoopSecurityEnabled()) {
// we allow this to pass through in "simple" security mode
// for mini cluster testing
throw new DoNotRetryIOException("User token cannot be null");
}
if (region.getCoprocessorHost() != null) {
region.getCoprocessorHost().preBulkLoadHFile(familyPaths);
}
Map> map = null;
try {
incrementUgiReference(ugi);
// Get the target fs (HBase region server fs) delegation token
// Since we have checked the permission via 'preBulkLoadHFile', now let's give
// the 'request user' necessary token to operate on the target fs.
// After this point the 'doAs' user will hold two tokens, one for the source fs
// ('request user'), another for the target fs (HBase region server principal).
if (userProvider.isHadoopSecurityEnabled()) {
FsDelegationToken targetfsDelegationToken = new FsDelegationToken(userProvider, "renewer");
targetfsDelegationToken.acquireDelegationToken(fs);
Token targetFsToken = targetfsDelegationToken.getUserToken();
if (
targetFsToken != null
&& (userToken == null || !targetFsToken.getService().equals(userToken.getService()))
) {
ugi.addToken(targetFsToken);
}
}
map = ugi.doAs(new PrivilegedAction>>() {
@Override
public Map> run() {
FileSystem fs = null;
try {
/*
* This is creating and caching a new FileSystem instance. Other code called "beneath"
* this method will rely on this FileSystem instance being in the cache. This is
* important as those methods make _no_ attempt to close this FileSystem instance. It is
* critical that here, in SecureBulkLoadManager, we are tracking the lifecycle and
* closing the FS when safe to do so.
*/
fs = FileSystem.get(conf);
for (Pair el : familyPaths) {
Path stageFamily = new Path(bulkToken, Bytes.toString(el.getFirst()));
if (!fs.exists(stageFamily)) {
fs.mkdirs(stageFamily);
fs.setPermission(stageFamily, PERM_ALL_ACCESS);
}
}
if (fsCreatedListener != null) {
fsCreatedListener.accept(region);
}
// We call bulkLoadHFiles as requesting user
// To enable access prior to staging
return region.bulkLoadHFiles(familyPaths, true,
new SecureBulkLoadListener(fs, bulkToken, conf), request.getCopyFile(), clusterIds,
request.getReplicate());
} catch (Exception e) {
LOG.error("Failed to complete bulk load", e);
}
return null;
}
});
} finally {
decrementUgiReference(ugi);
try {
if (!UserGroupInformation.getLoginUser().equals(ugi) && !isUserReferenced(ugi)) {
FileSystem.closeAllForUGI(ugi);
}
} catch (IOException e) {
LOG.error("Failed to close FileSystem for: {}", ugi, e);
}
if (region.getCoprocessorHost() != null) {
region.getCoprocessorHost().postBulkLoadHFile(familyPaths, map);
}
}
return map;
}
private Path createStagingDir(Path baseDir, User user, TableName tableName) throws IOException {
String tblName = tableName.getNameAsString().replace(":", "_");
String randomDir = user.getShortName() + "__" + tblName + "__"
+ (new BigInteger(RANDOM_WIDTH, random).toString(RANDOM_RADIX));
return createStagingDir(baseDir, user, randomDir);
}
private Path createStagingDir(Path baseDir, User user, String randomDir) throws IOException {
Path p = new Path(baseDir, randomDir);
fs.mkdirs(p, PERM_ALL_ACCESS);
fs.setPermission(p, PERM_ALL_ACCESS);
return p;
}
private User getActiveUser() throws IOException {
// for non-rpc handling, fallback to system user
User user = RpcServer.getRequestUser().orElse(userProvider.getCurrent());
// this is for testing
if (
userProvider.isHadoopSecurityEnabled()
&& "simple".equalsIgnoreCase(conf.get(User.HBASE_SECURITY_CONF_KEY))
) {
return User.createUserForTesting(conf, user.getShortName(), new String[] {});
}
return user;
}
// package-private for test purpose only
static class SecureBulkLoadListener implements BulkLoadListener {
// Target filesystem
private final FileSystem fs;
private final String stagingDir;
private final Configuration conf;
// Source filesystem
private FileSystem srcFs = null;
private Map origPermissions = null;
private Map origSources = null;
public SecureBulkLoadListener(FileSystem fs, String stagingDir, Configuration conf) {
this.fs = fs;
this.stagingDir = stagingDir;
this.conf = conf;
this.origPermissions = new HashMap<>();
this.origSources = new HashMap<>();
}
@Override
public String prepareBulkLoad(final byte[] family, final String srcPath, boolean copyFile,
String customStaging) throws IOException {
Path p = new Path(srcPath);
// store customStaging for failedBulkLoad
String currentStaging = stagingDir;
if (StringUtils.isNotEmpty(customStaging)) {
currentStaging = customStaging;
}
Path stageP = new Path(currentStaging, new Path(Bytes.toString(family), p.getName()));
// In case of Replication for bulk load files, hfiles are already copied in staging directory
if (p.equals(stageP)) {
LOG.debug(
p.getName() + " is already available in staging directory. Skipping copy or rename.");
return stageP.toString();
}
if (srcFs == null) {
srcFs = FileSystem.newInstance(p.toUri(), conf);
}
if (!isFile(p)) {
throw new IOException("Path does not reference a file: " + p);
}
// Check to see if the source and target filesystems are the same
if (!FSUtils.isSameHdfs(conf, srcFs, fs)) {
LOG.debug("Bulk-load file " + srcPath + " is on different filesystem than "
+ "the destination filesystem. Copying file over to destination staging dir.");
FileUtil.copy(srcFs, p, fs, stageP, false, conf);
} else if (copyFile) {
LOG.debug("Bulk-load file " + srcPath + " is copied to destination staging dir.");
FileUtil.copy(srcFs, p, fs, stageP, false, conf);
} else {
LOG.debug("Moving " + p + " to " + stageP);
FileStatus origFileStatus = fs.getFileStatus(p);
origPermissions.put(srcPath, origFileStatus.getPermission());
origSources.put(stageP.toString(), srcPath);
if (!fs.rename(p, stageP)) {
throw new IOException("Failed to move HFile: " + p + " to " + stageP);
}
}
fs.setPermission(stageP, PERM_ALL_ACCESS);
return stageP.toString();
}
@Override
public void doneBulkLoad(byte[] family, String srcPath) throws IOException {
LOG.debug("Bulk Load done for: " + srcPath);
closeSrcFs();
}
private void closeSrcFs() throws IOException {
if (srcFs != null) {
srcFs.close();
srcFs = null;
}
}
@Override
public void failedBulkLoad(final byte[] family, final String stagedPath) throws IOException {
try {
String src = origSources.get(stagedPath);
if (StringUtils.isEmpty(src)) {
LOG.debug(stagedPath + " was not moved to staging. No need to move back");
return;
}
Path stageP = new Path(stagedPath);
if (!fs.exists(stageP)) {
throw new IOException(
"Missing HFile: " + stageP + ", can't be moved back to it's original place");
}
// we should not move back files if the original exists
Path srcPath = new Path(src);
if (srcFs.exists(srcPath)) {
LOG.debug(src + " is already at it's original place. No need to move.");
return;
}
LOG.debug("Moving " + stageP + " back to " + srcPath);
if (!fs.rename(stageP, srcPath)) {
throw new IOException("Failed to move HFile: " + stageP + " to " + srcPath);
}
// restore original permission
if (origPermissions.containsKey(stagedPath)) {
fs.setPermission(srcPath, origPermissions.get(src));
} else {
LOG.warn("Can't find previous permission for path=" + stagedPath);
}
} finally {
closeSrcFs();
}
}
/**
* Check if the path is referencing a file. This is mainly needed to avoid symlinks.
* @return true if the p is a file
*/
private boolean isFile(Path p) throws IOException {
FileStatus status = srcFs.getFileStatus(p);
boolean isFile = !status.isDirectory();
try {
isFile =
isFile && !(Boolean) Methods.call(FileStatus.class, status, "isSymlink", null, null);
} catch (Exception e) {
}
return isFile;
}
}
}