org.apache.hadoop.fs.cosn.ranger.client.RangerQcloudObjectStorageClientImpl Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of hadoop-ranger-client-for-impala Show documentation
Show all versions of hadoop-ranger-client-for-impala Show documentation
Tencent Qcloud chdfs hadoop ranger client.
package org.apache.hadoop.fs.cosn.ranger.client;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.cosn.ranger.constants.ObjectStorageConstants;
import org.apache.hadoop.fs.cosn.ranger.protocol.ClientQcloudObjectStorageProtocol;
import org.apache.hadoop.fs.cosn.ranger.protocolpb.ClientQcloudObjectStorageProtocolPB;
import org.apache.hadoop.fs.cosn.ranger.protocolpb.client.ClientQcloudObjectStorageProtocolTranslatorPB;
import org.apache.hadoop.fs.cosn.ranger.security.authorization.PermissionRequest;
import org.apache.hadoop.fs.cosn.ranger.security.authorization.PermissionResponse;
import org.apache.hadoop.fs.cosn.ranger.security.authorization.RangerAuthPolicyResponse;
import org.apache.hadoop.fs.cosn.ranger.security.node.CosRangerNodeResponse;
import org.apache.hadoop.fs.cosn.ranger.security.sts.GetSTSRequest;
import org.apache.hadoop.fs.cosn.ranger.security.sts.GetSTSResponse;
import org.apache.hadoop.fs.cosn.ranger.security.token.DelegationTokenIdentifier;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.Arrays;
import java.util.Collection;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import static org.apache.hadoop.fs.cosn.ranger.constants.ObjectStorageConstants.DEFAULT_QCLOUD_OBJECT_STORAGE_ONLY_SEND_LEADER;
import static org.apache.hadoop.fs.cosn.ranger.constants.ObjectStorageConstants.DEFAULT_QCLOUD_OBJECT_STORAGE_RANGER_CLIENT_RETRY_MAX;
import static org.apache.hadoop.fs.cosn.ranger.constants.ObjectStorageConstants.QCLOUD_OBJECT_STORAGE_ONLY_SEND_LEADER_KEY;
import static org.apache.hadoop.fs.cosn.ranger.constants.ObjectStorageConstants.QCLOUD_OBJECT_STORAGE_RANGER_CLIENT_RETRY_MAX_KEY;
public class RangerQcloudObjectStorageClientImpl implements RangerQcloudObjectStorageClient {
public static final RangerQcloudObjectStorageClientImpl INSTANCE = new RangerQcloudObjectStorageClientImpl();
private static final Logger log = LoggerFactory.getLogger(RangerQcloudObjectStorageClientImpl.class);
private Configuration conf;
private String serviceName;
private AtomicReference cosRangerServiceLeaderIpAddr = new AtomicReference<>();
private ConcurrentHashMap allCosRangerServiceAddrMap = new ConcurrentHashMap<>();
private AtomicInteger cosRangerServiceAddrIndex = new AtomicInteger(0);
public RangerQcloudObjectStorageClientImpl() {
}
private ClientQcloudObjectStorageProtocol buildClient(InetSocketAddress targetAddr) throws IOException {
if (targetAddr == null) {
return null;
}
log.debug("build client for targetAddr: {}", targetAddr);
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
final long version = RPC.getProtocolVersion(ClientQcloudObjectStorageProtocolPB.class);
ClientQcloudObjectStorageProtocolPB proxy = RPC.getProtocolProxy(ClientQcloudObjectStorageProtocolPB.class,
version, targetAddr, ugi, conf, NetUtils.getDefaultSocketFactory(conf),
org.apache.hadoop.ipc.Client.getTimeout(conf), null, new AtomicBoolean(false)).getProxy();
return new ClientQcloudObjectStorageProtocolTranslatorPB(proxy);
}
private ClientQcloudObjectStorageProtocol getAnyMemberClient(int startClientIndex, int retryCnt)
throws IOException {
waitCosRangerServiceMemberInit();
Collection allAddrCollection = this.allCosRangerServiceAddrMap.keySet();
if (allAddrCollection.isEmpty()) {
return getLeaderClient();
}
InetSocketAddress[] allAddrArray = allAddrCollection.toArray(new InetSocketAddress[0]);
if (allAddrArray.length == 0) {
return getLeaderClient();
}
int addrIndex = (startClientIndex + retryCnt) % allAddrArray.length;
return buildClient(allAddrArray[addrIndex]);
}
private ClientQcloudObjectStorageProtocol getLeaderClient() throws IOException {
waitCosRangerServiceLeaderInit();
InetSocketAddress leaderAddrStr = this.cosRangerServiceLeaderIpAddr.get();
if (leaderAddrStr == null) {
// 主动查询一次
try {
updateCosRangerServiceAddr();
leaderAddrStr = this.cosRangerServiceLeaderIpAddr.get();
} catch (Exception e) {
throw new IOException(String.format("get data from cache for %s failed", leaderAddrStr.getHostString()), e);
}
}
return buildClient(leaderAddrStr);
}
private void waitCosRangerServiceMemberInit() {
if (!this.allCosRangerServiceAddrMap.isEmpty()) {
return;
}
synchronized (this.allCosRangerServiceAddrMap) {
if (!this.allCosRangerServiceAddrMap.isEmpty()) {
return;
}
try {
this.allCosRangerServiceAddrMap.wait(3000L);
} catch (InterruptedException e) {
}
}
}
private void waitCosRangerServiceLeaderInit() {
if (this.cosRangerServiceLeaderIpAddr.get() != null) {
return;
}
synchronized (this.cosRangerServiceLeaderIpAddr) {
if (this.cosRangerServiceLeaderIpAddr.get() != null) {
return;
}
try {
this.cosRangerServiceLeaderIpAddr.wait(3000L);
} catch (InterruptedException e) {
}
}
}
private ClientQcloudObjectStorageProtocol getRandomClient(int startClientIndex, int retryCnt) throws IOException {
// if use kerberos, we should always use leader addr
if (UserGroupInformation.isSecurityEnabled()) {
return getLeaderClient();
}
// if config set only leader
if (this.conf.getBoolean(QCLOUD_OBJECT_STORAGE_ONLY_SEND_LEADER_KEY,
DEFAULT_QCLOUD_OBJECT_STORAGE_ONLY_SEND_LEADER)) {
return getLeaderClient();
}
return getAnyMemberClient(startClientIndex, retryCnt);
}
private boolean backOff(int retryCnt) {
final int maxRetryCnt = this.conf.getInt(QCLOUD_OBJECT_STORAGE_RANGER_CLIENT_RETRY_MAX_KEY,
DEFAULT_QCLOUD_OBJECT_STORAGE_RANGER_CLIENT_RETRY_MAX);
log.warn("backOff, retryCnt: {}, maxRetry: {}, maxRetry: {}", retryCnt, maxRetryCnt);
if (retryCnt > maxRetryCnt) {
return false;
}
try {
Thread.sleep(ThreadLocalRandom.current().nextInt(500, 2000));
} catch (InterruptedException ignore) {
}
return true;
}
@Override
public String getCanonicalServiceName() {
return this.serviceName;
}
private int getCosRangerServiceStartIndex() {
int startIndex = this.cosRangerServiceAddrIndex.incrementAndGet();
// if over MAX
if (startIndex < 0) {
this.cosRangerServiceAddrIndex.set(0);
return 0;
}
return startIndex;
}
public synchronized void init(Configuration conf) throws IOException {
this.conf = conf;
RPC.setProtocolEngine(conf, ClientQcloudObjectStorageProtocolPB.class, ProtobufRpcEngine.class);
initServiceName();
updateCosRangerServiceAddr();
}
private void initServiceName() throws IOException {
String address = this.conf.get(ObjectStorageConstants.QCLOUD_OBJECT_STORAGE_RANGER_SERVICE_NAME_KEY,
ObjectStorageConstants.DEFAULT_QCLOUD_OBJECT_STORAGE_RANGER_SERVICE_NAME);
InetSocketAddress addr = NetUtils.createSocketAddr(address,
ObjectStorageConstants.QCLOUD_OBJECT_STORAGE_RPC_PORT_DEFAULT);
String host = addr.getAddress().getHostAddress();
this.serviceName = host + ":" + addr.getPort();
}
private void updateCosRangerServiceAddr() throws IOException {
String address = this.conf.get(ObjectStorageConstants.QCLOUD_OBJECT_STORANGE_RANGER_SERVICE_ADDRESS);
if (address == null || address.isEmpty()) {
throw new IOException(String.format("missing configuration for %s",
ObjectStorageConstants.QCLOUD_OBJECT_STORANGE_RANGER_SERVICE_ADDRESS));
}
String[] allCosRangerAddrs = address.split(",");
if (allCosRangerAddrs.length == 0) {
throw new IOException(String.format("wrong configuration for %s",
ObjectStorageConstants.QCLOUD_OBJECT_STORANGE_RANGER_SERVICE_ADDRESS));
}
int startIndex = getCosRangerServiceStartIndex();
int retryCnt = 0;
while (true) {
int addrIndex = (startIndex + retryCnt) % allCosRangerAddrs.length;
String cosRangerAdd = allCosRangerAddrs[addrIndex];
InetSocketAddress leaderIpAddr = NetUtils.createSocketAddr(cosRangerAdd);
try {
ClientQcloudObjectStorageProtocol rangerClient = buildClient(leaderIpAddr);
if (rangerClient == null) {
throw new IOException(
"ranger client is null, maybe ranger server for qcloud object storage is not deployed!");
}
CosRangerNodeResponse cosResp = rangerClient.getAvailableService();
if (cosResp == null) {
throw new IOException(
"cos ranger service is null, maybe ranger server for qcloud object storage is not " +
"deployed!");
}
this.cosRangerServiceLeaderIpAddr.set(NetUtils.createSocketAddr(cosResp.getCosRangerLeader()));
synchronized (this.cosRangerServiceLeaderIpAddr) {
this.cosRangerServiceLeaderIpAddr.notifyAll();
}
allCosRangerServiceAddrMap.clear();
for (String ip : cosResp.getAllCosRangers()) {
InetSocketAddress newAddAddrIp = NetUtils.createSocketAddr(
ip.substring(ip.lastIndexOf("/") + 1));
allCosRangerServiceAddrMap.putIfAbsent(newAddAddrIp, Boolean.TRUE);
}
synchronized (this.allCosRangerServiceAddrMap) {
this.allCosRangerServiceAddrMap.notifyAll();
}
log.info(String.format("cos ranger service leader address: %s, all cos ranger services: %s",
cosResp.getCosRangerLeader(),
Arrays.toString(cosResp.getAllCosRangers().toArray(new String[0]))));
return;
} catch (IOException e) {
if (!backOff(retryCnt)) {
log.error("get cos ranger service address failed", e);
throw e;
}
retryCnt++;
} catch (Exception e) {
log.error("get cos ranger service address failed", e);
throw e;
}
}
}
/**
* get delegation token
*
* @param renwer
* @return new token
* @throws IOException
*/
public Token getDelegationToken(String renwer) throws IOException {
int retryCnt = 0;
while (true) {
try {
ClientQcloudObjectStorageProtocol rangerClient = getLeaderClient();
if (rangerClient == null) {
throw new IOException(
"ranger client is null, maybe ranger server for qcloud object storage is not deployed!");
}
return rangerClient.getDelegationToken(new Text(renwer));
} catch (IOException e) {
if (!backOff(retryCnt)) {
log.error("getDelegationToken failed", e);
throw e;
}
retryCnt++;
// update cos ranger service again
updateCosRangerServiceAddr();
} catch (Exception e) {
log.error("get delegation token failed", e);
throw e;
}
}
}
/**
* renew token
*
* @param token token info
* @param configuration hadoop configuration
* @return the new expiration time
* @throws IOException
* @throws InterruptedException
*/
public long renew(Token token, Configuration configuration) throws IOException, InterruptedException {
if (this.cosRangerServiceLeaderIpAddr.get() == null) {
init(configuration);
}
int retryCnt = 0;
while (true) {
try {
ClientQcloudObjectStorageProtocol rangerClient = getLeaderClient();
if (rangerClient == null) {
throw new IOException(
"ranger client is null, maybe ranger server for qcloud object storage is not deployed!");
}
Token delToken = (Token) token;
return rangerClient.renewDelegationToken(delToken);
} catch (IOException e) {
if (!backOff(retryCnt)) {
log.error("renew token failed", e);
throw e;
}
retryCnt++;
// update cos ranger service again
updateCosRangerServiceAddr();
} catch (Exception e) {
log.error("renew token failed", e);
throw e;
}
}
}
/**
* cancel token
*
* @param token
* @param configuration
* @throws IOException
* @throws InterruptedException
*/
public void cancel(Token token, Configuration configuration) throws IOException, InterruptedException {
if (this.cosRangerServiceLeaderIpAddr.get() == null) {
init(configuration);
}
int retryCnt = 1;
while (true) {
try {
ClientQcloudObjectStorageProtocol rangerClient = getLeaderClient();
if (rangerClient == null) {
throw new IOException(
"ranger client is null, maybe ranger server for qcloud object storage is not deployed!");
}
Token delToken = (Token) token;
rangerClient.cancelDelegationToken(delToken);
return;
} catch (IOException e) {
if (!backOff(retryCnt)) {
log.error("cancel token failed", e);
throw e;
}
retryCnt++;
// update cos ranger service again
updateCosRangerServiceAddr();
} catch (Exception e) {
log.error("cancel token failed", e);
throw e;
}
}
}
/**
* check permission
*
* @param permissionRequest permission info
* @return PermissionResponse allowed and realUsername
* @throws IOException
*/
public PermissionResponse checkPermission(PermissionRequest permissionRequest) throws IOException {
int cosRangerServiceStartIndex = getCosRangerServiceStartIndex();
int retryCnt = 0;
while (true) {
try {
permissionRequest.setUserCred(this.getUserCredentials());
ClientQcloudObjectStorageProtocol rangerClient = getRandomClient(cosRangerServiceStartIndex, retryCnt);
if (rangerClient == null) {
throw new IOException(
"ranger client is null, maybe ranger server for qcloud object storage is not deployed!");
}
PermissionResponse permissionResp = rangerClient.checkPermission(permissionRequest);
return permissionResp;
} catch (IOException e) {
if (!backOff(retryCnt)) {
log.error("checkPermission failed", e);
throw e;
}
retryCnt++;
// update cos ranger service again
updateCosRangerServiceAddr();
} catch (Exception e) {
log.error("checkPermission failed", e);
throw e;
}
}
}
/**
* get sts for cos
*
* @param bucketRegion
* @param bucketName
* @return
* @throws IOException
*/
@Override
public GetSTSResponse getSTS(String bucketRegion, String bucketName) throws IOException {
GetSTSRequest getSTSRequest = new GetSTSRequest();
getSTSRequest.setBucketRegion(bucketRegion);
getSTSRequest.setBucketName(bucketName);
getSTSRequest.setAllowPrefix("");
int cosRangerServiceStartIndex = getCosRangerServiceStartIndex();
int retryCnt = 0;
while (true) {
try {
getSTSRequest.setUserCredentials(this.getUserCredentials());
ClientQcloudObjectStorageProtocol rangerClient = getRandomClient(cosRangerServiceStartIndex, retryCnt);
if (rangerClient == null) {
throw new IOException(
"ranger client is null, maybe ranger server for qcloud object storage is not deployed!");
}
return rangerClient.getSTS(getSTSRequest);
} catch (IOException e) {
if (!backOff(retryCnt)) {
log.error("get sts failed", e);
throw e;
}
retryCnt++;
// update cos ranger service again
updateCosRangerServiceAddr();
} catch (Exception e) {
log.error("get sts failed", e);
throw e;
}
}
}
/**
* get range auth policy
*
* @return
* @throws IOException
*/
@Override
public RangerAuthPolicyResponse getRangerAuthPolicy() throws IOException {
int cosRangerServiceStartIndex = getCosRangerServiceStartIndex();
int retryCnt = 0;
while (true) {
try {
ClientQcloudObjectStorageProtocol rangerClient = getRandomClient(cosRangerServiceStartIndex, retryCnt);
if (rangerClient == null) {
throw new IOException(
"ranger client is null, maybe ranger server for qcloud object storage is not deployed!");
}
return rangerClient.getRangerAuthPolicy();
} catch (IOException e) {
if (!backOff(retryCnt)) {
log.error("get ranger policy url failed", e);
throw e;
}
retryCnt++;
// update cos ranger service again
updateCosRangerServiceAddr();
} catch (Exception e) {
log.error("getRangerAuthPolicy failed", e);
throw e;
}
}
}
@Override
public void close() {
}
/**
* get user credentials in real time
*
* @return user credentials
* @throws IOException
*/
public String getUserCredentials() throws IOException {
// Initialize the configuration from a user-specified method first;
// If the user does not specify a method, the default value is used
String userCredentials = "";
String credentialGetWay = this.conf.get(ObjectStorageConstants.QCLOUD_OBJECT_STORAGE_USER_CREDENTIALS_INIT,
ObjectStorageConstants.QCLOUD_CONFIGURATION_FILE_VARIABLE);
String userCredentialsKey = this.conf.get(ObjectStorageConstants.QCLOUD_OBJECT_STORAGE_USER_CREDENTIALS_KEY,
ObjectStorageConstants.DEFAULT_QCLOUD_OBJECT_STORAGE_USER_CREDENTIALS_KEY);
switch (credentialGetWay) {
case ObjectStorageConstants.QCLOUD_CREDENTIALS_ENVIRONMENT_VARIABLE:
userCredentials = System.getenv(userCredentialsKey);
break;
case ObjectStorageConstants.QCLOUD_CONFIGURATION_FILE_VARIABLE:
userCredentials = this.conf.get(userCredentialsKey, "");
break;
default:
throw new IOException(String.format("can't get user credentials, " + "maybe the value of %s is not set",
ObjectStorageConstants.QCLOUD_OBJECT_STORAGE_USER_CREDENTIALS_INIT));
}
if (userCredentials == null) {
return "";
}
return userCredentials;
}
}