• Home
• com.xiaomi.infra.galaxy
• galaxy-fds-sdk-java
• 3.0.35-alpha1
• source code
• GalaxyFDSClient.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

com.xiaomi.infra.galaxy.fds.client.GalaxyFDSClient Maven / Gradle / Ivy

package com.xiaomi.infra.galaxy.fds.client;

import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.lang.reflect.Type;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.TimeZone;
import java.util.UUID;
import java.util.concurrent.TimeUnit;

import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.LinkedListMultimap;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
import com.google.gson.JsonParseException;
import com.xiaomi.infra.galaxy.fds.bean.ThirdPartyObjectBean;

import com.xiaomi.infra.galaxy.fds.client.model.InitiateMultipartUploadRequest;
import com.xiaomi.infra.galaxy.fds.client.network.SSLConnectionInfoRecorderSocketFactory;
import com.xiaomi.infra.galaxy.fds.model.StorageClass;
import com.xiaomi.infra.galaxy.fds.client.model.FDSBucket;
import com.xiaomi.infra.galaxy.fds.client.model.FDSCopyObjectRequest;
import com.xiaomi.infra.galaxy.fds.client.model.FDSMd5InputStream;
import com.xiaomi.infra.galaxy.fds.client.model.FDSObject;
import com.xiaomi.infra.galaxy.fds.client.model.FDSObjectInputStream;
import com.xiaomi.infra.galaxy.fds.client.model.FDSObjectListing;
import com.xiaomi.infra.galaxy.fds.client.model.FDSObjectSummary;
import com.xiaomi.infra.galaxy.fds.client.model.FDSPutObjectRequest;
import com.xiaomi.infra.galaxy.fds.client.model.Owner;
import com.xiaomi.infra.galaxy.fds.model.ThirdPartyObject;
import com.xiaomi.infra.galaxy.fds.model.TimestampAntiStealingLinkConfig;
import com.xiaomi.infra.galaxy.fds.result.AccessControlPolicy;
import com.xiaomi.infra.galaxy.fds.result.CopyObjectResult;
import com.xiaomi.infra.galaxy.fds.result.InitMultipartUploadResult;
import com.xiaomi.infra.galaxy.fds.result.ListAllAuthorizedBucketsResult;
import com.xiaomi.infra.galaxy.fds.result.ListAllBucketsResult;
import com.xiaomi.infra.galaxy.fds.result.ListDomainMappingsResult;
import com.xiaomi.infra.galaxy.fds.result.ListObjectsResult;
import com.xiaomi.infra.galaxy.fds.result.PutObjectResult;
import com.xiaomi.infra.galaxy.fds.result.QuotaPolicy;
import com.xiaomi.infra.galaxy.fds.result.UploadPartResult;
import com.xiaomi.infra.galaxy.fds.result.UploadPartResultList;
import com.xiaomi.infra.galaxy.fds.result.VersionListing;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.NTCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.config.SocketConfig;
import org.apache.http.conn.DnsResolver;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.entity.AbstractHttpEntity;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.FileEntity;
import org.apache.http.entity.InputStreamEntity;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.impl.conn.SystemDefaultDnsResolver;
import org.apache.http.ssl.SSLContexts;
import org.json.JSONException;

import com.xiaomi.infra.galaxy.fds.Action;
import com.xiaomi.infra.galaxy.fds.Common;
import com.xiaomi.infra.galaxy.fds.SubResource;
import com.xiaomi.infra.galaxy.fds.auth.signature.SignAlgorithm;
import com.xiaomi.infra.galaxy.fds.auth.signature.Signer;
import com.xiaomi.infra.galaxy.fds.auth.signature.XiaomiHeader;
import com.xiaomi.infra.galaxy.fds.bean.BucketBean;
import com.xiaomi.infra.galaxy.fds.bean.GrantBean;
import com.xiaomi.infra.galaxy.fds.bean.GranteeBean;
import com.xiaomi.infra.galaxy.fds.bean.ObjectBean;
import com.xiaomi.infra.galaxy.fds.bean.OwnerBean;
import com.xiaomi.infra.galaxy.fds.client.credential.GalaxyFDSCredential;
import com.xiaomi.infra.galaxy.fds.client.exception.GalaxyException;
import com.xiaomi.infra.galaxy.fds.client.exception.GalaxyFDSClientException;
import com.xiaomi.infra.galaxy.fds.client.filter.FDSClientLogFilter;
import com.xiaomi.infra.galaxy.fds.client.filter.MetricsRequestFilter;
import com.xiaomi.infra.galaxy.fds.client.filter.MetricsResponseFilter;
import com.xiaomi.infra.galaxy.fds.client.metrics.ClientMetrics;
import com.xiaomi.infra.galaxy.fds.client.metrics.MetricsCollector;
import com.xiaomi.infra.galaxy.fds.client.network.ConnectionInfoRecorderSocketFactory;
import com.xiaomi.infra.galaxy.fds.client.network.Constants;
import com.xiaomi.infra.galaxy.fds.client.network.FDSBlackListEnabledHostChecker;
import com.xiaomi.infra.galaxy.fds.client.network.HttpContextUtil;
import com.xiaomi.infra.galaxy.fds.client.network.InternalIpBlackListRetryHandler;
import com.xiaomi.infra.galaxy.fds.client.network.InternalSiteBlackListDNSResolver;
import com.xiaomi.infra.galaxy.fds.client.network.RoundRobinDNSResolver;
import com.xiaomi.infra.galaxy.fds.client.network.ServiceUnavailableDNSBlackListStrategy;
import com.xiaomi.infra.galaxy.fds.client.network.TimeBasedIpAddressBlackList;
import com.xiaomi.infra.galaxy.fds.model.AccessControlList;
import com.xiaomi.infra.galaxy.fds.model.AccessLogConfig;
import com.xiaomi.infra.galaxy.fds.model.FDSObjectMetadata;
import com.xiaomi.infra.galaxy.fds.model.HttpMethod;
import com.xiaomi.infra.galaxy.fds.model.LifecycleConfig;
import com.xiaomi.infra.galaxy.fds.model.CORSConfiguration;
import com.xiaomi.infra.galaxy.fds.model.CORSConfiguration.CORSRule;

import javax.net.ssl.SSLContext;

public class GalaxyFDSClient implements GalaxyFDS {

  public static final String ORG_ID_PARAM = "orgId";
  private static final String VERSION_ID = "versionId";
  private static final String ARCHIVE = "archive";
  private final GalaxyFDSCredential credential;
  private final FDSClientConfiguration fdsConfig;
  private final DnsResolver dnsResolver;
  private MetricsCollector metricsCollector;
  private String delimiter = "/";
  private final Random random = new Random();
  private final String clientId = UUID.randomUUID().toString().substring(0, 8);
  private HttpClient httpClient;
  private FDSClientLogFilter logFilter = new FDSClientLogFilter();
  private PoolingHttpClientConnectionManager connectionManager;
  private static final Set retryMethodSet = new HashSet();

  private BasicAuthCache authCache = null;
  private CredentialsProvider credentialsProvider = null;

  // TODO(wuzesheng) Make the authenticator configurable and let the
  // authenticator supply sign algorithm and generate signature
  private static SignAlgorithm SIGN_ALGORITHM = SignAlgorithm.HmacSHA1;

  public static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat(
      "EEE, dd MMM yyyy HH:mm:ss z", Locale.US);

  static {
    DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("GMT"));
    retryMethodSet.add("getBucketAcl");
    retryMethodSet.add("getObjectAcl");
    retryMethodSet.add("getAccessLogConfig");
    retryMethodSet.add("listNextBatchOfObjects");
    retryMethodSet.add("listObjects");
    retryMethodSet.add("listTrashObjects");
    retryMethodSet.add("getObjectMetadata");
    retryMethodSet.add("initMultipartUpload");
    retryMethodSet.add("getLifecycleConfig");
    retryMethodSet.add("listAuthorizedBuckets");
    retryMethodSet.add("listBuckets");
    retryMethodSet.add("listDomainMappings");
    retryMethodSet.add("getMirror");
    retryMethodSet.add("listNextBatchOfVersions");
    retryMethodSet.add("listVersions");
    retryMethodSet.add("doesBucketExist");
    retryMethodSet.add("doesObjectExist");
    retryMethodSet.add("prefetchObject");
    retryMethodSet.add("refreshObject");
    retryMethodSet.add("deleteDomainMapping");
    retryMethodSet.add("deleteMirror");
    retryMethodSet.add("deleteObjectAcl");
    retryMethodSet.add("getBucket");
    retryMethodSet.add("putDomainMapping");
    retryMethodSet.add("setBucketAcl");
    retryMethodSet.add("setMirror");
    retryMethodSet.add("setObjectAcl");
    retryMethodSet.add("setPublic");
    retryMethodSet.add("updateAccessLogConfig");
    retryMethodSet.add("updateLifecycleConfig");
  }

  private static final Log LOG = LogFactory.getLog(GalaxyFDSClient.class);
  protected TimeBasedIpAddressBlackList ipBlackList;
  protected InternalIpBlackListRetryHandler retryHandler;

  public GalaxyFDSClient(GalaxyFDSCredential credential,
      FDSClientConfiguration fdsConfig) {
    this(credential, fdsConfig, null);
  }

  public GalaxyFDSClient(GalaxyFDSCredential credential,
      FDSClientConfiguration fdsConfig,
      DnsResolver dnsResolver) {
    this.credential = credential;
    this.fdsConfig = fdsConfig;
    this.dnsResolver = dnsResolver;
    init();
  }

  private void init() {
    if (fdsConfig.isApacheConnectorEnabled()) {
      LOG.warn("Apache Connector not supported");
    }
    httpClient = createHttpClient(this.fdsConfig);
    if (fdsConfig.isMetricsEnabled()) {
      metricsCollector = new MetricsCollector(this);
    }
  }

  private HttpClient createHttpClient(FDSClientConfiguration config) {
    RequestConfig.Builder requestConfigBuilder = RequestConfig.custom()
        .setConnectTimeout(config.getConnectionTimeoutMs())
        .setSocketTimeout(config.getSocketTimeoutMs());

    String proxyHost = config.getProxyHost();

    int proxyPort = config.getProxyPort();

    if (proxyHost != null && proxyPort > 0) {
      HttpHost proxy = new HttpHost(proxyHost, proxyPort);
      requestConfigBuilder.setProxy(proxy);

      String proxyUsername = config.getProxyUsername();
      String proxyPassword = config.getProxyPassword();
      String proxyDomain = config.getProxyDomain();
      String proxyWorkstation = config.getProxyWorkstation();
      if (proxyUsername != null && proxyPassword != null) {
        credentialsProvider = new BasicCredentialsProvider();

        credentialsProvider.setCredentials(new AuthScope(proxyHost, proxyPort), new NTCredentials(proxyUsername, proxyPassword, proxyWorkstation, proxyDomain));

        authCache = new BasicAuthCache();
        authCache.put(proxy, new BasicScheme());
      }
    }

    RequestConfig requestConfig = requestConfigBuilder.build();

    SocketConfig socketConfig = SocketConfig.custom()
        .setSoTimeout(config.getSocketTimeoutMs())
        .build();

    RegistryBuilder registryBuilder = RegistryBuilder.create();
    registryBuilder.register("http", new ConnectionInfoRecorderSocketFactory(
        new PlainConnectionSocketFactory()));

    if (config.isHttpsEnabled()) {
      SSLContext sslContext = SSLContexts.createSystemDefault();
      SSLConnectionSocketFactory sslConnectionSocketFactory = new SSLConnectionInfoRecorderSocketFactory(
          sslContext,
          NoopHostnameVerifier.INSTANCE);
      registryBuilder.register("https", sslConnectionSocketFactory);
    }
    ipBlackList = new TimeBasedIpAddressBlackList(config.getIpAddressNegativeDurationMillsec());
    connectionManager = new PoolingHttpClientConnectionManager(registryBuilder.build(),
        null,
        null,
        new RoundRobinDNSResolver(new InternalSiteBlackListDNSResolver(ipBlackList,
            this.dnsResolver == null ?
                SystemDefaultDnsResolver.INSTANCE : this.dnsResolver)),
        config.getHTTPKeepAliveTimeoutMS(), TimeUnit.MILLISECONDS);
    connectionManager.setDefaultMaxPerRoute(config.getMaxConnection());
    connectionManager.setMaxTotal(config.getMaxConnection());
    connectionManager.setDefaultSocketConfig(socketConfig);
    FDSBlackListEnabledHostChecker fdsBlackListEnabledHostChecker = new FDSBlackListEnabledHostChecker();
    retryHandler = new InternalIpBlackListRetryHandler(config.getRetryCount(),
        ipBlackList, fdsBlackListEnabledHostChecker);

    return HttpClients.custom()
        .setRetryHandler(retryHandler)
        .setServiceUnavailableRetryStrategy(new ServiceUnavailableDNSBlackListStrategy(
            config.getRetryCount(),
            config.getRetryIntervalMilliSec(),
            ipBlackList,
            fdsBlackListEnabledHostChecker))
        .setConnectionManager(connectionManager)
        .setDefaultRequestConfig(requestConfig)
        .build();
  }

  @Override
  public void setDelimiter(String delimiter) {
    this.delimiter = delimiter;
  }

  private HttpUriRequest prepareRequestMethod(URI uri,
      HttpMethod method, ContentType contentType, FDSObjectMetadata metadata,
      HashMap params, Map> headers,
      HttpEntity requestEntity) throws GalaxyFDSClientException {
    if (params != null) {
      URIBuilder builder = new URIBuilder(uri).setCharset(Charset.forName("UTF-8"));
      for (Entry param : params.entrySet()) {
        builder.addParameter(param.getKey(), param.getValue());
      }
      try {
        uri = builder.build();
      } catch (URISyntaxException e) {
        throw new GalaxyFDSClientException("Invalid param: " + params.toString(), e);
      }
    }

    if (headers == null)
      headers = new HashMap>();
    Map h = prepareRequestHeader(uri, method, contentType, metadata);
    for (Entry hIte : h.entrySet()) {
      String key = hIte.getKey();
      if (!headers.containsKey(key)) {
        headers.put(key, new ArrayList