org.dasein.cloud.aws.AWSCloud Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of dasein-cloud-aws Show documentation
Show all versions of dasein-cloud-aws Show documentation
Implementation of the Dasein Cloud API for AWS.
/**
* Copyright (C) 2009-2015 Dell, Inc.
* See annotations for authorship information
*
* ====================================================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*/
package org.dasein.cloud.aws;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.codec.binary.Hex;
import org.apache.http.*;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.GzipDecompressingEntity;
import org.apache.http.client.utils.URIUtils;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.conn.params.ConnRoutePNames;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpParams;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.protocol.HttpContext;
import org.apache.log4j.Logger;
import org.dasein.cloud.*;
import org.dasein.cloud.aws.admin.AWSAdminServices;
import org.dasein.cloud.aws.compute.EC2ComputeServices;
import org.dasein.cloud.aws.compute.EC2Exception;
import org.dasein.cloud.aws.compute.EC2Method;
import org.dasein.cloud.aws.identity.AWSIdentityServices;
import org.dasein.cloud.aws.identity.IAMMethod;
import org.dasein.cloud.aws.network.EC2NetworkServices;
import org.dasein.cloud.aws.network.ELBMethod;
import org.dasein.cloud.aws.platform.AWSPlatformServices;
import org.dasein.cloud.aws.storage.AWSCloudStorageServices;
import org.dasein.cloud.aws.storage.S3Method;
import org.dasein.cloud.compute.ComputeServices;
import org.dasein.cloud.compute.VirtualMachineSupport;
import org.dasein.cloud.platform.KeyValuePair;
import org.dasein.cloud.storage.BlobStoreSupport;
import org.dasein.cloud.storage.StorageServices;
import org.dasein.cloud.util.APITrace;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.security.InvalidKeyException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
public class AWSCloud extends AbstractCloud {
private static final int MAX_RETRIES = 0;
static private String getLastItem( String name ) {
int idx = name.lastIndexOf('.');
if( idx < 0 ) {
return name;
}
else if( idx == ( name.length() - 1 ) ) {
return "";
}
return name.substring(idx + 1);
}
static public Logger getLogger( Class cls ) {
String pkg = getLastItem(cls.getPackage().getName());
if( pkg.equals("aws") ) {
pkg = "";
}
else {
pkg = pkg + ".";
}
return Logger.getLogger("dasein.cloud.aws.std." + pkg + getLastItem(cls.getName()));
}
static public Logger getWireLogger( Class cls ) {
return Logger.getLogger("dasein.cloud.aws.wire." + getLastItem(cls.getPackage().getName()) + "." + getLastItem(cls.getName()));
}
static private final Logger logger = getLogger(AWSCloud.class);
static public final String P_ACCESS = "AWSAccessKeyId";
static public final String P_ACTION = "Action";
static public final String P_CFAUTH = "Authorization";
static public final String P_AWS_DATE = "x-amz-date";
static public final String P_AWS_CONTENT_SHA256 = "x-amz-content-sha256";
static public final String P_GOOG_DATE = "x-goog-date";
static public final String P_SIGNATURE = "Signature";
static public final String P_SIGNATURE_METHOD = "SignatureMethod";
static public final String P_SIGNATURE_VERSION = "SignatureVersion";
static public final String P_TIMESTAMP = "Timestamp";
static public final String P_VERSION = "Version";
static public final String CLOUD_FRONT_ALGORITHM = "HmacSHA1";
static public final String EC2_ALGORITHM = "HmacSHA256";
static public final String S3_ALGORITHM = "HmacSHA1";
static public final String SIGNATURE_V2 = "2";
static public final String SIGNATURE_V4 = "4";
static public final String V4_ALGORITHM = "AWS4-HMAC-SHA256";
static public final String V4_TERMINATION = "aws4_request";
static public final String PLATFORM_EC2 = "EC2";
static public final String PLATFORM_VPC = "VPC";
static public @Nonnull String encode( @Nonnull String value, boolean encodePath ) throws InternalException {
String encoded;
try {
encoded = URLEncoder.encode(value, "utf-8").replace("+", "%20").replace("*", "%2A").replace("%7E", "~");
if( encodePath ) {
encoded = encoded.replace("%2F", "/");
}
}
catch( UnsupportedEncodingException e ) {
logger.error(e);
e.printStackTrace();
throw new InternalException(e);
}
return encoded;
}
static public String escapeXml( String nonxml ) {
StringBuilder str = new StringBuilder();
for( int i = 0; i < nonxml.length(); i++ ) {
char c = nonxml.charAt(i);
switch( c ) {
case '&':
str.append("&");
break;
case '>':
str.append(">");
break;
case '<':
str.append("<");
break;
case '"':
str.append(""");
break;
case '[':
str.append("[");
break;
case ']':
str.append("]");
break;
case '!':
str.append("!");
break;
default:
str.append(c);
}
}
return str.toString();
}
static public byte[] HmacSHA256( String data, byte[] key ) throws InternalException {
final String algorithm = "HmacSHA256";
Mac mac;
try {
mac = Mac.getInstance(algorithm);
mac.init(new SecretKeySpec(key, algorithm));
return mac.doFinal(data.getBytes("UTF-8"));
}
catch( NoSuchAlgorithmException e ) {
throw new InternalException(e);
}
catch( InvalidKeyException e ) {
throw new InternalException(e);
}
catch( UnsupportedEncodingException e ) {
throw new InternalException(e);
}
}
static public String computeSHA256Hash( String value ) throws InternalException {
try {
byte[] valueBytes = value.getBytes("utf-8");
BufferedInputStream inputStream = new BufferedInputStream(new ByteArrayInputStream(valueBytes));
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] buffer = new byte[4096];
int read;
while( ( read = inputStream.read(buffer, 0, buffer.length) ) != -1 ) {
digest.update(buffer, 0, read);
}
return new String(Hex.encodeHex(digest.digest(), true));
}
catch( NoSuchAlgorithmException e ) {
throw new InternalException(e);
}
catch( IOException e ) {
throw new InternalException(e);
}
}
public AWSCloud() {
}
private String buildEc2AuthString( String method, String serviceUrl, Map parameters ) throws InternalException {
StringBuilder authString = new StringBuilder();
TreeSet sortedKeys;
URI endpoint;
String tmp;
authString.append(method);
authString.append("\n");
try {
endpoint = new URI(serviceUrl);
}
catch( URISyntaxException e ) {
logger.error(e);
e.printStackTrace();
throw new InternalException(e);
}
authString.append(endpoint.getHost().toLowerCase());
authString.append("\n");
tmp = endpoint.getPath();
if( tmp == null || tmp.length() == 0 ) {
tmp = "/";
}
authString.append(encode(tmp, true));
authString.append("\n");
sortedKeys = new TreeSet();
sortedKeys.addAll(parameters.keySet());
boolean first = true;
for( String key : sortedKeys ) {
String value = parameters.get(key);
if( !first ) {
authString.append("&");
}
else {
first = false;
}
authString.append(encode(key, false));
authString.append("=");
if( value == null ) {
value = "";
}
authString.append(encode(value, false));
}
return authString.toString();
}
public boolean createTags( String service, String resourceId, Tag... keyValuePairs ) {
return createTags(service, new String[]{resourceId}, keyValuePairs);
}
public boolean createTags( final String service, final String[] resourceIds, final Tag... keyValuePairs ) {
// TODO(stas): de-async experiment
boolean async = false;
if( async ) {
hold();
Thread t = new Thread() {
public void run() {
try {
createTags(1, service, resourceIds, keyValuePairs);
}
finally {
release();
}
}
};
t.setName("Tag Setter");
t.setDaemon(true);
t.start();
} else {
createTags(1, service, resourceIds, keyValuePairs);
}
return true;
}
private void createTags( int attempt, String service, String[] resourceIds, Tag... keyValuePairs ) {
APITrace.begin(this, "Cloud.createTags");
try {
try {
Map parameters = null , tagParameters = null;
if (service.equalsIgnoreCase(ELBMethod.SERVICE_ID)) {
parameters = getElbParameters(getContext(), "AddTags");
addIndexedParameters(parameters, "LoadBalancerNames.member.", resourceIds);
tagParameters = getTagsFromKeyValuePairs("Tags.member.", keyValuePairs);
}
else if(service.equalsIgnoreCase("rds")) {
parameters = getStandardRdsParameters(getContext(), "AddTagsToResource");
// We can't tag multiple RDS resource at a time.
parameters.put("ResourceName", resourceIds[0]);
tagParameters = getTagsFromKeyValuePairs("Tags.member.", keyValuePairs);
}
else {
parameters = getStandardParameters(getContext(), "CreateTags");
addIndexedParameters(parameters, "ResourceId.", resourceIds);
tagParameters = getTagsFromKeyValuePairs("Tag.", keyValuePairs);
}
if( tagParameters.size() == 0 ) {
return;
}
addExtraParameters(parameters, tagParameters);
EC2Method method = new EC2Method(service, this, parameters);
try {
method.invoke();
} catch( EC2Exception e ) {
if( attempt > MAX_RETRIES ) {
logger.error("EC2 error setting tags for " + Arrays.toString(resourceIds) + ": " + e.getSummary());
return;
}
try {
Thread.sleep(5000L);
} catch( InterruptedException ignore ) {
}
logger.warn("Retry attempt "+ (attempt + 1) + " to create tags for ["+resourceIds+"]");
createTags(attempt + 1, service, resourceIds, keyValuePairs);
}
} catch( Throwable ignore ) {
logger.error("Error while creating tags for " + Arrays.toString(resourceIds) + ".", ignore);
}
} finally {
APITrace.end();
}
}
private Map getTagsFromKeyValuePairs(String tagPrefix, Tag... keyValuePairs) {
Map tagParameters = new HashMap();
for (int i = 0; i < keyValuePairs.length; i++) {
String key = keyValuePairs[i].getKey();
String value = keyValuePairs[i].getValue();
tagParameters.put(tagPrefix + (i + 1) + ".Key", key);
tagParameters.put(tagPrefix + (i + 1) + ".Value", value != null ? value : "" );
}
return tagParameters;
}
public void createTagsSynchronously(final String resourceId, final Tag... keyValuePairs) throws CloudException, InternalException {
createTagsSynchronously(new String[]{resourceId}, keyValuePairs);
}
public void createTagsSynchronously(final String[] resourceIds, final Tag... keyValuePairs) throws CloudException, InternalException {
APITrace.begin(this, "Cloud.createTagsSynchronously");
try {
Map parameters = getStandardParameters(getContext(), "CreateTags");
addIndexedParameters(parameters, "ResourceId.", resourceIds);
Map tagParameters = getTagsFromKeyValuePairs("Tag.", keyValuePairs);
if (tagParameters.size() == 0) {
return;
}
addExtraParameters(parameters, tagParameters);
new EC2Method(EC2Method.SERVICE_ID, this, parameters).invoke();
} finally {
APITrace.end();
}
}
public boolean removeTags( String service, String resourceId, Tag... keyValuePairs ) {
return removeTags(service, new String[]{resourceId}, keyValuePairs);
}
public boolean removeTags( String service, String[] resourceIds, Tag... keyValuePairs ) {
APITrace.begin(this, "Cloud.removeTags");
try {
try {
Map parameters, tagParameters = null ;
if (service.equalsIgnoreCase(ELBMethod.SERVICE_ID)){
parameters = getElbParameters(getContext(), "RemoveTags");
addIndexedParameters(parameters, "LoadBalancerNames.member.", resourceIds);
tagParameters = getTagsFromKeyValuePairs("Tags.member.", keyValuePairs);
} else if(service.equalsIgnoreCase("rds")){
parameters = getStandardRdsParameters(getContext(), "RemoveTagsFromResource");
parameters.put("ResourceName", resourceIds[0]);
for (int i = 0; i < keyValuePairs.length; i++)
parameters.put("TagKeys.member." + (i + 1) , keyValuePairs[i].getKey());
} else {
parameters = getStandardParameters(getContext(), "DeleteTags");
addIndexedParameters(parameters, "ResourceId.", resourceIds);
tagParameters = getTagsFromKeyValuePairs("Tag.", keyValuePairs);
}
addExtraParameters(parameters, tagParameters);
EC2Method method = new EC2Method(service, this, parameters);
method.invoke();
return true;
} catch( Throwable ignore ) {
logger.error("Error while removing tags for " + Arrays.toString(resourceIds) + ".", ignore);
return false;
}
} finally {
APITrace.end();
}
}
public Map getTagsFromTagSet( Node attr ) {
if( attr == null || !attr.hasChildNodes() ) {
return null;
}
Map tags = new HashMap();
NodeList tagNodes = attr.getChildNodes();
for( int j = 0; j < tagNodes.getLength(); j++ ) {
Tag t = toTag(tagNodes.item(j));
if (t != null) {
tags.put(t.getKey(), t.getValue());
}
}
return tags;
}
public Tag toTag(@Nonnull Node tag) {
if (tag.getNodeName().equals("item") && tag.hasChildNodes()) {
NodeList parts = tag.getChildNodes();
String key = null, value = null;
for (int k = 0; k < parts.getLength(); k++) {
Node part = parts.item(k);
if (part.getNodeName().equalsIgnoreCase("key")) {
if (part.hasChildNodes()) {
key = part.getFirstChild().getNodeValue().trim();
}
} else if (part.getNodeName().equalsIgnoreCase("value")) {
if (part.hasChildNodes()) {
value = part.getFirstChild().getNodeValue().trim();
}
}
}
if (key != null && value != null) {
return new Tag(key, value);
}
}
return null;
}
@Override
public AWSAdminServices getAdminServices() {
EC2Provider p = getEC2Provider();
if( p.isAWS() || p.isEnStratus() || p.isOpenStack() || p.isEucalyptus() ) {
return new AWSAdminServices(this);
}
return null;
}
private @Nonnull String[] getBootstrapUrls( @Nullable ProviderContext ctx ) {
String endpoint = ( ctx == null ? null : ctx.getCloud().getEndpoint() );
if( endpoint == null ) {
return new String[0];
}
if( !endpoint.contains(",") ) {
return new String[]{endpoint};
}
String[] endpoints = endpoint.split(",");
if( endpoints == null ) {
endpoints = new String[0];
}
if( endpoints.length > 1 ) {
String second = endpoints[1];
if( !second.startsWith("http") ) {
if( endpoints[0].startsWith("http") ) {
// likely a URL with a , in it
return new String[]{endpoint + ( getEC2Provider().isEucalyptus() ? "/Eucalyptus" : "" )};
}
}
}
for( int i = 0; i < endpoints.length; i++ ) {
if( !endpoints[i].startsWith("http") ) {
endpoints[i] = "https://" + endpoints[i] + ( getEC2Provider().isEucalyptus() ? "/Eucalyptus" : "" );
}
}
return endpoints;
}
@Override
public @Nonnull String getCloudName() {
ProviderContext ctx = getContext();
String name = ( ctx == null ? null : ctx.getCloud().getCloudName() );
return ( ( name == null ) ? "AWS" : name );
}
@Override
public EC2ComputeServices getComputeServices() {
if( getEC2Provider().isStorage() ) {
return null;
}
return new EC2ComputeServices(this);
}
static public final String DSN_ACCESS_KEY = "accessKey";
@Override
public @Nonnull ContextRequirements getContextRequirements() {
return new ContextRequirements(
new ContextRequirements.Field(DSN_ACCESS_KEY, "AWS API access keys", ContextRequirements.FieldType.KEYPAIR, ContextRequirements.Field.ACCESS_KEYS, true),
new ContextRequirements.Field("proxyHost", "Proxy host", ContextRequirements.FieldType.TEXT, false),
new ContextRequirements.Field("proxyPort", "Proxy port", ContextRequirements.FieldType.TEXT, false));
}
public byte[][] getAccessKey() {
return ( byte[][] ) getContext().getConfigurationValue(DSN_ACCESS_KEY);
}
@Override
public @Nonnull RegionsAndZones getDataCenterServices() {
return new RegionsAndZones(this);
}
private transient volatile EC2Provider provider;
public @Nonnull EC2Provider getEC2Provider() {
if( provider == null ) {
provider = EC2Provider.valueOf(getProviderName());
}
return provider;
}
public @Nullable String getEc2Url() {
ProviderContext ctx = getContext();
String url = getEc2Url(ctx == null ? null : ctx.getRegionId());
if( getEC2Provider().isEucalyptus() ) {
return url + "/Eucalyptus";
}
else {
return url;
}
}
public @Nullable String getEc2Url( @Nullable String regionId ) {
ProviderContext ctx = getContext();
String url;
if( regionId == null || regionId.isEmpty() ) {
return getBootstrapUrls(ctx)[0];
}
if( getEC2Provider().isAWS() ) {
url = ( ctx == null ? null : ctx.getCloud().getEndpoint() );
if( url != null && url.endsWith("amazonaws.com") ) {
return "https://ec2." + regionId + ".amazonaws.com";
}
return "https://ec2." + regionId + ".amazonaws.com";
}
else if( !getEC2Provider().isEucalyptus() ) {
url = ( ctx == null ? null : ctx.getCloud().getEndpoint() );
if( url == null ) {
return null;
}
if( !url.startsWith("http") ) {
String cloudUrl = ctx.getCloud().getEndpoint();
if( cloudUrl != null && cloudUrl.startsWith("http:") ) {
return "http://" + url + "/" + regionId;
}
return "https://" + url + "/" + regionId;
}
else {
return url + "/" + regionId;
}
}
url = ( ctx == null ? null : ctx.getCloud().getEndpoint() );
if( url == null ) {
return null;
}
if( !url.startsWith("http") ) {
String cloudUrl = ctx.getCloud().getEndpoint();
if( cloudUrl != null && cloudUrl.startsWith("http:") ) {
return "http://" + url;
}
return "https://" + url;
}
else {
return url;
}
}
public String getGlacierUrl() throws InternalException, CloudException {
ProviderContext ctx = getContext();
String regionId = ctx.getRegionId();
return "https://glacier." + regionId + ".amazonaws.com/-/";
}
public String getAutoScaleVersion() {
return "2011-01-01";
}
public String getCloudWatchVersion() {
return "2010-08-01";
}
public String getEc2Version() {
if (getEC2Provider().isAWS()) {
return "2014-05-01";
}
else if (getEC2Provider().isEucalyptus()) {
return "2010-11-15";
}
else if( getEC2Provider().isOpenStack() ) {
return "2009-11-30";
}
return "2012-07-20";
}
public String getElbVersion() {
return "2012-06-01";
}
public String getRdsVersion() {
return "2014-09-01";
}
public String getRoute53Version() {
return "2012-12-12";
}
public String getSdbVersion() {
return "2009-04-15";
}
public String getSnsVersion() {
return "2010-03-31";
}
public String getSqsVersion() {
return "2009-02-01";
}
@Override
public AWSIdentityServices getIdentityServices() {
if( getEC2Provider().isStorage() ) {
return null;
}
return new AWSIdentityServices(this);
}
@Override
public EC2NetworkServices getNetworkServices() {
if( getEC2Provider().isStorage() ) {
return null;
}
return new EC2NetworkServices(this);
}
@Override
public @Nullable AWSPlatformServices getPlatformServices() {
EC2Provider p = getEC2Provider();
if( p.isAWS() || p.isEnStratus() ) {
return new AWSPlatformServices(this);
}
return null;
}
@Override
public @Nonnull String getProviderName() {
ProviderContext ctx = getContext();
String name = ( ctx == null ? null : ctx.getCloud().getProviderName() );
return ( ( name == null ) ? EC2Provider.AWS.getName() : name );
}
public @Nullable String getProxyHost() {
ProviderContext ctx = getContext();
if( ctx == null ) {
return null;
}
Properties props = ctx.getCustomProperties();
return ( props == null ? null : props.getProperty("proxyHost") );
}
public int getProxyPort() {
ProviderContext ctx = getContext();
if( ctx == null ) {
return -1;
}
Properties props = ctx.getCustomProperties();
if( props == null ) {
return -1;
}
String port = props.getProperty("proxyPort");
if( port != null ) {
return Integer.parseInt(port);
}
return -1;
}
@Override
public @Nonnull AWSCloudStorageServices getStorageServices() {
return new AWSCloudStorageServices(this);
}
public Map getStandardParameters( ProviderContext ctx, String action ) throws InternalException {
return getStandardParameters(ctx, action, getEc2Version());
}
public Map getStandardParameters( ProviderContext ctx, String action, String version ) throws InternalException {
Map parameters = new HashMap();
parameters.put(P_ACTION, action);
// parameters.put(P_SIGNATURE_VERSION, SIGNATURE_V4);
// try {
// byte[][] keys = getAccessKey();
//
// parameters.put(P_ACCESS, new String(keys[0], "utf-8"));
// } catch( UnsupportedEncodingException e ) {
// logger.error(e);
// e.printStackTrace();
// throw new InternalException(e);
// }
// parameters.put(P_SIGNATURE_METHOD, EC2_ALGORITHM);
// parameters.put(P_TIMESTAMP, getTimestamp(System.currentTimeMillis(), true));
parameters.put(P_VERSION, version);
return parameters;
}
private @Nonnull Map getElbParameters( @Nonnull ProviderContext ctx, @Nonnull String action ) throws InternalException {
Map parameters = getStandardParameters(ctx, action);
parameters.put(P_VERSION, getElbVersion());
return parameters;
}
public Map getStandardCloudWatchParameters( ProviderContext ctx, String action ) throws InternalException {
Map parameters = getStandardParameters(ctx, action);
parameters.put(P_VERSION, getCloudWatchVersion());
return parameters;
}
public Map getStandardRdsParameters( ProviderContext ctx, String action ) throws InternalException {
Map parameters = getStandardParameters(ctx, action);
parameters.put(P_VERSION, getRdsVersion());
return parameters;
}
public Map getStandardSimpleDBParameters( ProviderContext ctx, String action ) throws InternalException {
Map parameters = getStandardParameters(ctx, action);
parameters.put(P_VERSION, getSdbVersion());
return parameters;
}
public Map getStandardSnsParameters( ProviderContext ctx, String action ) throws InternalException {
Map parameters = getStandardParameters(ctx, action);
parameters.put(P_VERSION, getSnsVersion());
return parameters;
}
public Map getStandardSqsParameters( ProviderContext ctx, String action ) throws InternalException {
Map parameters = getStandardParameters(ctx, action);
parameters.put(P_VERSION, getSqsVersion());
return parameters;
}
public static void addExtraParameters( Map parameters, Map extraParameters ) {
if( extraParameters == null || extraParameters.size() == 0 ) {
return;
}
if( parameters == null ) {
parameters = new HashMap();
}
parameters.putAll(extraParameters);
}
public static @Nullable Map getTagFilterParams( @Nullable Map tags ) {
return getTagFilterParams(tags, 1);
}
public static @Nullable Map getTagFilterParams( @Nullable Map tags, int startingFilterIndex ) {
if( tags == null || tags.size() == 0 ) {
return null;
}
Map filterParameters = new HashMap();
int i = startingFilterIndex;
for (Map.Entry parameter : tags.entrySet()) {
addFilterParameters(filterParameters, i, "tag:" + parameter.getKey(), Collections.singletonList(parameter.getValue()));
i++;
}
return filterParameters;
}
public static void addFilterParameters(Map filterParameters, int index, String filterName, Collection filterValues) {
if (filterValues == null || filterValues.isEmpty()) {
return;
}
filterParameters.put("Filter." + index + ".Name", filterName);
int valueIndex = 0;
for (Object filterValue : filterValues) {
// filter values must be in lower case
filterParameters.put("Filter." + index + ".Value." + valueIndex++, filterValue.toString().toLowerCase());
}
}
public static void addFilterParameters(Map filterParameters, int index, String filterName, Object ... filterValues) {
if (filterValues == null || filterValues.length == 0) {
return;
}
filterParameters.put("Filter." + index + ".Name", filterName);
int valueIndex = 0;
for (Object filterValue : filterValues) {
// filter values must be in lower case
filterParameters.put("Filter." + index + ".Value." + valueIndex++, filterValue.toString().toLowerCase());
}
}
public @Nonnull String getTimestamp(long timestamp, boolean withMillis) {
SimpleDateFormat fmt;
if( withMillis ) {
fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
}
else {
fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
}
fmt.setTimeZone(TimeZone.getTimeZone("UTC"));
return fmt.format(new Date(timestamp));
}
public long parseTime( @Nullable String time ) throws CloudException {
if( time == null ) {
return 0L;
}
SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
if( time.length() > 0 ) {
try {
return fmt.parse(time).getTime();
} catch( ParseException e ) {
fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
try {
return fmt.parse(time).getTime();
} catch( ParseException encore ) {
throw new CloudException("Could not parse date: " + time);
}
}
}
return 0L;
}
private String sign( byte[] key, String authString, String algorithm ) throws InternalException {
try {
Mac mac = Mac.getInstance(algorithm);
mac.init(new SecretKeySpec(key, algorithm));
return new String(Base64.encodeBase64(mac.doFinal(authString.getBytes("utf-8"))));
} catch( NoSuchAlgorithmException e ) {
logger.error(e);
e.printStackTrace();
throw new InternalException(e);
} catch( InvalidKeyException e ) {
logger.error(e);
e.printStackTrace();
throw new InternalException(e);
} catch( IllegalStateException e ) {
logger.error(e);
e.printStackTrace();
throw new InternalException(e);
} catch( UnsupportedEncodingException e ) {
logger.error(e);
e.printStackTrace();
throw new InternalException(e);
}
}
public String signUploadPolicy( String base64Policy ) throws InternalException {
ProviderContext ctx = getContext();
if( ctx == null ) {
throw new InternalException("No context for signing the request");
}
return sign(getAccessKey()[1], base64Policy, S3_ALGORITHM);
}
public String getRequestBodyHash(String bodyText) throws InternalException {
if (bodyText == null) {
// use hash of the empty string
return AWSCloud.computeSHA256Hash("");
} else {
return AWSCloud.computeSHA256Hash(bodyText);
}
}
public String signCloudFront( String accessKey, byte[] secretKey, String dateString ) throws InternalException {
String signature = sign(secretKey, dateString, CLOUD_FRONT_ALGORITHM);
if( getEC2Provider().isStorage() && "google".equalsIgnoreCase(getProviderName()) ) {
return ( "GOOG1 " + accessKey + ":" + signature );
}
else {
return ( "AWS " + accessKey + ":" + signature );
}
}
public String signEc2( byte[] key, String serviceUrl, Map parameters ) throws InternalException {
return sign(key, buildEc2AuthString("POST", serviceUrl, parameters), EC2_ALGORITHM);
}
public String signAWS3( String keyId, byte[] key, String dateString ) throws InternalException {
return ( "AWS3-HTTPS AWSAccessKeyId=" + keyId + ",Algorithm=" + EC2_ALGORITHM + ",Signature=" + sign(key, dateString, EC2_ALGORITHM) );
}
public String signS3( String accessKey, byte[] secretKey, String action, String hash, String contentType, Map headers, String bucket, String object ) throws InternalException {
StringBuilder toSign = new StringBuilder();
toSign.append(action);
toSign.append("\n");
if( hash != null ) {
toSign.append(hash);
}
toSign.append("\n");
if( contentType != null ) {
toSign.append(contentType);
}
toSign.append("\n\n");
ArrayList keys = new ArrayList();
keys.addAll(headers.keySet());
Collections.sort(keys);
for( String hkey : keys ) {
if( hkey.startsWith("x-amz") || ( getEC2Provider().isStorage() && hkey.startsWith("x-goog") ) ) {
String val = headers.get(hkey);
if( val != null ) {
toSign.append(hkey);
toSign.append(":");
toSign.append(headers.get(hkey).trim());
toSign.append("\n");
}
}
}
toSign.append("/");
if( getEC2Provider().isEucalyptus() ) {
toSign.append("services/Walrus/");
}
if( bucket != null ) {
toSign.append(bucket);
toSign.append("/");
}
if( object != null ) {
toSign.append(object.toLowerCase());
}
String signature = sign(secretKey, toSign.toString(), S3_ALGORITHM);
if( getEC2Provider().isStorage() && "google".equalsIgnoreCase(getProviderName()) ) {
return ( "GOOG1 " + accessKey + ":" + signature );
}
else {
return ( "AWS " + accessKey + ":" + signature );
}
}
/**
* Generates an AWS v4 signature authorization string
*
* @param accessKey Amazon credential
* @param secretKey Amazon credential
* @param action the HTTP method (GET, POST, etc)
* @param url the full URL for the request, including any query parameters
* @param serviceId the canonical name of the service targeted in the request (e.g. "glacier")
* @param headers map of headers of request. MUST include x-amz-date or date header.
* @param bodyHash a hex-encoded sha256 hash of the body of the request
* @return a string suitable for including as the HTTP Authorization header
* @throws InternalException
*/
public String getV4Authorization( String accessKey, String secretKey, String action, String url, String serviceId, Map headers, String bodyHash ) throws InternalException {
serviceId = serviceId.toLowerCase();
String regionId = "us-east-1"; // default for IAM
// if( ctx != null && ctx.getRegionId() != null && !ctx.getRegionId().isEmpty() &&
// !serviceId.equalsIgnoreCase(IAMMethod.SERVICE_ID) ) {
// regionId = getContext().getRegionId();
// } else {
String host = url.replaceAll("https?:\\/\\/", "");
if( host.indexOf('/') > 0 ) {
host = host.substring(0, host.indexOf('/', 1));
}
if( !IAMMethod.SERVICE_ID.equalsIgnoreCase(serviceId) ) {
String[] urlParts = host.split("\\."); // everywhere except s3 and iam this is: service.region.amazonaws.com
regionId = urlParts[urlParts.length-3];
if( regionId.startsWith("s3-") ) {
regionId = regionId.substring(3);
}
}
String amzDate = extractV4Date(headers);
String credentialScope = getV4CredentialScope(amzDate, regionId, serviceId);
String signedHeaders = getV4SignedHeaders(headers);
String signature = signV4(secretKey, action, url, regionId, serviceId, headers, bodyHash);
return V4_ALGORITHM + " " + "Credential=" + accessKey + "/" + credentialScope + ", " + "SignedHeaders=" + signedHeaders + ", " + "Signature=" + signature;
}
private String signV4( String secretKey, String action, String serviceUrl, String regionId, String serviceId, Map headers, String bodyHash ) throws InternalException {
final String canonicalRequest = getV4CanonicalRequest(action, serviceUrl, headers, bodyHash);
String amzDate = extractV4Date(headers);
final String stringToSign = getV4StringToSign(amzDate, regionId, serviceId, canonicalRequest);
// signature uses YYYYMMDD
String dateStamp = amzDate.substring(0, 8);
final byte[] signingKey = getV4SigningKey(secretKey, dateStamp, regionId, serviceId);
return new String(Hex.encodeHex(HmacSHA256(stringToSign, signingKey), true));
}
private String extractV4Date( Map headers ) throws InternalException {
Map lower = new HashMap();
for( Map.Entry entry : headers.entrySet() ) {
lower.put(entry.getKey().toLowerCase(), entry.getValue());
}
String amzDate = headers.get(P_AWS_DATE);
// expecting YYYYMMDDTHHMMSSZ
if( amzDate != null ) {
if( amzDate.length() != 16 ) {
throw new InternalException("request has invalid " + P_AWS_DATE);
}
return amzDate;
}
String date = lower.get("date");
if( date == null ) {
throw new InternalException("request is missing date header");
}
SimpleDateFormat parser = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz");
try {
return getV4HeaderDate(parser.parse(date));
} catch( ParseException e ) {
throw new InternalException("request has invalid date header format");
}
}
private byte[] getV4SigningKey( String secretKey, String dateStamp, String regionId, String serviceId ) throws InternalException {
byte[] withSecret = ( "AWS4" + secretKey ).getBytes();
byte[] withDate = HmacSHA256(dateStamp, withSecret);
byte[] withRegion = HmacSHA256(regionId, withDate);
byte[] withService = HmacSHA256(serviceId, withRegion);
return HmacSHA256("aws4_request", withService);
}
private String getV4StringToSign( String dateStamp, String regionId, String serviceId, String canonicalRequest ) throws InternalException {
return V4_ALGORITHM + "\n" + dateStamp + "\n" + getV4CredentialScope(dateStamp, regionId, serviceId) + "\n" + computeSHA256Hash(canonicalRequest);
}
private String getV4CredentialScope( String dateStamp, String regionId, String serviceId ) {
return dateStamp.substring(0, 8) + "/" + regionId + "/" + serviceId + "/" + V4_TERMINATION;
}
private String getV4CanonicalRequest( String action, String serviceUrl, Map headers, String bodyHash ) throws InternalException {
/*
CanonicalRequest =
HTTPRequestMethod + '\n' +
CanonicalURI + '\n' +
CanonicalQueryString + '\n' +
CanonicalHeaders + '\n' +
SignedHeaders + '\n' +
HexEncode(Hash(Payload))
*/
final URI endpoint;
try {
endpoint = new URI(serviceUrl.replace(" ", "%20")).normalize();
} catch( URISyntaxException e ) {
throw new InternalException(e);
}
final StringBuilder s = new StringBuilder();
s.append(action.toUpperCase()).append("\n");
String path = endpoint.getPath();
if( path == null || path.length() == 0 ) {
path = "/";
}
s.append(encode(path, true)).append("\n");
s.append(getV4CanonicalQueryString(endpoint)).append("\n");
List sortedHeaders = new ArrayList();
sortedHeaders.addAll(headers.keySet());
Collections.sort(sortedHeaders, String.CASE_INSENSITIVE_ORDER);
for( String header : sortedHeaders ) {
String value = headers.get(header).trim().replaceAll("\\s+", " ");
header = header.toLowerCase().replaceAll("\\s+", " ");
s.append(header).append(":").append(value).append("\n");
}
s.append("\n").append(getV4SignedHeaders(headers)).append("\n").append(bodyHash);
return s.toString();
}
private String getV4CanonicalQueryString( URI endpoint ) throws InternalException {
// parse query params and translate to another form of tuple that is comparable on both key and value
List parsedParams = URLEncodedUtils.parse(endpoint, "UTF-8");
List queryParams = new ArrayList(parsedParams.size());
for( NameValuePair param : parsedParams ) {
String key = encode(param.getName(), false);
String value = param.getValue() != null ? encode(param.getValue(), false) : "";
queryParams.add(new KeyValuePair(key, value));
}
// sort query parameters by key, then value
Collections.sort(queryParams);
StringBuilder sb = new StringBuilder();
for( KeyValuePair pair : queryParams ) {
if( sb.length() > 0 ) {
sb.append("&");
}
sb.append(pair.getKey()).append("=").append(pair.getValue());
}
return sb.toString();
}
private String getV4SignedHeaders( Map headers ) {
// move to set to lower case and remove dupes
Set sorted = new TreeSet();
for( String header : headers.keySet() ) {
sorted.add(header.toLowerCase());
}
StringBuilder sb = new StringBuilder();
for( String header : sorted ) {
if( sb.length() > 0 ) {
sb.append(";");
}
sb.append(header.toLowerCase());
}
return sb.toString();
}
public String getV4HeaderDate( Date date ) {
SimpleDateFormat fmt = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'");
Calendar cal = Calendar.getInstance(new SimpleTimeZone(0, "GMT"));
fmt.setCalendar(cal);
if( date == null ) {
return fmt.format(new Date());
}
else {
return fmt.format(date);
}
}
@Override
public String testContext() {
APITrace.begin(this, "Cloud.testContext");
try {
ProviderContext ctx = getContext();
if( ctx == null ) {
logger.warn("No context exists for testing");
return null;
}
try {
ComputeServices compute = getComputeServices();
if( compute != null ) {
VirtualMachineSupport support = compute.getVirtualMachineSupport();
if( support == null || !support.isSubscribed() ) {
logger.warn("Not subscribed to virtual machine support");
return null;
}
String actualAccountNumber = getOwnerId();
// Return actual account number as the number provided in configuration
// may have been incorrect
if( actualAccountNumber != null ) {
return actualAccountNumber;
}
}
else {
StorageServices storage = getStorageServices();
BlobStoreSupport support = storage.getOnlineStorageSupport();
if( support == null || !support.isSubscribed() ) {
logger.warn("No subscribed to storage services");
return null;
}
}
} catch( Throwable t ) {
logger.warn("Unable to connect to AWS for " + ctx.getAccountNumber() + ": " + t.getMessage());
return null;
}
return ctx.getAccountNumber();
} finally {
APITrace.end();
}
}
/**
* Retrieve current account number using DescribeSecurityGroups. May not always be reliable but is better than
* nothing.
*
* @return current account number or null if not found
*/
private String getOwnerId() {
APITrace.begin(this, "AWSCloud.getOwnerId");
try {
ProviderContext ctx = getContext();
if( ctx == null ) {
return null;
}
Map parameters = getStandardParameters(getContext(), EC2Method.DESCRIBE_SECURITY_GROUPS);
EC2Method method;
NodeList blocks;
Document doc;
method = new EC2Method(EC2Method.SERVICE_ID, this, parameters);
try {
doc = method.invoke();
} catch( EC2Exception e ) {
logger.error(e.getSummary());
throw new CloudException(e);
}
blocks = doc.getElementsByTagName("securityGroupInfo");
for( int i = 0; i < blocks.getLength(); i++ ) {
NodeList items = blocks.item(i).getChildNodes();
for( int j = 0; j < items.getLength(); j++ ) {
Node item = items.item(j);
if( item.getNodeName().equals("item") ) {
NodeList attrs = item.getChildNodes();
for( int k = 0; k < attrs.getLength(); k++ ) {
Node attr = attrs.item(k);
if( attr.getNodeName().equals("ownerId") ) {
return attr.getFirstChild().getNodeValue().trim();
}
}
}
}
}
return null;
} catch( InternalException e ) {
} catch( CloudException e ) {
} finally {
APITrace.end();
}
// Couldn't get the number for some reason
return null;
}
public void setTags( @Nonnull Node attr, @Nonnull Taggable item ) {
if( attr.hasChildNodes() ) {
NodeList tags = attr.getChildNodes();
for( int j = 0; j < tags.getLength(); j++ ) {
Tag t = toTag(tags.item(j));
if (t != null && t.getValue() != null) {
item.setTag(t.getKey(), t.getValue());
}
}
}
}
/**
* Gets the epoch form of the text value of the provided node.
*
* @param node the node to extact the value from
* @return the epoch time
* @throws CloudException
*/
public static long getTimestampValue( Node node ) throws CloudException {
SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
fmt.setTimeZone(TimeZone.getTimeZone("UTC"));
String value = getTextValue(node);
try {
return fmt.parse(value).getTime();
} catch( ParseException e ) {
logger.error(e);
e.printStackTrace();
throw new CloudException(e);
}
}
/**
* Returns the text from the given node.
*
* @param node the node to extract the value from
* @return the text from the node
*/
static public String getTextValue( Node node ) {
if( node.getChildNodes().getLength() == 0 ) {
return null;
}
return node.getFirstChild().getNodeValue();
}
/**
* Returns the boolean value of the given node.
*
* @param node the node to extract the value from
* @return the boolean value of the node
*/
static public boolean getBooleanValue( Node node ) {
return Boolean.valueOf(getTextValue(node));
}
/**
* Returns the int value of the given node.
*
* @param node the node to extract the value from
* @return the int value of the given node
*/
public static int getIntValue( Node node ) {
return Integer.valueOf(getTextValue(node));
}
/**
* Returns the double value of the given node.
*
* @param node the node to extract the value from
* @return the double value of the given node
*/
public static double getDoubleValue( Node node ) {
return Double.valueOf(getTextValue(node));
}
/**
* Returns the float value of the given node.
*
* @param node the node to extract the value from
* @return the float value of the given node
*/
public static float getFloatValue( Node node ) {
return Float.valueOf(getTextValue(node));
}
/**
* Helper method for adding indexed member parameters, e.g. AlarmNames.member.N. Will overwrite existing
* parameters if present. Assumes indexing starts at 1.
*
* @param parameters the existing parameters map to add to
* @param prefix the prefix value for each parameter key
* @param values the values to add
*/
public static void addIndexedParameters( @Nonnull Map parameters, @Nonnull String prefix, String ... values ) {
if( values == null || values.length == 0 ) {
return;
}
int i = 1;
if( !prefix.endsWith(".") ) {
prefix += ".";
}
for( String value : values ) {
parameters.put(String.format("%s%d", prefix, i), value);
i++;
}
}
/**
* Helper method for adding indexed member parameters, e.g. AlarmNames.member.N. Will overwrite existing
* parameters if present. Assumes indexing starts at 1.
*
* @param parameters the existing parameters map to add to
* @param prefix the prefix value for each parameter key
* @param extraParameters the values to add
*/
public static void addIndexedParameters( @Nonnull Map parameters, @Nonnull String prefix, Map extraParameters ) {
if( extraParameters == null || extraParameters.size() == 0 ) {
return;
}
int i = 1;
for( Map.Entry entry : extraParameters.entrySet() ) {
parameters.put(prefix + i + ".Name", entry.getKey());
if( entry.getValue() != null ) {
parameters.put(prefix + i + ".Value", entry.getValue());
}
i++;
}
}
/**
* Puts the given key/value into the given map only if the value is not null.
*
* @param parameters the map to add to
* @param key the key of the value
* @param value the value to add if not null
*/
public static void addValueIfNotNull( @Nonnull Map parameters, @Nonnull String key, Object value ) {
if( value == null ) {
return;
}
parameters.put(key, value.toString());
}
public @Nonnull HttpClient getClient() throws InternalException {
return getClient(false);
}
public @Nonnull HttpClient getClient(boolean multipart) throws InternalException {
ProviderContext ctx = getContext();
if( ctx == null ) {
throw new InternalException("No context was specified for this request");
}
final HttpParams params = new BasicHttpParams();
HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1);
if( !multipart ) {
HttpProtocolParams.setContentCharset(params, Consts.UTF_8.toString());
}
HttpProtocolParams.setUserAgent(params, "Dasein Cloud");
Properties p = ctx.getCustomProperties();
if( p != null ) {
String proxyHost = p.getProperty("proxyHost");
String proxyPortStr = p.getProperty("proxyPort");
int proxyPort = 0;
if( proxyPortStr != null ) {
proxyPort = Integer.parseInt(proxyPortStr);
}
if( proxyHost != null && proxyHost.length() > 0 && proxyPort > 0 ) {
params.setParameter(ConnRoutePNames.DEFAULT_PROXY,
new HttpHost(proxyHost, proxyPort)
);
}
}
DefaultHttpClient client = new DefaultHttpClient(params);
client.addRequestInterceptor(new HttpRequestInterceptor() {
public void process(
final HttpRequest request,
final HttpContext context) throws HttpException, IOException {
if( !request.containsHeader("Accept-Encoding") ) {
request.addHeader("Accept-Encoding", "gzip");
}
request.setParams(params);
}
});
client.addResponseInterceptor(new HttpResponseInterceptor() {
public void process(
final HttpResponse response,
final HttpContext context) throws HttpException, IOException {
HttpEntity entity = response.getEntity();
if( entity != null ) {
Header header = entity.getContentEncoding();
if( header != null ) {
for( HeaderElement codec : header.getElements() ) {
if( codec.getName().equalsIgnoreCase("gzip") ) {
response.setEntity(
new GzipDecompressingEntity(response.getEntity()));
break;
}
}
}
}
}
});
return client;
}
/**
* DEBUG_AWS should be specified as system properties, otherwise return false
*
* @return DEBUG_AWS properties value, or false if not specified
*/
public boolean isDebug() {
return Boolean.valueOf(System.getProperties().getProperty("DEBUG_AWS", "false"));
}
}