com.google.cloud.storage.StorageImpl Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of gcloud-java-storage Show documentation
Show all versions of gcloud-java-storage Show documentation
Java idiomatic client for Google Cloud Storage.
/*
* Copyright 2015 Google Inc. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.cloud.storage;
import static com.google.cloud.RetryHelper.runWithRetries;
import static com.google.cloud.storage.spi.StorageRpc.Option.DELIMITER;
import static com.google.cloud.storage.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.cloud.storage.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.cloud.storage.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.cloud.storage.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.cloud.storage.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.cloud.storage.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.cloud.storage.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.cloud.storage.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.common.base.MoreObjects.firstNonNull;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.api.services.storage.model.StorageObject;
import com.google.cloud.BaseService;
import com.google.cloud.BatchResult;
import com.google.cloud.Page;
import com.google.cloud.PageImpl;
import com.google.cloud.PageImpl.NextPageFetcher;
import com.google.cloud.ReadChannel;
import com.google.cloud.RetryHelper.RetryHelperException;
import com.google.cloud.ServiceAccountSigner;
import com.google.cloud.storage.spi.StorageRpc;
import com.google.cloud.storage.spi.StorageRpc.RewriteResponse;
import com.google.cloud.storage.spi.StorageRpc.Tuple;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.hash.Hashing;
import com.google.common.io.BaseEncoding;
import com.google.common.primitives.Ints;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
final class StorageImpl extends BaseService implements Storage {
private static final byte[] EMPTY_BYTE_ARRAY = {};
private static final String EMPTY_BYTE_ARRAY_MD5 = "1B2M2Y8AsgTpgAmY7PhCfg==";
private static final String EMPTY_BYTE_ARRAY_CRC32C = "AAAAAA==";
private static final String PATH_DELIMITER = "/";
private static final Function, Boolean> DELETE_FUNCTION =
new Function, Boolean>() {
@Override
public Boolean apply(Tuple tuple) {
return tuple.y();
}
};
private final StorageRpc storageRpc;
StorageImpl(StorageOptions options) {
super(options);
storageRpc = options.rpc();
}
@Override
public Bucket create(BucketInfo bucketInfo, BucketTargetOption... options) {
final com.google.api.services.storage.model.Bucket bucketPb = bucketInfo.toPb();
final Map optionsMap = optionMap(bucketInfo, options);
try {
return Bucket.fromPb(this, runWithRetries(
new Callable() {
@Override
public com.google.api.services.storage.model.Bucket call() {
return storageRpc.create(bucketPb, optionsMap);
}
}, options().retryParams(), EXCEPTION_HANDLER, options().clock()));
} catch (RetryHelperException e) {
throw StorageException.translateAndThrow(e);
}
}
@Override
public Blob create(BlobInfo blobInfo, BlobTargetOption... options) {
BlobInfo updatedInfo = blobInfo.toBuilder()
.md5(EMPTY_BYTE_ARRAY_MD5)
.crc32c(EMPTY_BYTE_ARRAY_CRC32C)
.build();
return create(updatedInfo, new ByteArrayInputStream(EMPTY_BYTE_ARRAY), options);
}
@Override
public Blob create(BlobInfo blobInfo, byte[] content, BlobTargetOption... options) {
content = firstNonNull(content, EMPTY_BYTE_ARRAY);
BlobInfo updatedInfo = blobInfo.toBuilder()
.md5(BaseEncoding.base64().encode(Hashing.md5().hashBytes(content).asBytes()))
.crc32c(BaseEncoding.base64().encode(
Ints.toByteArray(Hashing.crc32c().hashBytes(content).asInt())))
.build();
return create(updatedInfo, new ByteArrayInputStream(content), options);
}
@Override
public Blob create(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) {
Tuple targetOptions = BlobTargetOption.convert(blobInfo, options);
return create(targetOptions.x(), content, targetOptions.y());
}
private Blob create(BlobInfo info, final InputStream content, BlobTargetOption... options) {
final StorageObject blobPb = info.toPb();
final Map optionsMap = optionMap(info, options);
try {
return Blob.fromPb(this, runWithRetries(new Callable() {
@Override
public StorageObject call() {
return storageRpc.create(blobPb,
firstNonNull(content, new ByteArrayInputStream(EMPTY_BYTE_ARRAY)), optionsMap);
}
}, options().retryParams(), EXCEPTION_HANDLER, options().clock()));
} catch (RetryHelperException e) {
throw StorageException.translateAndThrow(e);
}
}
@Override
public Bucket get(String bucket, BucketGetOption... options) {
final com.google.api.services.storage.model.Bucket bucketPb = BucketInfo.of(bucket).toPb();
final Map optionsMap = optionMap(options);
try {
com.google.api.services.storage.model.Bucket answer = runWithRetries(
new Callable() {
@Override
public com.google.api.services.storage.model.Bucket call() {
return storageRpc.get(bucketPb, optionsMap);
}
}, options().retryParams(), EXCEPTION_HANDLER, options().clock());
return answer == null ? null : Bucket.fromPb(this, answer);
} catch (RetryHelperException e) {
throw StorageException.translateAndThrow(e);
}
}
@Override
public Blob get(String bucket, String blob, BlobGetOption... options) {
return get(BlobId.of(bucket, blob), options);
}
@Override
public Blob get(BlobId blob, BlobGetOption... options) {
final StorageObject storedObject = blob.toPb();
final Map optionsMap = optionMap(blob, options);
try {
StorageObject storageObject = runWithRetries(new Callable() {
@Override
public StorageObject call() {
return storageRpc.get(storedObject, optionsMap);
}
}, options().retryParams(), EXCEPTION_HANDLER, options().clock());
return storageObject == null ? null : Blob.fromPb(this, storageObject);
} catch (RetryHelperException e) {
throw StorageException.translateAndThrow(e);
}
}
@Override
public Blob get(BlobId blob) {
return get(blob, new BlobGetOption[0]);
}
private static class BucketPageFetcher implements NextPageFetcher {
private static final long serialVersionUID = 5850406828803613729L;
private final Map requestOptions;
private final StorageOptions serviceOptions;
BucketPageFetcher(
StorageOptions serviceOptions, String cursor,
Map optionMap) {
this.requestOptions =
PageImpl.nextRequestOptions(StorageRpc.Option.PAGE_TOKEN, cursor, optionMap);
this.serviceOptions = serviceOptions;
}
@Override
public Page nextPage() {
return listBuckets(serviceOptions, requestOptions);
}
}
private static class BlobPageFetcher implements NextPageFetcher {
private static final long serialVersionUID = 81807334445874098L;
private final Map requestOptions;
private final StorageOptions serviceOptions;
private final String bucket;
BlobPageFetcher(String bucket, StorageOptions serviceOptions, String cursor,
Map optionMap) {
this.requestOptions =
PageImpl.nextRequestOptions(StorageRpc.Option.PAGE_TOKEN, cursor, optionMap);
this.serviceOptions = serviceOptions;
this.bucket = bucket;
}
@Override
public Page nextPage() {
return listBlobs(bucket, serviceOptions, requestOptions);
}
}
@Override
public Page list(BucketListOption... options) {
return listBuckets(options(), optionMap(options));
}
@Override
public Page list(final String bucket, BlobListOption... options) {
return listBlobs(bucket, options(), optionMap(options));
}
private static Page listBuckets(final StorageOptions serviceOptions,
final Map optionsMap) {
try {
Tuple> result = runWithRetries(
new Callable>>() {
@Override
public Tuple> call() {
return serviceOptions.rpc().list(optionsMap);
}
}, serviceOptions.retryParams(), EXCEPTION_HANDLER, serviceOptions.clock());
String cursor = result.x();
Iterable buckets =
result.y() == null ? ImmutableList.of() : Iterables.transform(result.y(),
new Function() {
@Override
public Bucket apply(com.google.api.services.storage.model.Bucket bucketPb) {
return Bucket.fromPb(serviceOptions.service(), bucketPb);
}
});
return new PageImpl<>(
new BucketPageFetcher(serviceOptions, cursor, optionsMap), cursor,
buckets);
} catch (RetryHelperException e) {
throw StorageException.translateAndThrow(e);
}
}
private static Page listBlobs(final String bucket,
final StorageOptions serviceOptions, final Map optionsMap) {
try {
Tuple> result = runWithRetries(
new Callable>>() {
@Override
public Tuple> call() {
return serviceOptions.rpc().list(bucket, optionsMap);
}
}, serviceOptions.retryParams(), EXCEPTION_HANDLER, serviceOptions.clock());
String cursor = result.x();
Iterable blobs =
result.y() == null
? ImmutableList.of()
: Iterables.transform(result.y(), new Function() {
@Override
public Blob apply(StorageObject storageObject) {
return Blob.fromPb(serviceOptions.service(), storageObject);
}
});
return new PageImpl<>(
new BlobPageFetcher(bucket, serviceOptions, cursor, optionsMap),
cursor,
blobs);
} catch (RetryHelperException e) {
throw StorageException.translateAndThrow(e);
}
}
@Override
public Bucket update(BucketInfo bucketInfo, BucketTargetOption... options) {
final com.google.api.services.storage.model.Bucket bucketPb = bucketInfo.toPb();
final Map optionsMap = optionMap(bucketInfo, options);
try {
return Bucket.fromPb(this, runWithRetries(
new Callable() {
@Override
public com.google.api.services.storage.model.Bucket call() {
return storageRpc.patch(bucketPb, optionsMap);
}
}, options().retryParams(), EXCEPTION_HANDLER, options().clock()));
} catch (RetryHelperException e) {
throw StorageException.translateAndThrow(e);
}
}
@Override
public Blob update(BlobInfo blobInfo, BlobTargetOption... options) {
final StorageObject storageObject = blobInfo.toPb();
final Map optionsMap = optionMap(blobInfo, options);
try {
return Blob.fromPb(this, runWithRetries(new Callable() {
@Override
public StorageObject call() {
return storageRpc.patch(storageObject, optionsMap);
}
}, options().retryParams(), EXCEPTION_HANDLER, options().clock()));
} catch (RetryHelperException e) {
throw StorageException.translateAndThrow(e);
}
}
@Override
public Blob update(BlobInfo blobInfo) {
return update(blobInfo, new BlobTargetOption[0]);
}
@Override
public boolean delete(String bucket, BucketSourceOption... options) {
final com.google.api.services.storage.model.Bucket bucketPb = BucketInfo.of(bucket).toPb();
final Map optionsMap = optionMap(options);
try {
return runWithRetries(new Callable() {
@Override
public Boolean call() {
return storageRpc.delete(bucketPb, optionsMap);
}
}, options().retryParams(), EXCEPTION_HANDLER, options().clock());
} catch (RetryHelperException e) {
throw StorageException.translateAndThrow(e);
}
}
@Override
public boolean delete(String bucket, String blob, BlobSourceOption... options) {
return delete(BlobId.of(bucket, blob), options);
}
@Override
public boolean delete(BlobId blob, BlobSourceOption... options) {
final StorageObject storageObject = blob.toPb();
final Map optionsMap = optionMap(blob, options);
try {
return runWithRetries(new Callable() {
@Override
public Boolean call() {
return storageRpc.delete(storageObject, optionsMap);
}
}, options().retryParams(), EXCEPTION_HANDLER, options().clock());
} catch (RetryHelperException e) {
throw StorageException.translateAndThrow(e);
}
}
@Override
public boolean delete(BlobId blob) {
return delete(blob, new BlobSourceOption[0]);
}
@Override
public Blob compose(final ComposeRequest composeRequest) {
final List sources =
Lists.newArrayListWithCapacity(composeRequest.sourceBlobs().size());
for (ComposeRequest.SourceBlob sourceBlob : composeRequest.sourceBlobs()) {
sources.add(BlobInfo.builder(
BlobId.of(composeRequest.target().bucket(), sourceBlob.name(), sourceBlob.generation()))
.build().toPb());
}
final StorageObject target = composeRequest.target().toPb();
final Map targetOptions = optionMap(composeRequest.target().generation(),
composeRequest.target().metageneration(), composeRequest.targetOptions());
try {
return Blob.fromPb(this, runWithRetries(new Callable() {
@Override
public StorageObject call() {
return storageRpc.compose(sources, target, targetOptions);
}
}, options().retryParams(), EXCEPTION_HANDLER, options().clock()));
} catch (RetryHelperException e) {
throw StorageException.translateAndThrow(e);
}
}
@Override
public CopyWriter copy(final CopyRequest copyRequest) {
final StorageObject source = copyRequest.source().toPb();
final Map sourceOptions =
optionMap(copyRequest.source().generation(), null, copyRequest.sourceOptions(), true);
final StorageObject targetObject = copyRequest.target().toPb();
final Map targetOptions = optionMap(copyRequest.target().generation(),
copyRequest.target().metageneration(), copyRequest.targetOptions());
try {
RewriteResponse rewriteResponse = runWithRetries(new Callable() {
@Override
public RewriteResponse call() {
return storageRpc.openRewrite(new StorageRpc.RewriteRequest(source, sourceOptions,
copyRequest.overrideInfo(), targetObject, targetOptions,
copyRequest.megabytesCopiedPerChunk()));
}
}, options().retryParams(), EXCEPTION_HANDLER, options().clock());
return new CopyWriter(options(), rewriteResponse);
} catch (RetryHelperException e) {
throw StorageException.translateAndThrow(e);
}
}
@Override
public byte[] readAllBytes(String bucket, String blob, BlobSourceOption... options) {
return readAllBytes(BlobId.of(bucket, blob), options);
}
@Override
public byte[] readAllBytes(BlobId blob, BlobSourceOption... options) {
final StorageObject storageObject = blob.toPb();
final Map optionsMap = optionMap(blob, options);
try {
return runWithRetries(new Callable() {
@Override
public byte[] call() {
return storageRpc.load(storageObject, optionsMap);
}
}, options().retryParams(), EXCEPTION_HANDLER, options().clock());
} catch (RetryHelperException e) {
throw StorageException.translateAndThrow(e);
}
}
@Override
public StorageBatch batch() {
return new StorageBatch(this.options());
}
@Override
public ReadChannel reader(String bucket, String blob, BlobSourceOption... options) {
Map optionsMap = optionMap(options);
return new BlobReadChannel(options(), BlobId.of(bucket, blob), optionsMap);
}
@Override
public ReadChannel reader(BlobId blob, BlobSourceOption... options) {
Map optionsMap = optionMap(blob, options);
return new BlobReadChannel(options(), blob, optionsMap);
}
@Override
public BlobWriteChannel writer(BlobInfo blobInfo, BlobWriteOption... options) {
Tuple targetOptions = BlobTargetOption.convert(blobInfo, options);
return writer(targetOptions.x(), targetOptions.y());
}
private BlobWriteChannel writer(BlobInfo blobInfo, BlobTargetOption... options) {
final Map optionsMap = optionMap(blobInfo, options);
return new BlobWriteChannel(options(), blobInfo, optionsMap);
}
@Override
public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOption... options) {
EnumMap optionMap = Maps.newEnumMap(SignUrlOption.Option.class);
for (SignUrlOption option : options) {
optionMap.put(option.option(), option.value());
}
ServiceAccountSigner authCredentials =
(ServiceAccountSigner) optionMap.get(SignUrlOption.Option.SERVICE_ACCOUNT_CRED);
if (authCredentials == null) {
checkState(this.options().authCredentials() instanceof ServiceAccountSigner,
"Signing key was not provided and could not be derived");
authCredentials = (ServiceAccountSigner) this.options().authCredentials();
}
// construct signature - see https://cloud.google.com/storage/docs/access-control#Signed-URLs
StringBuilder stBuilder = new StringBuilder();
if (optionMap.containsKey(SignUrlOption.Option.HTTP_METHOD)) {
stBuilder.append(optionMap.get(SignUrlOption.Option.HTTP_METHOD));
} else {
stBuilder.append(HttpMethod.GET);
}
stBuilder.append('\n');
if (firstNonNull((Boolean) optionMap.get(SignUrlOption.Option.MD5), false)) {
checkArgument(blobInfo.md5() != null, "Blob is missing a value for md5");
stBuilder.append(blobInfo.md5());
}
stBuilder.append('\n');
if (firstNonNull((Boolean) optionMap.get(SignUrlOption.Option.CONTENT_TYPE), false)) {
checkArgument(blobInfo.contentType() != null, "Blob is missing a value for content-type");
stBuilder.append(blobInfo.contentType());
}
stBuilder.append('\n');
long expiration = TimeUnit.SECONDS.convert(
options().clock().millis() + unit.toMillis(duration), TimeUnit.MILLISECONDS);
stBuilder.append(expiration).append('\n');
StringBuilder path = new StringBuilder();
if (!blobInfo.bucket().startsWith("/")) {
path.append('/');
}
path.append(blobInfo.bucket());
if (!blobInfo.bucket().endsWith("/")) {
path.append('/');
}
if (blobInfo.name().startsWith("/")) {
path.setLength(path.length() - 1);
}
path.append(blobInfo.name());
stBuilder.append(path);
try {
byte[] signatureBytes = authCredentials.sign(stBuilder.toString().getBytes(UTF_8));
stBuilder = new StringBuilder("https://storage.googleapis.com").append(path);
String signature =
URLEncoder.encode(BaseEncoding.base64().encode(signatureBytes), UTF_8.name());
stBuilder.append("?GoogleAccessId=").append(authCredentials.account());
stBuilder.append("&Expires=").append(expiration);
stBuilder.append("&Signature=").append(signature);
return new URL(stBuilder.toString());
} catch (MalformedURLException | UnsupportedEncodingException ex) {
throw new IllegalStateException(ex);
}
}
@Override
public List get(BlobId... blobIds) {
return get(Arrays.asList(blobIds));
}
@Override
public List get(Iterable blobIds) {
StorageBatch batch = batch();
final List results = Lists.newArrayList();
for (BlobId blob : blobIds) {
batch.get(blob).notify(new BatchResult.Callback() {
@Override
public void success(Blob result) {
results.add(result);
}
@Override
public void error(StorageException exception) {
results.add(null);
}
});
}
batch.submit();
return Collections.unmodifiableList(results);
}
@Override
public List update(BlobInfo... blobInfos) {
return update(Arrays.asList(blobInfos));
}
@Override
public List update(Iterable blobInfos) {
StorageBatch batch = batch();
final List results = Lists.newArrayList();
for (BlobInfo blobInfo : blobInfos) {
batch.update(blobInfo).notify(new BatchResult.Callback() {
@Override
public void success(Blob result) {
results.add(result);
}
@Override
public void error(StorageException exception) {
results.add(null);
}
});
}
batch.submit();
return Collections.unmodifiableList(results);
}
@Override
public List delete(BlobId... blobIds) {
return delete(Arrays.asList(blobIds));
}
@Override
public List delete(Iterable blobIds) {
StorageBatch batch = batch();
final List results = Lists.newArrayList();
for (BlobId blob : blobIds) {
batch.delete(blob).notify(new BatchResult.Callback() {
@Override
public void success(Boolean result) {
results.add(result);
}
@Override
public void error(StorageException exception) {
results.add(Boolean.FALSE);
}
});
}
batch.submit();
return Collections.unmodifiableList(results);
}
private static void addToOptionMap(StorageRpc.Option option, T defaultValue,
Map map) {
addToOptionMap(option, option, defaultValue, map);
}
private static void addToOptionMap(StorageRpc.Option getOption, StorageRpc.Option putOption,
T defaultValue, Map map) {
if (map.containsKey(getOption)) {
@SuppressWarnings("unchecked")
T value = (T) map.remove(getOption);
checkArgument(value != null || defaultValue != null,
"Option " + getOption.value() + " is missing a value");
value = firstNonNull(value, defaultValue);
map.put(putOption, value);
}
}
private static Map optionMap(Long generation, Long metaGeneration,
Iterable options) {
return optionMap(generation, metaGeneration, options, false);
}
private static Map optionMap(Long generation, Long metaGeneration,
Iterable options, boolean useAsSource) {
Map temp = Maps.newEnumMap(StorageRpc.Option.class);
for (Option option : options) {
Object prev = temp.put(option.rpcOption(), option.value());
checkArgument(prev == null, "Duplicate option %s", option);
}
Boolean value = (Boolean) temp.remove(DELIMITER);
if (Boolean.TRUE.equals(value)) {
temp.put(DELIMITER, PATH_DELIMITER);
}
if (useAsSource) {
addToOptionMap(IF_GENERATION_MATCH, IF_SOURCE_GENERATION_MATCH, generation, temp);
addToOptionMap(IF_GENERATION_NOT_MATCH, IF_SOURCE_GENERATION_NOT_MATCH, generation, temp);
addToOptionMap(IF_METAGENERATION_MATCH, IF_SOURCE_METAGENERATION_MATCH, metaGeneration, temp);
addToOptionMap(IF_METAGENERATION_NOT_MATCH,
IF_SOURCE_METAGENERATION_NOT_MATCH, metaGeneration, temp);
} else {
addToOptionMap(IF_GENERATION_MATCH, generation, temp);
addToOptionMap(IF_GENERATION_NOT_MATCH, generation, temp);
addToOptionMap(IF_METAGENERATION_MATCH, metaGeneration, temp);
addToOptionMap(IF_METAGENERATION_NOT_MATCH, metaGeneration, temp);
}
return ImmutableMap.copyOf(temp);
}
private static Map optionMap(Option... options) {
return optionMap(null, null, Arrays.asList(options));
}
private static Map optionMap(Long generation, Long metaGeneration,
Option... options) {
return optionMap(generation, metaGeneration, Arrays.asList(options));
}
private static Map optionMap(BucketInfo bucketInfo, Option... options) {
return optionMap(null, bucketInfo.metageneration(), options);
}
static Map optionMap(BlobInfo blobInfo, Option... options) {
return optionMap(blobInfo.generation(), blobInfo.metageneration(), options);
}
static Map optionMap(BlobId blobId, Option... options) {
return optionMap(blobId.generation(), null, options);
}
}