• Home
• com.google.cloud
• google-cloud-storage
• 2.43.0
• source code
• GrpcStorageImpl.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

com.google.cloud.storage.GrpcStorageImpl Maven / Gradle / Ivy

/*
 * Copyright 2022 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.cloud.storage;

import static com.google.cloud.storage.ByteSizeConstants._16MiB;
import static com.google.cloud.storage.ByteSizeConstants._1MiB;
import static com.google.cloud.storage.ByteSizeConstants._256KiB;
import static com.google.cloud.storage.CrossTransportUtils.fmtMethodName;
import static com.google.cloud.storage.CrossTransportUtils.throwHttpJsonOnly;
import static com.google.cloud.storage.GrpcToHttpStatusCodeTranslation.resultRetryAlgorithmToCodes;
import static com.google.cloud.storage.StorageV2ProtoUtils.bucketAclEntityOrAltEq;
import static com.google.cloud.storage.StorageV2ProtoUtils.objectAclEntityOrAltEq;
import static com.google.cloud.storage.Utils.bucketNameCodec;
import static com.google.cloud.storage.Utils.ifNonNull;
import static com.google.cloud.storage.Utils.projectNameCodec;
import static com.google.common.base.MoreObjects.firstNonNull;
import static java.util.Objects.requireNonNull;

import com.google.api.core.ApiFuture;
import com.google.api.core.ApiFutures;
import com.google.api.core.BetaApi;
import com.google.api.gax.grpc.GrpcCallContext;
import com.google.api.gax.paging.AbstractPage;
import com.google.api.gax.paging.Page;
import com.google.api.gax.retrying.ResultRetryAlgorithm;
import com.google.api.gax.rpc.ApiException;
import com.google.api.gax.rpc.ApiExceptions;
import com.google.api.gax.rpc.ClientStreamingCallable;
import com.google.api.gax.rpc.NotFoundException;
import com.google.api.gax.rpc.StatusCode;
import com.google.api.gax.rpc.UnaryCallable;
import com.google.cloud.BaseService;
import com.google.cloud.Policy;
import com.google.cloud.WriteChannel;
import com.google.cloud.storage.Acl.Entity;
import com.google.cloud.storage.BlobWriteSessionConfig.WriterFactory;
import com.google.cloud.storage.BufferedWritableByteChannelSession.BufferedWritableByteChannel;
import com.google.cloud.storage.Conversions.Decoder;
import com.google.cloud.storage.HmacKey.HmacKeyMetadata;
import com.google.cloud.storage.HmacKey.HmacKeyState;
import com.google.cloud.storage.PostPolicyV4.PostConditionsV4;
import com.google.cloud.storage.PostPolicyV4.PostFieldsV4;
import com.google.cloud.storage.Storage.ComposeRequest.SourceBlob;
import com.google.cloud.storage.UnbufferedReadableByteChannelSession.UnbufferedReadableByteChannel;
import com.google.cloud.storage.UnbufferedWritableByteChannelSession.UnbufferedWritableByteChannel;
import com.google.cloud.storage.UnifiedOpts.BucketListOpt;
import com.google.cloud.storage.UnifiedOpts.BucketSourceOpt;
import com.google.cloud.storage.UnifiedOpts.BucketTargetOpt;
import com.google.cloud.storage.UnifiedOpts.Fields;
import com.google.cloud.storage.UnifiedOpts.HmacKeyListOpt;
import com.google.cloud.storage.UnifiedOpts.HmacKeySourceOpt;
import com.google.cloud.storage.UnifiedOpts.HmacKeyTargetOpt;
import com.google.cloud.storage.UnifiedOpts.Mapper;
import com.google.cloud.storage.UnifiedOpts.NamedField;
import com.google.cloud.storage.UnifiedOpts.ObjectListOpt;
import com.google.cloud.storage.UnifiedOpts.ObjectSourceOpt;
import com.google.cloud.storage.UnifiedOpts.ObjectTargetOpt;
import com.google.cloud.storage.UnifiedOpts.Opts;
import com.google.cloud.storage.UnifiedOpts.ProjectId;
import com.google.cloud.storage.UnifiedOpts.UserProject;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Streams;
import com.google.common.io.BaseEncoding;
import com.google.common.io.ByteStreams;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.iam.v1.GetIamPolicyRequest;
import com.google.iam.v1.SetIamPolicyRequest;
import com.google.iam.v1.TestIamPermissionsRequest;
import com.google.protobuf.ByteString;
import com.google.protobuf.FieldMask;
import com.google.storage.v2.BidiWriteObjectRequest;
import com.google.storage.v2.BucketAccessControl;
import com.google.storage.v2.ComposeObjectRequest;
import com.google.storage.v2.ComposeObjectRequest.SourceObject;
import com.google.storage.v2.CreateBucketRequest;
import com.google.storage.v2.CreateHmacKeyRequest;
import com.google.storage.v2.CreateNotificationConfigRequest;
import com.google.storage.v2.DeleteBucketRequest;
import com.google.storage.v2.DeleteHmacKeyRequest;
import com.google.storage.v2.DeleteNotificationConfigRequest;
import com.google.storage.v2.DeleteObjectRequest;
import com.google.storage.v2.GetBucketRequest;
import com.google.storage.v2.GetHmacKeyRequest;
import com.google.storage.v2.GetNotificationConfigRequest;
import com.google.storage.v2.GetObjectRequest;
import com.google.storage.v2.GetServiceAccountRequest;
import com.google.storage.v2.ListBucketsRequest;
import com.google.storage.v2.ListHmacKeysRequest;
import com.google.storage.v2.ListNotificationConfigsRequest;
import com.google.storage.v2.ListNotificationConfigsResponse;
import com.google.storage.v2.ListObjectsRequest;
import com.google.storage.v2.ListObjectsResponse;
import com.google.storage.v2.LockBucketRetentionPolicyRequest;
import com.google.storage.v2.NotificationConfig;
import com.google.storage.v2.NotificationConfigName;
import com.google.storage.v2.Object;
import com.google.storage.v2.ObjectAccessControl;
import com.google.storage.v2.ReadObjectRequest;
import com.google.storage.v2.RestoreObjectRequest;
import com.google.storage.v2.RewriteObjectRequest;
import com.google.storage.v2.RewriteResponse;
import com.google.storage.v2.StorageClient;
import com.google.storage.v2.StorageClient.ListNotificationConfigsPage;
import com.google.storage.v2.UpdateBucketRequest;
import com.google.storage.v2.UpdateHmacKeyRequest;
import com.google.storage.v2.UpdateObjectRequest;
import com.google.storage.v2.WriteObjectRequest;
import com.google.storage.v2.WriteObjectResponse;
import com.google.storage.v2.WriteObjectSpec;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.file.Files;
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.Spliterator;
import java.util.Spliterators.AbstractSpliterator;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.function.UnaryOperator;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;

@BetaApi
final class GrpcStorageImpl extends BaseService
    implements Storage, StorageInternal {

  private static final byte[] ZERO_BYTES = new byte[0];
  private static final Set READ_OPS = ImmutableSet.of(StandardOpenOption.READ);
  private static final Set WRITE_OPS =
      ImmutableSet.of(
          StandardOpenOption.WRITE,
          StandardOpenOption.CREATE,
          StandardOpenOption.TRUNCATE_EXISTING);
  private static final BucketSourceOption[] EMPTY_BUCKET_SOURCE_OPTIONS = new BucketSourceOption[0];

  private static final Opts ALL_BLOB_FIELDS =
      Opts.from(UnifiedOpts.fields(ImmutableSet.copyOf(BlobField.values())));
  private static final Opts ALL_BUCKET_FIELDS =
      // todo: b/308194853
      Opts.from(
          UnifiedOpts.fields(
              Arrays.stream(BucketField.values())
                  .filter(f -> !f.equals(BucketField.OBJECT_RETENTION))
                  .collect(ImmutableSet.toImmutableSet())));

  final StorageClient storageClient;
  final ResponseContentLifecycleManager responseContentLifecycleManager;
  final WriterFactory writerFactory;
  final GrpcConversions codecs;
  final GrpcRetryAlgorithmManager retryAlgorithmManager;
  final SyntaxDecoders syntaxDecoders;

  // workaround for https://github.com/googleapis/java-storage/issues/1736
  private final Opts defaultOpts;
  @Deprecated private final ProjectId defaultProjectId;

  GrpcStorageImpl(
      GrpcStorageOptions options,
      StorageClient storageClient,
      ResponseContentLifecycleManager responseContentLifecycleManager,
      WriterFactory writerFactory,
      Opts defaultOpts) {
    super(options);
    this.storageClient = storageClient;
    this.responseContentLifecycleManager = responseContentLifecycleManager;
    this.writerFactory = writerFactory;
    this.defaultOpts = defaultOpts;
    this.codecs = Conversions.grpc();
    this.retryAlgorithmManager = options.getRetryAlgorithmManager();
    this.syntaxDecoders = new SyntaxDecoders();
    this.defaultProjectId = UnifiedOpts.projectId(options.getProjectId());
  }

  @Override
  public void close() throws Exception {
    try (StorageClient s = storageClient) {
      s.shutdownNow();
      org.threeten.bp.Duration terminationAwaitDuration =
          getOptions().getTerminationAwaitDuration();
      s.awaitTermination(terminationAwaitDuration.toMillis(), TimeUnit.MILLISECONDS);
    }
  }

  @Override
  public Bucket create(BucketInfo bucketInfo, BucketTargetOption... options) {
    Opts opts = Opts.unwrap(options).resolveFrom(bucketInfo).prepend(defaultOpts);
    GrpcCallContext grpcCallContext =
        opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
    if (bucketInfo.getProject() == null || bucketInfo.getProject().trim().isEmpty()) {
      bucketInfo = bucketInfo.toBuilder().setProject(getOptions().getProjectId()).build();
    }
    com.google.storage.v2.Bucket bucket = codecs.bucketInfo().encode(bucketInfo);
    CreateBucketRequest.Builder builder =
        CreateBucketRequest.newBuilder()
            .setBucket(bucket)
            .setBucketId(bucketInfo.getName())
            .setParent("projects/_");
    CreateBucketRequest req = opts.createBucketsRequest().apply(builder).build();
    GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext());
    return Retrying.run(
        getOptions(),
        retryAlgorithmManager.getFor(req),
        () -> storageClient.createBucketCallable().call(req, merge),
        syntaxDecoders.bucket);
  }

  @Override
  public Blob create(BlobInfo blobInfo, BlobTargetOption... options) {
    return create(blobInfo, null, options);
  }

  @Override
  public Blob create(BlobInfo blobInfo, byte[] content, BlobTargetOption... options) {
    content = firstNonNull(content, ZERO_BYTES);
    return create(blobInfo, content, 0, content.length, options);
  }

  @Override
  public Blob create(
      BlobInfo blobInfo, byte[] content, int offset, int length, BlobTargetOption... options) {
    Opts opts = Opts.unwrap(options).resolveFrom(blobInfo);
    return internalDirectUpload(blobInfo, opts, ByteBuffer.wrap(content, offset, length))
        .asBlob(this);
  }

  @Override
  public Blob create(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) {
    try {
      requireNonNull(blobInfo, "blobInfo must be non null");
      InputStream inputStreamParam = firstNonNull(content, new ByteArrayInputStream(ZERO_BYTES));

      Opts optsWithDefaults = Opts.unwrap(options).prepend(defaultOpts);
      GrpcCallContext grpcCallContext =
          optsWithDefaults.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
      WriteObjectRequest req = getWriteObjectRequest(blobInfo, optsWithDefaults);
      Hasher hasher = Hasher.enabled();
      GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext());
      UnbufferedWritableByteChannelSession session =
          ResumableMedia.gapic()
              .write()
              .byteChannel(storageClient.writeObjectCallable().withDefaultCallContext(merge))
              .setByteStringStrategy(ByteStringStrategy.noCopy())
              .setHasher(hasher)
              .direct()
              .unbuffered()
              .setRequest(req)
              .build();

      try (UnbufferedWritableByteChannel c = session.open()) {
        ByteStreams.copy(Channels.newChannel(inputStreamParam), c);
      }
      ApiFuture responseApiFuture = session.getResult();
      return this.getBlob(responseApiFuture);
    } catch (IOException | ApiException e) {
      throw StorageException.coalesce(e);
    }
  }

  @Override
  public Blob createFrom(BlobInfo blobInfo, Path path, BlobWriteOption... options)
      throws IOException {
    return createFrom(blobInfo, path, _16MiB, options);
  }

  @Override
  public Blob createFrom(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options)
      throws IOException {
    Opts opts = Opts.unwrap(options).resolveFrom(blobInfo).prepend(defaultOpts);
    return internalCreateFrom(path, blobInfo, opts);
  }

  @Override
  public Blob internalCreateFrom(Path path, BlobInfo info, Opts opts)
      throws IOException {
    requireNonNull(path, "path must be non null");
    if (Files.isDirectory(path)) {
      throw new StorageException(0, path + " is a directory");
    }

    GrpcCallContext grpcCallContext =
        opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
    WriteObjectRequest req = getWriteObjectRequest(info, opts);

    ClientStreamingCallable write =
        storageClient.writeObjectCallable().withDefaultCallContext(grpcCallContext);

    ApiFuture start = startResumableWrite(grpcCallContext, req, opts);
    ApiFuture session2 =
        ApiFutures.transform(
            start,
            rw ->
                ResumableSession.grpc(
                    getOptions(),
                    retryAlgorithmManager.idempotent(),
                    write,
                    storageClient.queryWriteStatusCallable(),
                    rw,
                    Hasher.noop()),
            MoreExecutors.directExecutor());
    try {
      GrpcResumableSession got = session2.get();
      ResumableOperationResult<@Nullable Object> put = got.put(RewindableContent.of(path));
      Object object = put.getObject();
      if (object == null) {
        // if by some odd chance the put didn't get the Object, query for it
        ResumableOperationResult<@Nullable Object> query = got.query();
        object = query.getObject();
      }
      return codecs.blobInfo().decode(object).asBlob(this);
    } catch (InterruptedException | ExecutionException e) {
      throw StorageException.coalesce(e);
    }
  }

  @Override
  public Blob createFrom(BlobInfo blobInfo, InputStream content, BlobWriteOption... options)
      throws IOException {
    return createFrom(blobInfo, content, _16MiB, options);
  }

  @Override
  public Blob createFrom(
      BlobInfo blobInfo, InputStream in, int bufferSize, BlobWriteOption... options)
      throws IOException {
    requireNonNull(blobInfo, "blobInfo must be non null");

    Opts opts = Opts.unwrap(options).resolveFrom(blobInfo).prepend(defaultOpts);
    GrpcCallContext grpcCallContext =
        opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
    WriteObjectRequest req = getWriteObjectRequest(blobInfo, opts);

    ApiFuture start = startResumableWrite(grpcCallContext, req, opts);

    BufferedWritableByteChannelSession session =
        ResumableMedia.gapic()
            .write()
            .byteChannel(
                storageClient.writeObjectCallable().withDefaultCallContext(grpcCallContext))
            .setHasher(Hasher.noop())
            .setByteStringStrategy(ByteStringStrategy.noCopy())
            .resumable()
            .withRetryConfig(getOptions(), retryAlgorithmManager.idempotent())
            .buffered(Buffers.allocateAligned(bufferSize, _256KiB))
            .setStartAsync(start)
            .build();

    // Specifically not in the try-with, so we don't close the provided stream
    ReadableByteChannel src =
        Channels.newChannel(firstNonNull(in, new ByteArrayInputStream(ZERO_BYTES)));
    try (BufferedWritableByteChannel dst = session.open()) {
      ByteStreams.copy(src, dst);
    } catch (Exception e) {
      throw StorageException.coalesce(e);
    }
    return getBlob(session.getResult());
  }

  @Override
  public Bucket get(String bucket, BucketGetOption... options) {
    Opts unwrap = Opts.unwrap(options);
    return internalBucketGet(bucket, unwrap);
  }

  @Override
  public Bucket lockRetentionPolicy(BucketInfo bucket, BucketTargetOption... options) {
    Opts opts = Opts.unwrap(options).resolveFrom(bucket).prepend(defaultOpts);
    GrpcCallContext grpcCallContext =
        opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
    LockBucketRetentionPolicyRequest.Builder builder =
        LockBucketRetentionPolicyRequest.newBuilder()
            .setBucket(bucketNameCodec.encode(bucket.getName()));
    LockBucketRetentionPolicyRequest req =
        opts.lockBucketRetentionPolicyRequest().apply(builder).build();
    GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext());
    return Retrying.run(
        getOptions(),
        retryAlgorithmManager.getFor(req),
        () -> storageClient.lockBucketRetentionPolicyCallable().call(req, merge),
        syntaxDecoders.bucket);
  }

  @Override
  public Blob get(String bucket, String blob, BlobGetOption... options) {
    return get(BlobId.of(bucket, blob), options);
  }

  @Override
  public Blob get(BlobId blob, BlobGetOption... options) {
    Opts unwrap = Opts.unwrap(options);
    return internalBlobGet(blob, unwrap);
  }

  @Override
  public Blob get(BlobId blob) {
    return get(blob, new BlobGetOption[0]);
  }

  @Override
  public Blob restore(BlobId blob, BlobRestoreOption... options) {
    Opts unwrap = Opts.unwrap(options);
    return internalObjectRestore(blob, unwrap);
  }

  private Blob internalObjectRestore(BlobId blobId, Opts opts) {
    Opts finalOpts = opts.prepend(defaultOpts).prepend(ALL_BLOB_FIELDS);
    GrpcCallContext grpcCallContext =
        finalOpts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
    RestoreObjectRequest.Builder builder =
        RestoreObjectRequest.newBuilder()
            .setBucket(bucketNameCodec.encode(blobId.getBucket()))
            .setObject(blobId.getName());
    ifNonNull(blobId.getGeneration(), builder::setGeneration);
    RestoreObjectRequest req = finalOpts.restoreObjectRequest().apply(builder).build();
    GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext());
    return Retrying.run(
        getOptions(),
        retryAlgorithmManager.getFor(req),
        () -> storageClient.restoreObjectCallable().call(req, merge),
        resp -> {
          BlobInfo tmp = codecs.blobInfo().decode(resp);
          return finalOpts.clearBlobFields().decode(tmp).asBlob(this);
        });
  }

  @Override
  public Page list(BucketListOption... options) {
    Opts opts = Opts.unwrap(options).prepend(defaultOpts).prepend(ALL_BUCKET_FIELDS);
    GrpcCallContext grpcCallContext =
        opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
    ListBucketsRequest request =
        defaultProjectId
            .listBuckets()
            .andThen(opts.listBucketsRequest())
            .apply(ListBucketsRequest.newBuilder())
            .build();
    try {
      GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext());
      return Retrying.run(
          getOptions(),
          retryAlgorithmManager.getFor(request),
          () -> storageClient.listBucketsPagedCallable().call(request, merge),
          resp ->
              new TransformingPageDecorator<>(
                  resp.getPage(),
                  syntaxDecoders.bucket.andThen(opts.clearBucketFields()),
                  getOptions(),
                  retryAlgorithmManager.getFor(request)));
    } catch (Exception e) {
      throw StorageException.coalesce(e);
    }
  }

  @Override
  public Page list(String bucket, BlobListOption... options) {
    Opts opts = Opts.unwrap(options).prepend(defaultOpts).prepend(ALL_BLOB_FIELDS);
    GrpcCallContext grpcCallContext =
        opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
    ListObjectsRequest.Builder builder =
        ListObjectsRequest.newBuilder().setParent(bucketNameCodec.encode(bucket));
    ListObjectsRequest req = opts.listObjectsRequest().apply(builder).build();
    try {
      GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext());
      return Retrying.run(
          getOptions(),
          retryAlgorithmManager.getFor(req),
          () -> storageClient.listObjectsCallable().call(req, merge),
          resp -> new ListObjectsWithSyntheticDirectoriesPage(grpcCallContext, req, resp));
    } catch (Exception e) {
      throw StorageException.coalesce(e);
    }
  }

  @Override
  public Bucket update(BucketInfo bucketInfo, BucketTargetOption... options) {
    Opts unwrap = Opts.unwrap(options);
    if (bucketInfo.getModifiedFields().isEmpty()) {
      return internalBucketGet(bucketInfo.getName(), unwrap.constrainTo(BucketSourceOpt.class));
    }
    Opts opts = unwrap.resolveFrom(bucketInfo).prepend(defaultOpts);
    GrpcCallContext grpcCallContext =
        opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
    com.google.storage.v2.Bucket bucket = codecs.bucketInfo().encode(bucketInfo);
    UpdateBucketRequest.Builder builder =
        opts.updateBucketsRequest().apply(UpdateBucketRequest.newBuilder().setBucket(bucket));
    builder
        .getUpdateMaskBuilder()
        .addAllPaths(
            bucketInfo.getModifiedFields().stream()
                .map(NamedField::getGrpcName)
                .collect(ImmutableList.toImmutableList()));
    UpdateBucketRequest req = builder.build();
    GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext());
    return Retrying.run(
        getOptions(),
        retryAlgorithmManager.getFor(req),
        () -> storageClient.updateBucketCallable().call(req, merge),
        syntaxDecoders.bucket);
  }

  @Override
  public Blob update(BlobInfo blobInfo, BlobTargetOption... options) {
    Opts unwrap = Opts.unwrap(options);
    if (blobInfo.getModifiedFields().isEmpty()) {
      return internalBlobGet(blobInfo.getBlobId(), unwrap.constrainTo(ObjectSourceOpt.class));
    }
    Opts opts = unwrap.resolveFrom(blobInfo).prepend(defaultOpts);
    GrpcCallContext grpcCallContext =
        opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
    Object object = codecs.blobInfo().encode(blobInfo);
    UpdateObjectRequest.Builder builder =
        opts.updateObjectsRequest().apply(UpdateObjectRequest.newBuilder().setObject(object));
    builder
        .getUpdateMaskBuilder()
        .addAllPaths(
            blobInfo.getModifiedFields().stream()
                .map(NamedField::getGrpcName)
                .collect(ImmutableList.toImmutableList()));
    UpdateObjectRequest req = builder.build();
    GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext());
    return Retrying.run(
        getOptions(),
        retryAlgorithmManager.getFor(req),
        () -> storageClient.updateObjectCallable().call(req, merge),
        syntaxDecoders.blob);
  }

  @Override
  public Blob update(BlobInfo blobInfo) {
    return update(blobInfo, new BlobTargetOption[0]);
  }

  @Override
  public boolean delete(String bucket, BucketSourceOption... options) {
    Opts opts = Opts.unwrap(options).prepend(defaultOpts);
    GrpcCallContext grpcCallContext =
        opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
    DeleteBucketRequest.Builder builder =
        DeleteBucketRequest.newBuilder().setName(bucketNameCodec.encode(bucket));
    DeleteBucketRequest req = opts.deleteBucketsRequest().apply(builder).build();
    GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext());
    return Boolean.TRUE.equals(
        Retrying.run(
            getOptions(),
            retryAlgorithmManager.getFor(req),
            () -> {
              try {
                storageClient.deleteBucketCallable().call(req, merge);
                return true;
              } catch (NotFoundException e) {
                return false;
              }
            },
            Decoder.identity()));
  }

  @Override
  public boolean delete(String bucket, String blob, BlobSourceOption... options) {
    return delete(BlobId.of(bucket, blob), options);
  }

  @Override
  public boolean delete(BlobId blob, BlobSourceOption... options) {
    Opts opts = Opts.unwrap(options);
    try {
      internalObjectDelete(blob, opts);
      return true;
    } catch (NotFoundException e) {
      return false;
    } catch (StorageException e) {
      if (e.getCode() == 404) {
        return false;
      }
      throw e;
    }
  }

  @Override
  public boolean delete(BlobId blob) {
    return delete(blob, new BlobSourceOption[0]);
  }

  @Override
  public Void internalObjectDelete(BlobId id, Opts opts) {
    Opts finalOpts = opts.resolveFrom(id).prepend(defaultOpts);
    GrpcCallContext grpcCallContext =
        finalOpts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
    DeleteObjectRequest.Builder builder =
        DeleteObjectRequest.newBuilder()
            .setBucket(bucketNameCodec.encode(id.getBucket()))
            .setObject(id.getName());
    ifNonNull(id.getGeneration(), builder::setGeneration);
    DeleteObjectRequest req = finalOpts.deleteObjectsRequest().apply(builder).build();
    GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext());
    return Retrying.run(
        getOptions(),
        retryAlgorithmManager.getFor(req),
        () -> {
          storageClient.deleteObjectCallable().call(req, merge);
          return null;
        },
        Decoder.identity());
  }

  @Override
  public Blob compose(ComposeRequest composeRequest) {
    Opts opts = composeRequest.getTargetOpts().prepend(defaultOpts);
    GrpcCallContext grpcCallContext =
        opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
    ComposeObjectRequest.Builder builder = ComposeObjectRequest.newBuilder();
    composeRequest.getSourceBlobs().stream()
        .map(src -> sourceObjectEncode(src))
        .forEach(builder::addSourceObjects);
    final Object target = codecs.blobInfo().encode(composeRequest.getTarget());
    builder.setDestination(target);
    ComposeObjectRequest req = opts.composeObjectsRequest().apply(builder).build();
    GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext());
    return Retrying.run(
        getOptions(),
        retryAlgorithmManager.getFor(req),
        () -> storageClient.composeObjectCallable().call(req, merge),
        syntaxDecoders.blob);
  }

  @Override
  public CopyWriter copy(CopyRequest copyRequest) {
    BlobId src = copyRequest.getSource();
    BlobInfo dst = copyRequest.getTarget();
    Opts srcOpts =
        Opts.unwrap(copyRequest.getSourceOptions())
            .projectAsSource()
            .resolveFrom(src)
            .prepend(defaultOpts);
    Opts dstOpts =
        Opts.unwrap(copyRequest.getTargetOptions()).resolveFrom(dst).prepend(defaultOpts);

    Mapper mapper =
        srcOpts.rewriteObjectsRequest().andThen(dstOpts.rewriteObjectsRequest());

    Object srcProto = codecs.blobId().encode(src);
    Object dstProto = codecs.blobInfo().encode(dst);

    RewriteObjectRequest.Builder b =
        RewriteObjectRequest.newBuilder()
            .setDestinationName(dstProto.getName())
            .setDestinationBucket(dstProto.getBucket())
            // destination_kms_key comes from dstOpts
            // according to the docs in the protos, it is illegal to populate the following fields,
            // clear them out if they are set
            // destination_predefined_acl comes from dstOpts
            // if_*_match come from srcOpts and dstOpts
            // copy_source_encryption_* come from srcOpts
            // common_object_request_params come from dstOpts
            .setDestination(dstProto.toBuilder().clearName().clearBucket().clearKmsKey().build())
            .setSourceBucket(srcProto.getBucket())
            .setSourceObject(srcProto.getName());

    if (src.getGeneration() != null) {
      b.setSourceGeneration(src.getGeneration());
    }

    if (copyRequest.getMegabytesCopiedPerChunk() != null) {
      b.setMaxBytesRewrittenPerCall(copyRequest.getMegabytesCopiedPerChunk() * _1MiB);
    }

    RewriteObjectRequest req = mapper.apply(b).build();
    GrpcCallContext grpcCallContext =
        srcOpts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
    UnaryCallable callable =
        storageClient.rewriteObjectCallable().withDefaultCallContext(grpcCallContext);
    GrpcCallContext retryContext = Retrying.newCallContext();
    return Retrying.run(
        getOptions(),
        retryAlgorithmManager.getFor(req),
        () -> callable.call(req, retryContext),
        (resp) -> new GapicCopyWriter(this, callable, retryAlgorithmManager.idempotent(), resp));
  }

  @Override
  public byte[] readAllBytes(String bucket, String blob, BlobSourceOption... options) {
    return readAllBytes(BlobId.of(bucket, blob), options);
  }

  @Override
  public byte[] readAllBytes(BlobId blob, BlobSourceOption... options) {
    UnbufferedReadableByteChannelSession