org.apache.hadoop.fs.s3a.ProgressableProgressListener Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of hadoop-aws Show documentation

This module contains code to support integration with Amazon Web Services. It also declares the dependencies needed to work with AWS services.

There is a newer version: 3.4.0

Show newest version

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.fs.s3a;

import com.amazonaws.event.ProgressEvent;
import com.amazonaws.event.ProgressEventType;
import com.amazonaws.event.ProgressListener;
import com.amazonaws.services.s3.transfer.Upload;
import org.apache.hadoop.util.Progressable;
import org.slf4j.Logger;

import static com.amazonaws.event.ProgressEventType.TRANSFER_COMPLETED_EVENT;
import static com.amazonaws.event.ProgressEventType.TRANSFER_PART_STARTED_EVENT;

/**
 * Listener to progress from AWS regarding transfers.
 */
public class ProgressableProgressListener implements ProgressListener {
  private static final Logger LOG = S3AFileSystem.LOG;
  private final S3AFileSystem fs;
  private final String key;
  private final Progressable progress;
  private long lastBytesTransferred;
  private final Upload upload;

  /**
   * Instantiate.
   * @param fs filesystem: will be invoked with statistics updates
   * @param key key for the upload
   * @param upload source of events
   * @param progress optional callback for progress.
   */
  public ProgressableProgressListener(S3AFileSystem fs,
      String key,
      Upload upload,
      Progressable progress) {
    this.fs = fs;
    this.key = key;
    this.upload = upload;
    this.progress = progress;
    this.lastBytesTransferred = 0;
  }

  @Override
  public void progressChanged(ProgressEvent progressEvent) {
    if (progress != null) {
      progress.progress();
    }

    // There are 3 http ops here, but this should be close enough for now
    ProgressEventType pet = progressEvent.getEventType();
    if (pet == TRANSFER_PART_STARTED_EVENT ||
        pet == TRANSFER_COMPLETED_EVENT) {
      fs.incrementWriteOperations();
    }

    long transferred = upload.getProgress().getBytesTransferred();
    long delta = transferred - lastBytesTransferred;
    fs.incrementPutProgressStatistics(key, delta);
    lastBytesTransferred = transferred;
  }

  /**
   * Method to invoke after upload has completed.
   * This can handle race conditions in setup/teardown.
   * @return the number of bytes which were transferred after the notification
   */
  public long uploadCompleted() {
    long delta = upload.getProgress().getBytesTransferred() -
        lastBytesTransferred;
    if (delta > 0) {
      LOG.debug("S3A write delta changed after finished: {} bytes", delta);
      fs.incrementPutProgressStatistics(key, delta);
    }
    return delta;
  }

}