org.opencastproject.distribution.aws.s3.AwsS3DistributionServiceImpl Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of opencast-distribution-service-aws-s3 Show documentation
There is a newer version: 16.7
Show newest version
/*
 * Licensed to The Apereo Foundation under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional
 * information regarding copyright ownership.
 *
 *
 * The Apereo Foundation licenses this file to you under the Educational
 * Community License, Version 2.0 (the "License"); you may not use this file
 * except in compliance with the License. You may obtain a copy of the License
 * at:
 *
 *   http://opensource.org/licenses/ecl2.txt
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
 * License for the specific language governing permissions and limitations under
 * the License.
 *
 */
package org.opencastproject.distribution.aws.s3;

import static java.lang.String.format;
import static org.opencastproject.util.RequireUtil.notNull;

import org.opencastproject.distribution.api.AbstractDistributionService;
import org.opencastproject.distribution.api.DistributionException;
import org.opencastproject.distribution.api.DistributionService;
import org.opencastproject.distribution.api.DownloadDistributionService;
import org.opencastproject.distribution.aws.s3.api.AwsS3DistributionService;
import org.opencastproject.job.api.Job;
import org.opencastproject.mediapackage.AdaptivePlaylist;
import org.opencastproject.mediapackage.MediaPackage;
import org.opencastproject.mediapackage.MediaPackageElement;
import org.opencastproject.mediapackage.MediaPackageElementFlavor;
import org.opencastproject.mediapackage.MediaPackageElementParser;
import org.opencastproject.mediapackage.MediaPackageException;
import org.opencastproject.mediapackage.MediaPackageParser;
import org.opencastproject.mediapackage.Track;
import org.opencastproject.security.api.OrganizationDirectoryService;
import org.opencastproject.security.api.SecurityService;
import org.opencastproject.security.api.UserDirectoryService;
import org.opencastproject.serviceregistry.api.ServiceRegistry;
import org.opencastproject.serviceregistry.api.ServiceRegistryException;
import org.opencastproject.util.ConfigurationException;
import org.opencastproject.util.LoadUtil;
import org.opencastproject.util.NotFoundException;
import org.opencastproject.util.OsgiUtil;
import org.opencastproject.util.data.Option;
import org.opencastproject.workspace.api.Workspace;

import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.HttpMethod;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.auth.policy.Policy;
import com.amazonaws.auth.policy.Principal;
import com.amazonaws.auth.policy.Statement;
import com.amazonaws.auth.policy.actions.S3Actions;
import com.amazonaws.auth.policy.resources.S3ObjectResource;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.BucketWebsiteConfiguration;
import com.amazonaws.services.s3.model.SetBucketWebsiteConfigurationRequest;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;

import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.osgi.service.component.ComponentContext;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Deactivate;
import org.osgi.service.component.annotations.Reference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import javax.servlet.http.HttpServletResponse;

@Component(
    immediate = true,
    service = { DistributionService.class, DownloadDistributionService.class, AwsS3DistributionService.class },
    property = {
        "service.description=Distribution Service (AWS S3)",
        "distribution.channel=aws.s3"
    }
)
public class AwsS3DistributionServiceImpl extends AbstractDistributionService
        implements AwsS3DistributionService, DistributionService {

  /** Logging facility */
  private static final Logger logger = LoggerFactory.getLogger(AwsS3DistributionServiceImpl.class);

  /** Job type */
  public static final String JOB_TYPE = "org.opencastproject.distribution.aws.s3";

  /** List of available operations on jobs */
  public enum Operation {
    Distribute, Retract
  }

  // Service configuration
  public static final String AWS_S3_DISTRIBUTION_ENABLE = "org.opencastproject.distribution.aws.s3.distribution.enable";
  public static final String AWS_S3_DISTRIBUTION_BASE_CONFIG
          = "org.opencastproject.distribution.aws.s3.distribution.base";
  public static final String AWS_S3_ACCESS_KEY_ID_CONFIG = "org.opencastproject.distribution.aws.s3.access.id";
  public static final String AWS_S3_SECRET_ACCESS_KEY_CONFIG = "org.opencastproject.distribution.aws.s3.secret.key";
  public static final String AWS_S3_REGION_CONFIG = "org.opencastproject.distribution.aws.s3.region";
  public static final String AWS_S3_BUCKET_CONFIG = "org.opencastproject.distribution.aws.s3.bucket";
  public static final String AWS_S3_ENDPOINT_CONFIG = "org.opencastproject.distribution.aws.s3.endpoint";
  public static final String AWS_S3_PATH_STYLE_CONFIG = "org.opencastproject.distribution.aws.s3.path.style";
  public static final String AWS_S3_PRESIGNED_URL_CONFIG = "org.opencastproject.distribution.aws.s3.presigned.url";
  public static final String AWS_S3_PRESIGNED_URL_VALID_DURATION_CONFIG
      = "org.opencastproject.distribution.aws.s3.presigned.url.valid.duration";
  // S3 client configuration
  public static final String AWS_S3_MAX_CONNECTIONS = "org.opencastproject.distribution.aws.s3.max.connections";
  public static final String AWS_S3_CONNECTION_TIMEOUT = "org.opencastproject.distribution.aws.s3.connection.timeout";
  public static final String AWS_S3_MAX_RETRIES = "org.opencastproject.distribution.aws.s3.max.retries";
  // job loads
  public static final String DISTRIBUTE_JOB_LOAD_KEY = "job.load.aws.s3.distribute";
  public static final String RETRACT_JOB_LOAD_KEY = "job.load.aws.s3.retract";

  // config.properties
  public static final String OPENCAST_STORAGE_DIR = "org.opencastproject.storage.dir";
  public static final String DEFAULT_TEMP_DIR = "tmp/s3dist";

  // Defaults

  // S3 client config defaults
  public static final int DEFAULT_MAX_CONNECTIONS = 50;
  public static final int DEFAULT_CONNECTION_TIMEOUT = 10000;
  public static final int DEFAULT_MAX_RETRIES = 100;

  /** The load on the system introduced by creating a distribute job */
  public static final float DEFAULT_DISTRIBUTE_JOB_LOAD = 0.1f;

  /** The load on the system introduced by creating a retract job */
  public static final float DEFAULT_RETRACT_JOB_LOAD = 0.1f;

  /** Default expiration time for presigned URL in millis, 6 hours */
  public static final int DEFAULT_PRESIGNED_URL_EXPIRE_MILLIS = 6 * 60 * 60 * 1000;

  /** Max expiration time for presigned URL in millis, 7 days */
  private static final int MAXIMUM_PRESIGNED_URL_EXPIRE_MILLIS = 7 * 24 * 60 * 60 * 1000;

  /** The load on the system introduced by creating a distribute job */
  private float distributeJobLoad = DEFAULT_DISTRIBUTE_JOB_LOAD;

  /** The load on the system introduced by creating a retract job */
  private float retractJobLoad = DEFAULT_RETRACT_JOB_LOAD;

  /** Maximum number of tries for checking availability of distributed file */
  private static final int MAX_TRIES = 10;

  /** Interval time in millis to sleep between checks of availability */
  private static final long SLEEP_INTERVAL = 30000L;

  /** The AWS client and transfer manager */
  private AmazonS3 s3 = null;
  private TransferManager s3TransferManager = null;

  /** The AWS S3 bucket name */
  private String bucketName = null;
  private Path tmpPath = null;

  /** The AWS S3 endpoint */
  private String endpoint = null;

  /** path style enabled */
  private boolean pathStyle = false;

  /** whether use presigned URL */
  private boolean presignedUrl = false;

  /** valid duration for presigned URL in milliseconds */
  private int presignedUrlValidDuration = DEFAULT_PRESIGNED_URL_EXPIRE_MILLIS;

  /** The opencast download distribution url */
  private String opencastDistributionUrl = null;

  private Gson gson = new Gson();

  /**
   * Creates a new instance of the AWS S3 distribution service.
   */
  public AwsS3DistributionServiceImpl() {
    super(JOB_TYPE);
  }

  private String getAWSConfigKey(ComponentContext cc, String key) {
    try {
      return OsgiUtil.getComponentContextProperty(cc, key);
    } catch (RuntimeException e) {
      throw new ConfigurationException(key + " is missing or invalid", e);
    }
  }

  @Override
  @Activate
  public void activate(ComponentContext cc) {

    // Get the configuration
    if (cc != null) {

      if (!BooleanUtils.toBoolean(getAWSConfigKey(cc, AWS_S3_DISTRIBUTION_ENABLE))) {
        logger.info("AWS S3 distribution disabled");
        return;
      }

      tmpPath = Paths.get(cc.getBundleContext().getProperty(OPENCAST_STORAGE_DIR), DEFAULT_TEMP_DIR);

      // clean up old data and delete directory if it exists
      if (tmpPath.toFile().exists()) {
        try (Stream walk = Files.walk(tmpPath)) {
          walk.map(Path::toFile).sorted(Comparator.reverseOrder()).forEach(File::delete);
        } catch (IOException e) {
          logger.warn("Unable to delete {}", tmpPath, e);
        }
      }
      logger.info("AWS S3 Distribution uses temp storage in {}", tmpPath);
      try { // create a new temp directory
        Files.createDirectories(tmpPath);
      } catch (IOException e) {
        logger.error("Could not create temporary directory for AWS S3 Distribution : `{}`", tmpPath);
        throw new IllegalStateException(e);
      }

      // AWS S3 bucket name
      bucketName = getAWSConfigKey(cc, AWS_S3_BUCKET_CONFIG);
      logger.info("AWS S3 bucket name is {}", bucketName);

      // AWS region
      String regionStr = getAWSConfigKey(cc, AWS_S3_REGION_CONFIG);
      logger.info("AWS region is {}", regionStr);

      // AWS endpoint
      endpoint = OsgiUtil.getComponentContextProperty(cc, AWS_S3_ENDPOINT_CONFIG, "s3." + regionStr + ".amazonaws.com");
      logger.info("AWS S3 endpoint is {}", endpoint);

      // AWS path style
      pathStyle = BooleanUtils.toBoolean(OsgiUtil.getComponentContextProperty(cc, AWS_S3_PATH_STYLE_CONFIG, "false"));
      logger.info("AWS path style is {}", pathStyle);

      // AWS presigned URL
      String presignedUrlConfigValue = OsgiUtil.getComponentContextProperty(cc, AWS_S3_PRESIGNED_URL_CONFIG, "false");
      presignedUrl = StringUtils.equalsIgnoreCase("true", presignedUrlConfigValue);
      logger.info("AWS use presigned URL: {}", presignedUrl);

      // AWS presigned URL expiration time in millis
      String presignedUrlExpTimeMillisConfigValue = OsgiUtil.getComponentContextProperty(cc,
              AWS_S3_PRESIGNED_URL_VALID_DURATION_CONFIG, null);
      presignedUrlValidDuration = NumberUtils.toInt(presignedUrlExpTimeMillisConfigValue,
              DEFAULT_PRESIGNED_URL_EXPIRE_MILLIS);
      if (presignedUrlValidDuration > MAXIMUM_PRESIGNED_URL_EXPIRE_MILLIS) {
        logger.warn(
                "Valid duration of presigned URL is too large, MAXIMUM_PRESIGNED_URL_EXPIRE_MILLIS(7 days) is used");
        presignedUrlValidDuration = MAXIMUM_PRESIGNED_URL_EXPIRE_MILLIS;
      }

      opencastDistributionUrl = getAWSConfigKey(cc, AWS_S3_DISTRIBUTION_BASE_CONFIG);
      if (!opencastDistributionUrl.endsWith("/")) {
        opencastDistributionUrl = opencastDistributionUrl + "/";
      }
      logger.info("AWS distribution url is {}", opencastDistributionUrl);

      distributeJobLoad = LoadUtil.getConfiguredLoadValue(cc.getProperties(), DISTRIBUTE_JOB_LOAD_KEY,
              DEFAULT_DISTRIBUTE_JOB_LOAD, serviceRegistry);
      retractJobLoad = LoadUtil.getConfiguredLoadValue(cc.getProperties(), RETRACT_JOB_LOAD_KEY,
              DEFAULT_RETRACT_JOB_LOAD, serviceRegistry);

      // Explicit credentials are optional.
      AWSCredentialsProvider provider = null;
      Option accessKeyIdOpt = OsgiUtil.getOptCfg(cc.getProperties(), AWS_S3_ACCESS_KEY_ID_CONFIG);
      Option accessKeySecretOpt = OsgiUtil.getOptCfg(cc.getProperties(), AWS_S3_SECRET_ACCESS_KEY_CONFIG);

      // Keys not informed so use default credentials provider chain, which
      // will look at the environment variables, java system props, credential files, and instance
      // profile credentials
      if (accessKeyIdOpt.isNone() && accessKeySecretOpt.isNone()) {
        provider = new DefaultAWSCredentialsProviderChain();
      } else {
        provider = new AWSStaticCredentialsProvider(
                new BasicAWSCredentials(accessKeyIdOpt.get(), accessKeySecretOpt.get()));
      }

      // S3 client configuration
      ClientConfiguration clientConfiguration = new ClientConfiguration();

      int maxConnections = OsgiUtil.getOptCfgAsInt(cc.getProperties(), AWS_S3_MAX_CONNECTIONS)
              .getOrElse(DEFAULT_MAX_CONNECTIONS);
      logger.debug("Max Connections: {}", maxConnections);
      clientConfiguration.setMaxConnections(maxConnections);

      int connectionTimeout = OsgiUtil.getOptCfgAsInt(cc.getProperties(), AWS_S3_CONNECTION_TIMEOUT)
              .getOrElse(DEFAULT_CONNECTION_TIMEOUT);
      logger.debug("Connection Output: {}", connectionTimeout);
      clientConfiguration.setConnectionTimeout(connectionTimeout);

      int maxRetries = OsgiUtil.getOptCfgAsInt(cc.getProperties(), AWS_S3_MAX_RETRIES)
              .getOrElse(DEFAULT_MAX_RETRIES);
      logger.debug("Max Retry: {}", maxRetries);
      clientConfiguration.setMaxErrorRetry(maxRetries);

      // Create AWS client
      s3 = AmazonS3ClientBuilder.standard()
              .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, regionStr))
              .withClientConfiguration(clientConfiguration)
              .withPathStyleAccessEnabled(pathStyle).withCredentials(provider).build();

      s3TransferManager = new TransferManager(s3);

      // Create AWS S3 bucket if not there yet
      createAWSBucket();
      distributionChannel = OsgiUtil.getComponentContextProperty(cc, CONFIG_KEY_STORE_TYPE);

      logger.info("AwsS3DistributionService activated!");
    }
  }

  @Override
  public String getDistributionType() {
    return distributionChannel;
  }

  @Deactivate
  public void deactivate() {
    // Transfer manager is null if service disabled
    if (s3TransferManager != null) {
      s3TransferManager.shutdownNow();
    }

    logger.info("AwsS3DistributionService deactivated!");
  }

  @Override
  public Job distribute(String pubChannelId, MediaPackage mediaPackage, Set downloadIds,
          boolean checkAvailability, boolean preserveReference) throws DistributionException, MediaPackageException {
    throw new UnsupportedOperationException("Not supported yet.");
    // stub function
  }

  /**
   * {@inheritDoc}
   *
   * @see org.opencastproject.distribution.api.DownloadDistributionService#distribute(String,
   *      org.opencastproject.mediapackage.MediaPackage, String, boolean)
   */
  @Override
  public Job distribute(String channelId, MediaPackage mediaPackage, Set elementIds, boolean checkAvailability)
          throws DistributionException, MediaPackageException {
    notNull(mediaPackage, "mediapackage");
    notNull(elementIds, "elementIds");
    notNull(channelId, "channelId");
    try {
      return serviceRegistry.createJob(JOB_TYPE, Operation.Distribute.toString(), Arrays.asList(channelId,
              MediaPackageParser.getAsXml(mediaPackage), gson.toJson(elementIds), Boolean.toString(checkAvailability)),
              distributeJobLoad);
    } catch (ServiceRegistryException e) {
      throw new DistributionException("Unable to create a job", e);
    }
  }

  /**
   * {@inheritDoc}
   *
   * @see org.opencastproject.distribution.api.DistributionService#distribute(String,
   *      org.opencastproject.mediapackage.MediaPackage, String)
   */
  @Override
  public Job distribute(String channelId, MediaPackage mediapackage, String elementId)
          throws DistributionException, MediaPackageException {
    return distribute(channelId, mediapackage, elementId, true);
  }

  /**
   * {@inheritDoc}
   *
   * @see org.opencastproject.distribution.api.DownloadDistributionService#distribute(String,
   *      org.opencastproject.mediapackage.MediaPackage, String, boolean)
   */
  @Override
  public Job distribute(String channelId, MediaPackage mediaPackage, String elementId, boolean checkAvailability)
          throws DistributionException, MediaPackageException {
    Set elementIds = new HashSet<>();
    elementIds.add(elementId);
    return distribute(channelId, mediaPackage, elementIds, checkAvailability);
  }

  /**
   * Distribute Mediapackage elements to the download distribution service.
   *
   * @param channelId
   *          # The id of the publication channel to be distributed to.
   * @param mediapackage
   *          The media package that contains the elements to be distributed.
   * @param elementIds
   *          The ids of the elements that should be distributed contained within the media package.
   * @param checkAvailability
   *          Check the availability of the distributed element via http.
   * @return A reference to the MediaPackageElements that have been distributed.
   * @throws DistributionException
   *           Thrown if the parent directory of the MediaPackageElement cannot be created, if the MediaPackageElement
   *           cannot be copied or another unexpected exception occurs.
   */
  public MediaPackageElement[] distributeElements(String channelId, MediaPackage mediapackage, Set elementIds,
          boolean checkAvailability) throws DistributionException {
    notNull(mediapackage, "mediapackage");
    notNull(elementIds, "elementIds");
    notNull(channelId, "channelId");

    final Set elements = getElements(mediapackage, elementIds);
    List distributedElements = new ArrayList<>();

    if (AdaptivePlaylist.hasHLSPlaylist(elements)) {
      return distributeHLSElements(channelId, mediapackage, elements, checkAvailability);
    }

    for (MediaPackageElement element : elements) {
      MediaPackageElement distributedElement = distributeElement(channelId, mediapackage, element, checkAvailability);
      distributedElements.add(distributedElement);
    }
    return distributedElements.toArray(new MediaPackageElement[distributedElements.size()]);
  }

  private Set getElements(MediaPackage mediapackage, Set elementIds)
          throws IllegalStateException {
    final Set elements = new HashSet<>();
    for (String elementId : elementIds) {
      MediaPackageElement element = mediapackage.getElementById(elementId);
      if (element != null) {
        elements.add(element);
      } else {
        throw new IllegalStateException(
                format("No element %s found in mediapackage %s", elementId, mediapackage.getIdentifier()));
      }
    }
    return elements;
  }

  /**
   * Distribute a media package element to AWS S3.
   *
   * @param mediaPackage
   *          The media package that contains the element to distribute.
   * @param element
   *          The element that should be distributed contained within the media package.
   * @param checkAvailability
   *          Checks if the distributed element is available
   * @return A reference to the MediaPackageElement that has been distributed.
   * @throws DistributionException
   */
  public MediaPackageElement distributeElement(String channelId, final MediaPackage mediaPackage,
          MediaPackageElement element, boolean checkAvailability) throws DistributionException {
    notNull(channelId, "channelId");
    notNull(mediaPackage, "mediapackage");
    notNull(element, "element");

    try {
      return distributeElement(channelId, mediaPackage, element, checkAvailability, workspace.get(element.getURI()));
    } catch (NotFoundException e) {
      throw new DistributionException("Unable to find " + element.getURI() + " in the workspace", e);
    } catch (IOException e) {
      throw new DistributionException("Error loading " + element.getURI() + " from the workspace", e);
    }
  }

  private MediaPackageElement distributeElement(String channelId, final MediaPackage mediaPackage,
          MediaPackageElement element, boolean checkAvailability, File source) throws DistributionException {

    // Use TransferManager to take advantage of multipart upload.
    // TransferManager processes all transfers asynchronously, so this call will return immediately.
    try {
      String objectName = buildObjectName(channelId, mediaPackage.getIdentifier().toString(), element);
      logger.info("Uploading {} to bucket {}...", objectName, bucketName);
      Upload upload = s3TransferManager.upload(bucketName, objectName, source);
      long start = System.currentTimeMillis();

      try {
        // Block and wait for the upload to finish
        upload.waitForCompletion();
        logger.info("Upload of {} to bucket {} completed in {} seconds", objectName, bucketName,
                (System.currentTimeMillis() - start) / 1000);
      } catch (AmazonClientException e) {
        throw new DistributionException("AWS error: " + e.getMessage(), e);
      }

      // Create a representation of the distributed file in the media package
      MediaPackageElement distributedElement = (MediaPackageElement) element.clone();
      try {
        distributedElement.setURI(getDistributionUri(objectName));
      } catch (URISyntaxException e) {
        throw new DistributionException("Distributed element produces an invalid URI", e);
      }

      logger.info("Distributed element {}, object {}", element.getIdentifier(), objectName);

      if (checkAvailability) {
        URI uri = distributedElement.getURI();
        String distributedElementUriStr = uri.toString();
        int tries = 0;
        CloseableHttpResponse response = null;
        boolean success = false;
        while (tries < MAX_TRIES) {
          try {
            if (presignedUrl) {
              // 5 minutes should be enough for check availability for presigned URL.
              Date fiveMinutesLater = new Date(System.currentTimeMillis() + 5 * 60 * 1000);
              uri = s3.generatePresignedUrl(bucketName, objectName, fiveMinutesLater, HttpMethod.HEAD).toURI();
            }
            CloseableHttpClient httpClient = HttpClients.createDefault();
            logger.trace("Trying to access {}", uri);
            response = httpClient.execute(new HttpHead(uri));
            if (response.getStatusLine().getStatusCode() == HttpServletResponse.SC_OK) {
              logger.trace("Successfully got {}", uri);
              success = true;
              break; // Exit the loop, response is closed
            } else {
              logger.debug("Http status code when checking distributed element {} is {}", objectName,
                      response.getStatusLine().getStatusCode());
            }
          } catch (Exception e) {
            logger.info("Checking availability of {} threw exception {}. Trying again.", objectName, e.getMessage());
            // Just try again
          } finally {
            if (null != response) {
              response.close();
            }
          }
          tries++;
          logger.trace("Sleeping for {} seconds...", SLEEP_INTERVAL / 1000);
          Thread.sleep(SLEEP_INTERVAL);
        }
        if (!success) {
          logger.warn("Could not check availability of distributed file {}", uri);
          // throw new DistributionException("Unable to load distributed file " + uri.toString());
        }
      }

      return distributedElement;
    } catch (Exception e) {
      logger.warn("Error distributing element " + element.getIdentifier() + " of media package " + mediaPackage, e);
      if (e instanceof DistributionException) {
        throw (DistributionException) e;
      } else {
        throw new DistributionException(e);
      }
    }
  }

  @Override
  public Job retract(String channelId, MediaPackage mediapackage, String elementId) throws DistributionException {
    Set elementIds = new HashSet<>();
    elementIds.add(elementId);
    return retract(channelId, mediapackage, elementIds);
  }

  @Override
  public Job retract(String channelId, MediaPackage mediapackage, Set elementIds) throws DistributionException {
    notNull(mediapackage, "mediapackage");
    notNull(elementIds, "elementIds");
    notNull(channelId, "channelId");
    try {
      return serviceRegistry.createJob(JOB_TYPE, Operation.Retract.toString(),
              Arrays.asList(channelId, MediaPackageParser.getAsXml(mediapackage), gson.toJson(elementIds)),
              retractJobLoad);
    } catch (ServiceRegistryException e) {
      throw new DistributionException("Unable to create a job", e);
    }
  }

  @Override
  public List distributeSync(String channelId, MediaPackage mediapackage, String elementId)
          throws DistributionException, MediaPackageException {
    Set elementIds = new HashSet();
    elementIds.add(elementId);
    return distributeSync(channelId, mediapackage, elementIds, true);
  }

  @Override
  public List distributeSync(String channelId, MediaPackage mediapackage, Set elementIds,
          boolean checkAvailability) throws DistributionException {
    final MediaPackageElement[] distributedElements = distributeElements(channelId, mediapackage, elementIds,
            checkAvailability);
    if (distributedElements == null) {
      return null;
    }
    return Arrays.asList(distributedElements);
  }

  @Override
  public List retractSync(String channelId, MediaPackage mediapackage, String elementId)
          throws DistributionException {
    Set elementIds = new HashSet();
    elementIds.add(elementId);
    return retractSync(channelId, mediapackage, elementIds);
  }

  @Override
  public List retractSync(String channelId, MediaPackage mediaPackage, Set elementIds)
          throws DistributionException {
    final MediaPackageElement[] retractedElements = retractElements(channelId, mediaPackage, elementIds);
    if (retractedElements == null) {
      return null;
    }
    return Arrays.asList(retractedElements);
  }

  /**
   * Retracts the media package element with the given identifier from the distribution channel.
   *
   * @param channelId
   *          the channel id
   * @param mediaPackage
   *          the media package
   * @param element
   *          the element
   * @return the retracted element or null if the element was not retracted
   */
  protected MediaPackageElement retractElement(String channelId, MediaPackage mediaPackage, MediaPackageElement element)
          throws DistributionException {
    notNull(mediaPackage, "mediaPackage");
    notNull(element, "element");

    try {
      String objectName = getDistributedObjectName(element);
      if (objectName != null) {
        s3.deleteObject(bucketName, objectName);
        logger.info("Retracted element {}, object {}", element.getIdentifier(), objectName);
      }
      return element;
    } catch (AmazonClientException e) {
      throw new DistributionException("AWS error: " + e.getMessage(), e);
    }
  }

  /**
   * Retract a media package element from the distribution channel. The retracted element must not necessarily be the
   * one given as parameter elementId. Instead, the element's distribution URI will be calculated. This way
   * you are able to retract elements by providing the "original" element here.
   *
   * @param channelId
   *          the channel id
   * @param mediapackage
   *          the mediapackage
   * @param elementIds
   *          the element identifiers
   * @return the retracted element or null if the element was not retracted
   * @throws org.opencastproject.distribution.api.DistributionException
   *           in case of an error
   */
  protected MediaPackageElement[] retractElements(String channelId, MediaPackage mediapackage, Set elementIds)
          throws DistributionException {
    notNull(mediapackage, "mediapackage");
    notNull(elementIds, "elementIds");
    notNull(channelId, "channelId");

    Set elements = getElements(mediapackage, elementIds);
    List retractedElements = new ArrayList<>();

    for (MediaPackageElement element : elements) {
      MediaPackageElement retractedElement = retractElement(channelId, mediapackage, element);
      retractedElements.add(retractedElement);
    }
    return retractedElements.toArray(new MediaPackageElement[retractedElements.size()]);
  }

  /**
   * Builds the aws s3 object name.
   *
   * @param channelId
   * @param mpId
   * @param element
   * @return
   */
  protected String buildObjectName(String channelId, String mpId, MediaPackageElement element) {
    // Something like ORG_ID/CHANNEL_ID/MP_ID/ELEMENT_ID/FILE_NAME.EXTENSION
    final String orgId = securityService.getOrganization().getId();
    String uriString = element.getURI().toString();
    String fileName = FilenameUtils.getName(uriString);
    return buildObjectName(orgId, channelId, mpId, element.getIdentifier(), fileName);
  }

  /**
   * Builds the aws s3 object name using the raw elementID and filename
   *
   * @param orgId
   * @param channelId
   * @param mpId
   * @param elementId
   * @param fileName
   * @return
   */
  protected String buildObjectName(String orgId, String channelId, String mpId, String elementId, String fileName) {
    return StringUtils.join(new String[] { orgId, channelId, mpId, elementId, fileName }, "/");
  }

  /**
   * Gets the URI for the element to be distributed.
   *
   * @return The resulting URI after distribution
   * @throws URISyntaxException
   *           if the concrete implementation tries to create a malformed uri
   */
  protected URI getDistributionUri(String objectName) throws URISyntaxException {
    // Something like https://OPENCAST_DOWNLOAD_URL/ORG_ID/CHANNEL_ID/MP_ID/ELEMENT_ID/FILE_NAME.EXTENSION
    return new URI(opencastDistributionUrl + objectName);
  }

  /**
   * Gets the distributed object's name.
   *
   * @return The distributed object name
   */
  protected String getDistributedObjectName(MediaPackageElement element) {
    // Something like https://OPENCAST_DOWNLOAD_URL/ORG_ID/CHANNEL_ID/MP_ID/ORIGINAL_ELEMENT_ID/FILE_NAME.EXTENSION
    String uriString = element.getURI().toString();

    // String directoryName = distributionDirectory.getAbsolutePath();
    if (uriString.startsWith(opencastDistributionUrl) && uriString.length() > opencastDistributionUrl.length()) {
      return uriString.substring(opencastDistributionUrl.length());
    } else {
      // Cannot retract
      logger.warn(
          "Cannot retract {}. Uri must be in the format "
              + "https://host/bucketName/orgId/channelId/mpId/originalElementId/fileName.extension",
          uriString);
      return null;
    }
  }

  /**
   * Distribute static items, create a temp directory for playlists, modify them to fix references, then publish the new
   * list and then delete the temp files. This is used if there are any HLS playlists in the mediapackage, all the
   * videos in the publication should be HLS or progressive, but not both. However, If this is called with non HLS
   * files, it will distribute them anyway.
   *
   * @param channelId
   *          - distribution channel
   * @param mediapackage
   *          - that holds all the files
   * @param elements
   *          - all the elements for publication
   * @param checkAvailability
   *          - check before pub
   * @return distributed elements
   * @throws DistributionException
   * @throws IOException
   */
  private MediaPackageElement[] distributeHLSElements(String channelId, MediaPackage mediapackage,
          Set elements, boolean checkAvailability) throws DistributionException {

    List distributedElements = new ArrayList();
    List nontrackElements = elements.stream()
            .filter(e -> e.getElementType() != MediaPackageElement.Type.Track).collect(Collectors.toList());
    // Distribute non track items
    for (MediaPackageElement element : nontrackElements) {
      MediaPackageElement distributedElement = distributeElement(channelId, mediapackage, element, checkAvailability);
      distributedElements.add(distributedElement);
    }
    // Then get all tracks from mediapackage and sort them by flavor
    // Each flavor is one video with multiple renditions
    List trackElements = elements.stream().filter(e -> e.getElementType() == MediaPackageElement.Type.Track)
            .map(e -> (Track) e).collect(Collectors.toList());
    HashMap> trackElementsMap
        = new HashMap>();
    for (Track t : trackElements) {
      List l = trackElementsMap.get(t.getFlavor());
      if (l == null) {
        l = new ArrayList();
      }
      l.add(t);
      trackElementsMap.put(t.getFlavor(), l);
    }

    Path tmpDir = null;
    try {
      tmpDir = Files.createTempDirectory(tmpPath, mediapackage.getIdentifier().toString());
      // Run distribution one flavor at a time
      for (Entry> elementSet : trackElementsMap.entrySet()) {
        List tracks = elementSet.getValue();
        try {
          List transformedTracks = new ArrayList();
          // If there are playlists in this flavor
          if (tracks.stream().anyMatch(AdaptivePlaylist.isHLSTrackPred)) {
            // For each adaptive playlist, get all the HLS files from the track URI
            // and put them into a temporary directory
            List tmpTracks = new ArrayList();
            for (Track t : tracks) {

              Track tcopy = (Track) t.clone();
              String newName = "./" + t.getURI().getPath();
              Path newPath = tmpDir.resolve(newName).normalize();
              Files.createDirectories(newPath.getParent());
              // If this flavor is a HLS playlist and therefore has internal references
              if (AdaptivePlaylist.isPlaylist(t)) {
                File f = workspace.get(t.getURI()); // Get actual file
                Path plcopy = Files.copy(f.toPath(), newPath);
                tcopy.setURI(plcopy.toUri()); // make it into an URI from filesystem
              } else {
                Path plcopy = Files.createFile(newPath); // new Empty File, only care about the URI
                tcopy.setURI(plcopy.toUri());
              }
              tmpTracks.add(tcopy);
            }
            // The playlists' references are then replaced with relative links
            tmpTracks = AdaptivePlaylist.fixReferences(tmpTracks, tmpDir.toFile()); // replace with fixed elements
            // after fixing it, we retrieve the new playlist files and discard the old
            // we collect the mp4 tracks and the playlists and put them into transformedTracks
            tracks.stream().filter(AdaptivePlaylist.isHLSTrackPred.negate()).forEach(t -> transformedTracks.add(t));
            tmpTracks.stream().filter(AdaptivePlaylist.isHLSTrackPred).forEach(t -> transformedTracks.add(t));
          } else {
            transformedTracks.addAll(tracks); // not playlists, distribute anyway
          }
          for (Track track : transformedTracks) {
            MediaPackageElement distributedElement;
            if (AdaptivePlaylist.isPlaylist(track)) {
              distributedElement = distributeElement(channelId, mediapackage, track, checkAvailability,
                      new File(track.getURI()));
            } else {
              distributedElement = distributeElement(channelId, mediapackage, track, checkAvailability);
            }
            distributedElements.add(distributedElement);
          }
        } catch (MediaPackageException | NotFoundException | IOException e1) {
          logger.error("HLS Prepare failed for mediapackage {} in {}: {} ", elementSet.getKey(), mediapackage, e1);
          throw new DistributionException("Cannot distribute " + mediapackage);
        } catch (URISyntaxException e1) {
          logger.error("HLS Prepare failed - Bad URI syntax {} in {}: {} ", elementSet.getKey(), mediapackage, e1);
          throw new DistributionException("Cannot distribute - BAD URI syntax " + mediapackage);
        }
      }
    } catch (IOException e2) {
      throw new DistributionException("Cannot create tmp dir to process HLS:" + mediapackage + e2.getMessage());
    } finally {
      // Clean up temp dir
      try (Stream walk = Files.walk(tmpDir)) {
        walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
      } catch (IOException e) {
        logger.warn("Cannot delete tmp dir for processing HLS mp {}, path {}", mediapackage, tmpPath, e);
      }
    }
    return distributedElements.toArray(new MediaPackageElement[distributedElements.size()]);
  }

  /**
   * {@inheritDoc}
   *
   * @see org.opencastproject.job.api.AbstractJobProducer#process(org.opencastproject.job.api.Job)
   */
  @Override
  protected String process(Job job) throws Exception {
    Operation op = null;
    String operation = job.getOperation();
    List arguments = job.getArguments();
    try {
      op = Operation.valueOf(operation);
      String channelId = arguments.get(0);
      MediaPackage mediaPackage = MediaPackageParser.getFromXml(arguments.get(1));
      Set elementIds = gson.fromJson(arguments.get(2), new TypeToken>() {
      }.getType());
      switch (op) {
        case Distribute:
          Boolean checkAvailability = Boolean.parseBoolean(arguments.get(3));
          MediaPackageElement[] distributedElements = distributeElements(channelId, mediaPackage, elementIds,
                  checkAvailability);
          return (distributedElements != null)
                  ? MediaPackageElementParser.getArrayAsXml(Arrays.asList(distributedElements))
                  : null;
        case Retract:
          MediaPackageElement[] retractedElements = retractElements(channelId, mediaPackage, elementIds);
          return (retractedElements != null) ? MediaPackageElementParser.getArrayAsXml(Arrays.asList(retractedElements))
                  : null;
        default:
          throw new IllegalStateException("Don't know how to handle operation '" + operation + "'");
      }
    } catch (IllegalArgumentException e) {
      throw new ServiceRegistryException("This service can't handle operations of type '" + op + "'", e);
    } catch (IndexOutOfBoundsException e) {
      throw new ServiceRegistryException("This argument list for operation '" + op + "' does not meet expectations", e);
    } catch (Exception e) {
      throw new ServiceRegistryException("Error handling operation '" + op + "'", e);
    }
  }

  /**
   * Creates the AWS S3 bucket if it doesn't exist yet.
   */
  protected void createAWSBucket() {
    // Does bucket exist?
    try {
      s3.listObjects(bucketName);
    } catch (AmazonServiceException e) {
      if (e.getStatusCode() == 404) {
        // Create the bucket
        try {
          s3.createBucket(bucketName);
          // Allow public read
          Statement allowPublicReadStatement = new Statement(Statement.Effect.Allow).withPrincipals(Principal.AllUsers)
                  .withActions(S3Actions.GetObject).withResources(new S3ObjectResource(bucketName, "*"));
          Policy policy = new Policy().withStatements(allowPublicReadStatement);
          s3.setBucketPolicy(bucketName, policy.toJson());

          // Set the website configuration. This needs to be static-site-enabled currently.
          BucketWebsiteConfiguration defaultWebsite = new BucketWebsiteConfiguration();
          // These files don't actually exist, but that doesn't matter since no one should be looking around in the
          // bucket anyway.
          defaultWebsite.setIndexDocumentSuffix("index.html");
          defaultWebsite.setErrorDocument("error.html");
          s3.setBucketWebsiteConfiguration(new SetBucketWebsiteConfigurationRequest(bucketName, defaultWebsite));
          logger.info("AWS S3 bucket {} created", bucketName);
        } catch (Exception e2) {
          throw new ConfigurationException("Bucket " + bucketName + " cannot be created: " + e2.getMessage(), e2);
        }
      } else {
        throw new ConfigurationException("Bucket " + bucketName + " exists, but we can't access it: " + e.getMessage(),
                e);
      }
    }
  }

  public URI presignedURI(URI uri) throws URISyntaxException {
    if (!presignedUrl) {
      return uri;
    }
    String s3UrlPrefix = s3.getUrl(bucketName, "").toString();

    // Only handle URIs match s3 domain and bucket
    if (uri.toString().startsWith(s3UrlPrefix)) {
      String objectName = uri.toString().substring(s3UrlPrefix.length());
      Date validUntil = new Date(System.currentTimeMillis() + presignedUrlValidDuration);
      return s3.generatePresignedUrl(bucketName, objectName, validUntil).toURI();
    } else {
      return uri;
    }
  }

  /** The methods below are used by the test class */

  protected void setS3(AmazonS3 s3) {
    this.s3 = s3;
  }

  protected void setS3TransferManager(TransferManager s3TransferManager) {
    this.s3TransferManager = s3TransferManager;
  }

  protected void setBucketName(String bucketName) {
    this.bucketName = bucketName;
  }

  protected void setOpencastDistributionUrl(String distributionUrl) {
    opencastDistributionUrl = distributionUrl;
  }

  // Use by unit test
  protected void setStorageTmp(String path) {
    this.tmpPath = Paths.get(path, DEFAULT_TEMP_DIR);
    try {
      Files.createDirectories(tmpPath);
    } catch (IOException e) {
      logger.info("AWS S3 bucket cannot create {} ", tmpPath);
    }
  }

  @Reference
  @Override
  public void setWorkspace(Workspace workspace) {
    super.setWorkspace(workspace);
  }

  @Reference
  @Override
  public void setServiceRegistry(ServiceRegistry serviceRegistry) {
    super.setServiceRegistry(serviceRegistry);
  }

  @Reference
  @Override
  public void setSecurityService(SecurityService securityService) {
    super.setSecurityService(securityService);
  }

  @Reference
  @Override
  public void setUserDirectoryService(UserDirectoryService userDirectoryService) {
    super.setUserDirectoryService(userDirectoryService);
  }

  @Reference
  @Override
  public void setOrganizationDirectoryService(OrganizationDirectoryService organizationDirectoryService) {
    super.setOrganizationDirectoryService(organizationDirectoryService);
  }

}