org.apache.hadoop.fs.s3a.s3guard.S3GuardTableAccess Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of hadoop-aws Show documentation
This module contains code to support integration with Amazon Web Services. It also declares the dependencies needed to work with AWS services.
There is a newer version: 3.4.0
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.fs.s3a.s3guard;

import java.util.Collection;
import java.util.Iterator;

import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.ItemCollection;
import com.amazonaws.services.dynamodbv2.document.QueryOutcome;
import com.amazonaws.services.dynamodbv2.document.ScanOutcome;
import com.amazonaws.services.dynamodbv2.document.Table;
import com.amazonaws.services.dynamodbv2.document.internal.IteratorSupport;
import com.amazonaws.services.dynamodbv2.document.spec.QuerySpec;
import com.amazonaws.services.dynamodbv2.xspec.ExpressionSpecBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3a.Retries;
import org.apache.hadoop.fs.s3a.S3AFileStatus;

import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.VERSION_MARKER_ITEM_NAME;
import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.CHILD;
import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.PARENT;
import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.TABLE_VERSION;
import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.itemToPathMetadata;
import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.pathToKey;

/**
 * Package-scoped accessor to table state in S3Guard.
 * This is for maintenance, diagnostics and testing: it is not to
 * be used otherwise.
 * 
 *   
 *     Some of the operations here may dramatically alter the state of
 *     a table, so use carefully.
 *   
 *   
 *     Operations to assess consistency of a store are best executed
 *     against a table which is otherwise inactive.
 *   
 *   
 *     No retry/throttling or AWS to IOE logic here.
 *   
 *   
 *     If a scan or query includes the version marker in the result, it
 *     is converted to a {@link VersionMarker} instance.
 *   
 * 
 *
 */
@InterfaceAudience.Private
@InterfaceStability.Unstable
@Retries.OnceRaw
class S3GuardTableAccess {

  private static final Logger LOG =
      LoggerFactory.getLogger(S3GuardTableAccess.class);

  /**
   * Store instance to work with.
   */
  private final DynamoDBMetadataStore store;

  /**
   * Table; retrieved from the store.
   */
  private final Table table;

  /**
   * Construct.
   * @param store store to work with.
   */
  S3GuardTableAccess(final DynamoDBMetadataStore store) {
    this.store = checkNotNull(store);
    this.table = checkNotNull(store.getTable());
  }

  /**
   * Username of user in store.
   * @return a string.
   */
  private String getUsername() {
    return store.getUsername();
  }

  /**
   * Execute a query.
   * @param spec query spec.
   * @return the outcome.
   */
  @Retries.OnceRaw
  ItemCollection query(QuerySpec spec) {
    return table.query(spec);
  }

  /**
   * Issue a query where the result is to be an iterator over
   * the entries
   * of DDBPathMetadata instances.
   * @param spec query spec.
   * @return an iterator over path entries.
   */
  @Retries.OnceRaw
  Iterable queryMetadata(QuerySpec spec) {
    return new DDBPathMetadataCollection<>(query(spec));
  }

  @Retries.OnceRaw
  ItemCollection scan(ExpressionSpecBuilder spec) {
    return table.scan(spec.buildForScan());
  }

  @Retries.OnceRaw
  Iterable scanMetadata(ExpressionSpecBuilder spec) {
    return new DDBPathMetadataCollection<>(scan(spec));
  }

  @Retries.OnceRaw
  void delete(Collection paths) {
    paths.stream()
        .map(PathMetadataDynamoDBTranslation::pathToKey)
        .forEach(table::deleteItem);
  }

  @Retries.OnceRaw
  void delete(Path path) {
    table.deleteItem(pathToKey(path));
  }

  /**
   * A collection which wraps the result of a query or scan.
   * Important: iterate through this only once; the outcome
   * of repeating an iteration is "undefined"
   * @param  type of outcome.
   */
  private final class DDBPathMetadataCollection
      implements Iterable {

    /**
     * Query/scan result.
     */
    private final ItemCollection outcome;

    /**
     * Instantiate.
     * @param outcome query/scan outcome.
     */
    private DDBPathMetadataCollection(final ItemCollection outcome) {
      this.outcome = outcome;
    }

    /**
     * Get the iterator.
     * @return the iterator.
     */
    @Override
    public Iterator iterator() {
      return new DDBPathMetadataIterator<>(outcome.iterator());
    }

  }

  /**
   * An iterator which converts the iterated-over result of
   * a query or scan into a {@code DDBPathMetadataIterator} entry.
   * @param  type of source.
   */
  private final class DDBPathMetadataIterator implements
      Iterator {

    /**
     * Iterator to invoke.
     */
    private final IteratorSupport it;

    /**
     * Instantiate.
     * @param it Iterator to invoke.
     */
    private DDBPathMetadataIterator(final IteratorSupport it) {
      this.it = it;
    }

    @Override
    @Retries.OnceRaw
    public boolean hasNext() {
      return it.hasNext();
    }

    @Override
    @Retries.OnceRaw
    public DDBPathMetadata next() {
      Item item = it.next();
      Pair key = primaryKey(item);
      if (VERSION_MARKER_ITEM_NAME.equals(key.getLeft()) &&
          VERSION_MARKER_ITEM_NAME.equals(key.getRight())) {
        // a version marker is found, return the special type
        return new VersionMarker(item);
      } else {
        return itemToPathMetadata(item, getUsername());
      }
    }

  }

  /**
   * DDBPathMetadata subclass returned when a query returns
   * the version marker.
   * There is a FileStatus returned where the owner field contains
   * the table version; the path is always the unqualified path "/VERSION".
   * Because it is unqualified, operations which treat this as a normal
   * DDB metadata entry usually fail.
   */
  static final class VersionMarker extends DDBPathMetadata {

    /**
     * Instantiate.
     * @param versionMarker the version marker.
     */
    VersionMarker(Item versionMarker) {
      super(new S3AFileStatus(true, new Path("/VERSION"),
          "" + versionMarker.getString(TABLE_VERSION)));
    }
  }

  /**
   * Given an item, split it to the parent and child fields.
   * @param item item to split.
   * @return (parent, child).
   */
  private static Pair primaryKey(Item item) {
    return Pair.of(item.getString(PARENT), item.getString(CHILD));
  }
}