org.jets3t.samples.ThreadedObjectListing Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of jets3t Show documentation
Toolkit for Amazon S3, Amazon CloudFront, and Google Storage Service.
There is a newer version: 0.9.4
/*
 * JetS3t : Java S3 Toolkit
 * Project hosted at http://bitbucket.org/jmurty/jets3t/
 *
 * Copyright 2006-2010 James Murty
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.jets3t.samples;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;

import org.jets3t.service.S3ObjectsChunk;
import org.jets3t.service.S3Service;
import org.jets3t.service.S3ServiceException;
import org.jets3t.service.StorageObjectsChunk;
import org.jets3t.service.impl.rest.httpclient.RestS3Service;
import org.jets3t.service.multithread.ListObjectsEvent;
import org.jets3t.service.multithread.S3ServiceEventAdaptor;
import org.jets3t.service.multithread.S3ServiceMulti;
import org.jets3t.service.security.AWSCredentials;

/**
 * Demonstrates how to use the {@link S3ServiceMulti#listObjects(String, String[], String, long)}
 * method to list multiple "partitions" of an S3 bucket at once, using multiple
 * threads to list objects matching different prefixes.
 *
 * @author James Murty
 */
public class ThreadedObjectListing {

    public static String TEST_PROPERTIES_FILENAME = "test.properties";

    public static void main(String[] args) throws Exception {
        /*
         * Set these values to test the multi-theaded listing.
         *
         * If you set delimiter to null, only a standard single-threaded listing
         * will be performed. If you set the delimiter to a string that will
         * identify a number of "subdirectory" partitions in your bucket, a
         * threaded listing will be performed for each such partition.
         */
        final String bucketName = "jets3t";
        final String delimiter = "/";

        AWSCredentials awsCredentials = SamplesUtils.loadAWSCredentials();
        S3Service restService = new RestS3Service(awsCredentials);

        final List allObjects = Collections.synchronizedList(new ArrayList());
        final Exception s3ServiceExceptions[] = new Exception[1];

        /*
         * Identify top-level "subdirectory" names in a bucket by performing a
         * standard object listing with a delimiter string.
         */
        long startTime = System.currentTimeMillis();

        // Find all the objects and common prefixes at the top level.
        StorageObjectsChunk initialChunk = restService.listObjectsChunked(
            bucketName, null, delimiter, 1000, null, true);

        long totalElapsedTime = System.currentTimeMillis() - startTime;

        // We will use the common prefix strings, if any, to perform sub-listings
        final String[] commonPrefixes = initialChunk.getCommonPrefixes();

        if (commonPrefixes.length > 0) {
            System.out.println("Performing sub-listings for common prefixes: "
                + Arrays.asList(commonPrefixes));

            /*
             * Create a S3ServiceMulti object with an event listener that responds to
             * ListObjectsEvent notifications and populates a complete object listing.
             */
            final S3ServiceMulti s3Multi = new S3ServiceMulti(restService, new S3ServiceEventAdaptor() {
                @Override
                public void s3ServiceEventPerformed(ListObjectsEvent event) {
                    if (ListObjectsEvent.EVENT_IN_PROGRESS == event.getEventCode()) {
                        Iterator chunkIter = event.getChunkList().iterator();
                        while (chunkIter.hasNext()) {
                            StorageObjectsChunk chunk = (StorageObjectsChunk) chunkIter.next();

                            System.out.println("Listed " + chunk.getObjects().length
                                + " objects for sub-listing with prefix: '"
                                + chunk.getPrefix() + "'");

                            allObjects.addAll(Arrays.asList(chunk.getObjects()));
                        }
                    } else if (ListObjectsEvent.EVENT_ERROR == event.getEventCode()) {
                        s3ServiceExceptions[0] = new S3ServiceException(
                            "Failed to list all objects in S3 bucket",
                            event.getErrorCause());
                    }
                }
            });

            startTime = System.currentTimeMillis();

            /*
             * Perform a multi-threaded listing, where each common prefix string
             * will be used as the prefix for a separate listing thread.
             */
            (new Thread() {
                @Override
                public void run() {
                    s3Multi.listObjects(bucketName, commonPrefixes, null, 1000);
                };
            }).run();

            long threadedElapsedTime = System.currentTimeMillis() - startTime;
            totalElapsedTime += threadedElapsedTime;

            System.out.println("\nTime to list " + allObjects.size() + " objects with "
                + commonPrefixes.length + " sub-listings: " + threadedElapsedTime + " ms");
        }

        // Add top-level objects to the complete listing
        allObjects.addAll(Arrays.asList(initialChunk.getObjects()));

        System.out.println("\nTotal time to list " + allObjects.size()
            + " objects: " + totalElapsedTime + " ms");
    }


}