src.org.jets3t.samples.ThreadedObjectListing Maven / Gradle / Ivy

Go to download
package org.jets3t.samples;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;

import org.jets3t.service.S3ObjectsChunk;
import org.jets3t.service.S3Service;
import org.jets3t.service.S3ServiceException;
import org.jets3t.service.impl.rest.httpclient.RestS3Service;
import org.jets3t.service.multithread.ListObjectsEvent;
import org.jets3t.service.multithread.S3ServiceEventAdaptor;
import org.jets3t.service.multithread.S3ServiceMulti;
import org.jets3t.service.security.AWSCredentials;

/**
 * Demonstrates how to use the {@link S3ServiceMulti#listObjects(String, String[], String, long)}
 * method to list multiple "partitions" of an S3 bucket at once, using multiple 
 * threads to list objects matching different prefixes. 
 * 
 * @author James Murty
 */
public class ThreadedObjectListing {
    
    public static String TEST_PROPERTIES_FILENAME = "test.properties";

    public static void main(String[] args) throws Exception {
        /*
         * Set these values to test the multi-theaded listing.
         * 
         * If you set delimiter to null, only a standard single-threaded listing
         * will be performed. If you set the delimiter to a string that will 
         * identify a number of "subdirectory" partitions in your bucket, a
         * threaded listing will be performed for each such partition. 
         */
        final String bucketName = "jets3t";
        final String delimiter = "/"; 
        
        AWSCredentials awsCredentials = SamplesUtils.loadAWSCredentials();
        S3Service restService = new RestS3Service(awsCredentials);
        
        final List allObjects = Collections.synchronizedList(new ArrayList());
        final Exception s3ServiceExceptions[] = new Exception[1];        

        /*
         * Identify top-level "subdirectory" names in a bucket by performing a
         * standard object listing with a delimiter string.  
         */
        long startTime = System.currentTimeMillis();
        
        // Find all the objects and common prefixes at the top level.
        S3ObjectsChunk initialChunk = restService.listObjectsChunked(bucketName, null, delimiter, 1000, null, true);
        
        long totalElapsedTime = System.currentTimeMillis() - startTime;
                
        // We will use the common prefix strings, if any, to perform sub-listings
        final String[] commonPrefixes = initialChunk.getCommonPrefixes();

        if (commonPrefixes.length > 0) {
            System.out.println("Performing sub-listings for common prefixes: " 
                + Arrays.asList(commonPrefixes));

            /*
             * Create a S3ServiceMulti object with an event listener that responds to
             * ListObjectsEvent notifications and populates a complete object listing. 
             */
            final S3ServiceMulti s3Multi = new S3ServiceMulti(restService, new S3ServiceEventAdaptor() {
                public void s3ServiceEventPerformed(ListObjectsEvent event) {
                    if (ListObjectsEvent.EVENT_IN_PROGRESS == event.getEventCode()) {
                        Iterator chunkIter = event.getChunkList().iterator();
                        while (chunkIter.hasNext()) {
                            S3ObjectsChunk chunk = (S3ObjectsChunk) chunkIter.next();
                            
                            System.out.println("Listed " + chunk.getObjects().length 
                                + " objects for sub-listing with prefix: '" 
                                + chunk.getPrefix() + "'");
                            
                            allObjects.addAll(Arrays.asList(chunk.getObjects()));                            
                        }
                    } else if (ListObjectsEvent.EVENT_ERROR == event.getEventCode()) {
                        s3ServiceExceptions[0] = new S3ServiceException(
                            "Failed to list all objects in S3 bucket", 
                            event.getErrorCause());                    
                    }
                }
            });
                
            startTime = System.currentTimeMillis();

            /*
             * Perform a multi-threaded listing, where each common prefix string
             * will be used as the prefix for a separate listing thread.
             */ 
            (new Thread() {
                public void run() {
                    s3Multi.listObjects(bucketName, commonPrefixes, null, 1000);
                };
            }).run();
            
            long threadedElapsedTime = System.currentTimeMillis() - startTime;
            totalElapsedTime += threadedElapsedTime;

            System.out.println("\nTime to list " + allObjects.size() + " objects with " 
                + commonPrefixes.length + " sub-listings: " + threadedElapsedTime + " ms");
        }
        
        // Add top-level objects to the complete listing        
        allObjects.addAll(Arrays.asList(initialChunk.getObjects()));        

        System.out.println("\nTotal time to list " + allObjects.size() 
            + " objects: " + totalElapsedTime + " ms");
    }
    

}