com.foreach.common.filemanager.services.AmazonS3FolderResource Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of common-file-manager Show documentation
A number of general purpose libraries for Java. Dependencies are kept minimal but Spring framework is considered a base requirement for most libraries.
The newest version!
/*
 * Copyright 2014 the original author or authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.foreach.common.filemanager.services;

import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.*;
import com.foreach.common.filemanager.business.*;
import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.springframework.core.task.TaskExecutor;
import org.springframework.util.AntPathMatcher;

import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.function.BiPredicate;

/**
 * Represents a folder resource on Amazon S3 storage.
 * A folder on S3 is not necessarily an object but might simply be a common path prefix between
 * several other objects. A separate folder object will only be created when calling {@link #create()},
 * in which case an empty object with trailing / will be added. For the consumer it should not really
 * make much of a difference if a folder is explicitly created this way or not.
 * 
 * Likewise calling {@link #delete(boolean)} with {@code false} (not deleting child objects) will only
 * delete the folder object if it exists, but calls to {@link #exists()} will still return {@code true}.
 * 
 * Borrows code from {@link org.springframework.cloud.aws.core.io.s3.PathMatchingSimpleStorageResourcePatternResolver}
 * for finding resources based on ANT patterns.
 *
 * @author Arne Vandamme
 * @since 1.4.0
 */
@RequiredArgsConstructor
class AmazonS3FolderResource implements FolderResource
{
	@Getter
	private final FolderDescriptor descriptor;

	private final AmazonS3 amazonS3;
	private final String bucketName;
	private final String objectName;
	private final TaskExecutor taskExecutor;

	@Override
	public Optional getParentFolderResource() {
		return descriptor.getParentFolderDescriptor()
		                 .map( fd -> new AmazonS3FolderResource( fd, amazonS3, bucketName, extractParentObjectName(), taskExecutor ) );
	}

	private String extractParentObjectName() {
		Path parent = Paths.get( objectName ).getParent();
		return parent != null ? parent.toString() + "/" : "";
	}

	@Override
	public FileRepositoryResource getResource( @NonNull String relativePath ) {
		if ( relativePath.isEmpty() || "/".equals( relativePath ) ) {
			return this;
		}

		if ( relativePath.endsWith( "/" ) ) {
			FolderDescriptor folderDescriptor = descriptor.createFolderDescriptor( relativePath );
			String childPath = stripCurrentFolderId( folderDescriptor.getFolderId() );
			String childObjectName = Paths.get( objectName, childPath ).toString() + "/";
			return new AmazonS3FolderResource( folderDescriptor, amazonS3, bucketName, childObjectName, taskExecutor );
		}

		FileDescriptor fileDescriptor = descriptor.createFileDescriptor( relativePath );
		String childPath = stripCurrentFolderId( fileDescriptor.getFolderId() );
		String childObjectName = Paths.get( this.objectName, childPath, fileDescriptor.getFileId() ).toString();
		return new AmazonS3FileResource( fileDescriptor, amazonS3, bucketName, childObjectName, taskExecutor );
	}

	private String stripCurrentFolderId( String folderId ) {
		return StringUtils.defaultString( descriptor.getFolderId() != null ? StringUtils.removeStart( folderId, descriptor.getFolderId() ) : folderId );
	}

	@Override
	public Collection findResources( String pattern ) {
		if ( exists() ) {
			Set resources = new LinkedHashSet<>();
			AntPathMatcher pathMatcher = new AntPathMatcher( "/" );
			String p = StringUtils.startsWith( pattern, "/" ) ? pattern.substring( 1 ) : pattern;
			boolean matchOnlyDirectories = StringUtils.endsWith( p, "/" );

			if ( matchOnlyDirectories ) {
				p = p.substring( 0, p.length() - 1 );
			}

			BiPredicate keyMatcher = ( candidateObjectName, antPattern ) -> {
				if ( pathMatcher.match( antPattern, antPattern.endsWith( "/" ) ? candidateObjectName : StringUtils.removeEnd( candidateObjectName, "/" ) ) ) {
					return !matchOnlyDirectories || candidateObjectName.endsWith( "/" );
				}
				return false;
			};

			findResourcesWithMatchingKeys( keyMatcher, resources, objectName + getValidPrefix( p ), objectName + p );

			return resources;
		}

		return Collections.emptyList();
	}

	private void findResourcesWithMatchingKeys( BiPredicate keyMatcher,
	                                            Set resources,
	                                            String prefix,
	                                            String keyPattern ) {
		String remainingPatternPart = getRemainingPatternPart( keyPattern, prefix );
		if ( remainingPatternPart != null && remainingPatternPart.startsWith( "**" ) ) {
			findAllResourcesThatMatches( keyMatcher, resources, prefix, keyPattern );
		}
		else {
			findProgressivelyWithPartialMatch( keyMatcher, resources, prefix, keyPattern );
		}
	}

	private void findAllResourcesThatMatches( BiPredicate keyMatcher,
	                                          Set resources,
	                                          String prefix,
	                                          String keyPattern ) {
		ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName( bucketName ).withPrefix( prefix );
		ObjectListing objectListing = null;

		do {
			try {
				if ( objectListing == null ) {
					objectListing = this.amazonS3.listObjects( listObjectsRequest );
				}
				else {
					objectListing = this.amazonS3.listNextBatchOfObjects( objectListing );
				}
				addResourcesFromObjectSummaries( keyMatcher, keyPattern, objectListing.getObjectSummaries(), resources );
				extractFolderResources( keyMatcher, keyPattern, prefix, objectListing, resources );
			}
			catch ( AmazonS3Exception e ) {
				if ( 301 != e.getStatusCode() ) {
					throw e;
				}
			}
		}
		while ( objectListing != null && objectListing.isTruncated() );
	}

	private void extractFolderResources( BiPredicate keyMatcher,
	                                     String keyPattern, String prefix, ObjectListing objectListing, Set resources ) {
		objectListing.getObjectSummaries().forEach( objectSummary -> {
			String resultObjectName = StringUtils.removeEnd( objectSummary.getKey(), "/" );

			int last = resultObjectName.lastIndexOf( '/' );

			if ( last > 0 ) {
				int delim = resultObjectName.indexOf( '/', prefix.length() + 1 );
				while ( delim != -1 && delim <= last ) {
					String partial = resultObjectName.substring( 0, delim + 1 );
					if ( partial.length() > 0 && keyMatcher.test( partial, keyPattern ) ) {
						resources.add( toFileRepositoryResource( partial, null ) );
					}
					delim = resultObjectName.indexOf( '/', delim + 1 );
				}
			}
		} );
	}

	private void findProgressivelyWithPartialMatch( BiPredicate keyMatcher,
	                                                Set resources,
	                                                String prefix,
	                                                String keyPattern ) {
		ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName( bucketName ).withDelimiter( "/" ).withPrefix( prefix );
		ObjectListing objectListing = null;

		do {
			if ( objectListing == null ) {
				objectListing = amazonS3.listObjects( listObjectsRequest );
			}
			else {
				objectListing = amazonS3.listNextBatchOfObjects( objectListing );
			}

			addResourcesFromObjectSummaries( keyMatcher, keyPattern, objectListing.getObjectSummaries(), resources );

			for ( String commonPrefix : objectListing.getCommonPrefixes() ) {
				if ( keyMatcher.test( commonPrefix, keyPattern ) ) {
					resources.add( toFileRepositoryResource( commonPrefix, null ) );
				}

				if ( isKeyPathMatchesPartially( keyMatcher, keyPattern, commonPrefix ) ) {
					findResourcesWithMatchingKeys( keyMatcher, resources, commonPrefix, keyPattern );
				}
			}
		}
		while ( objectListing.isTruncated() );
	}

	private String getRemainingPatternPart( String keyPattern, String path ) {
		int numberOfSlashes = org.springframework.util.StringUtils.countOccurrencesOf( path, "/" );
		int indexOfNthSlash = getIndexOfNthOccurrence( keyPattern, numberOfSlashes );
		return indexOfNthSlash == -1 ? null : keyPattern.substring( indexOfNthSlash );
	}

	private boolean isKeyPathMatchesPartially( BiPredicate keyMatcher, String keyPattern, String keyPath ) {
		int numberOfSlashes = org.springframework.util.StringUtils.countOccurrencesOf( keyPath, "/" );
		int indexOfNthSlash = getIndexOfNthOccurrence( keyPattern, numberOfSlashes );
		if ( indexOfNthSlash != -1 ) {
			return keyMatcher.test( keyPath, keyPattern.substring( 0, indexOfNthSlash ) );
		}
		else {
			return false;
		}
	}

	private int getIndexOfNthOccurrence( String str, int pos ) {
		int result = 0;
		String subStr = str;
		for ( int i = 0; i < pos; i++ ) {
			int nthOccurrence = subStr.indexOf( '/' );
			if ( nthOccurrence == -1 ) {
				return -1;
			}
			else {
				result += nthOccurrence + 1;
				subStr = subStr.substring( nthOccurrence + 1 );
			}
		}
		return result;
	}

	private void addResourcesFromObjectSummaries( BiPredicate keyMatcher,
	                                              String keyPattern,
	                                              List objectSummaries,
	                                              Set resources ) {
		objectSummaries.forEach( candidate -> {
			String candidateObjectName = candidate.getKey();
			if ( !candidateObjectName.equals( objectName ) && keyMatcher.test( candidateObjectName, keyPattern ) ) {
				resources.add( toFileRepositoryResource( candidateObjectName, candidate ) );
			}
		} );
	}

	private FileRepositoryResource toFileRepositoryResource( String childObjectName, S3ObjectSummary childObjectSummary ) {
		String childPath = StringUtils.removeStart( childObjectName, objectName );
		if ( childObjectName.endsWith( "/" ) ) {
			return new AmazonS3FolderResource( descriptor.createFolderDescriptor( childPath ), amazonS3, bucketName, childObjectName, taskExecutor );
		}

		AmazonS3FileResource fileResource = new AmazonS3FileResource( descriptor.createFileDescriptor( childPath ), amazonS3, bucketName,
		                                                              childObjectName, taskExecutor );
		if ( childObjectSummary != null ) {
			fileResource.loadMetadata( childObjectSummary );
		}

		return fileResource;
	}

	@SuppressWarnings("Duplicates")
	private String getValidPrefix( String keyPattern ) {
		int starIndex = keyPattern.indexOf( '*' );
		int markIndex = keyPattern.indexOf( '?' );
		int index = Math.min(
				starIndex == -1 ? keyPattern.length() : starIndex,
				markIndex == -1 ? keyPattern.length() : markIndex
		);
		String beforeIndex = keyPattern.substring( 0, index );
		return beforeIndex.contains( "/" ) ? beforeIndex.substring( 0, beforeIndex.lastIndexOf( '/' ) + 1 ) : "";
	}

	@Override
	public boolean delete( boolean deleteChildren ) {
		boolean deleted = false;

		if ( deleteChildren ) {
			deleted = deleteChildren();
		}

		if ( isNotRootFolder() ) {
			amazonS3.deleteObject( bucketName, objectName );
			deleted = true;
		}

		return deleted;
	}

	@Override
	public boolean deleteChildren() {
		Collection resources = findResources( "*" );

		if ( !resources.isEmpty() ) {
			try {
				resources.forEach( r -> {
					if ( r instanceof FileResource ) {
						( (FileResource) r ).delete();
					}
					else {
						( (FolderResource) r ).delete( true );
					}
				} );
			}
			catch ( AmazonS3Exception ignore ) {
				return false;
			}
			return true;
		}

		return false;
	}

	@Override
	public boolean create() {
		if ( isNotRootFolder() && !amazonS3.doesObjectExist( bucketName, objectName ) ) {
			// explicitly create a folder object
			ObjectMetadata metadata = new ObjectMetadata();
			metadata.setContentLength( 0 );
			InputStream emptyContent = new ByteArrayInputStream( new byte[0] );
			PutObjectRequest putObjectRequest = new PutObjectRequest( bucketName, objectName, emptyContent, metadata );
			amazonS3.putObject( putObjectRequest );

			return true;
		}

		return false;
	}

	@Override
	public boolean exists() {
		// a folder concept in S3 always exists as it can contain resources
		return true;
	}

	private boolean isNotRootFolder() {
		return !"".equals( objectName );
	}

	@Override
	public String toString() {
		return "axfs [" + descriptor.toString() + "] -> Amazon s3 resource [bucket='" + this.bucketName + "' and object='" + this.objectName + "']";
	}

	@Override
	public boolean equals( Object obj ) {
		return obj == this || ( obj instanceof FolderResource && descriptor.equals( ( (FolderResource) obj ).getDescriptor() ) );
	}

	@Override
	public int hashCode() {
		return descriptor.hashCode();
	}
}