org.duracloud.retrieval.mgmt.RetrievalWorker Maven / Gradle / Ivy
/*
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://duracloud.org/license/
*/
package org.duracloud.retrieval.mgmt;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.duracloud.chunk.util.ChunkUtil;
import org.duracloud.client.ContentStore;
import org.duracloud.common.model.ContentItem;
import org.duracloud.common.util.ChecksumUtil;
import org.duracloud.common.util.DateUtil;
import org.duracloud.retrieval.source.ContentStream;
import org.duracloud.retrieval.source.RetrievalSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.attribute.BasicFileAttributeView;
import java.nio.file.attribute.FileTime;
import java.text.ParseException;
import java.util.Date;
import java.util.Map;
/**
* Handles the retrieving of a single file from DuraCloud.
*
* @author: Bill Branan
* Date: Oct 12, 2010
*/
public class RetrievalWorker implements Runnable {
private final Logger logger = LoggerFactory.getLogger(RetrievalWorker.class);
private static final int MAX_ATTEMPTS = 5;
private static final String COPY = "-copy";
private ContentItem contentItem;
private RetrievalSource source;
private File contentDir;
private boolean overwrite;
private OutputWriter outWriter;
private boolean createSpaceDir;
private boolean applyTimestamps;
private int attempts;
private File localFile;
private ContentStream contentStream;
private StatusManager statusManager;
/**
* Creates a Retrieval Worker to handle retrieving a file
*/
public RetrievalWorker(ContentItem contentItem,
RetrievalSource source,
File contentDir,
boolean overwrite,
OutputWriter outWriter,
boolean createSpaceDir,
boolean applyTimestamps) {
this.contentItem = contentItem;
this.source = source;
this.contentDir = contentDir;
this.overwrite = overwrite;
this.outWriter = outWriter;
this.createSpaceDir = createSpaceDir;
this.applyTimestamps = applyTimestamps;
this.statusManager = StatusManager.getInstance();
this.attempts = 0;
}
public void run() {
statusManager.startingWork();
retrieveFile();
}
public Map retrieveFile() {
attempts++;
File localFile = getLocalFile();
Map props = null;
try {
if(localFile.exists()) { // File already exists
props = getContentProperties();
if(checksumsMatch(localFile,
props.get(ContentStore.CONTENT_CHECKSUM))) {
noChangeNeeded(localFile.getAbsolutePath());
} else { // Different file in DuraStore
if(overwrite) {
deleteFile(localFile);
} else {
renameFile(localFile);
}
props = retrieveToFile(localFile);
succeed(localFile.getAbsolutePath());
}
} else { // File does not exist
File parentDir = localFile.getParentFile();
if(!parentDir.exists()) {
parentDir.mkdirs();
parentDir.setWritable(true);
}
props = retrieveToFile(localFile);
succeed(localFile.getAbsolutePath());
}
} catch(Exception e) {
logger.error("Exception retrieving remote file " +
contentItem.getContentId() + " as local file " +
localFile.getAbsolutePath() + ": " + e.getMessage(), e);
if(attempts < MAX_ATTEMPTS) {
props = retrieveFile();
} else {
fail(e.getMessage());
}
}
return props;
}
/*
* Gets the local storage file for the content item
*/
public File getLocalFile() {
if(this.localFile == null) {
ChunkUtil util = new ChunkUtil();
String contentId = contentItem.getContentId();
if (util.isChunkManifest(contentId)) {
contentId = util.preChunkedContentId(contentId);
}
if(createSpaceDir) {
File spaceDir = new File(contentDir, contentItem.getSpaceId());
this.localFile = new File(spaceDir, contentId);
} else {
this.localFile = new File(contentDir, contentId);
}
}
return this.localFile;
}
protected boolean checksumsMatch(File localFile) throws IOException {
return checksumsMatch(localFile, null);
}
/*
* Checks to see if the checksums of the local file and remote file match
*/
protected boolean checksumsMatch(File localFile, String remoteChecksum)
throws IOException {
if(remoteChecksum == null || "".equals(remoteChecksum)) {
if(contentStream != null) {
remoteChecksum = contentStream.getChecksum();
} else {
remoteChecksum = source.getSourceChecksum(contentItem);
}
}
String localChecksum = getChecksum(localFile);
return localChecksum.equals(remoteChecksum);
}
protected String getChecksum(File localFile) throws IOException {
ChecksumUtil checksumUtil =
new ChecksumUtil(ChecksumUtil.Algorithm.MD5);
String localChecksum = checksumUtil.generateChecksum(localFile);
return localChecksum;
}
/*
* Renames the given file, returns the copied file. Does not change
* the original passed in file path.
*/
protected File renameFile(File localFile) throws IOException {
File origFile = new File(localFile.getAbsolutePath());
File copiedFile = new File(localFile.getParent(),
localFile.getName() + COPY);
for(int i=2; copiedFile.exists(); i++) {
copiedFile = new File(localFile.getParent(),
localFile.getName() + COPY + "-" + i);
}
FileUtils.moveFile(origFile, copiedFile);
return copiedFile;
}
/*
* Deletes a local file
*/
protected void deleteFile(File localFile) throws IOException {
localFile.delete();
}
protected Map getContentProperties() {
Map properties = null;
if(contentStream != null) {
properties = contentStream.getProperties();
} else {
properties = source.getSourceProperties(contentItem);
}
return properties;
}
/*
* Transfers the remote file stream to the local file
* @returns the checksum of the File upon successful retrieval. Successful
* retrieval means the checksum of the local file and remote file match,
* otherwise an IOException is thrown.
*/
protected Map retrieveToFile(File localFile) throws IOException {
contentStream = source.getSourceContent(contentItem);
try (
InputStream inStream = contentStream.getStream();
OutputStream outStream = new FileOutputStream(localFile);
) {
IOUtils.copyLarge(inStream, outStream);
}
if(! checksumsMatch(localFile, contentStream.getChecksum())) {
deleteFile(localFile);
throw new IOException("Calculated checksum value for retrieved " +
"file does not match properties checksum.");
}
// Set time stamps
if(applyTimestamps) {
applyTimestamps(contentStream, localFile);
}
return contentStream.getProperties();
}
/*
* Applies timestamps which are found in the content item's properties
* to the retrieved file
*/
protected void applyTimestamps(ContentStream content, File localFile) {
FileTime createTime =
convertDateToFileTime(content.getDateCreated());
FileTime lastAccessTime =
convertDateToFileTime(content.getDateLastAccessed());
FileTime lastModTime =
convertDateToFileTime(content.getDateLastModified());
BasicFileAttributeView fileAttributeView =
Files.getFileAttributeView(localFile.toPath(),
BasicFileAttributeView.class,
LinkOption.NOFOLLOW_LINKS);
// If any time value is null, that value is left unchanged
try {
fileAttributeView.setTimes(lastModTime, lastAccessTime, createTime);
} catch(IOException e) {
logger.error("Error setting timestamps for local file " +
localFile.getAbsolutePath() + ": " + e.getMessage(),
e);
}
}
/*
* Converts a date in LONG string format to a FileTime object
*/
private FileTime convertDateToFileTime(String strDate) {
FileTime time = null;
if(null != strDate) {
Date date = null;
try {
date = DateUtil.convertToDate(strDate,
DateUtil.DateFormat.LONG_FORMAT);
} catch(ParseException e) {
date = null;
}
if(null != date) {
time = FileTime.fromMillis(date.getTime());
}
}
return time;
}
protected void noChangeNeeded(String localFilePath) {
if(logger.isDebugEnabled()) {
logger.debug("Local file " + localFilePath +
" matches remote file " + contentItem.toString() +
" no update needed");
}
statusManager.noChangeCompletion();
}
protected void succeed(String localFilePath) {
if(logger.isDebugEnabled()) {
logger.debug("Successfully retrieved " + contentItem.toString() +
" to local file " + localFilePath);
}
outWriter.writeSuccess(contentItem, localFilePath, attempts);
statusManager.successfulCompletion();
}
protected void fail(String errMsg) {
String error = "Failed to retrieve " + contentItem.toString() +
" after " + attempts +
" attempts. Last error message was: " + errMsg;
logger.error(error);
System.err.println(error);
outWriter.writeFailure(contentItem, error, attempts);
statusManager.failedCompletion();
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy