All Downloads are FREE. Search and download functionalities are using the official Maven repository.
Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.duracloud.syncui.service.SyncProcessManagerImpl Maven / Gradle / Ivy
/*
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://duracloud.org/license/
*/
package org.duracloud.syncui.service;
import java.io.File;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.event.EventListenerSupport;
import org.duracloud.client.ContentStore;
import org.duracloud.client.ContentStoreManager;
import org.duracloud.common.model.Credential;
import org.duracloud.error.ContentStoreException;
import org.duracloud.sync.backup.SyncBackupManager;
import org.duracloud.sync.endpoint.DuraStoreChunkSyncEndpoint;
import org.duracloud.sync.endpoint.EndPointLogger;
import org.duracloud.sync.endpoint.MonitoredFile;
import org.duracloud.sync.endpoint.SyncEndpoint;
import org.duracloud.sync.mgmt.ChangedList;
import org.duracloud.sync.mgmt.ChangedListListener;
import org.duracloud.sync.mgmt.FileExclusionManager;
import org.duracloud.sync.mgmt.StatusManager;
import org.duracloud.sync.mgmt.SyncManager;
import org.duracloud.sync.mgmt.SyncSummary;
import org.duracloud.sync.monitor.DirectoryUpdateMonitor;
import org.duracloud.sync.walker.DeleteChecker;
import org.duracloud.sync.walker.DirWalker;
import org.duracloud.sync.walker.RestartDirWalker;
import org.duracloud.syncui.domain.DirectoryConfigs;
import org.duracloud.syncui.domain.DuracloudConfiguration;
import org.duracloud.syncui.domain.SyncProcessState;
import org.duracloud.syncui.domain.SyncProcessStats;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
/**
* The SyncProcessManagerImpl is an implementation of the SyncProcessManager
* interface. It coordinates the various elements from synctool that perform the
* synchronization activites.
*
* @author Daniel Bernstein
*/
@Component("syncProcessManager")
public class SyncProcessManagerImpl implements SyncProcessManager {
private static final int CHANGE_LIST_MONITOR_FREQUENCY = 5000;
private static final int BACKUP_FREQUENCY = 5*60*1000;
private static Logger log =
LoggerFactory.getLogger(SyncProcessManagerImpl.class);
private InternalState currentState;
private SyncConfigurationManager syncConfigurationManager;
private StoppedState stoppedState = new StoppedState();
private StartingState startingState = new StartingState();
private RunningState runningState = new RunningState();
private StoppingState stoppingState = new StoppingState();
private PausingState pausingState = new PausingState();
private PausedState pausedState = new PausedState();
private ResumingState resumingState = new ResumingState();
private EventListenerSupport listeners;
private SyncProcessStateTransitionValidator syncProcessStateTransitionValidator;
private SyncManager syncManager;
private DirWalker dirWalker;
private DirectoryUpdateMonitor dirMonitor;
private DeleteChecker deleteChecker;
private SyncProcessError error;
private SyncOptimizeManager syncOptimizeManager;
private ContentStoreManagerFactory contentStoreManagerFactory;
private Date syncStartedDate = null;
private ChangedListListener changedListListener;
private SyncBackupManager syncBackupManager;
private File backupDir;
private class InternalChangedListListener implements ChangedListListener {
@Override
public void listChanged(final ChangedList list){
//ignore if there are items in the list
if(list.getListSize() > 0){
return;
}
//if appears to be empty remove listener
list.removeListener(InternalChangedListListener.this);
//in separate thread start shutdown only
//after list is absolutely empty (no reserved files)
//and sync manager has finished transferring files.
final SyncManager sm = syncManager;
new Thread(new Runnable(){
@Override
public void run() {
while (!allWorkComplete(0)) {
SyncProcessManagerImpl.this.sleep(2000);
}
SyncProcessManagerImpl.this.stop();
}
// Verify work is complete multiple times before giving
// the ok to shut everything down
private boolean allWorkComplete(int attempt) {
boolean workComplete = (sm == null || syncManager.getFilesInTransfer().isEmpty()) &&
list.getListSizeIncludingReservedFiles() <= 0;
if(!workComplete || (workComplete && attempt > 2)) {
return workComplete;
} else {
// Wait before another attempt
SyncProcessManagerImpl.this.sleep(2000);
return allWorkComplete(attempt+1);
}
}
}).start();
}
}
@Autowired
public SyncProcessManagerImpl(
SyncConfigurationManager syncConfigurationManager,
ContentStoreManagerFactory contentStoreManagerFactory,
SyncOptimizeManager syncOptimizeManager) {
this.syncConfigurationManager = syncConfigurationManager;
this.currentState = this.stoppedState;
this.listeners = new EventListenerSupport<>(SyncStateChangeListener.class);
this.syncProcessStateTransitionValidator =
new SyncProcessStateTransitionValidator();
this.contentStoreManagerFactory = contentStoreManagerFactory;
this.syncOptimizeManager = syncOptimizeManager;
this.backupDir = new File(syncConfigurationManager.getWorkDirectory(), "backup");
syncBackupManager =
new SyncBackupManager(this.backupDir,
BACKUP_FREQUENCY,
syncConfigurationManager.retrieveDirectoryConfigs().toFileList());
ChangedList.getInstance()
.addListener(this.changedListListener = new InternalChangedListListener());
}
@PostConstruct
public void init(){
automaticallyRestartIfAppropriate();
}
protected void automaticallyRestartIfAppropriate() {
RuntimeStateMemento m = RuntimeStateMemento.get();
if (this.syncConfigurationManager.isConfigurationComplete()
&& SyncProcessState.RUNNING.equals(m.getSyncProcessState())) {
try {
start();
} catch (SyncProcessException e) {
log.error("failed to automatically restart the sync process");
}
}
}
@Override
public SyncProcessError getError() {
return this.error;
}
@Override
public void clearError() {
this.error = null;
}
@Override
public void start() throws SyncProcessException {
this.currentState.start();
}
@Override
public void resume() throws SyncProcessException {
this.currentState.resume();
}
@Override
public void stop() {
this.currentState.stop();
}
@Override
public void pause() {
this.currentState.pause();
}
@Override
public void restart() {
this.currentState.restart();
}
@Override
public SyncProcessState getProcessState() {
return this.currentState.getProcessState();
}
@Override
public SyncProcessStats getProcessStats() {
return this.currentState.getProcessStats();
}
@Override
public void
addSyncStateChangeListener(SyncStateChangeListener syncStateChangeListener) {
this.listeners.addListener(syncStateChangeListener);
}
@Override
public void
removeSyncStateChangeListener(SyncStateChangeListener syncStateChangeListener) {
this.listeners.removeListener(syncStateChangeListener);
}
private void fireStateChanged(SyncProcessState state) {
SyncStateChangedEvent event = new SyncStateChangedEvent(state);
listeners.fire().stateChanged(event);
}
private synchronized void changeState(InternalState state) {
SyncProcessState current = this.currentState.getProcessState();
SyncProcessState incoming = state.getProcessState();
boolean validStateChange =
syncProcessStateTransitionValidator.validate(current, incoming);
if (validStateChange) {
this.currentState = state;
persistState(this.currentState);
fireStateChanged(this.currentState.getProcessState());
}
}
private void persistState(InternalState currentState) {
RuntimeStateMemento state = RuntimeStateMemento.get();
state.setSyncProcessState(currentState.getProcessState());
RuntimeStateMemento.persist(state);
}
private void startImpl() throws SyncProcessException {
if(syncOptimizeManager.isRunning()){
String errorMsg = "The transfer rate is currently being optimized.";
setError(new SyncProcessError(errorMsg));
return;
}
changeState(startingState);
this.syncStartedDate = new Date();
setError(null);
startAsynchronously();
}
private void startAsynchronously() {
new Thread(new Runnable(){
@Override
public void run() {
try {
startSyncProcess();
changeState(runningState);
} catch (SyncProcessException e) {
log.error("start failed: " + e.getMessage(), e);
changeState(stoppingState);
changeState(stoppedState);
}
}
}).start();
}
private void resumeImpl() throws SyncProcessException {
changeState(resumingState);
startAsynchronously();
}
private void startSyncProcess() throws SyncProcessException {
DirectoryConfigs directories =
this.syncConfigurationManager.retrieveDirectoryConfigs();
if(directories.isEmpty()){
throw new SyncProcessException("unable to start because "+
"no watch directories are configured.");
}
List dirs = directories.toFileList();
DuracloudConfiguration dc =
this.syncConfigurationManager.retrieveDuracloudConfiguration();
try {
ContentStoreManager csm = contentStoreManagerFactory.create();
String username = dc.getUsername();
String spaceId = dc.getSpaceId();
csm.login(new Credential(username, dc.getPassword()));
ContentStore contentStore = csm.getPrimaryContentStore();
boolean syncDeletes = this.syncConfigurationManager.isSyncDeletes();
String prefix = this.syncConfigurationManager.getPrefix();
SyncEndpoint syncEndpoint =
new DuraStoreChunkSyncEndpoint(contentStore,
username,
spaceId,
syncDeletes,
this.syncConfigurationManager.getMaxFileSizeInBytes(), // 1GB chunk size
this.syncConfigurationManager.isSyncUpdates(),
this.syncConfigurationManager.isRenameUpdates(),
this.syncConfigurationManager.isJumpStart(),
this.syncConfigurationManager.getUpdateSuffix(),
prefix);
syncEndpoint.addEndPointListener(new EndPointLogger());
this.backupDir.mkdirs();
syncBackupManager = new SyncBackupManager(this.backupDir,
BACKUP_FREQUENCY,
dirs);
long backup = -1;
if(syncBackupManager.hasBackups()){
backup = syncBackupManager.attemptRestart();
}
syncManager = new SyncManager(dirs, syncEndpoint,
this.syncConfigurationManager.getThreadCount(), // threads
CHANGE_LIST_MONITOR_FREQUENCY); // change list poll frequency
syncManager.beginSync();
RunMode mode = this.syncConfigurationManager.getMode();
if (backup < 0) {
dirWalker = DirWalker.start(dirs, new FileExclusionManager());
} else if (mode.equals(RunMode.CONTINUOUS)) {
dirWalker =
RestartDirWalker.start(dirs,
backup,
new FileExclusionManager());
}
startBackupsOnDirWalkerCompletion();
dirMonitor =
new DirectoryUpdateMonitor(dirs,
CHANGE_LIST_MONITOR_FREQUENCY,
syncDeletes);
configureMode(mode);
if(syncDeletes) {
deleteChecker = DeleteChecker.start(syncEndpoint,
spaceId,
dirs,
prefix);
}
} catch (ContentStoreException e) {
String message = StringUtils.abbreviate(e.getMessage(),100);
handleStartupException(message, e);
} catch (Exception e){
String message = StringUtils.abbreviate(e.getMessage(),100);
handleStartupException(message, e);
}
}
private void startBackupsOnDirWalkerCompletion() {
new Thread(new Runnable(){
@Override
public void run() {
while(dirWalker != null && !dirWalker.walkComplete()){
sleep(100);
}
log.info("Starting back up manager...");
syncBackupManager.startupBackups();
}
}, "walk-completion-checker thread").start();
}
protected void sleep() {
sleep(500);
}
private void sleep(long millis) {
try {
Thread.sleep(millis);
} catch (InterruptedException e) {
log.warn(e.getMessage(), e);
}
}
private void configureMode(RunMode mode) {
// only start the dirMonitor if the sync config manager is
// set to run continuously.
try {
if (mode.equals(RunMode.CONTINUOUS)) {
dirMonitor.startMonitor();
ChangedList.getInstance().removeListener(changedListListener);
} else {
ChangedList.getInstance().addListener(changedListListener);
dirMonitor.stopMonitor();
}
} catch (Exception ex) {
log.error(ex.getMessage(), ex);
}
}
private void handleStartupException(String message, Exception e)
throws SyncProcessException {
log.error(message, e);
setError(new SyncProcessError(message));
shutdownSyncProcess();
changeState(stoppingState);
changeState(stoppedState);
throw new SyncProcessException(message, e);
}
private void setError(SyncProcessError error) {
this.error = error;
}
private SyncProcessStats getProcessStatsImpl() {
int queueSize = ChangedList.getInstance().getListSize();
int errorSize = StatusManager.getInstance().getFailed().size();
return new SyncProcessStats(this.syncStartedDate,
null,
errorSize,
0,
0,
queueSize);
}
private void shutdownSyncProcess() {
if(this.syncManager != null){
this.syncManager.terminateSync();
}
try{
if(this.dirMonitor != null){
this.dirMonitor.stopMonitor();
}
}catch(Exception ex){
log.warn("stop monitor failed: " + ex.getMessage());
}
if(this.deleteChecker != null){
this.deleteChecker.stop();
}
if(this.dirWalker != null){
this.dirWalker.stopWalk();
}
}
private void resetChangeList() {
ChangedList.getInstance().clear();
syncBackupManager.endBackups();
syncBackupManager.clearBackups();
}
@SuppressWarnings("unused")
private class InternalListener implements SyncStateChangeListener {
private CountDownLatch latch = new CountDownLatch(1);
private SyncProcessState state;
public InternalListener(SyncProcessState state) {
this.state = state;
}
@Override
public void stateChanged(SyncStateChangedEvent event) {
if (event.getProcessState() == this.state) {
latch.countDown();
}
}
private void waitForStateChange() {
try {
latch.await();
} catch (InterruptedException e) {
log.warn(e.getMessage(), e);
}
}
}
private void stopImpl() {
changeState(stoppingState);
new Thread() {
@Override
public void run() {
shutdownSyncProcess();
syncStartedDate = null;
while (!syncManager.getFilesInTransfer().isEmpty()) {
SyncProcessManagerImpl.this.sleep();
}
resetChangeList();
changeState(stoppedState);
}
}.start();
}
private void pauseImpl() {
changeState(pausingState);
final Thread t = new Thread() {
@Override
public void run() {
shutdownSyncProcess();
SyncManager sm = SyncProcessManagerImpl.this.syncManager;
while (!sm.getFilesInTransfer().isEmpty()) {
SyncProcessManagerImpl.this.sleep();
}
changeState(pausedState);
}
};
t.start();
}
// internally the SyncProcessManagerImpl makes use of the Gof4 State
// pattern.
private abstract class InternalState implements SyncProcess {
@Override
public void start() throws SyncProcessException {
// do nothing by default
}
@Override
public void stop() {
// do nothing by default
}
@Override
public void resume() throws SyncProcessException{
// do nothing by default
}
@Override
public void pause() {
//do nothing by default
}
@Override
public void restart(){
//do nothing by default
}
@Override
public SyncProcessStats getProcessStats() {
return getProcessStatsImpl();
}
}
private class StoppedState extends InternalState {
@Override
public void start() throws SyncProcessException {
startImpl();
}
@Override
public SyncProcessState getProcessState() {
return SyncProcessState.STOPPED;
}
}
private class PausedState extends InternalState {
@Override
public void resume() throws SyncProcessException {
resumeImpl();
}
@Override
public void stop() {
stopImpl();
}
@Override
public SyncProcessState getProcessState() {
return SyncProcessState.PAUSED;
}
}
private class StartingState extends InternalState {
@Override
public SyncProcessState getProcessState() {
return SyncProcessState.STARTING;
}
}
private class ResumingState extends InternalState {
@Override
public SyncProcessState getProcessState() {
return SyncProcessState.RESUMING;
}
}
private class RunningState extends InternalState {
@Override
public void stop() {
stopImpl();
}
@Override
public void pause() {
pauseImpl();
}
@Override
public void restart() {
//register stopped listener
addSyncStateChangeListener(new SyncStateChangeListener() {
@Override
public void stateChanged(SyncStateChangedEvent event) {
if(event.getProcessState().equals(SyncProcessState.STOPPED)){
try {
removeSyncStateChangeListener(this);
startImpl();
} catch (SyncProcessException e) {
log.warn(e.getMessage(), e);
}
}
}
});
//invoke stop
stopImpl();
}
@Override
public SyncProcessState getProcessState() {
return SyncProcessState.RUNNING;
}
}
private class StoppingState extends InternalState {
@Override
public SyncProcessState getProcessState() {
return SyncProcessState.STOPPING;
}
}
private class PausingState extends InternalState {
@Override
public SyncProcessState getProcessState() {
return SyncProcessState.PAUSING;
}
}
@Override
public List getMonitoredFiles() {
SyncManager sm = this.syncManager;
if (sm != null) {
return sm.getFilesInTransfer();
}
return new LinkedList();
}
@Override
public List getFailures() {
if (this.syncManager != null) {
return StatusManager.getInstance().getFailed();
}
return new LinkedList();
}
@Override
public List getRecentlyCompleted() {
if (this.syncManager != null) {
return StatusManager.getInstance().getRecentlyCompleted();
}
return new LinkedList();
}
@Override
public List getQueuedFiles() {
return ChangedList.getInstance().peek(10);
}
@Override
public void clearFailures() {
StatusManager.getInstance().clearFailed();
}
@PreDestroy
public void shutdown(){
this.syncBackupManager.endBackups();
ChangedList.getInstance().shutdown();
}
}
