org.jumpmind.symmetric.service.impl.DataLoaderService Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of symmetric-ds Show documentation
Show all versions of symmetric-ds Show documentation
SymmetricDS is an open source database synchronization solution. It is platform-independent,
web-enabled, and database-agnostic. SymmetricDS was first built to replicate changes between 'retail store'
databases and ad centralized 'corporate' database.
The newest version!
/*
* Licensed to JumpMind Inc under one or more contributor
* license agreements. See the NOTICE file distributed
* with this work for additional information regarding
* copyright ownership. JumpMind Inc licenses this file
* to you under the GNU Lesser General Public License (the
* "License"); you may not use this file except in compliance
* with the License.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, see
* filters;
private IStatisticManager statisticManager;
private INodeService nodeService;
private Map> columnFilters = new HashMap>();
private List batchListeners;
/**
* Connect to the remote node and pull data. The acknowledgment of commit/error status is sent separately after the
* data is processed.
*/
public boolean loadData(Node remote, Node local) throws IOException {
boolean wasWorkDone = false;
try {
NodeSecurity localSecurity = nodeService.findNodeSecurity(local.getNodeId());
if (localSecurity != null) {
Map requestProperties = new HashMap();
ChannelMap suspendIgnoreChannels = configurationService.getSuspendIgnoreChannelLists();
requestProperties.put(WebConstants.SUSPENDED_CHANNELS, suspendIgnoreChannels.getSuspendChannelsAsString());
requestProperties.put(WebConstants.IGNORED_CHANNELS, suspendIgnoreChannels.getIgnoreChannelsAsString());
List list = loadDataAndReturnBatches(transportManager.getPullTransport(remote, local, localSecurity.getNodePassword(), requestProperties, parameterService.getRegistrationUrl()));
if (list.size() > 0) {
sendAck(remote, local, localSecurity, list);
wasWorkDone = true;
}
} else {
log.error("NodeSecurityMissing", local.getNodeId());
}
} catch (RegistrationRequiredException e) {
log.warn("RegistrationLost");
loadData(transportManager.getRegisterTransport(local, parameterService.getRegistrationUrl()));
nodeService.findIdentity(false);
wasWorkDone = true;
} catch (MalformedURLException e) {
log.error("URLConnectingFailure", remote.getNodeId(), remote.getSyncUrl());
}
return wasWorkDone;
}
/**
* Try a configured number of times to get the ACK through.
*/
private void sendAck(Node remote, Node local, NodeSecurity localSecurity, List list) throws IOException {
Exception error = null;
int sendAck = -1;
int numberOfStatusSendRetries = parameterService.getInt(ParameterConstants.DATA_LOADER_NUM_OF_ACK_RETRIES);
for (int i = 0; i < numberOfStatusSendRetries && sendAck != HttpURLConnection.HTTP_OK; i++) {
try {
sendAck = transportManager.sendAcknowledgement(remote, list, local, localSecurity.getNodePassword(), parameterService.getRegistrationUrl());
} catch (IOException ex) {
log.warn("AckSendingFailed", (i + 1), ex.getMessage());
error = ex;
} catch (RuntimeException ex) {
log.warn("AckSendingFailed", (i + 1), ex.getMessage());
error = ex;
}
if (sendAck != HttpURLConnection.HTTP_OK) {
if (i < numberOfStatusSendRetries - 1) {
AppUtils.sleep(parameterService.getLong(ParameterConstants.DATA_LOADER_TIME_BETWEEN_ACK_RETRIES));
} else if (error instanceof RuntimeException) {
throw (RuntimeException) error;
} else if (error instanceof IOException) {
throw (IOException) error;
} else {
throw new IOException(Integer.toString(sendAck));
}
}
}
}
public IDataLoader openDataLoader(BufferedReader reader) throws IOException {
IDataLoader dataLoader = (IDataLoader) beanFactory.getBean(Constants.DATALOADER);
dataLoader.open(reader, filters, columnFilters);
return dataLoader;
}
public IDataLoaderStatistics loadDataBatch(String batchData) throws IOException {
BufferedReader reader = new BufferedReader(new StringReader(batchData));
IDataLoader dataLoader = openDataLoader(reader);
IDataLoaderStatistics stats = null;
try {
while (dataLoader.hasNext()) {
dataLoader.load();
IncomingBatch history = new IncomingBatch(dataLoader.getContext());
history.setValues(dataLoader.getStatistics(), true);
fireBatchComplete(dataLoader, history);
}
} finally {
stats = dataLoader.getStatistics();
dataLoader.close();
}
return stats;
}
/**
* Load database from input stream and return a list of batch statuses. This is used for a pull request that
* responds with data, and the acknowledgment is sent later.
*/
protected List loadDataAndReturnBatches(IIncomingTransport transport) throws IOException {
List list = new ArrayList();
IncomingBatch batch = null;
IDataLoader dataLoader = null;
try {
long totalNetworkMillis = System.currentTimeMillis();
if (parameterService.is(ParameterConstants.STREAM_TO_FILE_ENABLED)) {
transport = writeToFile(transport);
totalNetworkMillis = System.currentTimeMillis() - totalNetworkMillis;
}
dataLoader = openDataLoader(transport.open());
IDataLoaderContext context = dataLoader.getContext();
while (dataLoader.hasNext()) {
batch = new IncomingBatch(context);
if (parameterService.is(ParameterConstants.DATA_LOADER_ENABLED) ||
(batch.getChannelId() != null && batch.getChannelId().equals(Constants.CHANNEL_CONFIG))) {
list.add(batch);
loadBatch(dataLoader, batch);
}
batch = null;
}
if (parameterService.is(ParameterConstants.STREAM_TO_FILE_ENABLED)) {
estimateNetworkMillis(list, totalNetworkMillis);
}
for (IncomingBatch incomingBatch : list) {
if (incomingBatch.isPersistable()) {
incomingBatchService.updateIncomingBatch(incomingBatch);
}
}
} catch (RegistrationRequiredException ex) {
throw ex;
} catch (ConnectException ex) {
throw ex;
} catch (UnknownHostException ex) {
log.warn("TransportFailedUnknownHost", ex.getMessage());
throw ex;
} catch (RegistrationNotOpenException ex) {
log.warn("RegistrationFailed");
} catch (ConnectionRejectedException ex) {
log.warn("TransportFailedConnectionBusy");
throw ex;
} catch (AuthenticationException ex) {
log.warn("AuthenticationFailed");
} catch (SyncDisabledException ex) {
log.warn("SyncDisabled");
throw ex;
} catch (Throwable e) {
if (dataLoader != null && dataLoader.getContext().getBatchId() > 0 && batch == null) {
batch = new IncomingBatch(dataLoader.getContext());
list.add(batch);
}
if (dataLoader != null && batch != null) {
statisticManager.incrementDataLoadedErrors(batch.getChannelId(), 1);
if (e instanceof IOException || e instanceof TransportException) {
log.warn("BatchLoadingFailed", batch.getNodeBatchId(), e.getMessage());
batch.setSqlMessage(e.getMessage());
} else {
log.error("BatchLoadingFailed", e, batch.getNodeBatchId(), e.getMessage());
SQLException se = unwrapSqlException(e);
if (se != null) {
batch.setSqlState(se.getSQLState());
batch.setSqlCode(se.getErrorCode());
batch.setSqlMessage(se.getMessage());
} else {
batch.setSqlMessage(e.getMessage());
}
}
batch.setValues(dataLoader.getStatistics(), false);
handleBatchError(batch);
} else {
if (e instanceof IOException) {
if (!e.getMessage().startsWith("http")) {
log.error("BatchReadingFailed", e.getMessage());
} else {
log.error("BatchReadingFailed", e.getMessage(), e);
}
} else {
log.error("BatchParsingFailed", e);
}
}
} finally {
if (dataLoader != null) {
dataLoader.close();
}
transport.close();
}
return list;
}
protected void estimateNetworkMillis(List list, long totalNetworkMillis) {
long totalNumberOfBytes = 0;
for (IncomingBatch incomingBatch : list) {
totalNumberOfBytes += incomingBatch.getByteCount();
}
for (IncomingBatch incomingBatch : list) {
if (totalNumberOfBytes > 0) {
double ratio = (double) incomingBatch.getByteCount() / (double) totalNumberOfBytes;
incomingBatch.setNetworkMillis((long) (totalNetworkMillis * ratio));
}
}
}
protected IIncomingTransport writeToFile(IIncomingTransport transport) throws IOException {
ThresholdFileWriter writer = null;
try {
writer = new ThresholdFileWriter(parameterService.getLong(ParameterConstants.STREAM_TO_FILE_THRESHOLD),
"load");
IOUtils.copy(transport.open(), writer);
} finally {
IOUtils.closeQuietly(writer);
transport.close();
}
return new FileIncomingTransport(writer);
}
public boolean loadData(IIncomingTransport transport) throws IOException {
boolean inError = false;
List list = loadDataAndReturnBatches(transport);
if (list != null && list.size() > 0) {
for (IncomingBatch incomingBatch : list) {
inError |= incomingBatch.getStatus() != org.jumpmind.symmetric.model.IncomingBatch.Status.OK;
}
} else {
inError = true;
}
return !inError;
}
private void fireEarlyCommit(IDataLoader loader, IncomingBatch batch) {
if (batchListeners != null) {
long ts = System.currentTimeMillis();
for (IBatchListener listener : batchListeners) {
listener.earlyCommit(loader, batch);
}
// update the filter milliseconds so batch listeners are also
// included
batch.setFilterMillis(batch.getFilterMillis() + (System.currentTimeMillis() - ts));
}
}
private void fireBatchComplete(IDataLoader loader, IncomingBatch batch) {
if (batchListeners != null) {
long ts = System.currentTimeMillis();
for (IBatchListener listener : batchListeners) {
listener.batchComplete(loader, batch);
}
// update the filter milliseconds so batch listeners are also
// included
batch.setFilterMillis(batch.getFilterMillis() + (System.currentTimeMillis() - ts));
}
}
private void fireBatchCommitted(IDataLoader loader, IncomingBatch batch) {
if (batchListeners != null) {
long ts = System.currentTimeMillis();
for (IBatchListener listener : batchListeners) {
listener.batchCommitted(loader, batch);
}
// update the filter milliseconds so batch listeners are also
// included
batch.setFilterMillis(batch.getFilterMillis() + (System.currentTimeMillis() - ts));
}
}
private void fireBatchRolledback(IDataLoader loader, IncomingBatch batch, Exception ex) {
if (batchListeners != null) {
long ts = System.currentTimeMillis();
for (IBatchListener listener : batchListeners) {
listener.batchRolledback(loader, batch, ex);
}
// update the filter milliseconds so batch listeners are also
// included
batch.setFilterMillis(batch.getFilterMillis() + (System.currentTimeMillis() - ts));
}
}
protected void handleBatchError(final IncomingBatch status) {
try {
if (!status.isRetry()) {
status.setStatus(IncomingBatch.Status.ER);
incomingBatchService.insertIncomingBatch(status);
}
} catch (Exception e) {
log.error("BatchStatusRecordFailed", status.getNodeBatchId());
}
}
/**
* Load database from input stream and write acknowledgment to output stream. This is used for a "push" request with
* a response of an acknowledgment.
*/
public void loadData(InputStream in, OutputStream out) throws IOException {
List list = loadDataAndReturnBatches(new InternalIncomingTransport(in));
Node local = nodeService.findIdentity();
NodeSecurity security = nodeService.findNodeSecurity(local.getNodeId());
transportManager.writeAcknowledgement(out, list, local, security != null ? security.getNodePassword() : null);
}
public void setDataLoaderFilters(List filters) {
this.filters = filters;
}
public void addDataLoaderFilter(IDataLoaderFilter filter) {
if (filters == null) {
filters = new ArrayList();
}
filters.add(filter);
}
public void removeDataLoaderFilter(IDataLoaderFilter filter) {
filters.remove(filter);
}
public void setTransportManager(ITransportManager remoteService) {
this.transportManager = remoteService;
}
public void setIncomingBatchService(IIncomingBatchService incomingBatchService) {
this.incomingBatchService = incomingBatchService;
}
public void setBeanFactory(BeanFactory beanFactory) throws BeansException {
this.beanFactory = beanFactory;
}
public void setDbDialect(IDbDialect dbDialect) {
this.dbDialect = dbDialect;
}
public void addColumnFilter(String tableName, IColumnFilter filter) {
List filters = this.columnFilters.get(tableName);
if (filters == null) {
filters = new ArrayList();
this.columnFilters.put(tableName, filters);
}
filters.add(filter);
}
/**
* @see IDataLoaderService#reRegisterColumnFilter(String[], IColumnFilter)
*/
public void reRegisterColumnFilter(String[] tableNames, IColumnFilter filter) {
Set>> entries = this.columnFilters.entrySet();
Iterator>> it = entries.iterator();
while (it.hasNext()) {
Entry> entry = it.next();
if (entry.getValue().contains(filter)) {
entry.getValue().remove(filter);
}
}
if (tableNames != null) {
for (String name : tableNames) {
this.addColumnFilter(name, filter);
}
}
}
public void setStatisticManager(IStatisticManager statisticManager) {
this.statisticManager = statisticManager;
}
public void addBatchListener(IBatchListener batchListener) {
if (this.batchListeners == null) {
this.batchListeners = new ArrayList();
}
this.batchListeners.add(batchListener);
}
public void setBatchListeners(List batchListeners) {
this.batchListeners = batchListeners;
}
public void setNodeService(INodeService nodeService) {
this.nodeService = nodeService;
}
public void setConfigurationService(IConfigurationService configurationService) {
this.configurationService = configurationService;
}
private enum LoadStatus {
CONTINUE, DONE
}
protected void loadBatch(final IDataLoader dataLoader, final IncomingBatch batch) {
try {
TransactionalLoadDelegate loadDelegate = new TransactionalLoadDelegate(batch, dataLoader);
LoadStatus loadStatus = loadDelegate.getLoadStatus();
do {
newTransactionTemplate.execute(loadDelegate);
loadStatus = loadDelegate.getLoadStatus();
if (loadStatus == LoadStatus.CONTINUE) {
// Chances are if SymmetricDS is configured to commit early in a batch we want to give other threads
// a chance to do work and access the database.
AppUtils.sleep(5);
}
} while (LoadStatus.CONTINUE == loadStatus);
fireBatchCommitted(dataLoader, batch);
} catch (RuntimeException ex) {
fireBatchRolledback(dataLoader, batch, ex);
throw ex;
}
}
class TransactionalLoadDelegate implements TransactionCallback {
IncomingBatch batch;
IDataLoader dataLoader;
LoadStatus loadStatus = LoadStatus.DONE;
public TransactionalLoadDelegate(IncomingBatch status, IDataLoader dataLoader) {
this.batch = status;
this.dataLoader = dataLoader;
}
public LoadStatus doInTransaction(TransactionStatus txStatus) {
try {
boolean done = true;
dbDialect.disableSyncTriggers(dataLoader.getContext().getSourceNodeId());
if (this.loadStatus == LoadStatus.CONTINUE || incomingBatchService.acquireIncomingBatch(batch)) {
done = dataLoader.load();
} else {
dataLoader.skip();
}
batch.setValues(dataLoader.getStatistics(), true);
if (done) {
fireBatchComplete(dataLoader, batch);
this.loadStatus = LoadStatus.DONE;
} else {
log.info("LoaderEarlyCommit", batch.getBatchId(), dataLoader.getContext().getTableName(), dataLoader.getStatistics().getLineCount());
fireEarlyCommit(dataLoader, batch);
this.loadStatus = LoadStatus.CONTINUE;
}
return this.loadStatus;
} catch (IOException e) {
throw new TransportException(e);
} finally {
dbDialect.enableSyncTriggers();
}
}
public LoadStatus getLoadStatus() {
return loadStatus;
}
}
}