com.aliyun.dms.subscribe.clients.DefaultDistributedDTSConsumer Maven / Gradle / Ivy
package com.aliyun.dms.subscribe.clients;
import com.aliyun.dts.subscribe.clients.ConsumerContext;
import com.aliyun.dts.subscribe.clients.DTSConsumer;
import com.aliyun.dts.subscribe.clients.RecordListener;
import com.aliyun.dts.subscribe.clients.common.Checkpoint;
import com.aliyun.dts.subscribe.clients.metastore.MetaStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
public class DefaultDistributedDTSConsumer implements DistributedDTSConsumer {
private static final Logger LOG = LoggerFactory.getLogger(DefaultDistributedDTSConsumer.class);
private List dtsConsumers = new ArrayList<>();
private int corePoolSize = 8;
private int maximumPoolSize = 8;
private ThreadPoolExecutor executor;
private volatile boolean isClosePoolExecutor = false;
public DefaultDistributedDTSConsumer() {}
public void addDTSConsumer(DTSConsumer consumer) {
dtsConsumers.add(consumer);
}
public void init(Map topic2checkpoint, DBMapper dbMapper, String dProxy, Map topic2Sid, String username, String password,
ConsumerContext.ConsumerSubscribeMode subscribeMode, boolean isForceUseInitCheckpoint,
MetaStore userRegisteredStore, Map recordListeners) {
this.init(topic2checkpoint, dbMapper, dProxy, topic2Sid, username, password, subscribeMode,
isForceUseInitCheckpoint, userRegisteredStore, recordListeners, new Properties());
}
public void init(Map topic2checkpoint, DBMapper dbMapper, String dProxy, Map topic2Sid, String username, String password,
ConsumerContext.ConsumerSubscribeMode subscribeMode, boolean isForceUseInitCheckpoint,
MetaStore userRegisteredStore, Map recordListeners, Properties properties) {
this.executor = new ThreadPoolExecutor(corePoolSize, maximumPoolSize, 1000 * 60,
TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>());
for (Map.Entry topicCheckpoint: topic2checkpoint.entrySet()) {
ConsumerContext consumerContext = new ConsumerContext(dbMapper, dProxy, topicCheckpoint.getKey(), topic2Sid.get(topicCheckpoint.getKey()), username, password,
topicCheckpoint.getValue(), subscribeMode, properties);
consumerContext.setUserRegisteredStore(userRegisteredStore);
consumerContext.setForceUseCheckpoint(isForceUseInitCheckpoint);
DTSConsumer dtsConsumer = new DTSConsumerWithDBMapping(consumerContext);
dtsConsumer.addRecordListeners(recordListeners);
addDTSConsumer(dtsConsumer);
}
}
@Override
public void start() {
for (DTSConsumer consumer: dtsConsumers) {
try {
executor.submit(consumer::start);
} catch (Exception e) {
LOG.error("error starting consumer:" + e);
shutdownGracefully(10, TimeUnit.SECONDS);
}
}
}
public void shutdownGracefully(long timeout, TimeUnit timeUnit) {
executor.shutdown();
try {
if (!executor.awaitTermination(timeout, timeUnit)) {
executor.shutdownNow();
}
} catch (InterruptedException e) {
executor.shutdownNow();
Thread.currentThread().interrupt();
} finally {
isClosePoolExecutor = true;
}
}
@Override
public void addRecordListeners(Map recordListeners) {
for (DTSConsumer dtsConsumer: dtsConsumers) {
dtsConsumer.addRecordListeners(recordListeners);
}
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy