Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
io.edurt.datacap.service.service.impl.SourceServiceImpl Maven / Gradle / Ivy
package io.edurt.datacap.service.service.impl;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Maps;
import com.google.common.io.Files;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.edurt.datacap.common.enums.NodeType;
import io.edurt.datacap.common.enums.ServiceState;
import io.edurt.datacap.common.enums.Type;
import io.edurt.datacap.common.response.CommonResponse;
import io.edurt.datacap.common.utils.CodeUtils;
import io.edurt.datacap.common.utils.JsonUtils;
import io.edurt.datacap.executor.common.RunState;
import io.edurt.datacap.plugin.Plugin;
import io.edurt.datacap.plugin.PluginManager;
import io.edurt.datacap.plugin.PluginType;
import io.edurt.datacap.service.adapter.PageRequestAdapter;
import io.edurt.datacap.service.body.FilterBody;
import io.edurt.datacap.service.body.SharedSourceBody;
import io.edurt.datacap.service.body.SourceBody;
import io.edurt.datacap.service.common.ConfigureUtils;
import io.edurt.datacap.service.common.PluginUtils;
import io.edurt.datacap.service.configure.IConfigure;
import io.edurt.datacap.service.configure.IConfigureField;
import io.edurt.datacap.service.entity.ColumnEntity;
import io.edurt.datacap.service.entity.DatabaseEntity;
import io.edurt.datacap.service.entity.PageEntity;
import io.edurt.datacap.service.entity.PluginEntity;
import io.edurt.datacap.service.entity.ScheduledEntity;
import io.edurt.datacap.service.entity.ScheduledHistoryEntity;
import io.edurt.datacap.service.entity.SourceEntity;
import io.edurt.datacap.service.entity.TableEntity;
import io.edurt.datacap.service.entity.TemplateEntity;
import io.edurt.datacap.service.entity.UserEntity;
import io.edurt.datacap.service.itransient.SqlConfigure;
import io.edurt.datacap.service.repository.BaseRepository;
import io.edurt.datacap.service.repository.ScheduledHistoryRepository;
import io.edurt.datacap.service.repository.SourceRepository;
import io.edurt.datacap.service.repository.TemplateSqlRepository;
import io.edurt.datacap.service.repository.UserRepository;
import io.edurt.datacap.service.repository.metadata.ColumnRepository;
import io.edurt.datacap.service.repository.metadata.DatabaseRepository;
import io.edurt.datacap.service.repository.metadata.TableRepository;
import io.edurt.datacap.service.security.UserDetailsService;
import io.edurt.datacap.service.service.SourceService;
import io.edurt.datacap.service.wrapper.ResponseWrapper;
import io.edurt.datacap.spi.PluginService;
import io.edurt.datacap.spi.model.Configure;
import io.edurt.datacap.spi.model.Response;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.compress.utils.Lists;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.springframework.core.env.Environment;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Service;
import java.io.File;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import static java.util.Objects.requireNonNull;
@Slf4j
@Service
@SuppressFBWarnings(value = {"RV_RETURN_VALUE_IGNORED_BAD_PRACTICE", "REC_CATCH_EXCEPTION", "NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE", "EI_EXPOSE_REP2"})
public class SourceServiceImpl
implements SourceService
{
private final SourceRepository sourceRepository;
private final UserRepository userRepository;
private final ScheduledHistoryRepository scheduledHistoryRepository;
private final DatabaseRepository databaseHandler;
private final TableRepository tableHandler;
private final ColumnRepository columnHandler;
private final TemplateSqlRepository templateHandler;
private final ScheduledHistoryRepository scheduledHistoryHandler;
private final PluginManager pluginManager;
private final Environment environment;
public SourceServiceImpl(SourceRepository sourceRepository, UserRepository userRepository, ScheduledHistoryRepository scheduledHistoryRepository, DatabaseRepository databaseHandler, TableRepository tableHandler, ColumnRepository columnHandler, TemplateSqlRepository templateHandler, ScheduledHistoryRepository scheduledHistoryHandler, PluginManager pluginManager, Environment environment)
{
this.sourceRepository = sourceRepository;
this.userRepository = userRepository;
this.scheduledHistoryRepository = scheduledHistoryRepository;
this.databaseHandler = databaseHandler;
this.tableHandler = tableHandler;
this.columnHandler = columnHandler;
this.templateHandler = templateHandler;
this.scheduledHistoryHandler = scheduledHistoryHandler;
this.pluginManager = pluginManager;
this.environment = environment;
}
@Override
public CommonResponse> getAll(BaseRepository repository, FilterBody filter)
{
UserEntity user = UserDetailsService.getUser();
Page page = this.sourceRepository.findAllByUserOrPublishIsTrue(user, PageRequestAdapter.of(filter));
// Populate pipeline configuration information
page.getContent().forEach(item -> {
IConfigure fromConfigure = PluginUtils.loadYamlConfigure(item.getType(), item.getType(), item.getType(), environment);
if (fromConfigure != null) {
item.setPipelines(fromConfigure.getPipelines());
}
});
return CommonResponse.success(PageEntity.build(page));
}
@SneakyThrows
@Override
public CommonResponse delete(Long id)
{
Optional entityOptional = this.sourceRepository.findById(id);
if (entityOptional.isPresent()) {
SourceEntity source = entityOptional.get();
if (source.isUsedConfig()) {
String configHome = environment.getProperty("datacap.config.data");
if (StringUtils.isEmpty(configHome)) {
configHome = String.join(File.separator, System.getProperty("user.dir"), "config");
}
String destination = String.join(File.separator, configHome, source.getUser().getUsername(), source.getType(), String.valueOf(source.getId()));
FileUtils.deleteDirectory(new File(destination));
}
this.sourceRepository.deleteById(id);
}
return CommonResponse.success(id);
}
@Override
public CommonResponse getByCode(BaseRepository repository, String code)
{
return repository.findByCode(code)
.map(entity -> {
SourceBody configure = new SourceBody();
configure.setId(entity.getId());
configure.setName(entity.getType());
configure.setType(entity.getProtocol());
// Load default configure
IConfigure iConfigure = PluginUtils.loadYamlConfigure(configure.getType(), configure.getName(), configure.getName(), environment);
configure.setConfigure(ConfigureUtils.preparedConfigure(iConfigure, entity));
entity.setSchema(iConfigure);
return CommonResponse.success(entity);
})
.orElseGet(() -> CommonResponse.failure(String.format("Source [ %s ] not found", code)));
}
@Override
public CommonResponse> getPlugins()
{
List plugins = Lists.newArrayList();
pluginManager.getPluginInfos()
.stream()
.filter(v -> v.getType().equals(PluginType.CONNECTOR))
.forEach(plugin -> {
PluginEntity entity = new PluginEntity();
entity.setName(plugin.getName());
entity.setDescription(plugin.getName());
entity.setType(plugin.getType().getName());
entity.setConfigure(PluginUtils.loadYamlConfigure("JDBC", plugin.getName(), plugin.getName(), environment));
plugins.add(entity);
});
return CommonResponse.success(plugins);
}
@Override
public CommonResponse count()
{
return CommonResponse.success(this.sourceRepository.countByUserOrPublishIsTrue(UserDetailsService.getUser()));
}
@Override
public CommonResponse shared(SharedSourceBody configure)
{
Optional sourceOptional = this.sourceRepository.findById(configure.getSourceId());
if (!sourceOptional.isPresent()) {
return CommonResponse.failure(ServiceState.SOURCE_NOT_FOUND);
}
Optional userOptional = this.userRepository.findById(configure.getUserId());
if (!userOptional.isPresent()) {
return CommonResponse.failure(ServiceState.USER_NOT_FOUND);
}
SourceEntity source = sourceOptional.get();
source.setUser(userOptional.get());
source.setPublish(configure.getPublish());
return CommonResponse.success(this.sourceRepository.save(source));
}
@Override
public CommonResponse testConnection(SourceBody configure)
{
return pluginManager.getPlugin(configure.getType())
.>map(plugin -> {
// Check configure
IConfigure iConfigure = PluginUtils.loadYamlConfigure(configure.getType(), configure.getType(), configure.getType(), environment);
if (ObjectUtils.isEmpty(iConfigure) || iConfigure.getConfigures().size() != configure.getConfigure().getConfigures().size()) {
return CommonResponse.failure(ServiceState.PLUGIN_CONFIGURE_MISMATCH);
}
// Filter required
List requiredMismatchConfigures = configure.getConfigure().getConfigures()
.stream()
.filter(IConfigureField::isRequired)
.filter(v -> ObjectUtils.isEmpty(v.getValue()))
.collect(Collectors.toList());
if (!requiredMismatchConfigures.isEmpty()) {
return CommonResponse.failure(ServiceState.PLUGIN_CONFIGURE_REQUIRED, ConfigureUtils.preparedMessage(requiredMismatchConfigures));
}
// The filter parameter value is null data
PluginService pluginService = plugin.getService(PluginService.class);
List applyConfigures = ConfigureUtils.filterNotEmpty(configure.getConfigure().getConfigures());
Configure _configure = ConfigureUtils.preparedConfigure(applyConfigures);
_configure.setPluginManager(pluginManager);
_configure.setPlugin(plugin);
// Adapter file configure
if (_configure.isUsedConfig()) {
String cacheHome = environment.getProperty("datacap.cache.data");
if (StringUtils.isEmpty(cacheHome)) {
cacheHome = String.join(File.separator, System.getProperty("user.dir"), "cache");
}
_configure.setHome(cacheHome);
_configure.setUsername(Optional.of(UserDetailsService.getUser().getUsername()));
}
Response response = pluginService.execute(_configure, pluginService.validator());
if (response.getIsSuccessful()) {
return CommonResponse.success(ResponseWrapper.from(response));
}
return CommonResponse.failure(ServiceState.PLUGIN_EXECUTE_FAILED, response.getMessage());
})
.orElse(CommonResponse.failure(ServiceState.PLUGIN_NOT_FOUND));
}
@Override
public CommonResponse saveOrUpdate(BaseRepository repository, SourceEntity configure)
{
return pluginManager.getPlugin(configure.getName())
.>map(plugin -> {
// Check configure
IConfigure iConfigure = PluginUtils.loadYamlConfigure(configure.getType(), configure.getType(), configure.getType(), environment);
if (ObjectUtils.isEmpty(iConfigure) || iConfigure.getConfigures().size() != configure.getConfigure().getConfigures().size()) {
return CommonResponse.failure(ServiceState.PLUGIN_CONFIGURE_MISMATCH);
}
// Filter required
List requiredMismatchConfigures = configure.getConfigure()
.getConfigures()
.stream()
.filter(IConfigureField::isRequired)
.filter(v -> ObjectUtils.isEmpty(v.getValue()))
.collect(Collectors.toList());
if (!requiredMismatchConfigures.isEmpty()) {
return CommonResponse.failure(ServiceState.PLUGIN_CONFIGURE_REQUIRED, ConfigureUtils.preparedMessage(requiredMismatchConfigures));
}
// The filter parameter value is null data
List applyConfigures = ConfigureUtils.filterNotEmpty(configure.getConfigure().getConfigures());
SourceEntity source = ConfigureUtils.preparedSourceEntity(applyConfigures);
source.setProtocol(configure.getType());
source.setType(configure.getName());
source.setUser(UserDetailsService.getUser());
source.setId(configure.getId());
source.setCode(configure.getCode());
if (StringUtils.isNotEmpty(configure.getVersion())) {
source.setVersion(configure.getVersion());
source.setAvailable(true);
}
else {
source.setAvailable(false);
}
this.sourceRepository.save(source);
// Copy file to local data
if (source.isUsedConfig()) {
String cacheHome = environment.getProperty("datacap.cache.data");
if (StringUtils.isEmpty(cacheHome)) {
cacheHome = String.join(File.separator, System.getProperty("user.dir"), "cache");
}
String configHome = environment.getProperty("datacap.config.data");
if (StringUtils.isEmpty(configHome)) {
configHome = String.join(File.separator, System.getProperty("user.dir"), "config");
}
File sourceFile = new File(String.join(File.separator, cacheHome, source.getUser().getUsername(), source.getType()));
String destination = String.join(File.separator, configHome, source.getUser().getUsername(), source.getType(), String.valueOf(source.getId()));
File directory = new File(destination);
try {
if (!directory.exists()) {
directory.mkdirs();
}
for (File file : requireNonNull(sourceFile.listFiles())) {
Files.copy(file, new File(String.join(File.separator, destination, file.getName())));
}
FileUtils.deleteDirectory(sourceFile);
}
catch (Exception e) {
return CommonResponse.failure("Copy failed: " + e.getMessage());
}
}
// Start sync metadata
this.syncMetadata(source.getCode());
return CommonResponse.success(source);
})
.orElse(CommonResponse.failure(ServiceState.PLUGIN_NOT_FOUND));
}
@Override
public CommonResponse> getHistory(String code, FilterBody filter)
{
return sourceRepository.findByCode(code)
.map(entity -> {
Pageable pageable = PageRequestAdapter.of(filter);
return CommonResponse.success(PageEntity.build(this.scheduledHistoryRepository.findAllBySource(entity, pageable)));
})
.orElse(CommonResponse.failure(String.format("Source [ %s ] not found", code)));
}
@Override
public CommonResponse syncMetadata(String code)
{
return this.sourceRepository.findByCode(code)
.map(entity -> {
Executors.newSingleThreadExecutor()
.submit(() -> startSyncMetadata(entity, null));
return CommonResponse.success(entity);
})
.orElseGet(() -> CommonResponse.failure(String.format("Source [ %s ] not found", code)));
}
private void startSyncMetadata(SourceEntity entity, ScheduledEntity scheduled)
{
AtomicInteger databaseAddedCount = new AtomicInteger(0);
AtomicInteger databaseUpdatedCount = new AtomicInteger(0);
AtomicInteger databaseRemovedCount = new AtomicInteger(0);
AtomicInteger tableAddedCount = new AtomicInteger(0);
AtomicInteger tableUpdatedCount = new AtomicInteger(0);
AtomicInteger tableRemovedCount = new AtomicInteger(0);
AtomicInteger columnAddedCount = new AtomicInteger(0);
AtomicInteger columnUpdatedCount = new AtomicInteger(0);
AtomicInteger columnRemovedCount = new AtomicInteger(0);
Map databaseCache = Maps.newHashMap();
Map> databaseTableCache = Maps.newHashMap();
Map tableCache = Maps.newHashMap();
Map> tableColumnCache = Maps.newHashMap();
ScheduledHistoryEntity scheduledHistory = ScheduledHistoryEntity.builder().name(String.format("Sync source [ %s ]", entity.getName())).scheduled(scheduled).source(entity).state(RunState.RUNNING).build();
scheduledHistoryHandler.save(scheduledHistory);
log.info("==================== Sync metadata [ {} ] started =================", entity.getName());
pluginManager.getPlugin(entity.getType())
.ifPresent(plugin -> {
try {
PluginService pluginService = plugin.getService(PluginService.class);
Configure pConfigure = entity.toConfigure(pluginManager, plugin);
pConfigure.setPluginManager(pluginManager);
Response response = pluginService.execute(pConfigure, pluginService.validator());
if (!response.getIsSuccessful()) {
log.error("The source [ {} ] not available", entity.getName());
}
else {
this.startSyncDatabase(
entity,
plugin,
pluginService,
databaseCache,
databaseTableCache,
tableCache,
tableColumnCache,
databaseAddedCount,
databaseUpdatedCount,
databaseRemovedCount,
tableAddedCount,
tableUpdatedCount,
tableRemovedCount,
columnAddedCount,
columnUpdatedCount,
columnRemovedCount
);
}
scheduledHistory.setState(RunState.SUCCESS);
}
catch (Exception e) {
log.error("The scheduled task [ {} ] source [ {} ] not available ", scheduled.getName(), entity.getName(), e);
scheduledHistory.setState(RunState.FAILURE);
scheduledHistory.setMessage(ExceptionUtils.getStackTrace(e));
}
});
log.info("==================== Sync metadata [ {} ] finished =================", entity.getName());
Properties info = new Properties();
info.put("databaseAddedCount", databaseAddedCount.get());
info.put("databaseUpdatedCount", databaseUpdatedCount.get());
info.put("databaseRemovedCount", databaseRemovedCount.get());
info.put("tableAddedCount", tableAddedCount.get());
info.put("tableUpdatedCount", tableUpdatedCount.get());
info.put("tableRemovedCount", tableRemovedCount.get());
info.put("columnAddedCount", columnAddedCount.get());
info.put("columnUpdatedCount", columnUpdatedCount.get());
info.put("columnRemovedCount", columnRemovedCount.get());
scheduledHistory.setInfo(info);
scheduledHistoryHandler.save(scheduledHistory);
databaseCache.clear();
databaseTableCache.clear();
tableCache.clear();
tableColumnCache.clear();
}
/**
* Retrieves the SQL context for the given TemplateSqlEntity and map of key-value pairs.
*
* @param entity the TemplateSqlEntity object containing the SQL configuration and content
* @param map the map of key-value pairs used for replacing expressions in the SQL content
* @return the SQL content with the expressions replaced by the corresponding values in the map
*/
private String getSqlContext(TemplateEntity entity, Map map)
{
try {
if (ObjectUtils.isNotEmpty(entity.getConfigure())) {
final String[] content = {entity.getContent()};
List configures = JsonUtils.objectmapper.readValue(entity.getConfigure(), List.class);
map.entrySet().forEach(value -> {
Optional sqlConfigure = configures.stream().filter(v -> String.valueOf(v.get("column")).equalsIgnoreCase(value.getKey())).map(v -> {
SqlConfigure configure = new SqlConfigure();
configure.setColumn(String.valueOf(v.get("column")));
configure.setType(Type.valueOf(String.valueOf(v.get("type"))));
configure.setExpression(String.valueOf(v.get("expression")));
return configure;
}).findFirst();
if (sqlConfigure.isPresent()) {
content[0] = content[0].replace(sqlConfigure.get().getExpression(), String.valueOf(value.getValue()));
}
});
return content[0];
}
}
catch (Exception e) {
log.warn("Failed to convert [ {} ] sql context", entity.getName());
return entity.getContent();
}
return entity.getContent();
}
/**
* Retrieves the text value of a specific node in a JSON object.
*
* @param json the JSON object to retrieve the node from
* @param key the type of node to retrieve
* @return the text value of the specified node, or null if the node does not exist
*/
private String getNodeText(Object json, NodeType key)
{
ObjectNode objectNode = (ObjectNode) json;
if (objectNode != null) {
JsonNode jsonNode = objectNode.get(String.join("_", key.getValue(), "NODE"));
if (jsonNode != null) {
return jsonNode.asText();
}
}
return null;
}
/**
* Retrieves a TemplateSqlEntity object based on the given template name and SourceEntity.
*
* @param templateName the name of the template to retrieve
* @param entity the SourceEntity object to use for filtering
* @return the TemplateSqlEntity object that matches the given criteria
*/
private TemplateEntity getTemplate(String templateName, SourceEntity entity)
{
return templateHandler.findByNameAndPluginContaining(templateName, entity.getType());
}
/**
* Starts the synchronization of the database.
*
* @param entity the SourceEntity object representing the entity
* @param plugin the Plugin object representing the plugin
* @param databaseCache the Map object representing the database cache
* @param databaseTableCache the Map object representing the database table cache
* @param tableCache the Map object representing the table cache
* @param tableColumnCache the Map object representing the table column cache
* @param databaseAddedCount the AtomicInteger object representing the count of added databases
* @param databaseUpdatedCount the AtomicInteger object representing the count of updated databases
* @param databaseRemovedCount the AtomicInteger object representing the count of removed databases
* @param tableAddedCount the AtomicInteger object representing the count of added tables
* @param tableUpdatedCount the AtomicInteger object representing the count of updated tables
* @param tableRemovedCount the AtomicInteger object representing the count of removed tables
* @param columnAddedCount the AtomicInteger object representing the count of added columns
* @param columnUpdatedCount the AtomicInteger object representing the count of updated columns
* @param columnRemovedCount the AtomicInteger object representing the count of removed columns
*/
private void startSyncDatabase(SourceEntity entity, Plugin plugin, PluginService pluginService, Map databaseCache, Map> databaseTableCache, Map tableCache, Map> tableColumnCache, AtomicInteger databaseAddedCount, AtomicInteger databaseUpdatedCount, AtomicInteger databaseRemovedCount, AtomicInteger tableAddedCount, AtomicInteger tableUpdatedCount, AtomicInteger tableRemovedCount, AtomicInteger columnAddedCount, AtomicInteger columnUpdatedCount, AtomicInteger columnRemovedCount)
{
String templateName = "SYSTEM_FOR_GET_ALL_DATABASES";
TemplateEntity template = getTemplate(templateName, entity);
if (template == null) {
log.warn("The source [ {} ] protocol [ {} ] template [ {} ] is not available, skip sync database", entity.getName(), entity.getProtocol(), templateName);
}
else {
Response response = pluginService.execute(entity.toConfigure(pluginManager, plugin), getSqlContext(template, null));
if (!response.getIsSuccessful()) {
log.error("The source [ {} ] protocol [ {} ] sync metadata [ {} ] failed", entity.getName(), entity.getProtocol(), response.getMessage());
}
else {
List origin = databaseHandler.findAllBySource(entity);
List entities = response.getColumns()
.stream()
.map(item -> {
DatabaseEntity database = DatabaseEntity.builder().name(getNodeText(item, NodeType.SCHEMA)).catalog(getNodeText(item, NodeType.CATALOG)).description(String.format("[ %s ] of [ %s ]", getNodeText(item, NodeType.SCHEMA), getNodeText(item, NodeType.CATALOG))).source(entity).build();
Optional optionalDatabase = origin.stream().filter(node -> node.getName().equals(database.getName())).findAny();
if (optionalDatabase.isPresent()) {
database.setId(optionalDatabase.get().getId());
database.setCreateTime(optionalDatabase.get().getCreateTime());
if (StringUtils.isEmpty(database.getCode())) {
database.setCode(CodeUtils.generateCode(false));
}
databaseUpdatedCount.addAndGet(1);
}
else {
databaseAddedCount.addAndGet(1);
}
return database;
})
.collect(Collectors.toList());
// Write the new data retrieved to the database
log.info("Added database size [ {} ] to source [ {} ]", entities.size(), entity.getName());
databaseHandler.saveAll(entities);
entities.forEach(item -> {
String key = String.format("%s_%s", item.getCatalog(), item.getName());
databaseCache.put(key, item);
databaseTableCache.put(key, this.tableHandler.findSimpleAllByDatabase(item));
});
// Delete invalid data that no longer exists
List deleteEntities = origin.stream()
.filter(node -> entities.stream().noneMatch(item -> node.getName().equals(item.getName())))
.collect(Collectors.toList());
log.info("Removed database size [ {} ] from source [ {} ]", deleteEntities.size(), entity.getName());
databaseHandler.deleteAll(deleteEntities);
databaseRemovedCount.addAndGet(deleteEntities.size());
}
this.startSyncTable(
entity,
plugin,
pluginService,
databaseCache,
databaseTableCache,
tableCache,
tableColumnCache,
tableAddedCount,
tableUpdatedCount,
tableRemovedCount,
columnAddedCount,
columnUpdatedCount,
columnRemovedCount
);
}
}
/**
* Starts the synchronization of a table.
*
* @param entity the source entity
* @param plugin the plugin
* @param databaseCache the database cache
* @param databaseTableCache the database table cache
* @param tableCache the table cache
* @param tableColumnCache the table column cache
* @param tableAddedCount the table added count
* @param tableUpdatedCount the table updated count
* @param tableRemovedCount the table removed count
* @param columnAddedCount the column added count
* @param columnUpdatedCount the column updated count
* @param columnRemovedCount the column removed count
*/
private void startSyncTable(SourceEntity entity, Plugin plugin, PluginService pluginService, Map databaseCache, Map> databaseTableCache, Map tableCache, Map> tableColumnCache, AtomicInteger tableAddedCount, AtomicInteger tableUpdatedCount, AtomicInteger tableRemovedCount, AtomicInteger columnAddedCount, AtomicInteger columnUpdatedCount, AtomicInteger columnRemovedCount)
{
String templateName = "SYSTEM_FOR_GET_ALL_TABLES";
TemplateEntity template = getTemplate(templateName, entity);
if (template == null) {
log.warn("The source [ {} ] protocol [ {} ] template [ {} ] is not available, skip sync table", entity.getName(), entity.getProtocol(), templateName);
}
else {
Response response = pluginService.execute(entity.toConfigure(pluginManager, plugin), getSqlContext(template, null));
if (!response.getIsSuccessful()) {
log.error("The source [ {} ] protocol [ {} ] sync metadata tables [ {} ] failed", entity.getName(), entity.getProtocol(), response.getMessage());
}
else {
List entities = response.getColumns()
.stream()
.map(item -> {
String key = String.format("%s_%s", getNodeText(item, NodeType.CATALOG), getNodeText(item, NodeType.SCHEMA));
DatabaseEntity database = databaseCache.get(key);
String name = getNodeText(item, NodeType.TABLE);
TableEntity configure = TableEntity.builder()
.name(name)
.description(String.format("Table [ %s ] of database [ %s ] ", name, getNodeText(item, NodeType.SCHEMA)))
.type(getNodeText(item, NodeType.TYPE)).engine(getNodeText(item, NodeType.ENGINE))
.format(getNodeText(item, NodeType.FORMAT))
.inCreateTime(getNodeText(item, NodeType.CREATE_TIME))
.inUpdateTime(getNodeText(item, NodeType.UPDATE_TIME))
.collation(getNodeText(item, NodeType.COLLATION))
.rows(getNodeText(item, NodeType.ROWS))
.comment(getNodeText(item, NodeType.COMMENT))
.avgRowLength(getNodeText(item, NodeType.AVG_ROW))
.dataLength(getNodeText(item, NodeType.DATA))
.indexLength(getNodeText(item, NodeType.INDEX))
.autoIncrement(getNodeText(item, NodeType.AUTO_INCREMENT))
.database(database)
.build();
if (StringUtils.isEmpty(configure.getCode())) {
configure.setCode(CodeUtils.generateCode(false));
}
return configure;
})
.collect(Collectors.toList());
Map> groupEntities = entities.stream().collect(Collectors.groupingBy(item -> String.format("%s_%s", item.getDatabase().getCatalog(), item.getDatabase().getName())));
groupEntities.forEach((key, groupItem) -> {
// Detect data that needs to be updated
List origin = databaseTableCache.get(key);
groupItem.forEach(item -> {
Optional optionalTable = origin.stream().filter(node -> node.getName().equals(item.getName())).findAny();
if (optionalTable.isPresent()) {
TableEntity node = optionalTable.get();
item.setId(node.getId());
item.setCreateTime(node.getCreateTime());
tableUpdatedCount.addAndGet(1);
}
else {
tableAddedCount.addAndGet(1);
}
});
log.info("Added table size [ {} ] to database [ {} ]", groupItem.size(), key);
tableHandler.saveAll(groupItem);
groupItem.forEach(item -> {
String tableCacheKey = String.format("%s_%s", item.getDatabase().getName(), item.getName());
tableCache.put(tableCacheKey, item);
tableColumnCache.put(tableCacheKey, this.columnHandler.findSimpleAllByTable(item));
});
List deleteEntities = origin.stream().filter(node -> groupItem.stream().noneMatch(item -> node.getName().equals(item.getName()))).collect(Collectors.toList());
log.info("Removed table size [ {} ] from database [ {} ]", deleteEntities.size(), key);
tableHandler.deleteAll(deleteEntities);
tableRemovedCount.addAndGet(deleteEntities.size());
});
this.startSyncColumn(
entity,
plugin,
pluginService,
tableCache,
tableColumnCache,
columnAddedCount,
columnUpdatedCount,
columnRemovedCount
);
}
}
}
/**
* Synchronizes the columns of a source entity with the corresponding table in the database.
*
* @param entity the source entity to sync the columns for
* @param plugin the plugin to connect to the database
* @param tableCache a cache of table entities for efficient lookup
* @param tableColumnCache a cache of column entities for efficient lookup
* @param columnAddedCount an atomic counter for tracking the number of columns added
* @param columnUpdatedCount an atomic counter for tracking the number of columns updated
* @param columnRemovedCount an atomic counter for tracking the number of columns removed
*/
private void startSyncColumn(SourceEntity entity, Plugin plugin, PluginService pluginService, Map tableCache, Map> tableColumnCache, AtomicInteger columnAddedCount, AtomicInteger columnUpdatedCount, AtomicInteger columnRemovedCount)
{
String templateName = "SYSTEM_FOR_GET_ALL_COLUMNS";
TemplateEntity template = getTemplate(templateName, entity);
if (template == null) {
log.warn("The source [ {} ] protocol [ {} ] template [ {} ] is not available, skip sync column", entity.getName(), entity.getProtocol(), templateName);
}
else {
Response response = pluginService.execute(entity.toConfigure(pluginManager, plugin), getSqlContext(template, null));
if (!response.getIsSuccessful()) {
log.error("The source [ {} ] protocol [ {} ] sync metadata columns [ {} ] failed", entity.getName(), entity.getProtocol(), response.getMessage());
}
else {
List entities = response.getColumns()
.stream()
.map(item -> {
String key = String.format("%s_%s", getNodeText(item, NodeType.CATALOG), getNodeText(item, NodeType.SCHEMA));
TableEntity table = tableCache.get(key);
String name = getNodeText(item, NodeType.COLUMN);
ColumnEntity configure = ColumnEntity.builder()
.name(name)
.description(String.format("Table [ %s ] of column [ %s ] ", table.getName(), name))
.type(getNodeText(item, NodeType.COLUMN_TYPE))
.comment(getNodeText(item, NodeType.COMMENT))
.defaultValue(getNodeText(item, NodeType.DEFAULT))
.position(getNodeText(item, NodeType.POSITION))
.maximumLength(getNodeText(item, NodeType.MAXIMUM_LENGTH))
.collation(getNodeText(item, NodeType.COLLATION))
.isKey(getNodeText(item, NodeType.KEY))
.privileges(getNodeText(item, NodeType.FORMAT))
.dataType(getNodeText(item, NodeType.DATA_TYPE))
.extra(getNodeText(item, NodeType.EXTRA))
.isNullable(getNodeText(item, NodeType.NULLABLE))
.table(table)
.build();
if (StringUtils.isEmpty(configure.getCode())) {
configure.setCode(CodeUtils.generateCode(false));
}
return configure;
})
.collect(Collectors.toList());
Map> groupEntities = entities.stream().collect(Collectors.groupingBy(item -> String.format("%s_%s", item.getTable().getDatabase().getName(), item.getTable().getName())));
groupEntities.forEach((key, groupItem) -> {
// Detect data that needs to be updated
List origin = tableColumnCache.get(key);
groupItem.forEach(item -> {
Optional optionalColumn = origin.stream().filter(node -> node.getName().equals(item.getName())).findAny();
if (optionalColumn.isPresent()) {
ColumnEntity node = optionalColumn.get();
item.setId(node.getId());
item.setCreateTime(node.getCreateTime());
columnUpdatedCount.addAndGet(1);
}
else {
columnAddedCount.addAndGet(1);
}
});
log.info("Added column size [ {} ] to table [ {} ]", groupItem.size(), key);
columnHandler.saveAll(groupItem);
List deleteEntities = origin.stream().filter(node -> groupItem.stream().noneMatch(item -> node.getName().equals(item.getName()))).collect(Collectors.toList());
log.info("Removed column size [ {} ] from table [ {} ]", deleteEntities.size(), key);
columnHandler.deleteAll(deleteEntities);
columnRemovedCount.addAndGet(deleteEntities.size());
});
}
}
}
}
