com.hecloud.runtime.database.engine.BatchEngine Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of hecloud.runtime.database Show documentation
There is a newer version: 1.0.8
Show newest version
package com.hecloud.runtime.database.engine;

import com.hecloud.runtime.common.collections.Lists;
import com.hecloud.runtime.common.exception.DatabaseException;
import com.hecloud.runtime.common.model.Result;
import com.hecloud.runtime.database.builder.BatchInsertBuilder;
import com.hecloud.runtime.database.builder.BatchSQLBuilder;
import com.hecloud.runtime.database.builder.BatchUpdateBuilder;
import com.hecloud.runtime.database.emuns.ColumnField;
import com.hecloud.runtime.database.emuns.Strategy;
import com.hecloud.runtime.database.task.SQLExecuteTask;
import com.hecloud.runtime.database.utils.EntityLoader;
import lombok.Data;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;

import java.util.List;
import java.util.concurrent.*;

/**
 * 批量处理引擎
 *
 * @author LoveinBJ
 */
@Data
public final class BatchEngine {

    private static Logger logger = LoggerFactory.getLogger(BatchEngine.class);
    private static Integer DEFAULT_SIZE = 1000;
    private static int maxPoolSize = Runtime.getRuntime().availableProcessors() * 2;
    private JdbcTemplate jdbcTemplate;
    private EntityLoader processor;

    public BatchEngine(JdbcTemplate jdbcTemplate, EntityLoader processor) {
        super();
        this.jdbcTemplate = jdbcTemplate;
        this.processor = processor;
    }

    /**
     * 批量插入对象方法
     * 主要思路就是构造insert into table () values(),()sql语句。利用多线程并发执行的思路。
     *
     * @param objects   对象列表
     * @param batchSize 批量大小
     * @return 结果
     * @throws DatabaseException 数操操作异常
     */
    public Result batchInsert(List objects, Integer batchSize) throws DatabaseException {
        checkNull(objects);
        if (null == batchSize || batchSize <= 0) {
            batchSize = DEFAULT_SIZE;
        }
        Object object = objects.get(0);
        Strategy strategy = processor.loadStrategy(object);
        List columnFields = processor.loadColumnField(object.getClass());
        String tableName = processor.loadTableName(object.getClass());
        BatchSQLBuilder batchSQLBuilder = new BatchInsertBuilder(objects.get(0).getClass(),null);

        int totalSize = objects.size();
        int remainder = totalSize % batchSize;
        int frequency = totalSize / batchSize;
        if (remainder > 0) {
            frequency = frequency + 1;
        }
        int poolSize = Math.min(frequency, maxPoolSize);
        int startIndex;
        int stopIndex;
        ExecutorService executor = new ThreadPoolExecutor(poolSize, poolSize, 30, TimeUnit.SECONDS,
                new LinkedBlockingDeque<>(), (ThreadFactory) Thread::new);
        CompletionService completionService = new ExecutorCompletionService<>(executor);
        String prefix = batchSQLBuilder.buildPrefix();
        String values;
        for (int i = 0; i < frequency; i++) {
            startIndex = i * batchSize;
            stopIndex = (i + 1) * batchSize;
            if (stopIndex >= totalSize) {
                stopIndex = totalSize;
            }
            values = batchSQLBuilder.buildValues(objects.subList(startIndex, stopIndex));
            if (StringUtils.isEmpty(values)) {
                logger.warn("No values exist！");
                continue;
            }
            String insertSql = prefix + values;
            completionService.submit(new SQLExecuteTask(insertSql, jdbcTemplate));
        }
        int index = 0;
        try {
            while (index < frequency) {
                if (completionService.take().get().isSuccess()) {
                    index++;
                } else {
                    return new Result(false, "批量添加失败！");
                }
            }
            return new Result(true, "批量添加成功！");
        } catch (InterruptedException | ExecutionException e) {
            throw new DatabaseException("批量添加异常", e);
        } finally {
            executor.shutdown();
        }
    }

    /**
     * 批量更新语句
     *
     * @param objects   实体类列表
     * @param fields    更新字段
     * @param batchSize 批量规模
     * @return 更新结果
     * @throws DatabaseException 数据操作异常
     */
    public Result batchUpdate(List objects, String[] fields, Integer batchSize) throws DatabaseException {
        checkNull(objects);
        if (null == batchSize || batchSize <= 0) {
            batchSize = DEFAULT_SIZE;
        }
        Object object = objects.get(0);
        Strategy strategy = processor.loadStrategy(object);
        List columnFields = processor.loadColumnField(object.getClass());
        String tableName = processor.loadTableName(object.getClass());
        ColumnField pk = this.getPK(columnFields);
        Assert.notNull(pk, "PK is null !");
        BatchSQLBuilder batchSQLBuilder = new BatchUpdateBuilder(object.getClass(),fields,"mysql");

        int totalSize = objects.size();
        int remainder = totalSize % batchSize;
        int frequency = totalSize / batchSize;
        if (remainder == 0) {
            frequency = frequency + 1;
        }
        int poolSize = frequency;
        if (frequency > maxPoolSize) {
            poolSize = maxPoolSize;
        }
        int startIndex;
        int stopIndex;
        ExecutorService executor = new ThreadPoolExecutor(poolSize, poolSize, 30, TimeUnit.SECONDS,
                new LinkedBlockingDeque<>(), (ThreadFactory) Thread::new);
        CompletionService completionService = new ExecutorCompletionService<>(executor);
        String prefix = batchSQLBuilder.buildPrefix();
        String values;
        for (int i = 0; i < frequency; i++) {
            startIndex = i * batchSize;
            stopIndex = (i + 1) * batchSize;
            if (stopIndex >= totalSize) {
                stopIndex = totalSize;
            }
            values = batchSQLBuilder.buildValues(objects.subList(startIndex, stopIndex));
            completionService.submit(new SQLExecuteTask(prefix + values, jdbcTemplate));
        }
        int index = 0;
        try {
            while (index < frequency) {
                if (completionService.take().get().isSuccess()) {
                    index++;
                } else {
                    return new Result(false, "批量更新失败！");
                }
            }
            return new Result(true, "批量更新成功！");
        } catch (InterruptedException | ExecutionException e) {
            throw new DatabaseException("批量更新异常！", e);
        } finally {
            executor.shutdown();
        }
    }

    private void checkNull(List objects) throws DatabaseException {
        if (Lists.allNull(objects)) {
            throw new DatabaseException("No Object need to be Insert!", new Throwable());
        }
    }

    private ColumnField getPK(List columnFields) {
        return columnFields.stream().filter(ColumnField::isPk).findFirst().orElse(null);
    }
}