com.xxdb.route.PartitionedTableAppender Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of dolphindb-javaapi Show documentation
Show all versions of dolphindb-javaapi Show documentation
The messaging and data conversion protocol between Java and DolphinDB server
package com.xxdb.route;
import com.xxdb.BasicDBTask;
import com.xxdb.DBConnection;
import com.xxdb.DBConnectionPool;
import com.xxdb.DBTask;
import com.xxdb.ExclusiveDBConnectionPool;
import com.xxdb.data.*;
import com.xxdb.data.Vector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.*;
/**
* PartitionedTableAppender is used to append a table to a partitioned table
* across a cluster of DolphinDB instances.
*/
public class PartitionedTableAppender {
private BasicDictionary tableInfo;
private Domain domain;
private int partitionColumnIdx;
private int cols;
private Entity.DATA_CATEGORY columnCategories[];
private Entity.DATA_TYPE columnTypes[];
private int threadCount;
private DBConnectionPool pool;
private List> chunkIndices;
private String appendScript;
private static final Logger log = LoggerFactory.getLogger(PartitionedTableAppender.class);
public PartitionedTableAppender(String dbUrl, String tableName, String partitionColName, DBConnectionPool pool) throws Exception {
this(dbUrl, tableName, partitionColName, null, pool);
}
public PartitionedTableAppender(String dbUrl, String tableName, String partitionColName, String appendFunction, DBConnectionPool pool) throws Exception {
this.pool = pool;
this.threadCount = pool.getConnectionCount();
chunkIndices = new ArrayList>(threadCount);
for(int i=0; i());
DBConnection conn = new DBConnection();
Entity partitionSchema;
BasicTable colDefs;
BasicIntVector typeInts;
int partitionType;
Entity.DATA_TYPE partitionColType;
try {
DBTask task;
if(dbUrl == null || dbUrl.isEmpty()){
task = new BasicDBTask("schema(" + tableName+ ")");
appendScript = "tableInsert{" + tableName + "}";
}
else{
task = new BasicDBTask("schema(loadTable(\"" + dbUrl + "\", \"" + tableName + "\"))");
appendScript = "tableInsert{loadTable('" + dbUrl + "', '" + tableName + "')}";
}
if(appendFunction !=null && !appendFunction.isEmpty()){
appendScript = appendFunction;
}
pool.execute(task);
if(!task.isSuccessful())
throw new RuntimeException(task.getErrorMsg());
tableInfo = (BasicDictionary) task.getResult();
Entity partColNames = tableInfo.get(new BasicString("partitionColumnName"));
if(partColNames == null)
throw new RuntimeException("Can't find specified partition column name.");
if(partColNames.isScalar()){
if(!((BasicString)partColNames).getString().equalsIgnoreCase(partitionColName))
throw new RuntimeException("Can't find specified partition column name.");
partitionColumnIdx = ((BasicInt)tableInfo.get(new BasicString("partitionColumnIndex"))).getInt();
partitionSchema = tableInfo.get(new BasicString("partitionSchema"));
partitionType = ((BasicInt) tableInfo.get(new BasicString("partitionType"))).getInt();
partitionColType = Entity.DATA_TYPE.valueOf(((BasicInt) tableInfo.get(new BasicString("partitionColumnType"))).getInt());
}
else{
BasicStringVector vec = (BasicStringVector)partColNames;
int dims = vec.rows();
int index = -1;
for(int i=0; i keys = domain.getPartitionKeys(table.getColumn(partitionColumnIdx));
int rows = keys.size();
for(int i=0; i= 0)
chunkIndices.get(key % threadCount).add(i);
}
List tasks = new ArrayList(threadCount);
for(int i=0; i chunk = chunkIndices.get(i);
if(chunk.isEmpty())
continue;
int count = chunk.size();
int[] array = new int[count];
for(int j=0; j args = new ArrayList(1);
args.add(subTable);
tasks.add(new BasicDBTask(appendScript, args));
}
pool.execute(tasks);
int affected = 0;
for(int i=0; i