com.xxdb.multithreadedtablewriter.MultithreadedTableWriter Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of dolphindb-javaapi Show documentation
Show all versions of dolphindb-javaapi Show documentation
The messaging and data conversion protocol between Java and DolphinDB server
package com.xxdb.multithreadedtablewriter;
import com.xxdb.DBConnection;
import com.xxdb.comm.ErrorCodeInfo;
import com.xxdb.data.*;
import com.xxdb.route.Domain;
import com.xxdb.route.DomainFactory;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import static com.xxdb.data.Entity.DATA_TYPE.*;
public class MultithreadedTableWriter {
private static final org.slf4j.Logger log = LoggerFactory.getLogger(MultithreadedTableWriter.class);
public static class ThreadStatus{
public long threadId;
public long sentRows, unsentRows, sendFailedRows;
public String toString(){
StringBuilder sb = new StringBuilder();
sb.append(String.format("%16s", threadId) + String.format("%16s", sentRows) + String.format("%16s", unsentRows) + String.format("%16s", sendFailedRows) + "\n");
return sb.toString();
}
};
static class ColInfo{
public Entity.DATA_TYPE type_;
public String name_;
public int extra_;
public ColInfo(){
type_ = Entity.DATA_TYPE.DT_OBJECT;
name_ = "";
extra_ = 0;
}
}
public enum Mode{
M_Append(0),
M_Upsert(1);
private int value;
Mode(int value){
this.value = value;
}
}
public static class Status extends ErrorCodeInfo{
public boolean isExiting;
public long sentRows, unsentRows, sendFailedRows;
public List threadStatusList=new ArrayList<>();
public String toString(){
StringBuilder sb = new StringBuilder();
sb.append("errorCode : " + getErrorCode() + "\n");
sb.append("errorInfo : " + getErrorInfo() + "\n");
sb.append("isExiting : " + isExiting + "\n");
sb.append("sentRows : " + sentRows + "\n");
sb.append("unsentRows : " + unsentRows + "\n");
sb.append("sendFailedRows: " + sendFailedRows + "\n");
sb.append("threadStatus :" + "\n");
sb.append(String.format("%16s", "threadId") + String.format("%16s", "sentRows")
+ String.format("%16s", "unsentRows") + String.format("%16s", "sendFailedRows") + "\n");
for (int i = 0; i < threadStatusList.size(); i++){
sb.append(threadStatusList.get(i).toString());
}
return sb.toString();
}
};
static class WriterThread implements Runnable{
WriterThread(MultithreadedTableWriter tableWriter, DBConnection conn, Callback callbackHandler) {
tableWriter_=tableWriter;
sentRows_=0;
conn_ = conn;
callbackHandler_ = callbackHandler;
exit_ = false;
isFinished_ = false;
writeQueue_.add(tableWriter_.createListVector());
writeThread_ = new Thread(this);
writeThread_.start();
}
@Override
public void run(){
if (!init())//init
return;
long batchWaitTimeout, diff;
try {
while (!isExiting()) {
synchronized (writeQueue_) {
//tableWriter_.logger_.info(writeThread_.getId()+" run wait0 start");
//if (!isExiting()&&writeQueue_.size()==0)
writeQueue_.wait();
if (!isExiting() && tableWriter_.batchSize_ > 1 && tableWriter_.throttleMilsecond_ > 0) {
batchWaitTimeout = System.currentTimeMillis() + tableWriter_.throttleMilsecond_;
while (!isExiting() && (writeQueue_.size() - 1) * vectorSize + writeQueue_.get(writeQueue_.size() - 1).get(0).rows() < tableWriter_.batchSize_) {//check batchsize
diff = batchWaitTimeout - System.currentTimeMillis();
if (diff > 0) {
writeQueue_.wait(diff);
} else {
break;
}
}
}
}
while (!isExiting() && writeAllData());
}
while (!tableWriter_.hasError_ && writeAllData());
if (tableWriter_.ifCallback_ && tableWriter_.hasError_){
List callbackList = new ArrayList<>();
BasicStringVector bs = new BasicStringVector(0);
callbackList.add(bs);
int allLength = 0;
synchronized (writeQueue_){
for (int i = 0; i < writeQueue_.size(); i++){
List notInsertV = writeQueue_.get(i);
Vector id = notInsertV.get(0);
callbackList.get(0).Append(id);
allLength += id.rows();
}
}
boolean[] bArray = new boolean[allLength];
for (int i = 0; i < allLength; i++){
bArray[i] = false;
}
BasicBooleanVector bv = new BasicBooleanVector(bArray);
callbackList.add(bv);
List callbackColNames = new ArrayList<>();
callbackColNames.add("callbackId");
callbackColNames.add("isSuccess");
callbackHandler_.writeCompletion(new BasicTable(callbackColNames, callbackList));
}
}catch (Exception e) {
e.printStackTrace();
tableWriter_.hasError_ = true;
tableWriter_.errorCodeInfo_.set(ErrorCodeInfo.Code.EC_None, e.getMessage());
}
synchronized (writeThread_){
conn_.close();
isFinished_ = true;
writeThread_.notify();
}
}
boolean writeAllData(){
synchronized (busyLock_) {
List items;
List callbackList = new ArrayList<>();
int callbackRows = 0;
int addRowCount = 0;
synchronized (writeQueue_) {
items = writeQueue_.get(0);
addRowCount = items.get(0).rows();
if (addRowCount < 1)
return false;
writeQueue_.remove(0);
if (writeQueue_.size() == 0)
writeQueue_.add(tableWriter_.createListVector());
}
boolean isWriteDone = true;
BasicTable writeTable = null;
List colNames = new ArrayList<>();
for (int i = 0; i < tableWriter_.colInfos_.length; i++){
colNames.add(tableWriter_.colInfos_[i].name_);
}
if(tableWriter_.ifCallback_){
callbackList.add(items.get(0));
items.remove(0);
colNames.remove(0);
}
try {
writeTable = new BasicTable(colNames, items);
}catch (Exception e){
e.printStackTrace();
tableWriter_.log.warn("threadid=" + writeThread_.getId() + " sendindex=" + sentRows_ + " create table error: " + e);
tableWriter_.setError(ErrorCodeInfo.Code.EC_Server, "Failed to createTable: " + e);
isWriteDone = false;
}
if (isWriteDone) {//may contain empty vector for exit
String runscript = "";
try {//save table
if (tableWriter_.enableActualSendTime_) {
long[] time = new long[writeTable.rows()];
long curTime = Utils.countDTNanoseconds(LocalDateTime.now());
for (int i = 0; i < writeTable.rows(); i++) {
time[i] = curTime;
}
BasicNanoTimestampVector basicNanoTimestampVector = new BasicNanoTimestampVector(time);
writeTable.replaceColumn(writeTable.getColumnName(writeTable.columns() - 1), basicNanoTimestampVector);
}
List args = new ArrayList<>();
args.add(writeTable);
runscript = scriptTableInsert_;
conn_.run(runscript, args);
if (scriptSaveTable_ != null && !scriptSaveTable_.isEmpty()) {
runscript = scriptSaveTable_;
conn_.run(runscript);
}
sentRows_ += addRowCount;
} catch (Exception e) {
e.printStackTrace();
tableWriter_.log.warn("threadid=" + writeThread_.getId() + " sendindex=" + sentRows_ + " Save table error: " + e + " script:" + runscript);
tableWriter_.setError(ErrorCodeInfo.Code.EC_Server, "Failed to save the inserted data: " + e + " script: " + runscript);
isWriteDone = false;
tableWriter_.hasError_ = true;
}
}
if (!isWriteDone) {
synchronized (failedQueue_) {
int cols = items.size();
int rows = items.get(0).rows();
for (int i = 0; i < rows; i++){
List tmp = new ArrayList<>();
if (tableWriter_.ifCallback_)
tmp.add(callbackList.get(0).get(i));
for (int j = 0; j < cols; j++){
tmp.add(items.get(j).get(i));
}
failedQueue_.add(tmp);
}
}
}
if (tableWriter_.ifCallback_){
callbackRows = callbackList.get(0).rows();
boolean[] bArray = new boolean[callbackRows];
if (!isWriteDone){
for (int i = 0; i < callbackRows; i++){
bArray[i] = false;
}
}else {
for (int i = 0; i < callbackRows; i++){
bArray[i] = true;
}
}
BasicBooleanVector bv = new BasicBooleanVector(bArray);
callbackList.add(bv);
List callbackColNames = new ArrayList<>();
callbackColNames.add("callbackId");
callbackColNames.add("isSuccess");
callbackHandler_.writeCompletion(new BasicTable(callbackColNames, callbackList));
}
boolean startgc = false;
synchronized (tableWriter_) {
tableWriter_.sentRowsAfterGc_ += addRowCount;
if (tableWriter_.sentRowsAfterGc_ > 10000) {//every sent 10000 rows, manual start gc to avoid out of memory
tableWriter_.sentRowsAfterGc_ = 0;
startgc = true;
}
}
if (startgc && Runtime.getRuntime().freeMemory() < 104857600)
System.gc();
}
return true;
}
boolean init(){
if (tableWriter_.mode_ == Mode.M_Append){
if (tableWriter_.dbName_.isEmpty()) {
scriptTableInsert_ = "tableInsert{\"" + tableWriter_.tableName_ + "\"}";
} else {// single partitioned table
scriptTableInsert_ = "tableInsert{loadTable(\"" + tableWriter_.dbName_ + "\",\"" + tableWriter_.tableName_ + "\")}";
}
}else if (tableWriter_.mode_ == Mode.M_Upsert){
StringBuilder sb = new StringBuilder();
if(tableWriter_.dbName_.isEmpty()){
sb.append("upsert!{" + tableWriter_.tableName_);
}else{
sb.append("upsert!{loadTable(\"" + tableWriter_.dbName_ + "\",\"" + tableWriter_.tableName_ + "\")");
}
sb.append(",");
if (tableWriter_.pModeOption_ != null && tableWriter_.pModeOption_.length != 0){
for (String one : tableWriter_.pModeOption_){
sb.append("," + one);
}
}
sb.append("}");
scriptTableInsert_ = sb.toString();
}
return true;
}
void getStatus(ThreadStatus status){
status.threadId = writeThread_.getId();
status.sentRows = sentRows_;
synchronized (writeQueue_){
status.unsentRows = (long) (writeQueue_.size() - 1) * vectorSize+ writeQueue_.get(writeQueue_.size() - 1).get(0).rows();
}
status.sendFailedRows = failedQueue_.size();
}
boolean isExiting(){
return exit_ || tableWriter_.hasError_;
}
void exit(){
synchronized (writeQueue_) {
exit_ = true;
writeQueue_.notify();
}
}
MultithreadedTableWriter tableWriter_;
DBConnection conn_;
Object busyLock_ = new Object();
Callback callbackHandler_;
String scriptTableInsert_,scriptSaveTable_;
List> writeQueue_ = new ArrayList<>();
List> failedQueue_=new ArrayList<>();
Thread writeThread_;
long sentRows_;
boolean exit_;//Only set when exit
boolean isFinished_;
public static int vectorSize = 65535;
};
private String dbName_;
private String tableName_;
private int batchSize_;
private int throttleMilsecond_;
private boolean isPartionedTable_;
private boolean hasError_;
private boolean isExiting_ = false;
private int[] compressTypes_=null;
private Domain partitionDomain_;
private int partitionColumnIdx_;
private int threadByColIndexForNonPartion_;
private int sentRowsAfterGc_;//manual start gc after sent some rows
private List threads_=new ArrayList<>();
private ErrorCodeInfo errorCodeInfo_ = new ErrorCodeInfo();
private Mode mode_;
private String[] pModeOption_;
private boolean ifCallback_ = false;
private ColInfo[] colInfos_;
private boolean enableActualSendTime_ = false;
public MultithreadedTableWriter(String hostName, int port, String userId, String password,
String dbName, String tableName, boolean useSSL,
boolean enableHighAvailability, String[] highAvailabilitySites,
int batchSize, float throttle,
int threadCount, String partitionCol,
int[] compressTypes, Mode mode, String[] pModeOption) throws Exception{
init(hostName,port,userId, password,dbName, tableName, useSSL,enableHighAvailability,highAvailabilitySites,
batchSize, throttle,threadCount,partitionCol,compressTypes, mode, pModeOption, null, false);
}
public MultithreadedTableWriter(String hostName, int port, String userId, String password,
String dbName, String tableName, boolean useSSL,
boolean enableHighAvailability, String[] highAvailabilitySites,
int batchSize, float throttle,
int threadCount, String partitionCol,
int[] compressTypes) throws Exception{
init(hostName,port,userId, password,dbName, tableName, useSSL,enableHighAvailability,highAvailabilitySites,
batchSize, throttle,threadCount,partitionCol,compressTypes, Mode.M_Append, null, null, false);
}
public MultithreadedTableWriter(String hostName, int port, String userId, String password,
String dbName, String tableName, boolean useSSL,
boolean enableHighAvailability, String[] highAvailabilitySites,
int batchSize, float throttle,
int threadCount, String partitionCol) throws Exception{
init(hostName,port,userId, password,dbName, tableName, useSSL,enableHighAvailability,highAvailabilitySites,
batchSize, throttle,threadCount,partitionCol,null, Mode.M_Append, null, null, false);
}
public MultithreadedTableWriter(String hostName, int port, String userId, String password,
String dbName, String tableName, boolean useSSL,
boolean enableHighAvailability, String[] highAvailabilitySites,
int batchSize, float throttle,
int threadCount, String partitionCol,
int[] compressTypes, Callback callbackHandler) throws Exception{
init(hostName,port,userId, password,dbName, tableName, useSSL,enableHighAvailability,highAvailabilitySites,
batchSize, throttle,threadCount,partitionCol,compressTypes, Mode.M_Append, null, callbackHandler, false);
}
public MultithreadedTableWriter(String hostName, int port, String userId, String password,
String dbName, String tableName, boolean useSSL,
boolean enableHighAvailability, String[] highAvailabilitySites,
int batchSize, float throttle,
int threadCount, String partitionCol,
int[] compressTypes, Mode mode, String[] pModeOption,
boolean enableActualSendTime) throws Exception {
init(hostName, port, userId, password, dbName, tableName, useSSL, enableHighAvailability, highAvailabilitySites,
batchSize, throttle, threadCount, partitionCol, compressTypes, mode, pModeOption, null, enableActualSendTime);
}
public MultithreadedTableWriter(String hostName, int port, String userId, String password,
String dbName, String tableName, boolean useSSL,
boolean enableHighAvailability, String[] highAvailabilitySites,
int batchSize, float throttle,
int threadCount, String partitionCol,
int[] compressTypes,
boolean enableActualSendTime) throws Exception {
init(hostName, port, userId, password, dbName, tableName, useSSL, enableHighAvailability, highAvailabilitySites,
batchSize, throttle, threadCount, partitionCol, compressTypes, Mode.M_Append, null, null, enableActualSendTime);
}
public MultithreadedTableWriter(String hostName, int port, String userId, String password,
String dbName, String tableName, boolean useSSL,
boolean enableHighAvailability, String[] highAvailabilitySites,
int batchSize, float throttle,
int threadCount, String partitionCol,
boolean enableActualSendTime) throws Exception {
init(hostName, port, userId, password, dbName, tableName, useSSL, enableHighAvailability, highAvailabilitySites,
batchSize, throttle, threadCount, partitionCol, null, Mode.M_Append, null, null, enableActualSendTime);
}
public MultithreadedTableWriter(String hostName, int port, String userId, String password,
String dbName, String tableName, boolean useSSL,
boolean enableHighAvailability, String[] highAvailabilitySites,
int batchSize, float throttle,
int threadCount, String partitionCol,
int[] compressTypes, Callback callbackHandler,
boolean enableActualSendTime) throws Exception {
init(hostName, port, userId, password, dbName, tableName, useSSL, enableHighAvailability, highAvailabilitySites,
batchSize, throttle, threadCount, partitionCol, compressTypes, Mode.M_Append, null, callbackHandler, enableActualSendTime);
}
private void init(String hostName, int port, String userId, String password,
String dbName, String tableName, boolean useSSL,
boolean enableHighAvailability, String[] highAvailabilitySites,
int batchSize, float throttle,
int threadCount, String partitionCol,
int[] compressTypes, Mode mode, String[] pModeOption, Callback callbackHandler,
boolean enableActualSendTime) throws Exception{
dbName_=dbName;
tableName_=tableName;
batchSize_=batchSize;
throttleMilsecond_=(int)throttle*1000;
hasError_=false;
enableActualSendTime_ = enableActualSendTime;
if (mode == null)
mode_ = Mode.M_Append;
else
mode_ = mode;
pModeOption_ = pModeOption;
if(threadCount < 1){
throw new RuntimeException("The parameter threadCount must be greater than or equal to 1.");
}
if(batchSize < 1){
throw new RuntimeException("The parameter batchSize must be greater than or equal to 1.");
}
if(throttle < 0){
throw new RuntimeException("The parameter throttle must be greater than or equal to 0.");
}
if (threadCount > 1 && partitionCol.length()<1) {
throw new RuntimeException("The parameter partitionCol must be specified when threadCount is greater than 1.");
}
boolean isCompress = false;
if (compressTypes != null && compressTypes.length > 0) {
for (int one : compressTypes) {
if (one != Vector.COMPRESS_LZ4 && one != Vector.COMPRESS_DELTA) {
throw new RuntimeException("Unsupported compress method " + one);
}
}
isCompress = true;
compressTypes_=new int[compressTypes.length];
System.arraycopy(compressTypes,0,compressTypes_,0,compressTypes.length);
}
DBConnection pConn = newConn(hostName,port,userId,password,dbName,tableName,useSSL,enableHighAvailability,highAvailabilitySites,isCompress);
if(pConn==null){
throw new RuntimeException("Failed to connect to server " + hostName + ":" + port);
}
BasicDictionary schema;
if(dbName.isEmpty()){
schema = (BasicDictionary)pConn.run("schema(" + tableName + ")");
}else{
schema = (BasicDictionary)pConn.run("schema(loadTable(\"" + dbName + "\",\"" + tableName + "\"))");
}
Entity partColNames = schema.get(new BasicString("partitionColumnName"));
if(partColNames!=null){//partitioned table
isPartionedTable_ = true;
}else{//没有分区
if(!dbName.isEmpty()){//Single partitioned table
if(threadCount > 1){
throw new RuntimeException("The parameter threadCount must be 1 for a dimension table.");
}
}
isPartionedTable_ = false;
}
BasicTable colDefs = (BasicTable)schema.get(new BasicString("colDefs"));
BasicIntVector colDefsTypeInt = (BasicIntVector)colDefs.getColumn("typeInt");
int columnSize = colDefs.rows();
if (Objects.nonNull(compressTypes_)) {
if (!enableActualSendTime && compressTypes_.length != columnSize)
throw new RuntimeException("The number of elements in parameter compressMethods does not match the column size " + columnSize);
if (enableActualSendTime_) {
if (compressTypes_.length != columnSize - 1)
throw new RuntimeException("The number of elements in parameter compressMethods does not match the column size " + (columnSize - 1));
int[] copy = new int[compressTypes_.length + 1];
System.arraycopy(compressTypes_, 0, copy, 0, compressTypes_.length);
copy[compressTypes_.length] = Vector.COMPRESS_LZ4;
compressTypes_ = copy;
}
}
if (callbackHandler != null){
ifCallback_ = true;
colInfos_ = new ColInfo[columnSize+1];
}else
colInfos_ = new ColInfo[columnSize];
for (int i = 0; i < colInfos_.length; i++){
ColInfo colInfo = new ColInfo();
colInfos_[i] = colInfo;
}
BasicIntVector colExtra= (BasicIntVector)colDefs.getColumn("extra");
BasicStringVector colDefsName = (BasicStringVector)colDefs.getColumn("name");
int colDefsIndex = 0;
for(int i = 0; i < colInfos_.length; i++){
if (i == 0 && ifCallback_){
colInfos_[i].type_ = Entity.DATA_TYPE.DT_STRING;
colInfos_[i].name_ = colDefsName.getString(0) + "_id";
colInfos_[i].extra_ = -1;
}else {
colInfos_[i].name_ = colDefsName.getString(colDefsIndex);
if (colExtra != null){
colInfos_[i].extra_ = colExtra.getInt(colDefsIndex);
}
if (compressTypes_ != null){
boolean check = AbstractVector.checkCompressedMethod(Entity.DATA_TYPE.valueOf(colDefsTypeInt.getInt(colDefsIndex)), compressTypes_[colDefsIndex]);
if (check)
colInfos_[i].type_ = Entity.DATA_TYPE.valueOf(colDefsTypeInt.getInt(colDefsIndex));
else
throw new RuntimeException("Compression Failed: only support integral and temporal data, not support " + Entity.DATA_TYPE.valueOf(colDefsTypeInt.getInt(colDefsIndex)));
}
else{
colInfos_[i].type_ = Entity.DATA_TYPE.valueOf(colDefsTypeInt.getInt(colDefsIndex));
}
colDefsIndex++;
}
}
if(isPartionedTable_){
Entity partitionSchema;
int partitionType;
if(partColNames.isScalar()){
if (!partColNames.getString().equals(partitionCol)) {
throw new RuntimeException("The parameter partionCol must be the partitioning column "+ partColNames.getString()+" in the table.");
}
partitionColumnIdx_ = ((BasicInt)schema.get(new BasicString("partitionColumnIndex"))).getInt();
partitionSchema = schema.get(new BasicString("partitionSchema"));
partitionType = ((BasicInt)schema.get(new BasicString("partitionType"))).getInt();
}else{
BasicStringVector partColNamesVec = (BasicStringVector)partColNames;
int dims = partColNamesVec.rows();
if(dims > 1 && partitionCol.isEmpty()){
throw new RuntimeException("The parameter partitionCol must be specified for a partitioned table.");
}
int index = -1;
for(int i=0; i> getUnwrittenData() {
if (ifCallback_)
throw new RuntimeException("getUnwrittenData is disabled when callback is enabled.");
List> unwrittenData = new ArrayList<>();
for(WriterThread writeThread : threads_){
synchronized (writeThread.busyLock_) {
synchronized (writeThread.failedQueue_) {
unwrittenData.addAll(writeThread.failedQueue_);
writeThread.failedQueue_.clear();
}
synchronized (writeThread.writeQueue_) {
int cols = colInfos_.length;
int size = writeThread.writeQueue_.size();
for (int i = 0; i < size; ++i)
{
int rows = writeThread.writeQueue_.get(i).get(0).rows();
for (int row = 0; row < rows; ++row)
{
List tmp = new ArrayList<>();
for (int j = 0; j < cols; ++j)
{
tmp.add(writeThread.writeQueue_.get(i).get(j).get(row));
}
unwrittenData.add(tmp);
}
}
writeThread.writeQueue_.clear();
writeThread.writeQueue_.add(createListVector());
}
}
}
return unwrittenData;
}
public List> getFailedData() throws InterruptedException{
List> failedData = new ArrayList<>();
for (WriterThread writeThread : threads_){
synchronized (writeThread.busyLock_){
synchronized (writeThread.failedQueue_){
failedData.addAll(writeThread.failedQueue_);
writeThread.failedQueue_.clear();
}
}
}
return failedData;
}
public Status getStatus(){
Status status = new Status();
status.setErrorCode(errorCodeInfo_.getErrorCode());
status.setErrorInfo(errorCodeInfo_.getErrorInfo());
status.sendFailedRows=status.sentRows=status.unsentRows=0;
status.isExiting=isExiting();
for(WriterThread writeThread : threads_){
ThreadStatus threadStatus=new ThreadStatus();
writeThread.getStatus(threadStatus);
status.threadStatusList.add(threadStatus);
status.sentRows += threadStatus.sentRows;
status.unsentRows += threadStatus.unsentRows;
status.sendFailedRows += threadStatus.sendFailedRows;
}
return status;
}
public void waitForThreadCompletion() throws InterruptedException{
isExiting_ = true;
for(WriterThread one:threads_){
one.exit();
}
for (WriterThread one : threads_) {
synchronized (one.writeThread_) {
if(!one.isFinished_) {
one.writeThread_.wait();
}
}
one.conn_ = null;
}
setError(ErrorCodeInfo.Code.EC_None,"");
}
public ErrorCodeInfo insertUnwrittenData(List> records){
if(isExiting()){
throw new RuntimeException("Thread is exiting. ");
}
if(threads_.size() > 1){
if(isPartionedTable_){
Vector pvector=BasicEntityFactory.instance().createVectorWithDefaultValue(colInfos_[partitionColumnIdx_].type_,records.size(), -1);
int rowindex=0;
try {
for (List row : records) {
if (row.size() != colInfos_.length) {
return new ErrorCodeInfo(ErrorCodeInfo.Code.EC_InvalidParameter, "Column counts don't match.");
}
if (row.get(partitionColumnIdx_) != null) {
Scalar scalar=(Scalar) row.get(partitionColumnIdx_);
if(scalar!=null)
pvector.set(rowindex, scalar);
else
pvector.setNull(rowindex);
} else {
pvector.setNull(rowindex);
}
rowindex++;
}
}catch (Exception e){
e.printStackTrace();
return new ErrorCodeInfo(ErrorCodeInfo.Code.EC_InvalidParameter, "Row in records " + rowindex +
" mismatch type " + colInfos_[partitionColumnIdx_].type_);
}
List threadindexes = partitionDomain_.getPartitionKeys(pvector);
try {
for(int row = 0; row < threadindexes.size(); row++){
insertThreadWrite(threadindexes.get(row), records.get(row));
}
}catch (Exception e){
}
}else{
Vector partionvector=BasicEntityFactory.instance().createVectorWithDefaultValue(colInfos_[threadByColIndexForNonPartion_].type_,records.size(), -1);
int rowindex=0;
try{
for(List row : records) {
if (row.size() != colInfos_.length) {
return new ErrorCodeInfo(ErrorCodeInfo.Code.EC_InvalidParameter, "Column counts don't match.");
}
Scalar scalar=(Scalar) row.get(threadByColIndexForNonPartion_);
if(scalar!=null)
partionvector.set(rowindex, scalar);
else
partionvector.setNull(rowindex);
rowindex++;
}
}catch (Exception e){
e.printStackTrace();
return new ErrorCodeInfo(ErrorCodeInfo.Code.EC_InvalidParameter, "Row in records " + rowindex +
" mismatch type " + colInfos_[partitionColumnIdx_].type_);
}
int threadindex;
try {
for(rowindex=0;rowindex row : records){
insertThreadWrite(0, row);
}
}catch (Exception e){
}
}
return new ErrorCodeInfo();
}
private void insertThreadWrite(int threadhashkey, List row) throws Exception{
if(threadhashkey < 0)
threadhashkey = 0;
int threadIndex = threadhashkey % threads_.size();
WriterThread writerThread=threads_.get(threadIndex);
int blockSize = Math.max(this.batchSize_, WriterThread.vectorSize);
synchronized (writerThread.writeQueue_) {
int rows = writerThread.writeQueue_.get(writerThread.writeQueue_.size() - 1).get(0).rows();
if (rows > blockSize){
writerThread.writeQueue_.add(createListVector());
}
int size = row.size();
for (int i = 0; i < size; i++){
if (colInfos_[i].type_.getValue() < 65){
writerThread.writeQueue_.get(writerThread.writeQueue_.size()-1).get(i).Append((Scalar) row.get(i));
}else {
writerThread.writeQueue_.get(writerThread.writeQueue_.size()-1).get(i).Append((Vector) row.get(i));
}
}
writerThread.writeQueue_.notify();
}
}
public ErrorCodeInfo insert(Object... args){
if(isExiting()){
throw new RuntimeException("Thread is exiting. ");
}
if ((!enableActualSendTime_ && args.length != colInfos_.length) ||
(enableActualSendTime_ && args.length != colInfos_.length - 1)) {
return new ErrorCodeInfo(ErrorCodeInfo.Code.EC_InvalidParameter, "Column counts don't match.");
}
List prow=new ArrayList<>();
int colindex = 0;
Entity.DATA_TYPE dataType;
boolean isAllNull = true;
for (Object one : args) {
dataType = colInfos_[colindex].type_;
Entity entity;
isAllNull = false;
try {
entity = BasicEntityFactory.createScalar(dataType, one, colInfos_[colindex].extra_);
} catch (Exception e) {
String errorMsg = "Invalid object error when create scalar for column '" + colInfos_[colindex].name_ + "': " + e.getMessage();
log.error(errorMsg);
return new ErrorCodeInfo(ErrorCodeInfo.Code.EC_InvalidObject, errorMsg);
}
if (entity == null) {
return new ErrorCodeInfo(ErrorCodeInfo.Code.EC_InvalidObject, "Data conversion error: " + dataType);
}
prow.add(entity);
colindex++;
}
if (enableActualSendTime_) {
dataType = colInfos_[colindex].type_;
if (dataType != DT_NANOTIMESTAMP)
return new ErrorCodeInfo(ErrorCodeInfo.Code.EC_InvalidObject, String.format("Data type error: %s,should be NANOTIMESTAMP.", dataType));
Entity entity;
isAllNull = false;
try {
entity = BasicEntityFactory.createScalar(dataType, null, colInfos_[colindex].extra_);
} catch (Exception e) {
String errorMsg = "Invalid object error when create scalar for column '" + colInfos_[colindex].name_ + "': " + e.getMessage();
log.error(errorMsg);
return new ErrorCodeInfo(ErrorCodeInfo.Code.EC_InvalidObject, errorMsg);
}
prow.add(entity);
}
try {
if(isAllNull)
return new ErrorCodeInfo(ErrorCodeInfo.Code.EC_InvalidObject, "Can't insert a Null row.");
int threadindex;
if(threads_.size() > 1){
if(isPartionedTable_){
try {
threadindex = partitionDomain_.getPartitionKey((Scalar) prow.get(partitionColumnIdx_));
}catch (Exception e){
return new ErrorCodeInfo(ErrorCodeInfo.Code.EC_InvalidObject, e.getMessage());
}
}else{
if (prow.get(threadByColIndexForNonPartion_) != null) {
threadindex = ((Scalar) prow.get(threadByColIndexForNonPartion_)).hashBucket(threads_.size());
}
else {
threadindex = 0;
}
}
}else{
threadindex = 0;
}
insertThreadWrite(threadindex, prow);
return new ErrorCodeInfo();
}catch (Exception e){
log.error(e.getMessage());
return new ErrorCodeInfo(ErrorCodeInfo.Code.EC_InvalidObject, "Invalid object error " + e);
}
}
private List createListVector(){
List tmp = new ArrayList<>();
int cols = colInfos_.length;
for (int i = 0; i < cols; i++){
Entity.DATA_TYPE type = colInfos_[i].type_;
if (type.getValue() >= 65){
tmp.add(new BasicArrayVector(type, 1, colInfos_[i].extra_));
}
else{
Vector value = BasicEntityFactory.instance().createVectorWithDefaultValue(type, 0, colInfos_[i].extra_);
if (type == DT_DECIMAL32 || type == DT_DECIMAL64 || type == DT_DECIMAL128) {
((AbstractVector)value).setExtraParamForType(colInfos_[i].extra_);
}
tmp.add(value);
}
}
return tmp;
}
private boolean isExiting() { return hasError_ || isExiting_; }
private DBConnection newConn(String hostName, int port, String userId, String password,
String dbName, String tableName, boolean useSSL,
boolean enableHighAvailability, String[] highAvailabilitySites,boolean compress) throws IOException {
DBConnection pConn = new DBConnection(false,useSSL,compress);
//String hostName, int port, String userId, String password, String initialScript, boolean enableHighAvailability, String[] highAvailabilitySites
boolean ret = pConn.connect(hostName, port, userId, password, null,enableHighAvailability,highAvailabilitySites);
if (!ret)
return null;
return pConn;
}
private void setError(ErrorCodeInfo.Code code,String info) {
if (hasError_)
return;
hasError_ = true;
errorCodeInfo_ = new ErrorCodeInfo(code, info);
}
}