org.frameworkset.tran.plugin.mongodb.output.MongoDBMultiTargetTaskCommandImpl Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of bboss-datatran-mongodb Show documentation
bboss etl,datastream,elasticsearch client with restful and java api without elasticsearch jar dependended.
There is a newer version: 7.3.0
Show newest version
package org.frameworkset.tran.plugin.mongodb.output;
/**
 * Copyright 2008 biaoping.yin
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 

 * http://www.apache.org/licenses/LICENSE-2.0
 * 

 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import com.mongodb.bulk.BulkWriteResult;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import org.frameworkset.nosql.mongodb.MongoDB;
import org.frameworkset.nosql.mongodb.MongoDBHelper;
import org.frameworkset.tran.CommonRecord;
import org.frameworkset.tran.cdc.TableMapping;
import org.frameworkset.tran.exception.ImportExceptionUtil;
import org.frameworkset.tran.task.TaskCommandContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;

/**
 * 
Description: import datas to database task command
 * 
 * Copyright (c) 2018
 * @Date 2019/3/1 11:32
 * @author biaoping.yin
 * @version 1.0
 */
public class MongoDBMultiTargetTaskCommandImpl extends MongoDBTaskCommandImpl {

	private static final Logger logger = LoggerFactory.getLogger(MongoDBMultiTargetTaskCommandImpl.class);
	public MongoDBMultiTargetTaskCommandImpl(TaskCommandContext taskCommandContext) {
		super(    taskCommandContext);

	}




	protected Map> splitRecords(){
		String objectIdField = dbOutputConfig.getObjectIdField();
		if(objectIdField == null){
			objectIdField = "_id";
		}
		String defualtTargetDatasource = dbOutputConfig.getName();
		Map> datasourceDataMapMap = new LinkedHashMap<>();
		String defaultTable = dbOutputConfig.getDBCollection();
		String defaultDb = dbOutputConfig.getDB();
		StringBuilder defaultKey = new StringBuilder();
		defaultKey.append(defaultDb).append(":").append(defaultTable);
		String dk = defaultKey.toString();
		String mapKey = null;
		for(CommonRecord dbRecord:records){
			TableMapping tableMapping = dbRecord.getTableMapping();
			String tdb = tableMapping != null?tableMapping.getTargetDatabase():null;
			String tds = tableMapping != null?tableMapping.getTargetDatasource():null;
			String ttb = tableMapping != null?tableMapping.getTargetCollection():null;
			String table = null;
			if(ttb != null) {
				table = ttb ;
			}
			else{
				if(defaultTable != null)
				{
					table = defaultTable;
				}
				else{
					table = (String) dbRecord.getMetaValue("table");
				}

			}
			String database = null;
			if(tdb != null) {
				database = tdb ;
			}
			else{
				if(defaultDb != null)
				{
					database = defaultDb;
				}
				else{
					database = (String) dbRecord.getMetaValue("database");
				}

			}
			String targetDatasource = tds != null? tds:defualtTargetDatasource;
			if(targetDatasource == null || database == null || table == null){
				throw ImportExceptionUtil.buildDataImportException(importContext,new StringBuilder().append("Setting check failed:targetDatasource == ").append(targetDatasource)
						.append(" || database == ").append(database)
						.append(" || table == ").append(table)
						.append("").toString()
						);
			}
			Map dataMapMap = datasourceDataMapMap.get(targetDatasource);
			if(dataMapMap == null){
				dataMapMap = new LinkedHashMap<>();
				datasourceDataMapMap.put(targetDatasource,dataMapMap);
			}
			if(table != null && database != null){
				defaultKey.setLength(0);
				defaultKey.append(database).append(":").append(table);
				mapKey = defaultKey.toString();
			}
			else{
				table = defaultTable;
				database = defaultDb;
				mapKey = dk;
			}
			DataMap dataMap = dataMapMap.get(mapKey);
			if(dataMap == null){
				dataMap = new DataMap();
				dataMap.setDatabase(database);
				dataMap.setCollection(table);
				dataMapMap.put(mapKey,dataMap);
			}
			dataMap.addRecord(dbRecord,objectIdField);

		}
		return datasourceDataMapMap;
	}
	@Override
	public Object _execute( ){

		List results = null;
		try {
			Map> dsDataMapMap = splitRecords();
			if(dsDataMapMap != null && dsDataMapMap.size() > 0){
				Iterator>> dsiterator = dsDataMapMap.entrySet().iterator();
				while (dsiterator.hasNext()){
					Map.Entry> entry = dsiterator.next();

					Map dataMapMap = entry.getValue();
					if(dataMapMap != null && dataMapMap.size() > 0){
						String ds = entry.getKey();
						MongoDB mogodb = MongoDBHelper.getMongoDB(ds);
						Iterator> iterator = dataMapMap.entrySet().iterator();
						while (iterator.hasNext()){
							DataMap dataMap = iterator.next().getValue();
							MongoDatabase db = mogodb.getDB(dataMap.getDatabase());
							MongoCollection dbCollection = db.getCollection(dataMap.getCollection());
							BulkWriteResult bulkWriteResult = dbCollection.bulkWrite(dataMap.getBulkOperations());
							if(results == null){
								results = new ArrayList<>();
							}
							results.add(bulkWriteResult);
						}
					}
				}
			}


		}
		catch (Exception e) {
			throw ImportExceptionUtil.buildDataImportException(importContext,taskInfo,e);
		}

		catch (Throwable e) {
			throw ImportExceptionUtil.buildDataImportException(importContext,taskInfo,e);
		} 
		return results;
	}




}