org.frameworkset.tran.kafka.input.KafkaTranBatchConsumer2ndStore Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of bboss-datatran-kafka2x Show documentation
bboss elasticsearch client with restful and java api without elasticsearch jar dependended.
The newest version!
package org.frameworkset.tran.kafka.input;
/**
 * Copyright 2008 biaoping.yin
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 

 * http://www.apache.org/licenses/LICENSE-2.0
 * 

 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.frameworkset.plugin.kafka.KafkaBatchConsumer2ndStore;
import org.frameworkset.tran.BaseDataTran;
import org.frameworkset.tran.Record;
import org.frameworkset.tran.kafka.KafkaData;
import org.frameworkset.tran.kafka.KafkaMapRecord;
import org.frameworkset.tran.kafka.KafkaStringRecord;
import org.frameworkset.tran.plugin.kafka.input.Kafka2InputConfig;

import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

/**
 * 
Description: 
 * 
 * Copyright (c) 2018
 * @Date 2019/9/28 10:41
 * @author biaoping.yin
 * @version 1.0
 */
public class KafkaTranBatchConsumer2ndStore extends KafkaBatchConsumer2ndStore {
	private Kafka2InputConfig kafka2InputConfig;
	public KafkaTranBatchConsumer2ndStore(BaseDataTran asynESOutPutDataTran, Kafka2InputConfig kafka2InputConfig) {
		this.asynESOutPutDataTran = asynESOutPutDataTran;
		this.kafka2InputConfig = kafka2InputConfig;
	}

	private BaseDataTran asynESOutPutDataTran;
	@Override
	public void store(ConsumerRecords messages) throws Exception {

		List records = parserData(messages);
		asynESOutPutDataTran.appendData(new KafkaData(records));
	}

	@Override
	public void store(ConsumerRecord message) throws Exception {

	}
    private Map buildMetas(ConsumerRecord consumerRecord){
        Map metas = new LinkedHashMap<>();
        metas.put("topic",consumerRecord.topic());
        metas.put("offset",consumerRecord.offset());
        metas.put("key",consumerRecord.key());
        metas.put("partition",consumerRecord.partition());
        metas.put("timestamp",consumerRecord.timestamp());
        return metas;
    }
	private void deserializeData(ConsumerRecord consumerRecord,List results){
		Object value = consumerRecord.value();
        Map metas = buildMetas( consumerRecord);
		if (value instanceof List) {
			List rs = (List) value;

			for (int i = 0; i < rs.size(); i++) {
				Object v = rs.get(i);
				if (v instanceof Map) {
					results.add(new KafkaMapRecord(asynESOutPutDataTran.getTaskContext(),asynESOutPutDataTran.getImportContext(),consumerRecord.key(), (Map) v,consumerRecord.offset(),metas));
				} else {
					results.add(new KafkaStringRecord(asynESOutPutDataTran.getTaskContext(),asynESOutPutDataTran.getImportContext(),consumerRecord.key(), (String) v,consumerRecord.offset(),metas));
				}
			}
			//return new KafkaMapRecord((ConsumerRecord>>) data);
		} else if (value instanceof Map) {
			results.add( new KafkaMapRecord(asynESOutPutDataTran.getTaskContext(),asynESOutPutDataTran.getImportContext(),consumerRecord.key(), (Map) value,consumerRecord.offset(),metas));
		} else if (value instanceof String) {
			results.add(new KafkaStringRecord(asynESOutPutDataTran.getTaskContext(),asynESOutPutDataTran.getImportContext(),consumerRecord.key(), (String) value,consumerRecord.offset(),metas));
		}
		else{
			if(logger.isWarnEnabled()){
				logger.warn("unknown value type:{}",value.getClass().getName());
			}
			results.add(new KafkaStringRecord(asynESOutPutDataTran.getTaskContext(),asynESOutPutDataTran.getImportContext(),consumerRecord.key(), String.valueOf( value),consumerRecord.offset(),metas));
		}
//		throw new IllegalArgumentException(new StringBuilder().append("unknown consumerRecord").append(consumerRecord.toString()).toString());
	}
	protected List parserData(ConsumerRecords messages) {
		List results = new ArrayList();
		for(ConsumerRecord consumerRecord:messages) {
			deserializeData(consumerRecord,results);

		}
		return results;
	}



}