org.frameworkset.tran.kafka.input.KafkaTranBatchConsumer2ndStore Maven / Gradle / Ivy
Show all versions of bboss-datatran-kafka2x Show documentation
package org.frameworkset.tran.kafka.input;
/**
* Copyright 2008 biaoping.yin
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.frameworkset.plugin.kafka.KafkaBatchConsumer2ndStore;
import org.frameworkset.tran.BaseDataTran;
import org.frameworkset.tran.Record;
import org.frameworkset.tran.kafka.KafkaData;
import org.frameworkset.tran.kafka.KafkaMapRecord;
import org.frameworkset.tran.kafka.KafkaStringRecord;
import org.frameworkset.tran.plugin.kafka.input.Kafka2InputConfig;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
*
Description:
*
* Copyright (c) 2018
* @Date 2019/9/28 10:41
* @author biaoping.yin
* @version 1.0
*/
public class KafkaTranBatchConsumer2ndStore extends KafkaBatchConsumer2ndStore {
private Kafka2InputConfig kafka2InputConfig;
public KafkaTranBatchConsumer2ndStore(BaseDataTran asynESOutPutDataTran, Kafka2InputConfig kafka2InputConfig) {
this.asynESOutPutDataTran = asynESOutPutDataTran;
this.kafka2InputConfig = kafka2InputConfig;
}
private BaseDataTran asynESOutPutDataTran;
@Override
public void store(ConsumerRecords messages) throws Exception {
List records = parserData(messages);
asynESOutPutDataTran.appendData(new KafkaData(records));
}
@Override
public void store(ConsumerRecord message) throws Exception {
}
private void deserializeData(ConsumerRecord consumerRecord,List results){
Object value = consumerRecord.value();
if (value instanceof List) {
List rs = (List) value;
for (int i = 0; i < rs.size(); i++) {
Object v = rs.get(i);
if (v instanceof Map) {
results.add(new KafkaMapRecord(asynESOutPutDataTran.getTaskContext(),consumerRecord.key(), (Map) v,consumerRecord.offset()));
} else {
results.add(new KafkaStringRecord(asynESOutPutDataTran.getTaskContext(),consumerRecord.key(), (String) v,consumerRecord.offset()));
}
}
//return new KafkaMapRecord((ConsumerRecord>>) data);
} else if (value instanceof Map) {
results.add( new KafkaMapRecord(asynESOutPutDataTran.getTaskContext(),consumerRecord.key(), (Map) value,consumerRecord.offset()));
} else if (value instanceof String) {
results.add(new KafkaStringRecord(asynESOutPutDataTran.getTaskContext(),consumerRecord.key(), (String) value,consumerRecord.offset()));
}
else{
if(logger.isWarnEnabled()){
logger.warn("unknown value type:{}",value.getClass().getName());
}
results.add(new KafkaStringRecord(asynESOutPutDataTran.getTaskContext(),consumerRecord.key(), String.valueOf( value),consumerRecord.offset()));
}
// throw new IllegalArgumentException(new StringBuilder().append("unknown consumerRecord").append(consumerRecord.toString()).toString());
}
protected List parserData(ConsumerRecords messages) {
List results = new ArrayList();
for(ConsumerRecord consumerRecord:messages) {
deserializeData(consumerRecord,results);
}
return results;
}
}