io.rtdi.bigdata.connector.pipeline.foundation.AvroSerializer Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of pipelinefoundation Show documentation

The abstraction of an Apache Kafka-like Pipeline and common classes/interfaces.

There is a newer version: 0.10.20

package io.rtdi.bigdata.connector.pipeline.foundation;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;

import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.io.BinaryEncoder;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.io.EncoderFactory;

import io.rtdi.bigdata.connector.pipeline.foundation.avro.JexlGenericData.JexlRecord;
import io.rtdi.bigdata.kafka.avro.datatypes.LogicalDataTypesRegistry;
import io.rtdi.bigdata.connector.pipeline.foundation.utils.IOUtils;

/**
 * Creates a a binary representation of an Avro GenericRecord in the Kafka format.
 * Hence all data put into Kafka can be read by other tools like Kafka Connect.  
 *
 */
public class AvroSerializer {
	
	private static EncoderFactory encoderFactory = EncoderFactory.get();
	static {
		LogicalDataTypesRegistry.registerAll();
	}


	/**
	 * Convert an AvroRecord into a Kafka payload
	 * 
	 * @param schemaid ID of the Kafka Schema
	 * @param data AvroRecord
	 * @return binary representation of the AvroRecord
	 * @throws IOException in case the AvroRecord cannot be serialized
	 */
	public static byte[] serialize(int schemaid, GenericRecord data) throws IOException {
		try ( ByteArrayOutputStream out = new ByteArrayOutputStream(); ) {
			out.write(IOUtils.MAGIC_BYTE);
			out.write(ByteBuffer.allocate(Integer.BYTES).putInt(schemaid).array());
			BinaryEncoder encoder = encoderFactory.directBinaryEncoder(out, null);
			DatumWriter writer = new GenericDatumWriter<>(data.getSchema());
			writer.write(data, encoder);
			encoder.flush();
			byte[] bytes = out.toByteArray();
			return bytes;
		} catch (Exception e) {
			if (e instanceof IOException) {
				throw e;
			} else {
				throw new IOException(e);
			}
		}
	}

	public static byte[] serialize(JexlRecord data) throws IOException {
		return serialize(data.getSchemaId(), data);
	}

}