All Downloads are FREE. Search and download functionalities are using the official Maven repository.
  • Log In
  • Register

    • com.netflix.zeno.fastblob.FastBlobFrameworkSerializer Maven / Gradle / Ivy

    There is a newer version: 2.22.3
    Show newest version
    /*
 *
 *  Copyright 2013 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.zeno.fastblob;

import com.netflix.zeno.fastblob.record.ByteDataBuffer;
import com.netflix.zeno.fastblob.record.FastBlobSchema.FieldType;
import com.netflix.zeno.fastblob.record.FastBlobSerializationRecord;
import com.netflix.zeno.fastblob.record.VarInt;
import com.netflix.zeno.fastblob.state.FastBlobTypeSerializationState;
import com.netflix.zeno.serializer.FrameworkSerializer;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
import java.util.Set;

/**
 * Defines the binary serialized representation for each of the "Zeno native" elements in a FastBlob
 *
 * @author dkoszewnik
 *
 */
public class FastBlobFrameworkSerializer extends FrameworkSerializer {

    static final int NULL_FLOAT_BITS = Float.floatToIntBits(Float.NaN) + 1;
    static final long NULL_DOUBLE_BITS = Double.doubleToLongBits(Double.NaN) + 1;

    public FastBlobFrameworkSerializer(FastBlobStateEngine framework) {
        super(framework);
    }

    /**
     * Serialize a primitive element
     */
    @Override
    public void serializePrimitive(FastBlobSerializationRecord rec, String fieldName, Object value) {
        if (value == null) {
            return;
        }

        if (value instanceof Integer) {
            serializePrimitive(rec, fieldName, ((Integer) value).intValue());
        } else if (value instanceof Long) {
            serializePrimitive(rec, fieldName, ((Long) value).longValue());
        } else if (value instanceof Float) {
            serializePrimitive(rec, fieldName, ((Float) value).floatValue());
        } else if (value instanceof Double) {
            serializePrimitive(rec, fieldName, ((Double) value).doubleValue());
        } else if (value instanceof Boolean) {
            serializePrimitive(rec, fieldName, ((Boolean) value).booleanValue());
        } else if (value instanceof String) {
            serializeString(rec, fieldName, (String) value);
        } else if (value instanceof byte[]){
            serializeBytes(rec, fieldName, (byte[]) value);
        } else {
            throw new RuntimeException("Primitive type " + value.getClass().getSimpleName() + " not supported!");
        }

    }

    /**
     * Serialize a string as the UTF-8 value
     */
    public void serializeString(FastBlobSerializationRecord rec, String fieldName, String value) {
        if(value == null)
            return;

        int position = rec.getSchema().getPosition(fieldName);
        FieldType fieldType = rec.getSchema().getFieldType(position);

        if(fieldType != FieldType.STRING)
            throw new IllegalArgumentException("Attempting to serialize a String as " + fieldType + " in field " + fieldName + ".  Carefully check your schema for type " + rec.getSchema().getName() + ".");

        ByteDataBuffer fieldBuffer = rec.getFieldBuffer(position);

        writeString(value, fieldBuffer);
    }

    /**
     * Serialize an integer, use zig-zag encoding to (probably) get a small positive value, then encode the result as a variable-byte integer.
     */
    @Override
    public void serializePrimitive(FastBlobSerializationRecord rec, String fieldName, int value) {
        int position = rec.getSchema().getPosition(fieldName);
        FieldType fieldType = rec.getSchema().getFieldType(position);

        if(fieldType != FieldType.INT && fieldType != FieldType.LONG)
            throw new IllegalArgumentException("Attempting to serialize an int as " + fieldType + " in field " + fieldName + ".  Carefully check your schema for type " + rec.getSchema().getName() + ".");


        ByteDataBuffer fieldBuffer = rec.getFieldBuffer(position);

        // zig zag encoding
        VarInt.writeVInt(fieldBuffer, (value << 1) ^ (value >> 31));
    }

    /**
     * Serialize a long, use zig-zag encoding to (probably) get a small positive value, then encode the result as a variable-byte long.
     */
    @Override
    public void serializePrimitive(FastBlobSerializationRecord rec, String fieldName, long value) {
        int position = rec.getSchema().getPosition(fieldName);
        FieldType fieldType = rec.getSchema().getFieldType(position);

        if(fieldType != FieldType.LONG)
            throw new IllegalArgumentException("Attempting to serialize a long as " + fieldType + " in field " + fieldName + ".  Carefully check your schema for type " + rec.getSchema().getName() + ".");


        ByteDataBuffer fieldBuffer = rec.getFieldBuffer(position);

        // zig zag encoding
        VarInt.writeVLong(fieldBuffer, (value << 1) ^ (value >> 63));
    }

    /**
     * Serialize a float into 4 consecutive bytes
     */
    @Override
    public void serializePrimitive(FastBlobSerializationRecord rec, String fieldName, float value) {
        int position = rec.getSchema().getPosition(fieldName);
        FieldType fieldType = rec.getSchema().getFieldType(position);

        if(fieldType != FieldType.FLOAT) {
            throw new IllegalArgumentException("Attempting to serialize a float as " + fieldType + " in field " + fieldName + ".  Carefully check your schema for type " + rec.getSchema().getName() + ".");
        }

        ByteDataBuffer fieldBuffer = rec.getFieldBuffer(position);
        int intBits = Float.floatToIntBits(value);
        writeFixedLengthInt(fieldBuffer, intBits);
    }

    /**
     * Serialize a special 4-byte long sequence indicating a null Float value.
     */
    public static void writeNullFloat(final ByteDataBuffer fieldBuffer) {
        writeFixedLengthInt(fieldBuffer, NULL_FLOAT_BITS);
    }

    /**
     * Write 4 consecutive bytes
     */
    private static void writeFixedLengthInt(ByteDataBuffer fieldBuffer, int intBits) {
        fieldBuffer.write((byte) (intBits >>> 24));
        fieldBuffer.write((byte) (intBits >>> 16));
        fieldBuffer.write((byte) (intBits >>> 8));
        fieldBuffer.write((byte) (intBits));
    }

    /**
     * Serialize a double into 8 consecutive bytes
     */
    @Override
    public void serializePrimitive(FastBlobSerializationRecord rec, String fieldName, double value) {
        int position = rec.getSchema().getPosition(fieldName);
        FieldType fieldType = rec.getSchema().getFieldType(position);

        if(fieldType != FieldType.DOUBLE)
            throw new IllegalArgumentException("Attempting to serialize a double as " + fieldType + " in field " + fieldName + ".  Carefully check your schema for type " + rec.getSchema().getName() + ".");

        ByteDataBuffer fieldBuffer = rec.getFieldBuffer(position);
        long intBits = Double.doubleToLongBits(value);
        writeFixedLengthLong(fieldBuffer, intBits);
    }

    /**
     * Serialize a special 8-byte long sequence indicating a null Double value.
     */
    public static void writeNullDouble(ByteDataBuffer fieldBuffer) {
        writeFixedLengthLong(fieldBuffer, NULL_DOUBLE_BITS);
    }

    /**
     * Write 8 consecutive bytes
     */
    private static void writeFixedLengthLong(ByteDataBuffer fieldBuffer, long intBits) {
        fieldBuffer.write((byte) (intBits >>> 56));
        fieldBuffer.write((byte) (intBits >>> 48));
        fieldBuffer.write((byte) (intBits >>> 40));
        fieldBuffer.write((byte) (intBits >>> 32));
        fieldBuffer.write((byte) (intBits >>> 24));
        fieldBuffer.write((byte) (intBits >>> 16));
        fieldBuffer.write((byte) (intBits >>> 8));
        fieldBuffer.write((byte) (intBits));
    }

    /**
     * Serialize a boolean as a single byte
     */
    @Override
    public void serializePrimitive(FastBlobSerializationRecord rec, String fieldName, boolean value) {
        int position = rec.getSchema().getPosition(fieldName);
        FieldType fieldType = rec.getSchema().getFieldType(position);

        if(fieldType != FieldType.BOOLEAN)
            throw new IllegalArgumentException("Attempting to serialize a boolean as " + fieldType + " in field " + fieldName + ".  Carefully check your schema for type " + rec.getSchema().getName() + ".");

        ByteDataBuffer fieldBuffer = rec.getFieldBuffer(position);
        byte byteValue = value ? (byte) 1 : (byte) 0;
        fieldBuffer.write(byteValue);
    }

    /**
     * Serialize a sequence of bytes
     */
    @Override
    public void serializeBytes(FastBlobSerializationRecord rec, String fieldName, byte[] value) {
        if(value == null)
            return;

        int position = rec.getSchema().getPosition(fieldName);
        FieldType fieldType = rec.getSchema().getFieldType(position);

        if(fieldType != FieldType.BYTES)
            throw new IllegalArgumentException("Attempting to serialize a byte array as " + fieldType + " in field " + fieldName + ".  Carefully check your schema for type " + rec.getSchema().getName() + ".");

        ByteDataBuffer fieldBuffer = rec.getFieldBuffer(position);

        for (int i = 0; i < value.length; i++) {
            fieldBuffer.write(value[i]);
        }
    }

    /**
     * Recursively call the framework to serialize the speicfied Object, then serialize the resulting ordinal as a variable-byte integer.
     */
    @Override
    public void serializeObject(FastBlobSerializationRecord rec, String fieldName, String typeName, Object obj) {
        if(obj == null)
            return;

        int position = rec.getSchema().getPosition(fieldName);
        FieldType fieldType = rec.getSchema().getFieldType(position);

        if(fieldType != FieldType.OBJECT)
            throw new IllegalArgumentException("Attempting to serialize an Object as " + fieldType + " in field " + fieldName + ".  Carefully check your schema for type " + rec.getSchema().getName() + ".");

        ByteDataBuffer fieldBuffer = rec.getFieldBuffer(position);

        FastBlobTypeSerializationState typeSerializationState = ((FastBlobStateEngine) framework).getTypeSerializationState(typeName);

        int ordinal = typeSerializationState.add(obj, rec.getImageMembershipsFlags());

        VarInt.writeVInt(fieldBuffer, ordinal);
    }

    /**
     * Serialize a list.
     *
     * The framework is used to recursively serialize each of the list's elements, then
     * the ordinals are encoded as a sequence of variable-byte integers.
     */
    @Override
    public  void serializeList(FastBlobSerializationRecord rec, String fieldName, String typeName, Collection collection) {
        if(collection == null)
            return;

        int position = rec.getSchema().getPosition(fieldName);
        FieldType fieldType = rec.getSchema().getFieldType(position);

        if(fieldType != FieldType.COLLECTION)
            throw new IllegalArgumentException("Attempting to serialize a List as " + fieldType + " in field " + fieldName + ".  Carefully check your schema for type " + rec.getSchema().getName() + ".");

        ByteDataBuffer fieldBuffer = rec.getFieldBuffer(position);

        FastBlobTypeSerializationState typeSerializationState = ((FastBlobStateEngine) framework).getTypeSerializationState(typeName);

        for (T obj : collection) {
            if(obj == null) {
                VarInt.writeVNull(fieldBuffer);
            } else {
                int ordinal = typeSerializationState.add(obj, rec.getImageMembershipsFlags());
                VarInt.writeVInt(fieldBuffer, ordinal);
            }
        }
    }

    /**
     * Serialize a set.
     *
     *  The framework is used to recursively serialize each of the set's elements, then
     *  the ordinals are encoded as a sequence of gap-encoded variable-byte integers.
     */
    @Override
    public  void serializeSet(FastBlobSerializationRecord rec, String fieldName, String typeName, Set set) {
        if(set == null)
            return;

        int position = rec.getSchema().getPosition(fieldName);
        FieldType fieldType = rec.getSchema().getFieldType(position);

        if(fieldType != FieldType.COLLECTION)
            throw new IllegalArgumentException("Attempting to serialize a Set as " + fieldType + " in field " + fieldName + ".  Carefully check your schema for type " + rec.getSchema().getName() + ".");

        ByteDataBuffer fieldBuffer = rec.getFieldBuffer(position);
        FastBlobTypeSerializationState typeSerializationState = ((FastBlobStateEngine) framework).getTypeSerializationState(typeName);
        int setOrdinals[] = new int[set.size()];

        int i = 0;
        for (T obj : set) {
            if(obj == null) {
                setOrdinals[i++] = -1;
            } else {
                setOrdinals[i++] = typeSerializationState.add(obj, rec.getImageMembershipsFlags());
            }
        }

        if(setOrdinals.length > i)
            setOrdinals = Arrays.copyOf(setOrdinals, i);

        Arrays.sort(setOrdinals);

        int currentOrdinal = 0;

        for (i = 0; i < setOrdinals.length; i++) {
            if(setOrdinals[i] == -1) {
                VarInt.writeVNull(fieldBuffer);
            } else {
                VarInt.writeVInt(fieldBuffer, setOrdinals[i] - currentOrdinal);
                currentOrdinal = setOrdinals[i];
            }
        }
    }

    /**
     * Serialize a Map.
     *
     * The framework is used to recursively serialize the map's keys and values, then
     * the Map's entries are each encoded as a variable-byte integer for the key's ordinal, and a gap-encoded variable-byte integer for the value's ordinal.
     */
    @Override
    public  void serializeMap(FastBlobSerializationRecord rec, String fieldName, String keyTypeName, String valueTypeName, Map map) {
        if(map == null)
            return;

        int position = rec.getSchema().getPosition(fieldName);
        FieldType fieldType = rec.getSchema().getFieldType(position);

        if(fieldType != FieldType.MAP)
            throw new IllegalArgumentException("Attempting to serialize a Map as " + fieldType + " in field " + fieldName + ".  Carefully check your schema for type " + rec.getSchema().getName() + ".");

        ByteDataBuffer fieldBuffer = rec.getFieldBuffer(position);
        FastBlobTypeSerializationState keySerializationState = ((FastBlobStateEngine) framework).getTypeSerializationState(keyTypeName);
        FastBlobTypeSerializationState valueSerializationState = ((FastBlobStateEngine) framework).getTypeSerializationState(valueTypeName);
        long mapEntries[] = new long[map.size()];

        int i = 0;
        for (Map.Entry entry : map.entrySet()) {
            int keyOrdinal = -1;
            int valueOrdinal = -1;

            if(entry.getKey() != null)
                keyOrdinal = keySerializationState.add(entry.getKey(), rec.getImageMembershipsFlags());
            if(entry.getValue() != null)
                valueOrdinal = valueSerializationState.add(entry.getValue(), rec.getImageMembershipsFlags());

            mapEntries[i++] = ((long)valueOrdinal << 32) | (keyOrdinal & 0xFFFFFFFFL);
        }

        if(mapEntries.length > i)
            mapEntries = Arrays.copyOf(mapEntries, i);

        Arrays.sort(mapEntries);

        int currentValueOrdinal = 0;

        for (i = 0; i < mapEntries.length ; i++) {
            int keyOrdinal = (int) mapEntries[i];
            int valueOrdinal = (int) (mapEntries[i] >> 32);

            if(keyOrdinal == -1)
                VarInt.writeVNull(fieldBuffer);
            else
                VarInt.writeVInt(fieldBuffer, keyOrdinal);

            if(valueOrdinal == -1) {
                VarInt.writeVNull(fieldBuffer);
            } else {
                VarInt.writeVInt(fieldBuffer, valueOrdinal - currentValueOrdinal);
                currentValueOrdinal = valueOrdinal;
            }
        }
    }


    /**
     * Encode a String as a series of VarInts, one per character.
    
     *
     * @param str
     * @param out
     * @return
     * @throws IOException
     */
    protected void writeString(String str, ByteDataBuffer out) {
        for(int i=0;i    
    
    
    

    




    
    
    © 2015 - 2025 Weber Informatics LLC | Privacy Policy