com.netease.arctic.utils.SerializationUtils Maven / Gradle / Ivy
The newest version!
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.netease.arctic.utils;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Serializer;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import com.netease.arctic.data.file.ContentFileWithSequence;
import com.netease.arctic.iceberg.optimize.StructLikeWrapper;
import com.netease.arctic.iceberg.optimize.StructLikeWrapperFactory;
import com.netease.arctic.shade.org.apache.avro.util.Utf8;
import com.netease.arctic.shade.org.apache.iceberg.ContentFile;
import com.netease.arctic.shade.org.apache.iceberg.StructLike;
import com.netease.arctic.shade.org.apache.iceberg.types.Types;
import com.netease.arctic.shade.org.apache.iceberg.util.ByteBuffers;
import org.objenesis.strategy.StdInstantiatorStrategy;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.nio.ByteBuffer;
import static com.netease.arctic.shade.org.apache.iceberg.relocated.com.google.common.base.Preconditions.checkNotNull;
public class SerializationUtils {
private static final ThreadLocal SERIALIZER =
ThreadLocal.withInitial(KryoSerializerInstance::new);
public static ByteBuffer toByteBuffer(Object obj) {
try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
try (ObjectOutputStream oos = new ObjectOutputStream(bos)) {
oos.writeObject(obj);
oos.flush();
return ByteBuffer.wrap(bos.toByteArray());
}
} catch (IOException e) {
throw new IllegalArgumentException("serialization error of " + obj, e);
}
}
public static ByteBuffer byteArrayToByteBuffer(byte[] bytes) {
return ByteBuffer.wrap(bytes);
}
public static byte[] byteBufferToByteArray(ByteBuffer buffer) {
return ByteBuffers.toByteArray(buffer);
}
public static Object toObject(ByteBuffer buffer) {
byte[] bytes = ByteBuffers.toByteArray(buffer);
try (ByteArrayInputStream bis = new ByteArrayInputStream(bytes)) {
try (ObjectInputStream ois = new ObjectInputStream(bis)) {
return ois.readObject();
}
} catch (IOException | ClassNotFoundException e) {
throw new IllegalArgumentException("deserialization error ", e);
}
}
public static Object toObject(byte[] bytes) {
try (ByteArrayInputStream bis = new ByteArrayInputStream(bytes)) {
try (ObjectInputStream ois = new ObjectInputStream(bis)) {
return ois.readObject();
}
} catch (IOException | ClassNotFoundException e) {
throw new IllegalArgumentException("deserialization error ", e);
}
}
public static ContentFileWithSequence> toInternalTableFile(ByteBuffer buffer) {
return (ContentFileWithSequence>) toObject(buffer);
}
public static ContentFile> toContentFile(ByteBuffer buffer) {
return (ContentFile>) toObject(buffer);
}
public static ContentFileWithSequence> toIcebergContentFile(ByteBuffer buffer) {
return (ContentFileWithSequence>) toObject(buffer);
}
public static byte[] serialize(final Object obj) throws IOException {
return SERIALIZER.get().serialize(obj);
}
public static T deserialize(final byte[] objectData) {
if (objectData == null) {
throw new IllegalArgumentException("The byte[] must not be null");
}
return (T) SERIALIZER.get().deserialize(objectData);
}
public static SimpleSerializer createJavaSimpleSerializer() {
return JavaSerializer.INSTANT;
}
public static SimpleSerializer createStructLikeWrapperSerializer(
StructLikeWrapperFactory structLikeWrapperFactory) {
return new StructLikeWrapperSerializer(structLikeWrapperFactory);
}
private static class KryoSerializerInstance implements Serializable {
public static final int KRYO_SERIALIZER_INITIAL_BUFFER_SIZE = 1048576;
private final Kryo kryo;
private final ByteArrayOutputStream baos;
KryoSerializerInstance() {
KryoInstantiator kryoInstantiator = new KryoInstantiator();
kryo = kryoInstantiator.newKryo();
baos = new ByteArrayOutputStream(KRYO_SERIALIZER_INITIAL_BUFFER_SIZE);
kryo.setRegistrationRequired(false);
}
byte[] serialize(Object obj) {
kryo.reset();
baos.reset();
Output output = new Output(baos);
this.kryo.writeClassAndObject(output, obj);
output.close();
return baos.toByteArray();
}
Object deserialize(byte[] objectData) {
return this.kryo.readClassAndObject(new Input(objectData));
}
}
private static class KryoInstantiator implements Serializable {
public Kryo newKryo() {
Kryo kryo = new Kryo();
// This instance of Kryo should not require prior registration of classes
kryo.setRegistrationRequired(false);
Kryo.DefaultInstantiatorStrategy instantiatorStrategy = new Kryo.DefaultInstantiatorStrategy();
instantiatorStrategy.setFallbackInstantiatorStrategy(new StdInstantiatorStrategy());
kryo.setInstantiatorStrategy(instantiatorStrategy);
// Handle cases where we may have an odd classloader setup like with libjars
// for hadoop
kryo.setClassLoader(Thread.currentThread().getContextClassLoader());
// Register serializers
kryo.register(Utf8.class, new AvroUtf8Serializer());
return kryo;
}
}
private static class AvroUtf8Serializer extends Serializer {
@SuppressWarnings("unchecked")
@Override
public void write(Kryo kryo, Output output, Utf8 utf8String) {
Serializer bytesSerializer = kryo.getDefaultSerializer(byte[].class);
bytesSerializer.write(kryo, output, utf8String.getBytes());
}
@SuppressWarnings("unchecked")
@Override
public Utf8 read(Kryo kryo, Input input, Class type) {
Serializer bytesSerializer = kryo.getDefaultSerializer(byte[].class);
byte[] bytes = bytesSerializer.read(kryo, input, byte[].class);
return new Utf8(bytes);
}
}
public interface SimpleSerializer {
byte[] serialize(T t);
T deserialize(byte[] bytes);
}
public static class StructLikeWrapperSerializer implements SimpleSerializer {
protected final StructLikeWrapperFactory structLikeWrapperFactory;
public StructLikeWrapperSerializer(StructLikeWrapperFactory structLikeWrapperFactory) {
this.structLikeWrapperFactory = structLikeWrapperFactory;
}
public StructLikeWrapperSerializer(Types.StructType type) {
this.structLikeWrapperFactory = new StructLikeWrapperFactory(type);
}
@Override
public byte[] serialize(StructLikeWrapper structLikeWrapper) {
checkNotNull(structLikeWrapper);
StructLike copy = SerializationUtils.StructLikeCopy.copy(structLikeWrapper.get());
try {
return SerializationUtils.serialize(copy);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Override
public StructLikeWrapper deserialize(byte[] bytes) {
if (bytes == null) {
return null;
}
SerializationUtils.StructLikeCopy structLike = SerializationUtils.deserialize(bytes);
return structLikeWrapperFactory.create().set(structLike);
}
}
public static class JavaSerializer implements SimpleSerializer {
public static final JavaSerializer INSTANT = new JavaSerializer();
@Override
public byte[] serialize(T t) {
try {
checkNotNull(t);
return SerializationUtils.serialize(t);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Override
public T deserialize(byte[] bytes) {
if (bytes == null) {
return null;
}
return SerializationUtils.deserialize(bytes);
}
}
private static class StructLikeCopy implements StructLike {
public static StructLike copy(StructLike struct) {
return struct != null ? new StructLikeCopy(struct) : null;
}
private final Object[] values;
private StructLikeCopy(StructLike toCopy) {
this.values = new Object[toCopy.size()];
for (int i = 0; i < values.length; i += 1) {
Object value = toCopy.get(i, Object.class);
if (value instanceof StructLike) {
values[i] = copy((StructLike) value);
} else {
values[i] = value;
}
}
}
@Override
public int size() {
return values.length;
}
@Override
public T get(int pos, Class javaClass) {
return javaClass.cast(values[pos]);
}
@Override
public void set(int pos, T value) {
throw new UnsupportedOperationException("Struct copy cannot be modified");
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy