org.apache.flink.streaming.api.functions.source.SerializedCheckpointData Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.streaming.api.functions.source;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.core.memory.DataInputDeserializer;
import org.apache.flink.core.memory.DataOutputSerializer;
import java.io.IOException;
import java.util.ArrayDeque;
import java.util.HashSet;
import java.util.Set;
/**
* This class represents serialized checkpoint data for a collection of elements.
*/
class SerializedCheckpointData implements java.io.Serializable {
private static final long serialVersionUID = -8783744683896503488L;
/** ID of the checkpoint for which the IDs are stored. */
private final long checkpointId;
/** The serialized elements. */
private final byte[] serializedData;
/** The number of elements in the checkpoint. */
private final int numIds;
/**
* Creates a SerializedCheckpointData object for the given serialized data.
*
* @param checkpointId The checkpointId of the checkpoint.
* @param serializedData The serialized IDs in this checkpoint.
* @param numIds The number of IDs in the checkpoint.
*/
public SerializedCheckpointData(long checkpointId, byte[] serializedData, int numIds) {
this.checkpointId = checkpointId;
this.serializedData = serializedData;
this.numIds = numIds;
}
/**
* Gets the checkpointId of the checkpoint.
* @return The checkpointId of the checkpoint.
*/
public long getCheckpointId() {
return checkpointId;
}
/**
* Gets the binary data for the serialized elements.
* @return The binary data for the serialized elements.
*/
public byte[] getSerializedData() {
return serializedData;
}
/**
* Gets the number of IDs in the checkpoint.
* @return The number of IDs in the checkpoint.
*/
public int getNumIds() {
return numIds;
}
// ------------------------------------------------------------------------
// Serialize to Checkpoint
// ------------------------------------------------------------------------
/**
* Converts a list of checkpoints with elements into an array of SerializedCheckpointData.
*
* @param checkpoints The checkpoints to be converted into IdsCheckpointData.
* @param serializer The serializer to serialize the IDs.
* @param The type of the ID.
* @return An array of serializable SerializedCheckpointData, one per entry in the queue.
*
* @throws IOException Thrown, if the serialization fails.
*/
public static SerializedCheckpointData[] fromDeque(
ArrayDeque>> checkpoints,
TypeSerializer serializer) throws IOException {
return fromDeque(checkpoints, serializer, new DataOutputSerializer(128));
}
/**
* Converts a list of checkpoints into an array of SerializedCheckpointData.
*
* @param checkpoints The checkpoints to be converted into IdsCheckpointData.
* @param serializer The serializer to serialize the IDs.
* @param outputBuffer The reusable serialization buffer.
* @param The type of the ID.
* @return An array of serializable SerializedCheckpointData, one per entry in the queue.
*
* @throws IOException Thrown, if the serialization fails.
*/
public static SerializedCheckpointData[] fromDeque(
ArrayDeque>> checkpoints,
TypeSerializer serializer,
DataOutputSerializer outputBuffer) throws IOException {
SerializedCheckpointData[] serializedCheckpoints = new SerializedCheckpointData[checkpoints.size()];
int pos = 0;
for (Tuple2> checkpoint : checkpoints) {
outputBuffer.clear();
Set checkpointIds = checkpoint.f1;
for (T id : checkpointIds) {
serializer.serialize(id, outputBuffer);
}
serializedCheckpoints[pos++] = new SerializedCheckpointData(
checkpoint.f0, outputBuffer.getCopyOfBuffer(), checkpointIds.size());
}
return serializedCheckpoints;
}
// ------------------------------------------------------------------------
// De-Serialize from Checkpoint
// ------------------------------------------------------------------------
/**
* De-serializes an array of SerializedCheckpointData back into an ArrayDeque of element checkpoints.
*
* @param data The data to be deserialized.
* @param serializer The serializer used to deserialize the data.
* @param The type of the elements.
* @return An ArrayDeque of element checkpoints.
*
* @throws IOException Thrown, if the serialization fails.
*/
public static ArrayDeque>> toDeque(
SerializedCheckpointData[] data,
TypeSerializer serializer) throws IOException {
ArrayDeque>> deque = new ArrayDeque<>(data.length);
DataInputDeserializer deser = null;
for (SerializedCheckpointData checkpoint : data) {
byte[] serializedData = checkpoint.getSerializedData();
if (deser == null) {
deser = new DataInputDeserializer(serializedData, 0, serializedData.length);
}
else {
deser.setBuffer(serializedData, 0, serializedData.length);
}
final Set ids = new HashSet<>(checkpoint.getNumIds());
final int numIds = checkpoint.getNumIds();
for (int i = 0; i < numIds; i++) {
ids.add(serializer.deserialize(deser));
}
deque.addLast(new Tuple2>(checkpoint.checkpointId, ids));
}
return deque;
}
}