com.staros.journal.Journal Maven / Gradle / Ivy
// Copyright 2021-present StarRocks, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.staros.journal;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.Message;
import com.staros.exception.ExceptionCode;
import com.staros.exception.StarException;
import com.staros.proto.CreateShardJournalInfo;
import com.staros.proto.DeleteShardGroupInfo;
import com.staros.proto.JournalEntry;
import com.staros.proto.JournalHeader;
import com.staros.proto.MetaGroupJournalInfo;
import com.staros.proto.OperationType;
import com.staros.service.Service;
import com.staros.service.ServiceTemplate;
import com.staros.shard.Shard;
import com.staros.shard.ShardGroup;
import com.staros.util.AbstractParser;
import com.staros.util.Constant;
import com.staros.util.Text;
import com.staros.util.Writable;
import com.staros.worker.Worker;
import org.apache.commons.lang3.tuple.Pair;
import java.io.ByteArrayOutputStream;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
// a journal wraps an operation as log
public class Journal implements Writable {
private JournalEntry entry;
public Journal(JournalEntry entry) {
this.entry = entry;
}
public JournalEntry getEntry() {
return entry;
}
@Override
public void write(DataOutput out) throws IOException {
Text.writeBytes(out, entry.toByteArray());
}
public static Journal read(DataInput in) throws IOException {
return new Journal(JournalEntry.parseFrom(Text.readBytes(in)));
}
// Base class to help serialize to Journal and deserialize from Journal
private abstract static class AbstractJournalOp extends AbstractParser {
public Journal toJournal(OperationType type, String serviceId, T data) throws StarException {
assert type != OperationType.OP_INVALID;
ByteArrayOutputStream out = new ByteArrayOutputStream(128);
try {
this.write(new DataOutputStream(out), data);
} catch (IOException e) {
throw new StarException(ExceptionCode.IO, e.getMessage());
}
JournalHeader header = JournalHeader.newBuilder()
.setServiceId(serviceId)
.setOperationType(type)
.build();
JournalEntry entry = JournalEntry.newBuilder()
.setHeader(header)
.setBody(ByteString.copyFrom(out.toByteArray()))
.build();
return new Journal(entry);
}
// Determine how to write `T data` into DataOutput
public abstract void write(DataOutput out, T data) throws IOException;
// parseFrom(DataInput in)
//
// Determines how to read bytes from `DataInput @in`, and construct `T`
// @Note `write(DataOutput, T)` and `T parseFrom(DataInput)` must
// implement the `T` serialization and deserialization symmetrically.
public T parseFromJournal(OperationType type, Journal journal) throws StarException {
try {
assert journal.entry.getHeader().getOperationType() == type;
return this.parseFrom(journal.entry.getBody().toByteArray());
} catch (IOException e) {
throw new StarException(ExceptionCode.IO, e.getMessage());
}
}
}
// Protobuf Message specialization: AbstractJournalOp
private abstract static class ProtoJournalOp extends AbstractJournalOp {
// Protobuf Message turns to toByteArray() and write to @out
@Override
public void write(DataOutput out, T data) throws IOException {
Text.writeBytes(out, data.toByteArray());
}
// @bytes raw bytes to construct the Protobuf Message
// The protobuf message is written to DataOutput by `Message.toByteArray()`
public abstract T read(byte[] bytes) throws InvalidProtocolBufferException;
// bytes are read from @in and pass to read(byte[]) interface to create the Protobuf object
@Override
public T parseFrom(DataInput in) throws IOException {
return this.read(Text.readBytes(in));
}
}
// Writable object specialization: AbstractJournalOp
private abstract static class WritableJournalOp extends AbstractJournalOp {
// data written to DataOutput using its Writable::write() interface
@Override
public void write(DataOutput out, T data) throws IOException {
data.write(out);
}
}
// List object specialization: AbstractJournalOp>
private abstract static class AbstractListJournalOp extends AbstractJournalOp> {
@Override
public void write(DataOutput out, List data) throws IOException {
out.writeInt(data.size());
for (T t : data) {
writeElement(out, t);
}
}
// Single Element write/parse implementation should be symmetric.
public abstract void writeElement(DataOutput out, T data) throws IOException;
public abstract T parseElement(DataInput in) throws IOException;
@Override
public List parseFrom(DataInput in) throws IOException {
int size = in.readInt();
List result = new ArrayList<>(size);
for (int i = 0; i < size; ++i) {
result.add(parseElement(in));
}
return result;
}
}
// List of Writable objects specialization: AbstractJournalOp>
private abstract static class AbstractListWritableJournalOp extends AbstractListJournalOp {
// Use Writable::write for list of Writable objects
@Override
public void writeElement(DataOutput out, T data) throws IOException {
data.write(out);
}
}
// serialize/deserialize of Worker
private static final AbstractJournalOp workerOp = new WritableJournalOp() {
// serialization using Worker::write(out);
@Override
public Worker parseFrom(DataInput in) throws IOException {
return Worker.read(in);
}
};
// serialize/deserialize of ServiceTemplate
private static final AbstractJournalOp serviceTmplOp = new WritableJournalOp() {
// serialization using ServiceTemplate::write(out);
@Override
public ServiceTemplate parseFrom(DataInput in) throws IOException {
return ServiceTemplate.read(in);
}
};
// serialize/deserialize of Service
private static final AbstractJournalOp serviceOp = new WritableJournalOp() {
// serialization using Service::write(out);
@Override
public Service parseFrom(DataInput in) throws IOException {
return Service.read(in);
}
};
// serialize/deserialize of String
private static final AbstractJournalOp stringOp = new AbstractJournalOp() {
@Override
public void write(DataOutput out, String data) throws IOException {
Text.writeBytes(out, data.getBytes(StandardCharsets.UTF_8));
}
@Override
public String parseFrom(DataInput in) throws IOException {
return new String(Text.readBytes(in), StandardCharsets.UTF_8);
}
};
// serialize/deserialize of Long
private static final AbstractJournalOp longOp = new AbstractJournalOp() {
@Override
public void write(DataOutput out, Long data) throws IOException {
out.writeLong(data);
}
@Override
public Long parseFrom(DataInput in) throws IOException {
return in.readLong();
}
};
// serialize/deserialize of List
private static final AbstractListWritableJournalOp shardListOp = new AbstractListWritableJournalOp() {
// serialization of writeElement using Shard::write(out);
@Override
public Shard parseElement(DataInput in) throws IOException {
return Shard.read(in);
}
};
// serialize/deserialize of List
private static final AbstractListWritableJournalOp shardGroupListOp =
new AbstractListWritableJournalOp() {
// serialization of writeElement using ShardGroup::write(out);
@Override
public ShardGroup parseElement(DataInput in) throws IOException {
return ShardGroup.read(in);
}
};
// serialize/deserialize of List
private static final AbstractListWritableJournalOp workerListOp = new AbstractListWritableJournalOp() {
// serialization of writeElement using Worker::write(out);
@Override
public Worker parseElement(DataInput in) throws IOException {
return Worker.read(in);
}
};
// serialize/deserialize of List
private static final AbstractListJournalOp longListOp = new AbstractListJournalOp() {
@Override
public void writeElement(DataOutput out, Long data) throws IOException {
out.writeLong(data);
}
@Override
public Long parseElement(DataInput in) throws IOException {
return in.readLong();
}
};
// serialize/deserialize of DeleteShardGroupInfo
private static final ProtoJournalOp deleteShardGroupsOp = new ProtoJournalOp() {
@Override
public DeleteShardGroupInfo read(byte[] bytes) {
try {
return DeleteShardGroupInfo.parseFrom(bytes);
} catch (InvalidProtocolBufferException e) {
throw new StarException(ExceptionCode.JOURNAL, e.getMessage());
}
};
};
// serialize/deserialize of MetaGroupJournalInfo
private static final ProtoJournalOp metaGroupOp = new ProtoJournalOp() {
@Override
public MetaGroupJournalInfo read(byte[] bytes) {
try {
return MetaGroupJournalInfo.parseFrom(bytes);
} catch (InvalidProtocolBufferException e) {
throw new StarException(ExceptionCode.JOURNAL, e.getMessage());
}
};
};
// serialize/deserialize of CreateShardJournalInfo
private static final ProtoJournalOp createShardOp = new ProtoJournalOp() {
@Override
public CreateShardJournalInfo read(byte[] bytes) {
try {
return CreateShardJournalInfo.parseFrom(bytes);
} catch (InvalidProtocolBufferException e) {
throw new StarException(ExceptionCode.JOURNAL, e.getMessage());
}
};
};
public static Journal logCreateShard(String serviceId, CreateShardJournalInfo info) throws StarException {
return createShardOp.toJournal(OperationType.OP_CREATE_SHARD, serviceId, info);
}
public static Journal logDeleteShard(String serviceId, List shardIds) throws StarException {
return longListOp.toJournal(OperationType.OP_DELETE_SHARD, serviceId, shardIds);
}
public static Journal logUpdateShard(String serviceId, List shards) throws StarException {
return shardListOp.toJournal(OperationType.OP_UPDATE_SHARD, serviceId, shards);
}
public static Journal logCreateShardGroup(String serviceId, List shardGroups) throws StarException {
return shardGroupListOp.toJournal(OperationType.OP_CREATE_SHARD_GROUP, serviceId, shardGroups);
}
public static Journal logDeleteShardGroup(String serviceId, DeleteShardGroupInfo deleteGroupInfo) throws StarException {
return deleteShardGroupsOp.toJournal(OperationType.OP_DELETE_SHARD_GROUP, serviceId, deleteGroupInfo);
}
public static Journal logCreateMetaGroup(String serviceId, MetaGroupJournalInfo metaGroupJournalInfo) throws StarException {
return metaGroupOp.toJournal(OperationType.OP_CREATE_META_GROUP, serviceId, metaGroupJournalInfo);
}
public static Journal logDeleteMetaGroup(String serviceId, MetaGroupJournalInfo metaGroupJournalInfo) throws StarException {
return metaGroupOp.toJournal(OperationType.OP_DELETE_META_GROUP, serviceId, metaGroupJournalInfo);
}
public static Journal logUpdateMetaGroup(String serviceId, MetaGroupJournalInfo metaGroupJournalInfo) throws StarException {
return metaGroupOp.toJournal(OperationType.OP_UPDATE_META_GROUP, serviceId, metaGroupJournalInfo);
}
public static Journal logAddWorker(Worker worker) throws StarException {
return workerOp.toJournal(OperationType.OP_ADD_WORKER, worker.getServiceId(), worker);
}
public static Journal logRemoveWorker(String serviceId, long groupId, long workerId) throws StarException {
List todo = new ArrayList<>(2);
todo.add(groupId);
todo.add(workerId);
return longListOp.toJournal(OperationType.OP_REMOVE_WORKER, serviceId, todo);
}
public static Journal logRegisterService(ServiceTemplate serviceTemplate) throws StarException {
return serviceTmplOp.toJournal(OperationType.OP_REGISTER_SERVICE, Constant.EMPTY_SERVICE_ID, serviceTemplate);
}
public static Journal logDeregisterService(String serviceTemplateName) throws StarException {
return stringOp.toJournal(OperationType.OP_DEREGISTER_SERVICE, Constant.EMPTY_SERVICE_ID, serviceTemplateName);
}
public static Journal logBootstrapService(Service service) throws StarException {
return serviceOp.toJournal(OperationType.OP_BOOTSTRAP_SERVICE, service.getServiceId(), service);
}
public static Journal logShutdownService(Service service) throws StarException {
return serviceOp.toJournal(OperationType.OP_SHUTDOWN_SERVICE, service.getServiceId(), service);
}
public static Journal logUpdateWorker(String serviceId, List workers) throws StarException {
assert !workers.isEmpty();
return workerListOp.toJournal(OperationType.OP_UPDATE_WORKER, serviceId, workers);
}
public static Journal logSetId(long id) throws StarException {
return longOp.toJournal(OperationType.OP_SET_ID, Constant.EMPTY_SERVICE_ID, id);
}
public static CreateShardJournalInfo parseLogCreateShard(Journal journal) throws IOException {
return createShardOp.parseFromJournal(OperationType.OP_CREATE_SHARD, journal);
}
public static List parseLogDeleteShard(Journal journal) throws IOException {
return longListOp.parseFromJournal(OperationType.OP_DELETE_SHARD, journal);
}
public static List parseLogUpdateShard(Journal journal) throws IOException {
return shardListOp.parseFromJournal(OperationType.OP_UPDATE_SHARD, journal);
}
public static List parseLogCreateShardGroup(Journal journal) throws IOException {
return shardGroupListOp.parseFromJournal(OperationType.OP_CREATE_SHARD_GROUP, journal);
}
public static DeleteShardGroupInfo parseLogDeleteShardGroup(Journal journal) throws IOException {
return deleteShardGroupsOp.parseFromJournal(OperationType.OP_DELETE_SHARD_GROUP, journal);
}
public static MetaGroupJournalInfo parseLogCreateMetaGroup(Journal journal) throws IOException {
return metaGroupOp.parseFromJournal(OperationType.OP_CREATE_META_GROUP, journal);
}
public static MetaGroupJournalInfo parseLogDeleteMetaGroup(Journal journal) throws IOException {
return metaGroupOp.parseFromJournal(OperationType.OP_DELETE_META_GROUP, journal);
}
public static MetaGroupJournalInfo parseLogUpdateMetaGroup(Journal journal) throws IOException {
return metaGroupOp.parseFromJournal(OperationType.OP_UPDATE_META_GROUP, journal);
}
public static Worker parseLogAddWorker(Journal journal) throws IOException {
return workerOp.parseFromJournal(OperationType.OP_ADD_WORKER, journal);
}
public static List parseLogUpdateWorker(Journal journal) throws IOException {
return workerListOp.parseFromJournal(OperationType.OP_UPDATE_WORKER, journal);
}
public static Pair parseLogRemoveWorker(Journal journal) throws IOException {
List data = longListOp.parseFromJournal(OperationType.OP_REMOVE_WORKER, journal);
assert data.size() == 2;
long groupId = data.get(0);
long workerId = data.get(1);
return Pair.of(groupId, workerId);
}
public static ServiceTemplate parseLogRegisterService(Journal journal) throws IOException {
return serviceTmplOp.parseFromJournal(OperationType.OP_REGISTER_SERVICE, journal);
}
public static String parseLogDeregisterService(Journal journal) throws IOException {
return stringOp.parseFromJournal(OperationType.OP_DEREGISTER_SERVICE, journal);
}
public static Service parseLogBootstrapService(Journal journal) throws IOException {
return serviceOp.parseFromJournal(OperationType.OP_BOOTSTRAP_SERVICE, journal);
}
public static Service parseLogShutdownService(Journal journal) throws IOException {
return serviceOp.parseFromJournal(OperationType.OP_SHUTDOWN_SERVICE, journal);
}
public static long parseLogSetId(Journal journal) throws IOException {
return longOp.parseFromJournal(OperationType.OP_SET_ID, journal);
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy