com.kolibrifx.plovercrest.server.internal.folds.EngineFoldManager Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of plovercrest-server Show documentation
Plovercrest server library.
The newest version!
/*
 * Copyright (c) 2010-2017, KolibriFX AS. Licensed under the Apache License, version 2.0.
 */

package com.kolibrifx.plovercrest.server.internal.folds;

import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import com.kolibrifx.common.Disposable;
import com.kolibrifx.lancebill.folds.CombineLatestFold;
import com.kolibrifx.plovercrest.client.TableMetadata;
import com.kolibrifx.plovercrest.client.TableSerializer;
import com.kolibrifx.plovercrest.client.TableSerializerFactory;
import com.kolibrifx.plovercrest.client.internal.shared.TableMetadataUtils;
import com.kolibrifx.plovercrest.server.TableInfo;
import com.kolibrifx.plovercrest.server.folds.FoldCombinator;
import com.kolibrifx.plovercrest.server.folds.FoldManager;
import com.kolibrifx.plovercrest.server.streams.Stream;
import com.kolibrifx.plovercrest.server.streams.StreamEngine;
import com.kolibrifx.plovercrest.server.streams.StreamWriter;
import com.kolibrifx.plovercrest.server.streams.folds.CombinatorStrategy;

public class EngineFoldManager implements FoldManager {
    private static final Logger log = Logger.getLogger(EngineFoldManager.class);

    private static final String METADATA_INPUT_TABLES_KEY = "com.kolibrifx.plovercrest.folds.input-tables";

    private final StreamEngine engine;
    private final TableSerializerFactory serializerFactory;

    public EngineFoldManager(final StreamEngine engine, final TableSerializerFactory serializerFactory) {
        this.engine = engine;
        this.serializerFactory = serializerFactory;
    }

    private  TableSerializer getInputTableSerializer(final String tableName, final Class elementClass) {
        final Stream table = engine.openRaw(tableName);
        if (table == null) {
            return null;
        }
        final TableMetadata metadata = table.getMetadata();
        return serializerFactory.createSerializer(metadata, elementClass);
    }

     Disposable registerFold(final Collection inputTables, final long fromTimestamp,
                                   final CombineLatestFold fold, final CombinatorStrategy combinatorStrategy,
                                   final FoldWriter writer) {
        final List latestValues = new ArrayList<>(inputTables.size());
        final List> inputSerializers = new ArrayList<>();
        final Map tableIndexes = new HashMap<>();
        int i = 0;
        for (final String tableName : inputTables) {
            latestValues.add(null);
            inputSerializers.add(null);
            tableIndexes.put(tableName, i++);
        }
        final FoldCallback callback = new FoldCallback() {
            private long latestTimestamp = Long.MIN_VALUE;

            @Override
            public void onNext(final String tableName, final long timestamp, final byte[] data) {
                latestTimestamp = Math.max(latestTimestamp, timestamp);
                final int index = tableIndexes.get(tableName);
                TableSerializer serializer = inputSerializers.get(index);
                if (serializer == null) {
                    serializer = getInputTableSerializer(tableName, fold.getInputClass());
                    inputSerializers.set(index, serializer);
                }
                latestValues.set(index, serializer.unserialize(timestamp, ByteBuffer.wrap(data)));
                if (!latestValues.contains(null)) {
                    final T result = fold.apply(latestTimestamp, Collections.unmodifiableList(latestValues));
                    if (result != null) {
                        writer.write(timestamp, result);
                    }
                }
            }
        };
        return new EngineFoldTriggerer(engine, inputTables, fromTimestamp, combinatorStrategy, callback);
    }

    @Override
    public  Disposable registerFold(final String outputTable, final Collection inputTables,
                                          final CombineLatestFold fold, final FoldCombinator combinator) {
        if (!(combinator instanceof CombinatorStrategy)) {
            throw new IllegalArgumentException("Unsupported combinator: " + combinator);
        }
        Stream output = engine.openRaw(outputTable);
        final TableSerializer outputSerializer;
        final String inputTablesSpec = StringUtils.join(inputTables, ',');
        final long fromTimestamp;
        if (output == null) {
            outputSerializer = serializerFactory.createSerializer(fold.getOutputClass());
            TableMetadata metadata = TableMetadataUtils.createFromSerializer(outputSerializer, serializerFactory);

            // Store a comma-separated list of input tables in the metadata.
            // This can for instance be used for detecting that this is a fold table.
            metadata = metadata.addFields(Collections.singletonMap(METADATA_INPUT_TABLES_KEY, inputTablesSpec));
            engine.create(new TableInfo(outputTable, metadata));
            output = engine.openRaw(outputTable);
            fromTimestamp = 0;
        } else {
            outputSerializer = serializerFactory.createSerializer(output.getMetadata(), fold.getOutputClass());
            if (!output.getMetadata().containsKey(METADATA_INPUT_TABLES_KEY)) {
                log.warn("Expected key " + METADATA_INPUT_TABLES_KEY + " not found in metadata for table "
                        + outputTable);
            } else {
                final String metadataValue = output.getMetadata().get(METADATA_INPUT_TABLES_KEY);
                if (!inputTablesSpec.equals(metadataValue)) {
                    log.warn("Input table mismatch in metadata. Expected '" + inputTablesSpec + "', was '"
                            + metadataValue + "'");
                }
            }
            // We assume that we can start reading input tables from the last output timestamp.
            // This works for some cases, but probably has to be pluggable somehow?
            fromTimestamp = output.getLastTimestamp();
        }
        final StreamWriter outputWriter = output.getWriter();
        final ByteBuffer buffer = ByteBuffer.allocate(1024 * 10);
        final long lastTimestamp = output.getLastTimestamp();
        final FoldWriter writer = new FoldWriter() {
            @Override
            public void write(final long timestamp, final T element) {
                if (timestamp <= lastTimestamp) {
                    // Skip earlier timestamps, needed due to resume corner cases.
                    // (could have special handling of timestamp == lastTimestamp?)
                    return;
                }
                buffer.clear();
                outputSerializer.serialize(buffer, element);
                buffer.flip();
                outputWriter.write(timestamp, buffer);
            }
        };
        return registerFold(inputTables, fromTimestamp, fold, (CombinatorStrategy) combinator, writer);
    }
}