All Downloads are FREE. Search and download functionalities are using the official Maven repository.
Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
com.kolibrifx.plovercrest.server.streams.folds.MasterSlaveFoldSetup Maven / Gradle / Ivy
/*
* Copyright (c) 2010-2017, KolibriFX AS. Licensed under the Apache License, version 2.0.
*/
package com.kolibrifx.plovercrest.server.streams.folds;
import java.util.List;
import com.kolibrifx.lancebill.folds.MasterSlaveFold;
import com.kolibrifx.plovercrest.server.internal.folds.FoldCallback;
import com.kolibrifx.plovercrest.server.internal.folds.FoldReaderInputs;
import com.kolibrifx.plovercrest.server.internal.folds.FoldReaderSetup;
import com.kolibrifx.plovercrest.server.internal.folds.FoldWriter;
import com.kolibrifx.plovercrest.server.streams.Stream;
public class MasterSlaveFoldSetup implements FoldSetup {
private final MasterSlaveFold fold;
public MasterSlaveFoldSetup(final MasterSlaveFold fold) {
this.fold = fold;
}
@Override
public FoldReaderSetup setupReader(final long timestamp, final FoldWriter foldOutput,
final List> inputStreams) {
final Stream primaryInput = inputStreams.get(0);
final String primaryName = primaryInput.getName();
final FoldCallback foldCallback = new FoldCallback() {
@SuppressWarnings("unchecked")
@Override
public void onNext(final String tableName, final long timestamp, final Object element) {
if (tableName.equals(primaryName)) {
final T output = fold.apply((U) element);
if (output != null) {
// re-timestamping is done by passing the primary input's timestamp here
foldOutput.write(timestamp, output);
}
} else {
fold.onSecondaryInput(tableName, (V) element);
}
}
};
final List> secondaryInputs = inputStreams.subList(1, inputStreams.size());
// reorder the streams so that the primary input is read after the secondary ones, if they have
// elements at the same timestamp.
long adjustedTimestamp = timestamp;
for (final Stream s : inputStreams) {
// clamp adjustedTimestamp between start and end for each input stream
final long first = s.getFirstTimestamp();
final long last = s.getLastTimestamp();
if (first > adjustedTimestamp) {
adjustedTimestamp = first;
}
if (last >= 0 && last < adjustedTimestamp) {
adjustedTimestamp = last;
}
}
adjustedTimestamp = getPrimaryReaderTimestamp(primaryInput, adjustedTimestamp);
final FoldReaderInputs inputs = new FoldReaderInputs<>();
inputs.addInputs(secondaryInputs,
getSecondaryReadersTimestamp(primaryInput, secondaryInputs, adjustedTimestamp));
inputs.addInput(primaryInput, adjustedTimestamp);
return new FoldReaderSetup() {
@Override
public FoldCallback getFoldCallback() {
return foldCallback;
}
@Override
public FoldReaderInputs getInputs() {
return inputs;
}
};
}
/**
* Can be overridden to specify which timestamp the primary input should start fetching data
* from. The default implementation calls seekPreviousTimestamp(). The result from this function
* is passed to {@link #getSecondaryReadersTimestamp(Stream, List, long)}.
*/
protected long getPrimaryReaderTimestamp(final Stream primaryInput, final long timestamp) {
return primaryInput.seekPreviousTimestamp(timestamp);
}
/**
* This can be overridden to specify that secondary inputs should get data from different
* (typically earlier) timestamps than the primary input.
*/
protected long getSecondaryReadersTimestamp(final Stream primaryInput, final List> secondaryInputs,
final long primaryReaderTimestamp) {
return primaryReaderTimestamp;
}
private boolean secondaryInputsHaveEnoughData(final long primaryTimestamp, final Stream primaryInput,
final List> secondaryInputs) {
final long adjustedTimestamp = getSecondaryReadersTimestamp(primaryInput, secondaryInputs, primaryTimestamp);
if (adjustedTimestamp < 0) {
return false;
}
for (final Stream stream : secondaryInputs) {
final long first = stream.getFirstTimestamp();
if (first < 0 || first > adjustedTimestamp) {
return false;
}
}
return true;
}
@Override
public long getLastOutputTimestamp(final List> inputStreams,
final CombinatorStrategy combinatorStrategy) {
// The instanceof checks are not elegant, should use some nicer multiple dispatch pattern...
final Stream primaryInput = inputStreams.get(0);
final List> secondaryInputs = inputStreams.subList(1, inputStreams.size());
if (combinatorStrategy instanceof LiveCombinatorStrategy) {
// In a live scenario, we only have to consider the primary input,
// assuming all the secondary inputs have enough data.
final long primaryLast = primaryInput.getLastTimestamp();
if (primaryLast == -1) {
return -1;
}
if (secondaryInputsHaveEnoughData(primaryLast, primaryInput, secondaryInputs)) {
return primaryLast;
} else {
return -1;
}
} else if (combinatorStrategy instanceof HistoricalCombinatorStrategy) {
// In a historical scenario, we combine the secondary inputs, then find the last master timestamp based on the result
long minimumSecondary = Long.MAX_VALUE;
for (final Stream secondaryInput : secondaryInputs) {
final long tmp = secondaryInput.getLastTimestamp();
if (tmp == -1) {
return -1;
}
minimumSecondary = Math.min(minimumSecondary, tmp);
}
final long candidate = primaryInput.seekPreviousTimestamp(minimumSecondary);
if (secondaryInputsHaveEnoughData(candidate, primaryInput, secondaryInputs)) {
return candidate;
} else {
return -1;
}
} else {
throw new IllegalStateException("Unknown combinator strategy " + combinatorStrategy);
}
}
}
