org.apache.kafka.streams.kstream.internals.KStreamImpl Maven / Gradle / Ivy
The newest version!
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kafka.streams.kstream.internals;
import org.apache.kafka.common.serialization.Serde;
import org.apache.kafka.common.utils.Bytes;
import org.apache.kafka.streams.KeyValue;
import org.apache.kafka.streams.internals.ApiUtils;
import org.apache.kafka.streams.kstream.BranchedKStream;
import org.apache.kafka.streams.kstream.ForeachAction;
import org.apache.kafka.streams.kstream.ForeachProcessor;
import org.apache.kafka.streams.kstream.GlobalKTable;
import org.apache.kafka.streams.kstream.Grouped;
import org.apache.kafka.streams.kstream.JoinWindows;
import org.apache.kafka.streams.kstream.Joined;
import org.apache.kafka.streams.kstream.KGroupedStream;
import org.apache.kafka.streams.kstream.KStream;
import org.apache.kafka.streams.kstream.KTable;
import org.apache.kafka.streams.kstream.KeyValueMapper;
import org.apache.kafka.streams.kstream.Materialized;
import org.apache.kafka.streams.kstream.Named;
import org.apache.kafka.streams.kstream.Predicate;
import org.apache.kafka.streams.kstream.Printed;
import org.apache.kafka.streams.kstream.Produced;
import org.apache.kafka.streams.kstream.Repartitioned;
import org.apache.kafka.streams.kstream.StreamJoined;
import org.apache.kafka.streams.kstream.TransformerSupplier;
import org.apache.kafka.streams.kstream.ValueJoiner;
import org.apache.kafka.streams.kstream.ValueJoinerWithKey;
import org.apache.kafka.streams.kstream.ValueMapper;
import org.apache.kafka.streams.kstream.ValueMapperWithKey;
import org.apache.kafka.streams.kstream.ValueTransformerSupplier;
import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier;
import org.apache.kafka.streams.kstream.internals.graph.BaseRepartitionNode;
import org.apache.kafka.streams.kstream.internals.graph.BaseRepartitionNode.BaseRepartitionNodeBuilder;
import org.apache.kafka.streams.kstream.internals.graph.GraphNode;
import org.apache.kafka.streams.kstream.internals.graph.OptimizableRepartitionNode;
import org.apache.kafka.streams.kstream.internals.graph.OptimizableRepartitionNode.OptimizableRepartitionNodeBuilder;
import org.apache.kafka.streams.kstream.internals.graph.ProcessorGraphNode;
import org.apache.kafka.streams.kstream.internals.graph.ProcessorParameters;
import org.apache.kafka.streams.kstream.internals.graph.StatefulProcessorNode;
import org.apache.kafka.streams.kstream.internals.graph.StreamSinkNode;
import org.apache.kafka.streams.kstream.internals.graph.StreamTableJoinNode;
import org.apache.kafka.streams.kstream.internals.graph.StreamToTableNode;
import org.apache.kafka.streams.kstream.internals.graph.UnoptimizableRepartitionNode;
import org.apache.kafka.streams.kstream.internals.graph.UnoptimizableRepartitionNode.UnoptimizableRepartitionNodeBuilder;
import org.apache.kafka.streams.processor.FailOnInvalidTimestamp;
import org.apache.kafka.streams.processor.StreamPartitioner;
import org.apache.kafka.streams.processor.TopicNameExtractor;
import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier;
import org.apache.kafka.streams.processor.api.ProcessorSupplier;
import org.apache.kafka.streams.processor.internals.InternalTopicProperties;
import org.apache.kafka.streams.processor.internals.StaticTopicNameExtractor;
import org.apache.kafka.streams.processor.internals.StoreBuilderWrapper;
import org.apache.kafka.streams.state.KeyValueStore;
import org.apache.kafka.streams.state.VersionedBytesStoreSupplier;
import org.apache.kafka.streams.state.internals.RocksDBTimeOrderedKeyValueBuffer;
import java.lang.reflect.Array;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import static org.apache.kafka.streams.kstream.internals.graph.OptimizableRepartitionNode.optimizableRepartitionNodeBuilder;
public class KStreamImpl extends AbstractStream implements KStream {
static final String JOINTHIS_NAME = "KSTREAM-JOINTHIS-";
static final String JOINOTHER_NAME = "KSTREAM-JOINOTHER-";
static final String JOIN_NAME = "KSTREAM-JOIN-";
static final String LEFTJOIN_NAME = "KSTREAM-LEFTJOIN-";
static final String MERGE_NAME = "KSTREAM-MERGE-";
static final String OUTERTHIS_NAME = "KSTREAM-OUTERTHIS-";
static final String OUTEROTHER_NAME = "KSTREAM-OUTEROTHER-";
static final String WINDOWED_NAME = "KSTREAM-WINDOWED-";
static final String OUTERSHARED_NAME = "KSTREAM-OUTERSHARED-";
static final String SOURCE_NAME = "KSTREAM-SOURCE-";
static final String SINK_NAME = "KSTREAM-SINK-";
static final String REPARTITION_TOPIC_SUFFIX = "-repartition";
private static final String BRANCH_NAME = "KSTREAM-BRANCH-";
private static final String BRANCHCHILD_NAME = "KSTREAM-BRANCHCHILD-";
private static final String FILTER_NAME = "KSTREAM-FILTER-";
private static final String PEEK_NAME = "KSTREAM-PEEK-";
private static final String FLATMAP_NAME = "KSTREAM-FLATMAP-";
private static final String FLATMAPVALUES_NAME = "KSTREAM-FLATMAPVALUES-";
private static final String MAP_NAME = "KSTREAM-MAP-";
private static final String MAPVALUES_NAME = "KSTREAM-MAPVALUES-";
private static final String PROCESSOR_NAME = "KSTREAM-PROCESSOR-";
private static final String PROCESSVALUES_NAME = "KSTREAM-PROCESSVALUES-";
private static final String PRINTING_NAME = "KSTREAM-PRINTER-";
private static final String KEY_SELECT_NAME = "KSTREAM-KEY-SELECT-";
private static final String TRANSFORM_NAME = "KSTREAM-TRANSFORM-";
private static final String TRANSFORMVALUES_NAME = "KSTREAM-TRANSFORMVALUES-";
private static final String FOREACH_NAME = "KSTREAM-FOREACH-";
private static final String TO_KTABLE_NAME = "KSTREAM-TOTABLE-";
private static final String REPARTITION_NAME = "KSTREAM-REPARTITION-";
private final boolean repartitionRequired;
private OptimizableRepartitionNode repartitionNode;
KStreamImpl(final String name,
final Serde keySerde,
final Serde valueSerde,
final Set subTopologySourceNodes,
final boolean repartitionRequired,
final GraphNode graphNode,
final InternalStreamsBuilder builder) {
super(name, keySerde, valueSerde, subTopologySourceNodes, graphNode, builder);
this.repartitionRequired = repartitionRequired;
}
@Override
public KStream filter(final Predicate predicate) {
return filter(predicate, NamedInternal.empty());
}
@Override
public KStream filter(final Predicate predicate,
final Named named) {
Objects.requireNonNull(predicate, "predicate can't be null");
Objects.requireNonNull(named, "named can't be null");
final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, FILTER_NAME);
final ProcessorParameters processorParameters =
new ProcessorParameters<>(new KStreamFilter<>(predicate, false), name);
final ProcessorGraphNode filterProcessorNode =
new ProcessorGraphNode<>(name, processorParameters);
builder.addGraphNode(graphNode, filterProcessorNode);
return new KStreamImpl<>(
name,
keySerde,
valueSerde,
subTopologySourceNodes,
repartitionRequired,
filterProcessorNode,
builder);
}
@Override
public KStream filterNot(final Predicate predicate) {
return filterNot(predicate, NamedInternal.empty());
}
@Override
public KStream filterNot(final Predicate predicate,
final Named named) {
Objects.requireNonNull(predicate, "predicate can't be null");
Objects.requireNonNull(named, "named can't be null");
final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, FILTER_NAME);
final ProcessorParameters processorParameters =
new ProcessorParameters<>(new KStreamFilter<>(predicate, true), name);
final ProcessorGraphNode filterNotProcessorNode =
new ProcessorGraphNode<>(name, processorParameters);
builder.addGraphNode(graphNode, filterNotProcessorNode);
return new KStreamImpl<>(
name,
keySerde,
valueSerde,
subTopologySourceNodes,
repartitionRequired,
filterNotProcessorNode,
builder);
}
@Override
public KStream selectKey(final KeyValueMapper mapper) {
return selectKey(mapper, NamedInternal.empty());
}
@Override
public KStream selectKey(final KeyValueMapper mapper,
final Named named) {
Objects.requireNonNull(mapper, "mapper can't be null");
Objects.requireNonNull(named, "named can't be null");
final ProcessorGraphNode selectKeyProcessorNode = internalSelectKey(mapper, new NamedInternal(named));
selectKeyProcessorNode.keyChangingOperation(true);
builder.addGraphNode(graphNode, selectKeyProcessorNode);
// key serde cannot be preserved
return new KStreamImpl<>(
selectKeyProcessorNode.nodeName(),
null,
valueSerde,
subTopologySourceNodes,
true,
selectKeyProcessorNode,
builder);
}
private ProcessorGraphNode internalSelectKey(final KeyValueMapper mapper,
final NamedInternal named) {
final String name = named.orElseGenerateWithPrefix(builder, KEY_SELECT_NAME);
final KStreamMap kStreamMap =
new KStreamMap<>((key, value) -> new KeyValue<>(mapper.apply(key, value), value));
final ProcessorParameters processorParameters = new ProcessorParameters<>(kStreamMap, name);
return new ProcessorGraphNode<>(name, processorParameters);
}
@Override
public KStream map(final KeyValueMapper> mapper) {
return map(mapper, NamedInternal.empty());
}
@Override
public KStream map(final KeyValueMapper> mapper,
final Named named) {
Objects.requireNonNull(mapper, "mapper can't be null");
Objects.requireNonNull(named, "named can't be null");
final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, MAP_NAME);
final ProcessorParameters processorParameters =
new ProcessorParameters<>(new KStreamMap<>(mapper), name);
final ProcessorGraphNode mapProcessorNode =
new ProcessorGraphNode<>(name, processorParameters);
mapProcessorNode.keyChangingOperation(true);
builder.addGraphNode(graphNode, mapProcessorNode);
// key and value serde cannot be preserved
return new KStreamImpl<>(
name,
null,
null,
subTopologySourceNodes,
true,
mapProcessorNode,
builder);
}
@Override
public KStream mapValues(final ValueMapper valueMapper) {
return mapValues(withKey(valueMapper));
}
@Override
public KStream mapValues(final ValueMapper mapper,
final Named named) {
return mapValues(withKey(mapper), named);
}
@Override
public KStream mapValues(final ValueMapperWithKey valueMapperWithKey) {
return mapValues(valueMapperWithKey, NamedInternal.empty());
}
@Override
public KStream mapValues(final ValueMapperWithKey valueMapperWithKey,
final Named named) {
Objects.requireNonNull(valueMapperWithKey, "valueMapperWithKey can't be null");
Objects.requireNonNull(named, "named can't be null");
final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, MAPVALUES_NAME);
final ProcessorParameters processorParameters =
new ProcessorParameters<>(new KStreamMapValues<>(valueMapperWithKey), name);
final ProcessorGraphNode mapValuesProcessorNode =
new ProcessorGraphNode<>(name, processorParameters);
mapValuesProcessorNode.setValueChangingOperation(true);
builder.addGraphNode(graphNode, mapValuesProcessorNode);
// value serde cannot be preserved
return new KStreamImpl<>(
name,
keySerde,
null,
subTopologySourceNodes,
repartitionRequired,
mapValuesProcessorNode,
builder);
}
@Override
public KStream flatMap(final KeyValueMapper>> mapper) {
return flatMap(mapper, NamedInternal.empty());
}
@Override
public KStream flatMap(final KeyValueMapper>> mapper,
final Named named) {
Objects.requireNonNull(mapper, "mapper can't be null");
Objects.requireNonNull(named, "named can't be null");
final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, FLATMAP_NAME);
final ProcessorParameters processorParameters =
new ProcessorParameters<>(new KStreamFlatMap<>(mapper), name);
final ProcessorGraphNode flatMapNode =
new ProcessorGraphNode<>(name, processorParameters);
flatMapNode.keyChangingOperation(true);
builder.addGraphNode(graphNode, flatMapNode);
// key and value serde cannot be preserved
return new KStreamImpl<>(name, null, null, subTopologySourceNodes, true, flatMapNode, builder);
}
@Override
public KStream flatMapValues(final ValueMapper> mapper) {
return flatMapValues(withKey(mapper));
}
@Override
public KStream flatMapValues(final ValueMapper> mapper,
final Named named) {
return flatMapValues(withKey(mapper), named);
}
@Override
public KStream flatMapValues(final ValueMapperWithKey> mapper) {
return flatMapValues(mapper, NamedInternal.empty());
}
@Override
public KStream flatMapValues(final ValueMapperWithKey> valueMapper,
final Named named) {
Objects.requireNonNull(valueMapper, "valueMapper can't be null");
Objects.requireNonNull(named, "named can't be null");
final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, FLATMAPVALUES_NAME);
final ProcessorParameters processorParameters =
new ProcessorParameters<>(new KStreamFlatMapValues<>(valueMapper), name);
final ProcessorGraphNode flatMapValuesNode =
new ProcessorGraphNode<>(name, processorParameters);
flatMapValuesNode.setValueChangingOperation(true);
builder.addGraphNode(graphNode, flatMapValuesNode);
// value serde cannot be preserved
return new KStreamImpl<>(
name,
keySerde,
null,
subTopologySourceNodes,
repartitionRequired,
flatMapValuesNode,
builder);
}
@Override
public void print(final Printed printed) {
Objects.requireNonNull(printed, "printed can't be null");
final PrintedInternal printedInternal = new PrintedInternal<>(printed);
final String name = new NamedInternal(printedInternal.name()).orElseGenerateWithPrefix(builder, PRINTING_NAME);
final ProcessorParameters processorParameters =
new ProcessorParameters<>(printedInternal.build(this.name), name);
final ProcessorGraphNode printNode =
new ProcessorGraphNode<>(name, processorParameters);
builder.addGraphNode(graphNode, printNode);
}
@Override
public void foreach(final ForeachAction action) {
foreach(action, NamedInternal.empty());
}
@Override
public void foreach(final ForeachAction action,
final Named named) {
Objects.requireNonNull(action, "action can't be null");
Objects.requireNonNull(named, "named can't be null");
final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, FOREACH_NAME);
final ProcessorParameters processorParameters =
new ProcessorParameters<>(() -> new ForeachProcessor<>(action), name);
final ProcessorGraphNode foreachNode =
new ProcessorGraphNode<>(name, processorParameters);
builder.addGraphNode(graphNode, foreachNode);
}
@Override
public KStream peek(final ForeachAction action) {
return peek(action, NamedInternal.empty());
}
@Override
public KStream peek(final ForeachAction action,
final Named named) {
Objects.requireNonNull(action, "action can't be null");
Objects.requireNonNull(named, "named can't be null");
final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, PEEK_NAME);
final ProcessorParameters processorParameters =
new ProcessorParameters<>(new KStreamPeek<>(action), name);
final ProcessorGraphNode peekNode =
new ProcessorGraphNode<>(name, processorParameters);
builder.addGraphNode(graphNode, peekNode);
return new KStreamImpl<>(
name,
keySerde,
valueSerde,
subTopologySourceNodes,
repartitionRequired,
peekNode,
builder);
}
@Deprecated
@Override
@SuppressWarnings("unchecked")
public KStream[] branch(final Predicate... predicates) {
return doBranch(NamedInternal.empty(), predicates);
}
@Deprecated
@Override
@SuppressWarnings("unchecked")
public KStream[] branch(final Named named,
final Predicate... predicates) {
Objects.requireNonNull(named, "named can't be null");
return doBranch(new NamedInternal(named), predicates);
}
@SuppressWarnings({"unchecked", "rawtypes"})
private KStream[] doBranch(final NamedInternal named,
final Predicate... predicates) {
Objects.requireNonNull(predicates, "predicates can't be a null array");
if (predicates.length == 0) {
throw new IllegalArgumentException("branch() requires at least one predicate");
}
for (final Predicate predicate : predicates) {
Objects.requireNonNull(predicate, "predicates can't be null");
}
final String branchName = named.orElseGenerateWithPrefix(builder, BRANCH_NAME);
final String[] childNames = new String[predicates.length];
for (int i = 0; i < predicates.length; i++) {
childNames[i] = named.suffixWithOrElseGet("-predicate-" + i, builder, BRANCHCHILD_NAME);
}
final ProcessorParameters processorParameters =
new ProcessorParameters<>(new KStreamBranch(Arrays.asList(predicates.clone()),
Arrays.asList(childNames)), branchName);
final ProcessorGraphNode branchNode =
new ProcessorGraphNode<>(branchName, processorParameters);
builder.addGraphNode(graphNode, branchNode);
final KStream[] branchChildren = (KStream[]) Array.newInstance(KStream.class, predicates.length);
for (int i = 0; i < predicates.length; i++) {
final ProcessorParameters innerProcessorParameters =
new ProcessorParameters<>(new PassThrough(), childNames[i]);
final ProcessorGraphNode branchChildNode =
new ProcessorGraphNode<>(childNames[i], innerProcessorParameters);
builder.addGraphNode(branchNode, branchChildNode);
branchChildren[i] = new KStreamImpl<>(childNames[i], keySerde, valueSerde, subTopologySourceNodes, repartitionRequired, branchChildNode, builder);
}
return branchChildren;
}
@Override
public BranchedKStream split() {
return new BranchedKStreamImpl<>(this, repartitionRequired, NamedInternal.empty());
}
@Override
public BranchedKStream split(final Named named) {
Objects.requireNonNull(named, "named can't be null");
return new BranchedKStreamImpl<>(this, repartitionRequired, new NamedInternal(named));
}
@Override
public KStream merge(final KStream stream) {
return merge(stream, NamedInternal.empty());
}
@Override
public KStream merge(final KStream stream,
final Named named) {
Objects.requireNonNull(stream, "stream can't be null");
Objects.requireNonNull(named, "named can't be null");
return merge(builder, stream, new NamedInternal(named));
}
private KStream merge(final InternalStreamsBuilder builder,
final KStream stream,
final NamedInternal named) {
final KStreamImpl streamImpl = (KStreamImpl) stream;
final boolean requireRepartitioning = streamImpl.repartitionRequired || repartitionRequired;
final String name = named.orElseGenerateWithPrefix(builder, MERGE_NAME);
final Set allSubTopologySourceNodes = new HashSet<>();
allSubTopologySourceNodes.addAll(subTopologySourceNodes);
allSubTopologySourceNodes.addAll(streamImpl.subTopologySourceNodes);
final ProcessorParameters processorParameters =
new ProcessorParameters<>(new PassThrough<>(), name);
final ProcessorGraphNode mergeNode =
new ProcessorGraphNode<>(name, processorParameters);
mergeNode.setMergeNode(true);
builder.addGraphNode(Arrays.asList(graphNode, streamImpl.graphNode), mergeNode);
// drop the serde as we cannot safely use either one to represent both streams
return new KStreamImpl<>(
name,
null,
null,
allSubTopologySourceNodes,
requireRepartitioning,
mergeNode,
builder);
}
@Deprecated
@Override
public KStream through(final String topic) {
return through(topic, Produced.with(keySerde, valueSerde, null));
}
@Deprecated
@Override
public KStream through(final String topic,
final Produced produced) {
Objects.requireNonNull(topic, "topic can't be null");
Objects.requireNonNull(produced, "produced can't be null");
final ProducedInternal producedInternal = new ProducedInternal<>(produced);
if (producedInternal.keySerde() == null) {
producedInternal.withKeySerde(keySerde);
}
if (producedInternal.valueSerde() == null) {
producedInternal.withValueSerde(valueSerde);
}
to(topic, producedInternal);
return builder.stream(
Collections.singleton(topic),
new ConsumedInternal<>(
producedInternal.keySerde(),
producedInternal.valueSerde(),
new FailOnInvalidTimestamp(),
null
)
);
}
@Override
public KStream repartition() {
return doRepartition(Repartitioned.as(null));
}
@Override
public KStream repartition(final Repartitioned repartitioned) {
return doRepartition(repartitioned);
}
private KStream doRepartition(final Repartitioned repartitioned) {
Objects.requireNonNull(repartitioned, "repartitioned can't be null");
final RepartitionedInternal repartitionedInternal = new RepartitionedInternal<>(repartitioned);
final String name = repartitionedInternal.name() != null ? repartitionedInternal.name() : builder
.newProcessorName(REPARTITION_NAME);
final Serde valueSerde = repartitionedInternal.valueSerde() == null ? this.valueSerde : repartitionedInternal.valueSerde();
final Serde keySerde = repartitionedInternal.keySerde() == null ? this.keySerde : repartitionedInternal.keySerde();
final UnoptimizableRepartitionNodeBuilder unoptimizableRepartitionNodeBuilder = UnoptimizableRepartitionNode
.unoptimizableRepartitionNodeBuilder();
final InternalTopicProperties internalTopicProperties = repartitionedInternal.toInternalTopicProperties();
final String repartitionSourceName = createRepartitionedSource(
builder,
repartitionedInternal.keySerde(),
valueSerde,
name,
repartitionedInternal.streamPartitioner(),
unoptimizableRepartitionNodeBuilder.withInternalTopicProperties(internalTopicProperties)
);
final UnoptimizableRepartitionNode unoptimizableRepartitionNode = unoptimizableRepartitionNodeBuilder.build();
builder.addGraphNode(graphNode, unoptimizableRepartitionNode);
final Set sourceNodes = new HashSet<>();
sourceNodes.add(unoptimizableRepartitionNode.nodeName());
return new KStreamImpl<>(
repartitionSourceName,
keySerde,
valueSerde,
Collections.unmodifiableSet(sourceNodes),
false,
unoptimizableRepartitionNode,
builder
);
}
@Override
public void to(final String topic) {
to(topic, Produced.with(keySerde, valueSerde, null));
}
@Override
public void to(final String topic,
final Produced produced) {
Objects.requireNonNull(topic, "topic can't be null");
Objects.requireNonNull(produced, "produced can't be null");
final ProducedInternal producedInternal = new ProducedInternal<>(produced);
if (producedInternal.keySerde() == null) {
producedInternal.withKeySerde(keySerde);
}
if (producedInternal.valueSerde() == null) {
producedInternal.withValueSerde(valueSerde);
}
to(new StaticTopicNameExtractor<>(topic), producedInternal);
}
@Override
public void to(final TopicNameExtractor topicExtractor) {
to(topicExtractor, Produced.with(keySerde, valueSerde, null));
}
@Override
public void to(final TopicNameExtractor topicExtractor,
final Produced produced) {
Objects.requireNonNull(topicExtractor, "topicExtractor can't be null");
Objects.requireNonNull(produced, "produced can't be null");
final ProducedInternal producedInternal = new ProducedInternal<>(produced);
if (producedInternal.keySerde() == null) {
producedInternal.withKeySerde(keySerde);
}
if (producedInternal.valueSerde() == null) {
producedInternal.withValueSerde(valueSerde);
}
to(topicExtractor, producedInternal);
}
private void to(final TopicNameExtractor topicExtractor,
final ProducedInternal produced) {
final String name = new NamedInternal(produced.name()).orElseGenerateWithPrefix(builder, SINK_NAME);
final StreamSinkNode sinkNode = new StreamSinkNode<>(
name,
topicExtractor,
produced
);
builder.addGraphNode(graphNode, sinkNode);
}
@Override
public KTable toTable() {
return toTable(NamedInternal.empty(), Materialized.with(keySerde, valueSerde));
}
@Override
public KTable toTable(final Named named) {
return toTable(named, Materialized.with(keySerde, valueSerde));
}
@Override
public KTable toTable(final Materialized> materialized) {
return toTable(NamedInternal.empty(), materialized);
}
@Override
public KTable toTable(final Named named,
final Materialized> materialized) {
Objects.requireNonNull(named, "named can't be null");
Objects.requireNonNull(materialized, "materialized can't be null");
final NamedInternal namedInternal = new NamedInternal(named);
final String name = namedInternal.orElseGenerateWithPrefix(builder, TO_KTABLE_NAME);
final MaterializedInternal> materializedInternal =
new MaterializedInternal<>(materialized, builder, TO_KTABLE_NAME);
final Serde keySerdeOverride = materializedInternal.keySerde() == null
? keySerde
: materializedInternal.keySerde();
final Serde valueSerdeOverride = materializedInternal.valueSerde() == null
? valueSerde
: materializedInternal.valueSerde();
final Set subTopologySourceNodes;
final GraphNode tableParentNode;
if (repartitionRequired) {
final OptimizableRepartitionNodeBuilder repartitionNodeBuilder = optimizableRepartitionNodeBuilder();
final String sourceName = createRepartitionedSource(
builder,
keySerdeOverride,
valueSerdeOverride,
name,
null,
repartitionNodeBuilder
);
tableParentNode = repartitionNodeBuilder.build();
builder.addGraphNode(graphNode, tableParentNode);
subTopologySourceNodes = Collections.singleton(sourceName);
} else {
tableParentNode = graphNode;
subTopologySourceNodes = this.subTopologySourceNodes;
}
final KTableSource tableSource = new KTableSource<>(
materializedInternal.storeName(),
materializedInternal.queryableStoreName()
);
final ProcessorParameters processorParameters = new ProcessorParameters<>(tableSource, name);
final GraphNode tableNode = new StreamToTableNode<>(
name,
processorParameters,
materializedInternal
);
tableNode.setOutputVersioned(materializedInternal.storeSupplier() instanceof VersionedBytesStoreSupplier);
builder.addGraphNode(tableParentNode, tableNode);
return new KTableImpl(
name,
keySerdeOverride,
valueSerdeOverride,
subTopologySourceNodes,
materializedInternal.queryableStoreName(),
tableSource,
tableNode,
builder
);
}
@Override
public KGroupedStream groupBy(final KeyValueMapper keySelector) {
return groupBy(keySelector, Grouped.with(null, valueSerde));
}
@Override
public KGroupedStream groupBy(final KeyValueMapper keySelector,
final Grouped grouped) {
Objects.requireNonNull(keySelector, "keySelector can't be null");
Objects.requireNonNull(grouped, "grouped can't be null");
final GroupedInternal groupedInternal = new GroupedInternal<>(grouped);
final ProcessorGraphNode selectKeyMapNode = internalSelectKey(keySelector, new NamedInternal(groupedInternal.name()));
selectKeyMapNode.keyChangingOperation(true);
builder.addGraphNode(graphNode, selectKeyMapNode);
return new KGroupedStreamImpl<>(
selectKeyMapNode.nodeName(),
subTopologySourceNodes,
groupedInternal,
true,
selectKeyMapNode,
builder);
}
@Override
public KGroupedStream groupByKey() {
return groupByKey(Grouped.with(keySerde, valueSerde));
}
@Override
public KGroupedStream groupByKey(final Grouped grouped) {
Objects.requireNonNull(grouped, "grouped can't be null");
final GroupedInternal groupedInternal = new GroupedInternal<>(grouped);
return new KGroupedStreamImpl<>(
name,
subTopologySourceNodes,
groupedInternal,
repartitionRequired,
graphNode,
builder);
}
@Override
public KStream join(final KStream otherStream,
final ValueJoiner joiner,
final JoinWindows windows) {
return join(otherStream, toValueJoinerWithKey(joiner), windows);
}
@Override
public KStream join(final KStream otherStream,
final ValueJoinerWithKey joiner,
final JoinWindows windows) {
return join(otherStream, joiner, windows, StreamJoined.with(null, null, null));
}
@Override
public KStream join(final KStream otherStream,
final ValueJoiner joiner,
final JoinWindows windows,
final StreamJoined streamJoined) {
return join(otherStream, toValueJoinerWithKey(joiner), windows, streamJoined);
}
@Override
public KStream join(final KStream otherStream,
final ValueJoinerWithKey joiner,
final JoinWindows windows,
final StreamJoined streamJoined) {
return doJoin(
otherStream,
joiner,
windows,
streamJoined,
new KStreamImplJoin(builder, false, false));
}
@Override
public KStream leftJoin(final KStream otherStream,
final ValueJoiner joiner,
final JoinWindows windows) {
return leftJoin(otherStream, toValueJoinerWithKey(joiner), windows);
}
@Override
public KStream leftJoin(final KStream otherStream,
final ValueJoinerWithKey joiner,
final JoinWindows windows) {
return leftJoin(otherStream, joiner, windows, StreamJoined.with(null, null, null));
}
@Override
public KStream leftJoin(final KStream otherStream,
final ValueJoiner joiner,
final JoinWindows windows,
final StreamJoined streamJoined) {
return doJoin(
otherStream,
toValueJoinerWithKey(joiner),
windows,
streamJoined,
new KStreamImplJoin(builder, true, false));
}
@Override
public KStream leftJoin(final KStream otherStream,
final ValueJoinerWithKey joiner,
final JoinWindows windows,
final StreamJoined streamJoined) {
return doJoin(
otherStream,
joiner,
windows,
streamJoined,
new KStreamImplJoin(builder, true, false));
}
@Override
public KStream outerJoin(final KStream otherStream,
final ValueJoiner joiner,
final JoinWindows windows) {
return outerJoin(otherStream, toValueJoinerWithKey(joiner), windows);
}
@Override
public KStream outerJoin(final KStream otherStream,
final ValueJoinerWithKey joiner,
final JoinWindows windows) {
return outerJoin(otherStream, joiner, windows, StreamJoined.with(null, null, null));
}
@Override
public KStream outerJoin(final KStream otherStream,
final ValueJoiner joiner,
final JoinWindows windows,
final StreamJoined streamJoined) {
return outerJoin(otherStream, toValueJoinerWithKey(joiner), windows, streamJoined);
}
@Override
public KStream outerJoin(final KStream otherStream,
final ValueJoinerWithKey joiner,
final JoinWindows windows,
final StreamJoined streamJoined) {
return doJoin(otherStream, joiner, windows, streamJoined, new KStreamImplJoin(builder, true, true));
}
private KStream doJoin(final KStream otherStream,
final ValueJoinerWithKey joiner,
final JoinWindows windows,
final StreamJoined streamJoined,
final KStreamImplJoin join) {
Objects.requireNonNull(otherStream, "otherStream can't be null");
Objects.requireNonNull(joiner, "joiner can't be null");
Objects.requireNonNull(windows, "windows can't be null");
Objects.requireNonNull(streamJoined, "streamJoined can't be null");
KStreamImpl joinThis = this;
KStreamImpl joinOther = (KStreamImpl) otherStream;
final StreamJoinedInternal streamJoinedInternal = new StreamJoinedInternal<>(streamJoined, builder);
final NamedInternal name = new NamedInternal(streamJoinedInternal.name());
if (joinThis.repartitionRequired) {
final String joinThisName = joinThis.name;
final String leftJoinRepartitionTopicName = name.suffixWithOrElseGet("-left", joinThisName);
joinThis = joinThis.repartitionForJoin(leftJoinRepartitionTopicName, streamJoinedInternal.keySerde(), streamJoinedInternal.valueSerde());
}
if (joinOther.repartitionRequired) {
final String joinOtherName = joinOther.name;
final String rightJoinRepartitionTopicName = name.suffixWithOrElseGet("-right", joinOtherName);
joinOther = joinOther.repartitionForJoin(rightJoinRepartitionTopicName, streamJoinedInternal.keySerde(), streamJoinedInternal.otherValueSerde());
}
joinThis.ensureCopartitionWith(Collections.singleton(joinOther));
return join.join(
joinThis,
joinOther,
joiner,
windows,
streamJoined);
}
/**
* Repartition a stream. This is required on join operations occurring after
* an operation that changes the key, i.e, selectKey, map(..), flatMap(..).
*/
private KStreamImpl repartitionForJoin(final String repartitionName,
final Serde keySerdeOverride,
final Serde valueSerdeOverride) {
final Serde repartitionKeySerde = keySerdeOverride != null ? keySerdeOverride : keySerde;
final Serde repartitionValueSerde = valueSerdeOverride != null ? valueSerdeOverride : valueSerde;
final OptimizableRepartitionNodeBuilder optimizableRepartitionNodeBuilder =
OptimizableRepartitionNode.optimizableRepartitionNodeBuilder();
// we still need to create the repartitioned source each time
// as it increments the counter which
// is needed to maintain topology compatibility
final String repartitionedSourceName = createRepartitionedSource(
builder,
repartitionKeySerde,
repartitionValueSerde,
repartitionName,
null,
optimizableRepartitionNodeBuilder);
if (repartitionNode == null || !name.equals(repartitionName)) {
repartitionNode = optimizableRepartitionNodeBuilder.build();
builder.addGraphNode(graphNode, repartitionNode);
}
return new KStreamImpl<>(
repartitionedSourceName,
repartitionKeySerde,
repartitionValueSerde,
Collections.singleton(repartitionedSourceName),
false,
repartitionNode,
builder);
}
static > String createRepartitionedSource(final InternalStreamsBuilder builder,
final Serde keySerde,
final Serde valueSerde,
final String repartitionTopicNamePrefix,
final StreamPartitioner streamPartitioner,
final BaseRepartitionNodeBuilder baseRepartitionNodeBuilder) {
final String repartitionTopicName = repartitionTopicNamePrefix.endsWith(REPARTITION_TOPIC_SUFFIX) ?
repartitionTopicNamePrefix :
repartitionTopicNamePrefix + REPARTITION_TOPIC_SUFFIX;
// Always need to generate the names to burn index counter for compatibility
final String genSinkName = builder.newProcessorName(SINK_NAME);
final String genNullKeyFilterProcessorName = builder.newProcessorName(FILTER_NAME);
final String genSourceName = builder.newProcessorName(SOURCE_NAME);
final String sinkName;
final String sourceName;
final String nullKeyFilterProcessorName;
if (repartitionTopicNamePrefix.matches("KSTREAM.*-[0-9]{10}")) {
sinkName = genSinkName;
sourceName = genSourceName;
nullKeyFilterProcessorName = genNullKeyFilterProcessorName;
} else {
sinkName = repartitionTopicName + "-sink";
sourceName = repartitionTopicName + "-source";
nullKeyFilterProcessorName = repartitionTopicName + "-filter";
}
final ProcessorParameters processorParameters = new ProcessorParameters<>(
new KStreamFilter<>((k, v) -> true, false),
nullKeyFilterProcessorName
);
baseRepartitionNodeBuilder.withKeySerde(keySerde)
.withValueSerde(valueSerde)
.withSourceName(sourceName)
.withRepartitionTopic(repartitionTopicName)
.withSinkName(sinkName)
.withProcessorParameters(processorParameters)
.withStreamPartitioner(streamPartitioner)
// reusing the source name for the graph node name
// adding explicit variable as it simplifies logic
.withNodeName(sourceName);
return sourceName;
}
@Override
public KStream join(final KTable table,
final ValueJoiner joiner) {
return join(table, toValueJoinerWithKey(joiner));
}
@Override
public KStream join(final KTable table,
final ValueJoinerWithKey joiner) {
return join(table, joiner, Joined.with(null, null, null));
}
@Override
public KStream join(final KTable table,
final ValueJoiner joiner,
final Joined joined) {
Objects.requireNonNull(table, "table can't be null");
Objects.requireNonNull(joiner, "joiner can't be null");
Objects.requireNonNull(joined, "joined can't be null");
return join(table, toValueJoinerWithKey(joiner), joined);
}
@Override
public KStream join(final KTable table,
final ValueJoinerWithKey joiner,
final Joined joined) {
Objects.requireNonNull(table, "table can't be null");
Objects.requireNonNull(joiner, "joiner can't be null");
Objects.requireNonNull(joined, "joined can't be null");
final JoinedInternal joinedInternal = new JoinedInternal<>(joined);
final String name = joinedInternal.name();
if (repartitionRequired) {
final KStreamImpl thisStreamRepartitioned = repartitionForJoin(
name != null ? name : this.name,
joined.keySerde(),
joined.valueSerde()
);
return thisStreamRepartitioned.doStreamTableJoin(table, joiner, joined, false);
} else {
return doStreamTableJoin(table, joiner, joined, false);
}
}
@Override
public KStream leftJoin(final KTable table, final ValueJoiner joiner) {
return leftJoin(table, toValueJoinerWithKey(joiner));
}
@Override
public KStream leftJoin(final KTable table, final ValueJoinerWithKey joiner) {
return leftJoin(table, joiner, Joined.with(null, null, null));
}
@Override
public KStream leftJoin(final KTable table,
final ValueJoiner joiner,
final Joined joined) {
Objects.requireNonNull(table, "table can't be null");
Objects.requireNonNull(joiner, "joiner can't be null");
Objects.requireNonNull(joined, "joined can't be null");
return leftJoin(table, toValueJoinerWithKey(joiner), joined);
}
@Override
public KStream leftJoin(final KTable table,
final ValueJoinerWithKey joiner,
final Joined joined) {
Objects.requireNonNull(table, "table can't be null");
Objects.requireNonNull(joiner, "joiner can't be null");
Objects.requireNonNull(joined, "joined can't be null");
final JoinedInternal joinedInternal = new JoinedInternal<>(joined);
final String name = joinedInternal.name();
if (repartitionRequired) {
final KStreamImpl thisStreamRepartitioned = repartitionForJoin(
name != null ? name : this.name,
joined.keySerde(),
joined.valueSerde()
);
return thisStreamRepartitioned.doStreamTableJoin(table, joiner, joined, true);
} else {
return doStreamTableJoin(table, joiner, joined, true);
}
}
@Override
public KStream join(final GlobalKTable globalTable,
final KeyValueMapper keySelector,
final ValueJoiner joiner) {
return join(globalTable, keySelector, toValueJoinerWithKey(joiner));
}
@Override
public KStream join(final GlobalKTable globalTable,
final KeyValueMapper keySelector,
final ValueJoinerWithKey joiner) {
return globalTableJoin(globalTable, keySelector, joiner, false, NamedInternal.empty());
}
@Override
public KStream join(final GlobalKTable globalTable,
final KeyValueMapper keySelector,
final ValueJoiner joiner,
final Named named) {
return join(globalTable, keySelector, toValueJoinerWithKey(joiner), named);
}
@Override
public KStream join(final GlobalKTable globalTable,
final KeyValueMapper keySelector,
final ValueJoinerWithKey joiner,
final Named named) {
return globalTableJoin(globalTable, keySelector, joiner, false, named);
}
@Override
public KStream leftJoin(final GlobalKTable globalTable,
final KeyValueMapper keySelector,
final ValueJoiner joiner) {
return leftJoin(globalTable, keySelector, toValueJoinerWithKey(joiner));
}
@Override
public KStream leftJoin(final GlobalKTable globalTable,
final KeyValueMapper keySelector,
final ValueJoinerWithKey joiner) {
return globalTableJoin(globalTable, keySelector, joiner, true, NamedInternal.empty());
}
@Override
public KStream leftJoin(final GlobalKTable globalTable,
final KeyValueMapper keySelector,
final ValueJoiner joiner,
final Named named) {
return leftJoin(globalTable, keySelector, toValueJoinerWithKey(joiner), named);
}
@Override
public KStream leftJoin(final GlobalKTable globalTable,
final KeyValueMapper keySelector,
final ValueJoinerWithKey joiner,
final Named named) {
return globalTableJoin(globalTable, keySelector, joiner, true, named);
}
private KStream globalTableJoin(final GlobalKTable globalTable,
final KeyValueMapper keySelector,
final ValueJoinerWithKey joiner,
final boolean leftJoin,
final Named named) {
Objects.requireNonNull(globalTable, "globalTable can't be null");
Objects.requireNonNull(keySelector, "keySelector can't be null");
Objects.requireNonNull(joiner, "joiner can't be null");
Objects.requireNonNull(named, "named can't be null");
final KTableValueGetterSupplier valueGetterSupplier =
((GlobalKTableImpl) globalTable).valueGetterSupplier();
final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, LEFTJOIN_NAME);
final ProcessorSupplier processorSupplier = new KStreamGlobalKTableJoin<>(
valueGetterSupplier,
joiner,
keySelector,
leftJoin);
final ProcessorParameters processorParameters = new ProcessorParameters<>(processorSupplier, name);
final StreamTableJoinNode streamTableJoinNode =
new StreamTableJoinNode<>(name, processorParameters, new String[] {}, null, null, Optional.empty());
if (leftJoin) {
streamTableJoinNode.labels().add(GraphNode.Label.NULL_KEY_RELAXED_JOIN);
}
builder.addGraphNode(graphNode, streamTableJoinNode);
// do not have serde for joined result
return new KStreamImpl<>(
name,
keySerde,
null,
subTopologySourceNodes,
repartitionRequired,
streamTableJoinNode,
builder);
}
@SuppressWarnings("unchecked")
private KStream doStreamTableJoin(final KTable table,
final ValueJoinerWithKey joiner,
final Joined joined,
final boolean leftJoin) {
Objects.requireNonNull(table, "table can't be null");
Objects.requireNonNull(joiner, "joiner can't be null");
final Set allSourceNodes = ensureCopartitionWith(Collections.singleton((AbstractStream) table));
final JoinedInternal joinedInternal = new JoinedInternal<>(joined);
final NamedInternal renamed = new NamedInternal(joinedInternal.name());
final String name = renamed.orElseGenerateWithPrefix(builder, leftJoin ? LEFTJOIN_NAME : JOIN_NAME);
Optional bufferStoreName = Optional.empty();
if (joined.gracePeriod() != null) {
if (!((KTableImpl