org.apache.kafka.streams.kstream.internals.CogroupedStreamAggregateBuilder Maven / Gradle / Ivy
The newest version!
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kafka.streams.kstream.internals;
import org.apache.kafka.common.serialization.Serde;
import org.apache.kafka.streams.kstream.Aggregator;
import org.apache.kafka.streams.kstream.EmitStrategy;
import org.apache.kafka.streams.kstream.Initializer;
import org.apache.kafka.streams.kstream.KTable;
import org.apache.kafka.streams.kstream.Merger;
import org.apache.kafka.streams.kstream.SessionWindows;
import org.apache.kafka.streams.kstream.SlidingWindows;
import org.apache.kafka.streams.kstream.Window;
import org.apache.kafka.streams.kstream.Windows;
import org.apache.kafka.streams.kstream.internals.graph.GraphNode;
import org.apache.kafka.streams.kstream.internals.graph.OptimizableRepartitionNode.OptimizableRepartitionNodeBuilder;
import org.apache.kafka.streams.kstream.internals.graph.ProcessorGraphNode;
import org.apache.kafka.streams.kstream.internals.graph.ProcessorParameters;
import org.apache.kafka.streams.kstream.internals.graph.StatefulProcessorNode;
import org.apache.kafka.streams.processor.api.ProcessorSupplier;
import org.apache.kafka.streams.processor.internals.StoreFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Map.Entry;
import static org.apache.kafka.streams.kstream.internals.graph.OptimizableRepartitionNode.optimizableRepartitionNodeBuilder;
class CogroupedStreamAggregateBuilder {
private final InternalStreamsBuilder builder;
private final Map, GraphNode> parentNodes = new LinkedHashMap<>();
CogroupedStreamAggregateBuilder(final InternalStreamsBuilder builder) {
this.builder = builder;
}
KTable build(final Map, Aggregator> groupPatterns,
final Initializer initializer,
final NamedInternal named,
final StoreFactory storeFactory,
final Serde keySerde,
final Serde valueSerde,
final String queryableName,
final boolean isOutputVersioned) {
processRepartitions(groupPatterns, storeFactory);
final Collection processors = new ArrayList<>();
final Collection parentProcessors = new ArrayList<>();
boolean stateCreated = false;
int counter = 0;
for (final Entry, Aggregator> kGroupedStream : groupPatterns.entrySet()) {
final KStreamAggProcessorSupplier parentProcessor =
new KStreamAggregate<>(storeFactory.name(), initializer, kGroupedStream.getValue());
parentProcessors.add(parentProcessor);
final StatefulProcessorNode statefulProcessorNode = getStatefulProcessorNode(
named.suffixWithOrElseGet(
"-cogroup-agg-" + counter++,
builder,
CogroupedKStreamImpl.AGGREGATE_NAME),
stateCreated,
storeFactory,
parentProcessor);
statefulProcessorNode.setOutputVersioned(isOutputVersioned);
stateCreated = true;
processors.add(statefulProcessorNode);
builder.addGraphNode(parentNodes.get(kGroupedStream.getKey()), statefulProcessorNode);
}
return createTable(processors, parentProcessors, named, keySerde, valueSerde, queryableName, storeFactory.name());
}
@SuppressWarnings("unchecked")
KTable build(final Map, Aggregator> groupPatterns,
final Initializer initializer,
final NamedInternal named,
final StoreFactory storeFactory,
final Serde keySerde,
final Serde valueSerde,
final String queryableName,
final Windows windows) {
processRepartitions(groupPatterns, storeFactory);
final Collection processors = new ArrayList<>();
final Collection parentProcessors = new ArrayList<>();
boolean stateCreated = false;
int counter = 0;
for (final Entry, Aggregator> kGroupedStream : groupPatterns.entrySet()) {
final KStreamAggProcessorSupplier parentProcessor =
(KStreamAggProcessorSupplier) new KStreamWindowAggregate(
windows,
storeFactory.name(),
EmitStrategy.onWindowUpdate(),
initializer,
kGroupedStream.getValue());
parentProcessors.add(parentProcessor);
final StatefulProcessorNode statefulProcessorNode = getStatefulProcessorNode(
named.suffixWithOrElseGet(
"-cogroup-agg-" + counter++,
builder,
CogroupedKStreamImpl.AGGREGATE_NAME),
stateCreated,
storeFactory,
parentProcessor);
stateCreated = true;
processors.add(statefulProcessorNode);
builder.addGraphNode(parentNodes.get(kGroupedStream.getKey()), statefulProcessorNode);
}
return createTable(processors, parentProcessors, named, keySerde, valueSerde, queryableName, storeFactory.name());
}
@SuppressWarnings("unchecked")
KTable build(final Map, Aggregator> groupPatterns,
final Initializer initializer,
final NamedInternal named,
final StoreFactory storeFactory,
final Serde keySerde,
final Serde valueSerde,
final String queryableName,
final SessionWindows sessionWindows,
final Merger sessionMerger) {
processRepartitions(groupPatterns, storeFactory);
final Collection processors = new ArrayList<>();
final Collection parentProcessors = new ArrayList<>();
boolean stateCreated = false;
int counter = 0;
for (final Entry, Aggregator> kGroupedStream : groupPatterns.entrySet()) {
final KStreamAggProcessorSupplier parentProcessor =
(KStreamAggProcessorSupplier) new KStreamSessionWindowAggregate(
sessionWindows,
storeFactory.name(),
EmitStrategy.onWindowUpdate(),
initializer,
kGroupedStream.getValue(),
sessionMerger);
parentProcessors.add(parentProcessor);
final StatefulProcessorNode statefulProcessorNode = getStatefulProcessorNode(
named.suffixWithOrElseGet(
"-cogroup-agg-" + counter++,
builder,
CogroupedKStreamImpl.AGGREGATE_NAME),
stateCreated,
storeFactory,
parentProcessor);
stateCreated = true;
processors.add(statefulProcessorNode);
builder.addGraphNode(parentNodes.get(kGroupedStream.getKey()), statefulProcessorNode);
}
return createTable(processors, parentProcessors, named, keySerde, valueSerde, queryableName, storeFactory.name());
}
@SuppressWarnings("unchecked")
KTable build(final Map, Aggregator> groupPatterns,
final Initializer initializer,
final NamedInternal named,
final StoreFactory storeFactory,
final Serde keySerde,
final Serde valueSerde,
final String queryableName,
final SlidingWindows slidingWindows) {
processRepartitions(groupPatterns, storeFactory);
final Collection parentProcessors = new ArrayList<>();
final Collection processors = new ArrayList<>();
boolean stateCreated = false;
int counter = 0;
for (final Entry, Aggregator> kGroupedStream : groupPatterns.entrySet()) {
final KStreamAggProcessorSupplier parentProcessor =
(KStreamAggProcessorSupplier) new KStreamSlidingWindowAggregate(
slidingWindows,
storeFactory.name(),
// TODO: We do not have other emit policies for co-group yet
EmitStrategy.onWindowUpdate(),
initializer,
kGroupedStream.getValue());
parentProcessors.add(parentProcessor);
final StatefulProcessorNode statefulProcessorNode = getStatefulProcessorNode(
named.suffixWithOrElseGet(
"-cogroup-agg-" + counter++,
builder,
CogroupedKStreamImpl.AGGREGATE_NAME),
stateCreated,
storeFactory,
parentProcessor);
stateCreated = true;
processors.add(statefulProcessorNode);
builder.addGraphNode(parentNodes.get(kGroupedStream.getKey()), statefulProcessorNode);
}
return createTable(processors, parentProcessors, named, keySerde, valueSerde, queryableName, storeFactory.name());
}
private void processRepartitions(final Map, Aggregator> groupPatterns,
final StoreFactory storeFactory) {
for (final KGroupedStreamImpl repartitionReqs : groupPatterns.keySet()) {
if (repartitionReqs.repartitionRequired) {
final OptimizableRepartitionNodeBuilder repartitionNodeBuilder = optimizableRepartitionNodeBuilder();
final String repartitionNamePrefix = repartitionReqs.userProvidedRepartitionTopicName != null ?
repartitionReqs.userProvidedRepartitionTopicName : storeFactory.name();
createRepartitionSource(repartitionNamePrefix, repartitionNodeBuilder, repartitionReqs.keySerde, repartitionReqs.valueSerde);
if (!parentNodes.containsKey(repartitionReqs)) {
final GraphNode repartitionNode = repartitionNodeBuilder.build();
builder.addGraphNode(repartitionReqs.graphNode, repartitionNode);
parentNodes.put(repartitionReqs, repartitionNode);
}
} else {
parentNodes.put(repartitionReqs, repartitionReqs.graphNode);
}
}
final Collection> groupedStreams = new ArrayList<>(parentNodes.keySet());
final AbstractStream kGrouped = groupedStreams.iterator().next();
groupedStreams.remove(kGrouped);
kGrouped.ensureCopartitionWith(groupedStreams);
}
@SuppressWarnings("unchecked")
KTable createTable(final Collection processors,
final Collection parentProcessors,
final NamedInternal named,
final Serde keySerde,
final Serde valueSerde,
final String queryableName,
final String storeName) {
final String mergeProcessorName = named.suffixWithOrElseGet(
"-cogroup-merge",
builder,
CogroupedKStreamImpl.MERGE_NAME);
final KTableProcessorSupplier passThrough = new KTablePassThrough<>(parentProcessors, storeName);
final ProcessorParameters processorParameters = new ProcessorParameters(passThrough, mergeProcessorName);
final ProcessorGraphNode mergeNode =
new ProcessorGraphNode<>(mergeProcessorName, processorParameters);
builder.addGraphNode(processors, mergeNode);
return new KTableImpl(
mergeProcessorName,
keySerde,
valueSerde,
Collections.singleton(mergeNode.nodeName()),
queryableName,
passThrough,
mergeNode,
builder);
}
private StatefulProcessorNode getStatefulProcessorNode(final String processorName,
final boolean stateCreated,
final StoreFactory storeFactory,
final ProcessorSupplier kStreamAggregate) {
final StatefulProcessorNode statefulProcessorNode;
if (!stateCreated) {
statefulProcessorNode =
new StatefulProcessorNode<>(
processorName,
new ProcessorParameters<>(kStreamAggregate, processorName),
storeFactory
);
} else {
statefulProcessorNode =
new StatefulProcessorNode<>(
processorName,
new ProcessorParameters<>(kStreamAggregate, processorName),
new String[]{storeFactory.name()}
);
}
return statefulProcessorNode;
}
@SuppressWarnings("unchecked")
private void createRepartitionSource(final String repartitionTopicNamePrefix,
final OptimizableRepartitionNodeBuilder optimizableRepartitionNodeBuilder,
final Serde keySerde,
final Serde valueSerde) {
KStreamImpl.createRepartitionedSource(builder,
keySerde,
(Serde) valueSerde,
repartitionTopicNamePrefix,
null,
(OptimizableRepartitionNodeBuilder) optimizableRepartitionNodeBuilder);
}
}