Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kafka.streams.kstream.internals;
import org.apache.kafka.common.serialization.Serde;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.common.utils.Bytes;
import org.apache.kafka.streams.KeyValue;
import org.apache.kafka.streams.kstream.Consumed;
import org.apache.kafka.streams.kstream.Grouped;
import org.apache.kafka.streams.kstream.KGroupedTable;
import org.apache.kafka.streams.kstream.KStream;
import org.apache.kafka.streams.kstream.KTable;
import org.apache.kafka.streams.kstream.KeyValueMapper;
import org.apache.kafka.streams.kstream.Materialized;
import org.apache.kafka.streams.kstream.Named;
import org.apache.kafka.streams.kstream.Predicate;
import org.apache.kafka.streams.kstream.Produced;
import org.apache.kafka.streams.kstream.Suppressed;
import org.apache.kafka.streams.kstream.ValueJoiner;
import org.apache.kafka.streams.kstream.ValueMapper;
import org.apache.kafka.streams.kstream.ValueMapperWithKey;
import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier;
import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.CombinedKey;
import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.CombinedKeySchema;
import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.ForeignJoinSubscriptionProcessorSupplier;
import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.ForeignJoinSubscriptionSendProcessorSupplier;
import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionJoinForeignProcessorSupplier;
import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionResolverJoinProcessorSupplier;
import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionResponseWrapper;
import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionResponseWrapperSerde;
import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionStoreReceiveProcessorSupplier;
import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionWrapper;
import org.apache.kafka.streams.kstream.internals.foreignkeyjoin.SubscriptionWrapperSerde;
import org.apache.kafka.streams.kstream.internals.graph.KTableKTableJoinNode;
import org.apache.kafka.streams.kstream.internals.graph.ProcessorGraphNode;
import org.apache.kafka.streams.kstream.internals.graph.ProcessorParameters;
import org.apache.kafka.streams.kstream.internals.graph.StatefulProcessorNode;
import org.apache.kafka.streams.kstream.internals.graph.StreamSinkNode;
import org.apache.kafka.streams.kstream.internals.graph.StreamSourceNode;
import org.apache.kafka.streams.kstream.internals.graph.GraphNode;
import org.apache.kafka.streams.kstream.internals.graph.TableProcessorNode;
import org.apache.kafka.streams.kstream.internals.suppress.FinalResultsSuppressionBuilder;
import org.apache.kafka.streams.kstream.internals.suppress.KTableSuppressProcessorSupplier;
import org.apache.kafka.streams.kstream.internals.suppress.NamedSuppressed;
import org.apache.kafka.streams.kstream.internals.suppress.SuppressedInternal;
import org.apache.kafka.streams.processor.internals.InternalTopicProperties;
import org.apache.kafka.streams.processor.internals.StaticTopicNameExtractor;
import org.apache.kafka.streams.state.KeyValueStore;
import org.apache.kafka.streams.state.StoreBuilder;
import org.apache.kafka.streams.state.Stores;
import org.apache.kafka.streams.state.TimestampedKeyValueStore;
import org.apache.kafka.streams.state.ValueAndTimestamp;
import org.apache.kafka.streams.state.internals.InMemoryTimeOrderedKeyValueBuffer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.time.Duration;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Supplier;
import static org.apache.kafka.streams.kstream.internals.graph.GraphGraceSearchUtil.findAndVerifyWindowGrace;
/**
* The implementation class of {@link KTable}.
*
* @param the key type
* @param the source's (parent's) value type
* @param the value type
*/
public class KTableImpl extends AbstractStream implements KTable {
private static final Logger LOG = LoggerFactory.getLogger(KTableImpl.class);
static final String SOURCE_NAME = "KTABLE-SOURCE-";
static final String STATE_STORE_NAME = "STATE-STORE-";
private static final String FILTER_NAME = "KTABLE-FILTER-";
private static final String JOINTHIS_NAME = "KTABLE-JOINTHIS-";
private static final String JOINOTHER_NAME = "KTABLE-JOINOTHER-";
private static final String MAPVALUES_NAME = "KTABLE-MAPVALUES-";
private static final String MERGE_NAME = "KTABLE-MERGE-";
private static final String SELECT_NAME = "KTABLE-SELECT-";
private static final String SUPPRESS_NAME = "KTABLE-SUPPRESS-";
private static final String TOSTREAM_NAME = "KTABLE-TOSTREAM-";
private static final String TRANSFORMVALUES_NAME = "KTABLE-TRANSFORMVALUES-";
private static final String FK_JOIN = "KTABLE-FK-JOIN-";
private static final String FK_JOIN_STATE_STORE_NAME = FK_JOIN + "SUBSCRIPTION-STATE-STORE-";
private static final String SUBSCRIPTION_REGISTRATION = FK_JOIN + "SUBSCRIPTION-REGISTRATION-";
private static final String SUBSCRIPTION_RESPONSE = FK_JOIN + "SUBSCRIPTION-RESPONSE-";
private static final String SUBSCRIPTION_PROCESSOR = FK_JOIN + "SUBSCRIPTION-PROCESSOR-";
private static final String SUBSCRIPTION_RESPONSE_RESOLVER_PROCESSOR = FK_JOIN + "SUBSCRIPTION-RESPONSE-RESOLVER-PROCESSOR-";
private static final String FK_JOIN_OUTPUT_NAME = FK_JOIN + "OUTPUT-";
private static final String TOPIC_SUFFIX = "-topic";
private static final String SINK_NAME = "KTABLE-SINK-";
// Temporarily setting the processorSupplier to type Object so that we can transition from the
// old ProcessorSupplier to the new api.ProcessorSupplier. This works because all accesses to
// this field are guarded by typechecks anyway.
private final Object processorSupplier;
private final String queryableStoreName;
private boolean sendOldValues = false;
@SuppressWarnings("deprecation") // Old PAPI compatibility.
public KTableImpl(final String name,
final Serde keySerde,
final Serde valueSerde,
final Set subTopologySourceNodes,
final String queryableStoreName,
final org.apache.kafka.streams.processor.ProcessorSupplier processorSupplier,
final GraphNode graphNode,
final InternalStreamsBuilder builder) {
super(name, keySerde, valueSerde, subTopologySourceNodes, graphNode, builder);
this.processorSupplier = processorSupplier;
this.queryableStoreName = queryableStoreName;
}
public KTableImpl(final String name,
final Serde keySerde,
final Serde valueSerde,
final Set subTopologySourceNodes,
final String queryableStoreName,
final org.apache.kafka.streams.processor.api.ProcessorSupplier newProcessorSupplier,
final GraphNode graphNode,
final InternalStreamsBuilder builder) {
super(name, keySerde, valueSerde, subTopologySourceNodes, graphNode, builder);
this.processorSupplier = newProcessorSupplier;
this.queryableStoreName = queryableStoreName;
}
@Override
public String queryableStoreName() {
return queryableStoreName;
}
private KTable doFilter(final Predicate predicate,
final Named named,
final MaterializedInternal> materializedInternal,
final boolean filterNot) {
final Serde keySerde;
final Serde valueSerde;
final String queryableStoreName;
final StoreBuilder> storeBuilder;
if (materializedInternal != null) {
// we actually do not need to generate store names at all since if it is not specified, we will not
// materialize the store; but we still need to burn one index BEFORE generating the processor to keep compatibility.
if (materializedInternal.storeName() == null) {
builder.newStoreName(FILTER_NAME);
}
// we can inherit parent key and value serde if user do not provide specific overrides, more specifically:
// we preserve the key following the order of 1) materialized, 2) parent
keySerde = materializedInternal.keySerde() != null ? materializedInternal.keySerde() : this.keySerde;
// we preserve the value following the order of 1) materialized, 2) parent
valueSerde = materializedInternal.valueSerde() != null ? materializedInternal.valueSerde() : this.valueSerde;
queryableStoreName = materializedInternal.queryableStoreName();
// only materialize if materialized is specified and it has queryable name
storeBuilder = queryableStoreName != null ? (new TimestampedKeyValueStoreMaterializer<>(materializedInternal)).materialize() : null;
} else {
keySerde = this.keySerde;
valueSerde = this.valueSerde;
queryableStoreName = null;
storeBuilder = null;
}
final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, FILTER_NAME);
final KTableNewProcessorSupplier processorSupplier =
new KTableFilter<>(this, predicate, filterNot, queryableStoreName);
final ProcessorParameters processorParameters = unsafeCastProcessorParametersToCompletelyDifferentType(
new ProcessorParameters<>(processorSupplier, name)
);
final GraphNode tableNode = new TableProcessorNode<>(
name,
processorParameters,
storeBuilder
);
builder.addGraphNode(this.graphNode, tableNode);
return new KTableImpl(
name,
keySerde,
valueSerde,
subTopologySourceNodes,
queryableStoreName,
processorSupplier,
tableNode,
builder);
}
@Override
public KTable filter(final Predicate predicate) {
Objects.requireNonNull(predicate, "predicate can't be null");
return doFilter(predicate, NamedInternal.empty(), null, false);
}
@Override
public KTable filter(final Predicate predicate, final Named named) {
Objects.requireNonNull(predicate, "predicate can't be null");
return doFilter(predicate, named, null, false);
}
@Override
public KTable filter(final Predicate predicate,
final Named named,
final Materialized> materialized) {
Objects.requireNonNull(predicate, "predicate can't be null");
Objects.requireNonNull(materialized, "materialized can't be null");
final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized);
return doFilter(predicate, named, materializedInternal, false);
}
@Override
public KTable filter(final Predicate predicate,
final Materialized> materialized) {
return filter(predicate, NamedInternal.empty(), materialized);
}
@Override
public KTable filterNot(final Predicate predicate) {
Objects.requireNonNull(predicate, "predicate can't be null");
return doFilter(predicate, NamedInternal.empty(), null, true);
}
@Override
public KTable filterNot(final Predicate predicate,
final Named named) {
Objects.requireNonNull(predicate, "predicate can't be null");
return doFilter(predicate, named, null, true);
}
@Override
public KTable filterNot(final Predicate predicate,
final Materialized> materialized) {
return filterNot(predicate, NamedInternal.empty(), materialized);
}
@Override
public KTable filterNot(final Predicate predicate,
final Named named,
final Materialized> materialized) {
Objects.requireNonNull(predicate, "predicate can't be null");
Objects.requireNonNull(materialized, "materialized can't be null");
final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized);
final NamedInternal renamed = new NamedInternal(named);
return doFilter(predicate, renamed, materializedInternal, true);
}
private KTable doMapValues(final ValueMapperWithKey mapper,
final Named named,
final MaterializedInternal> materializedInternal) {
final Serde keySerde;
final Serde valueSerde;
final String queryableStoreName;
final StoreBuilder> storeBuilder;
if (materializedInternal != null) {
// we actually do not need to generate store names at all since if it is not specified, we will not
// materialize the store; but we still need to burn one index BEFORE generating the processor to keep compatibility.
if (materializedInternal.storeName() == null) {
builder.newStoreName(MAPVALUES_NAME);
}
keySerde = materializedInternal.keySerde() != null ? materializedInternal.keySerde() : this.keySerde;
valueSerde = materializedInternal.valueSerde();
queryableStoreName = materializedInternal.queryableStoreName();
// only materialize if materialized is specified and it has queryable name
storeBuilder = queryableStoreName != null ? (new TimestampedKeyValueStoreMaterializer<>(materializedInternal)).materialize() : null;
} else {
keySerde = this.keySerde;
valueSerde = null;
queryableStoreName = null;
storeBuilder = null;
}
final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, MAPVALUES_NAME);
final KTableProcessorSupplier processorSupplier = new KTableMapValues<>(this, mapper, queryableStoreName);
// leaving in calls to ITB until building topology with graph
final ProcessorParameters processorParameters = unsafeCastProcessorParametersToCompletelyDifferentType(
new ProcessorParameters<>(processorSupplier, name)
);
final GraphNode tableNode = new TableProcessorNode<>(
name,
processorParameters,
storeBuilder
);
builder.addGraphNode(this.graphNode, tableNode);
// don't inherit parent value serde, since this operation may change the value type, more specifically:
// we preserve the key following the order of 1) materialized, 2) parent, 3) null
// we preserve the value following the order of 1) materialized, 2) null
return new KTableImpl<>(
name,
keySerde,
valueSerde,
subTopologySourceNodes,
queryableStoreName,
processorSupplier,
tableNode,
builder
);
}
@Override
public KTable mapValues(final ValueMapper mapper) {
Objects.requireNonNull(mapper, "mapper can't be null");
return doMapValues(withKey(mapper), NamedInternal.empty(), null);
}
@Override
public KTable mapValues(final ValueMapper mapper,
final Named named) {
Objects.requireNonNull(mapper, "mapper can't be null");
return doMapValues(withKey(mapper), named, null);
}
@Override
public KTable mapValues(final ValueMapperWithKey mapper) {
Objects.requireNonNull(mapper, "mapper can't be null");
return doMapValues(mapper, NamedInternal.empty(), null);
}
@Override
public KTable mapValues(final ValueMapperWithKey mapper,
final Named named) {
Objects.requireNonNull(mapper, "mapper can't be null");
return doMapValues(mapper, named, null);
}
@Override
public KTable mapValues(final ValueMapper mapper,
final Materialized> materialized) {
return mapValues(mapper, NamedInternal.empty(), materialized);
}
@Override
public KTable mapValues(final ValueMapper mapper,
final Named named,
final Materialized> materialized) {
Objects.requireNonNull(mapper, "mapper can't be null");
Objects.requireNonNull(materialized, "materialized can't be null");
final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized);
return doMapValues(withKey(mapper), named, materializedInternal);
}
@Override
public KTable mapValues(final ValueMapperWithKey mapper,
final Materialized> materialized) {
return mapValues(mapper, NamedInternal.empty(), materialized);
}
@Override
public KTable mapValues(final ValueMapperWithKey mapper,
final Named named,
final Materialized> materialized) {
Objects.requireNonNull(mapper, "mapper can't be null");
Objects.requireNonNull(materialized, "materialized can't be null");
final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized);
return doMapValues(mapper, named, materializedInternal);
}
@Override
public KTable transformValues(final ValueTransformerWithKeySupplier transformerSupplier,
final String... stateStoreNames) {
return doTransformValues(transformerSupplier, null, NamedInternal.empty(), stateStoreNames);
}
@Override
public KTable transformValues(final ValueTransformerWithKeySupplier transformerSupplier,
final Named named,
final String... stateStoreNames) {
Objects.requireNonNull(named, "processorName can't be null");
return doTransformValues(transformerSupplier, null, new NamedInternal(named), stateStoreNames);
}
@Override
public KTable transformValues(final ValueTransformerWithKeySupplier transformerSupplier,
final Materialized> materialized,
final String... stateStoreNames) {
return transformValues(transformerSupplier, materialized, NamedInternal.empty(), stateStoreNames);
}
@Override
public KTable transformValues(final ValueTransformerWithKeySupplier transformerSupplier,
final Materialized> materialized,
final Named named,
final String... stateStoreNames) {
Objects.requireNonNull(materialized, "materialized can't be null");
Objects.requireNonNull(named, "named can't be null");
final MaterializedInternal> materializedInternal = new MaterializedInternal<>(materialized);
return doTransformValues(transformerSupplier, materializedInternal, new NamedInternal(named), stateStoreNames);
}
private KTable doTransformValues(final ValueTransformerWithKeySupplier transformerSupplier,
final MaterializedInternal> materializedInternal,
final NamedInternal namedInternal,
final String... stateStoreNames) {
Objects.requireNonNull(stateStoreNames, "stateStoreNames");
final Serde keySerde;
final Serde valueSerde;
final String queryableStoreName;
final StoreBuilder> storeBuilder;
if (materializedInternal != null) {
// don't inherit parent value serde, since this operation may change the value type, more specifically:
// we preserve the key following the order of 1) materialized, 2) parent, 3) null
keySerde = materializedInternal.keySerde() != null ? materializedInternal.keySerde() : this.keySerde;
// we preserve the value following the order of 1) materialized, 2) null
valueSerde = materializedInternal.valueSerde();
queryableStoreName = materializedInternal.queryableStoreName();
// only materialize if materialized is specified and it has queryable name
storeBuilder = queryableStoreName != null ? (new TimestampedKeyValueStoreMaterializer<>(materializedInternal)).materialize() : null;
} else {
keySerde = this.keySerde;
valueSerde = null;
queryableStoreName = null;
storeBuilder = null;
}
final String name = namedInternal.orElseGenerateWithPrefix(builder, TRANSFORMVALUES_NAME);
final KTableProcessorSupplier processorSupplier = new KTableTransformValues<>(
this,
transformerSupplier,
queryableStoreName);
final ProcessorParameters processorParameters = unsafeCastProcessorParametersToCompletelyDifferentType(
new ProcessorParameters<>(processorSupplier, name)
);
final GraphNode tableNode = new TableProcessorNode<>(
name,
processorParameters,
storeBuilder,
stateStoreNames
);
builder.addGraphNode(this.graphNode, tableNode);
return new KTableImpl<>(
name,
keySerde,
valueSerde,
subTopologySourceNodes,
queryableStoreName,
processorSupplier,
tableNode,
builder);
}
@Override
public KStream toStream() {
return toStream(NamedInternal.empty());
}
@Override
public KStream toStream(final Named named) {
Objects.requireNonNull(named, "named can't be null");
final String name = new NamedInternal(named).orElseGenerateWithPrefix(builder, TOSTREAM_NAME);
final KStreamMapValues, V> kStreamMapValues = new KStreamMapValues<>((key, change) -> change.newValue);
final ProcessorParameters processorParameters = unsafeCastProcessorParametersToCompletelyDifferentType(
new ProcessorParameters<>(kStreamMapValues, name)
);
final ProcessorGraphNode toStreamNode = new ProcessorGraphNode<>(
name,
processorParameters
);
builder.addGraphNode(this.graphNode, toStreamNode);
// we can inherit parent key and value serde
return new KStreamImpl<>(name, keySerde, valueSerde, subTopologySourceNodes, false, toStreamNode, builder);
}
@Override
public KStream toStream(final KeyValueMapper mapper) {
return toStream().selectKey(mapper);
}
@Override
public KStream toStream(final KeyValueMapper mapper,
final Named named) {
return toStream(named).selectKey(mapper);
}
@Override
public KTable suppress(final Suppressed suppressed) {
final String name;
if (suppressed instanceof NamedSuppressed) {
final String givenName = ((NamedSuppressed) suppressed).name();
name = givenName != null ? givenName : builder.newProcessorName(SUPPRESS_NAME);
} else {
throw new IllegalArgumentException("Custom subclasses of Suppressed are not supported.");
}
final SuppressedInternal suppressedInternal = buildSuppress(suppressed, name);
final String storeName =
suppressedInternal.name() != null ? suppressedInternal.name() + "-store" : builder.newStoreName(SUPPRESS_NAME);
@SuppressWarnings("deprecation") // Old PAPI. Needs to be migrated.
final org.apache.kafka.streams.processor.ProcessorSupplier> suppressionSupplier = new KTableSuppressProcessorSupplier<>(
suppressedInternal,
storeName,
this
);
final StoreBuilder> storeBuilder;
if (suppressedInternal.bufferConfig().isLoggingEnabled()) {
final Map topicConfig = suppressedInternal.bufferConfig().getLogConfig();
storeBuilder = new InMemoryTimeOrderedKeyValueBuffer.Builder<>(
storeName,
keySerde,
valueSerde)
.withLoggingEnabled(topicConfig);
} else {
storeBuilder = new InMemoryTimeOrderedKeyValueBuffer.Builder<>(
storeName,
keySerde,
valueSerde)
.withLoggingDisabled();
}
final ProcessorGraphNode> node = new StatefulProcessorNode<>(
name,
new ProcessorParameters<>(suppressionSupplier, name),
storeBuilder
);
builder.addGraphNode(graphNode, node);
return new KTableImpl(
name,
keySerde,
valueSerde,
Collections.singleton(this.name),
null,
suppressionSupplier,
node,
builder
);
}
@SuppressWarnings("unchecked")
private SuppressedInternal buildSuppress(final Suppressed suppress, final String name) {
if (suppress instanceof FinalResultsSuppressionBuilder) {
final long grace = findAndVerifyWindowGrace(graphNode);
LOG.info("Using grace period of [{}] as the suppress duration for node [{}].",
Duration.ofMillis(grace), name);
final FinalResultsSuppressionBuilder builder = (FinalResultsSuppressionBuilder) suppress;
final SuppressedInternal finalResultsSuppression =
builder.buildFinalResultsSuppression(Duration.ofMillis(grace));
return (SuppressedInternal) finalResultsSuppression;
} else if (suppress instanceof SuppressedInternal) {
return (SuppressedInternal) suppress;
} else {
throw new IllegalArgumentException("Custom subclasses of Suppressed are not allowed.");
}
}
@Override
public KTable join(final KTable other,
final ValueJoiner joiner) {
return doJoin(other, joiner, NamedInternal.empty(), null, false, false);
}
@Override
public KTable join(final KTable other,
final ValueJoiner joiner,
final Named named) {
return doJoin(other, joiner, named, null, false, false);
}
@Override
public KTable join(final KTable other,
final ValueJoiner joiner,
final Materialized> materialized) {
return join(other, joiner, NamedInternal.empty(), materialized);
}
@Override
public KTable join(final KTable other,
final ValueJoiner joiner,
final Named named,
final Materialized> materialized) {
Objects.requireNonNull(materialized, "materialized can't be null");
final MaterializedInternal> materializedInternal =
new MaterializedInternal<>(materialized, builder, MERGE_NAME);
return doJoin(other, joiner, named, materializedInternal, false, false);
}
@Override
public KTable outerJoin(final KTable other,
final ValueJoiner joiner) {
return outerJoin(other, joiner, NamedInternal.empty());
}
@Override
public KTable outerJoin(final KTable other,
final ValueJoiner joiner,
final Named named) {
return doJoin(other, joiner, named, null, true, true);
}
@Override
public KTable outerJoin(final KTable other,
final ValueJoiner joiner,
final Materialized> materialized) {
return outerJoin(other, joiner, NamedInternal.empty(), materialized);
}
@Override
public KTable outerJoin(final KTable other,
final ValueJoiner joiner,
final Named named,
final Materialized> materialized) {
Objects.requireNonNull(materialized, "materialized can't be null");
final MaterializedInternal> materializedInternal =
new MaterializedInternal<>(materialized, builder, MERGE_NAME);
return doJoin(other, joiner, named, materializedInternal, true, true);
}
@Override
public KTable leftJoin(final KTable other,
final ValueJoiner joiner) {
return leftJoin(other, joiner, NamedInternal.empty());
}
@Override
public KTable leftJoin(final KTable other,
final ValueJoiner joiner,
final Named named) {
return doJoin(other, joiner, named, null, true, false);
}
@Override
public KTable leftJoin(final KTable other,
final ValueJoiner joiner,
final Materialized> materialized) {
return leftJoin(other, joiner, NamedInternal.empty(), materialized);
}
@Override
public KTable leftJoin(final KTable other,
final ValueJoiner joiner,
final Named named,
final Materialized> materialized) {
Objects.requireNonNull(materialized, "materialized can't be null");
final MaterializedInternal> materializedInternal =
new MaterializedInternal<>(materialized, builder, MERGE_NAME);
return doJoin(other, joiner, named, materializedInternal, true, false);
}
@SuppressWarnings("unchecked")
private KTable doJoin(final KTable other,
final ValueJoiner joiner,
final Named joinName,
final MaterializedInternal> materializedInternal,
final boolean leftOuter,
final boolean rightOuter) {
Objects.requireNonNull(other, "other can't be null");
Objects.requireNonNull(joiner, "joiner can't be null");
Objects.requireNonNull(joinName, "joinName can't be null");
final NamedInternal renamed = new NamedInternal(joinName);
final String joinMergeName = renamed.orElseGenerateWithPrefix(builder, MERGE_NAME);
final Set allSourceNodes = ensureCopartitionWith(Collections.singleton((AbstractStream) other));
if (leftOuter) {
enableSendingOldValues(true);
}
if (rightOuter) {
((KTableImpl) other).enableSendingOldValues(true);
}
final KTableKTableAbstractJoin joinThis;
final KTableKTableAbstractJoin joinOther;
if (!leftOuter) { // inner
joinThis = new KTableKTableInnerJoin<>(this, (KTableImpl) other, joiner);
joinOther = new KTableKTableInnerJoin<>((KTableImpl) other, this, reverseJoiner(joiner));
} else if (!rightOuter) { // left
joinThis = new KTableKTableLeftJoin<>(this, (KTableImpl) other, joiner);
joinOther = new KTableKTableRightJoin<>((KTableImpl) other, this, reverseJoiner(joiner));
} else { // outer
joinThis = new KTableKTableOuterJoin<>(this, (KTableImpl) other, joiner);
joinOther = new KTableKTableOuterJoin<>((KTableImpl) other, this, reverseJoiner(joiner));
}
final String joinThisName = renamed.suffixWithOrElseGet("-join-this", builder, JOINTHIS_NAME);
final String joinOtherName = renamed.suffixWithOrElseGet("-join-other", builder, JOINOTHER_NAME);
final ProcessorParameters, ?, ?> joinThisProcessorParameters = new ProcessorParameters<>(joinThis, joinThisName);
final ProcessorParameters, ?, ?> joinOtherProcessorParameters = new ProcessorParameters<>(joinOther, joinOtherName);
final Serde keySerde;
final Serde valueSerde;
final String queryableStoreName;
final StoreBuilder> storeBuilder;
if (materializedInternal != null) {
if (materializedInternal.keySerde() == null) {
materializedInternal.withKeySerde(this.keySerde);
}
keySerde = materializedInternal.keySerde();
valueSerde = materializedInternal.valueSerde();
queryableStoreName = materializedInternal.storeName();
storeBuilder = new TimestampedKeyValueStoreMaterializer<>(materializedInternal).materialize();
} else {
keySerde = this.keySerde;
valueSerde = null;
queryableStoreName = null;
storeBuilder = null;
}
final KTableKTableJoinNode kTableKTableJoinNode =
KTableKTableJoinNode.kTableKTableJoinNodeBuilder()
.withNodeName(joinMergeName)
.withJoinThisProcessorParameters(joinThisProcessorParameters)
.withJoinOtherProcessorParameters(joinOtherProcessorParameters)
.withThisJoinSideNodeName(name)
.withOtherJoinSideNodeName(((KTableImpl) other).name)
.withJoinThisStoreNames(valueGetterSupplier().storeNames())
.withJoinOtherStoreNames(((KTableImpl) other).valueGetterSupplier().storeNames())
.withKeySerde(keySerde)
.withValueSerde(valueSerde)
.withQueryableStoreName(queryableStoreName)
.withStoreBuilder(storeBuilder)
.build();
builder.addGraphNode(this.graphNode, kTableKTableJoinNode);
// we can inherit parent key serde if user do not provide specific overrides
return new KTableImpl, VR>(
kTableKTableJoinNode.nodeName(),
kTableKTableJoinNode.keySerde(),
kTableKTableJoinNode.valueSerde(),
allSourceNodes,
kTableKTableJoinNode.queryableStoreName(),
kTableKTableJoinNode.joinMerger(),
kTableKTableJoinNode,
builder
);
}
@Override
public KGroupedTable groupBy(final KeyValueMapper> selector) {
return groupBy(selector, Grouped.with(null, null));
}
@Override
public KGroupedTable groupBy(final KeyValueMapper> selector,
final Grouped grouped) {
Objects.requireNonNull(selector, "selector can't be null");
Objects.requireNonNull(grouped, "grouped can't be null");
final GroupedInternal groupedInternal = new GroupedInternal<>(grouped);
final String selectName = new NamedInternal(groupedInternal.name()).orElseGenerateWithPrefix(builder, SELECT_NAME);
final KTableProcessorSupplier> selectSupplier = new KTableRepartitionMap<>(this, selector);
final ProcessorParameters, ?, ?> processorParameters = new ProcessorParameters<>(selectSupplier, selectName);
// select the aggregate key and values (old and new), it would require parent to send old values
final ProcessorGraphNode> groupByMapNode = new ProcessorGraphNode<>(selectName, processorParameters);
builder.addGraphNode(this.graphNode, groupByMapNode);
this.enableSendingOldValues(true);
return new KGroupedTableImpl<>(
builder,
selectName,
subTopologySourceNodes,
groupedInternal,
groupByMapNode
);
}
@SuppressWarnings("unchecked")
public KTableValueGetterSupplier valueGetterSupplier() {
if (processorSupplier instanceof KTableSource) {
final KTableSource source = (KTableSource) processorSupplier;
// whenever a source ktable is required for getter, it should be materialized
source.materialize();
return new KTableSourceValueGetterSupplier<>(source.queryableName());
} else if (processorSupplier instanceof KStreamAggProcessorSupplier) {
return ((KStreamAggProcessorSupplier) processorSupplier).view();
} else if (processorSupplier instanceof KTableNewProcessorSupplier) {
return ((KTableNewProcessorSupplier) processorSupplier).view();
} else {
return ((KTableProcessorSupplier) processorSupplier).view();
}
}
@SuppressWarnings("unchecked")
public boolean enableSendingOldValues(final boolean forceMaterialization) {
if (!sendOldValues) {
if (processorSupplier instanceof KTableSource) {
final KTableSource source = (KTableSource) processorSupplier;
if (!forceMaterialization && !source.materialized()) {
return false;
}
source.enableSendingOldValues();
} else if (processorSupplier instanceof KStreamAggProcessorSupplier) {
((KStreamAggProcessorSupplier) processorSupplier).enableSendingOldValues();
} else if (processorSupplier instanceof KTableNewProcessorSupplier) {
final KTableNewProcessorSupplier tableProcessorSupplier =
(KTableNewProcessorSupplier) processorSupplier;
if (!tableProcessorSupplier.enableSendingOldValues(forceMaterialization)) {
return false;
}
} else {
final KTableProcessorSupplier tableProcessorSupplier = (KTableProcessorSupplier) processorSupplier;
if (!tableProcessorSupplier.enableSendingOldValues(forceMaterialization)) {
return false;
}
}
sendOldValues = true;
}
return true;
}
boolean sendingOldValueEnabled() {
return sendOldValues;
}
/**
* We conflate V with Change in many places. This will get fixed in the implementation of KIP-478.
* For now, I'm just explicitly lying about the parameterized type.
*/
@SuppressWarnings("unchecked")
private ProcessorParameters unsafeCastProcessorParametersToCompletelyDifferentType(final ProcessorParameters, ?, ?> kObjectProcessorParameters) {
return (ProcessorParameters) kObjectProcessorParameters;
}
@Override
public KTable join(final KTable other,
final Function foreignKeyExtractor,
final ValueJoiner joiner) {
return doJoinOnForeignKey(
other,
foreignKeyExtractor,
joiner,
NamedInternal.empty(),
Materialized.with(null, null),
false
);
}
@Override
public KTable join(final KTable other,
final Function foreignKeyExtractor,
final ValueJoiner joiner,
final Named named) {
return doJoinOnForeignKey(
other,
foreignKeyExtractor,
joiner,
named,
Materialized.with(null, null),
false
);
}
@Override
public KTable join(final KTable other,
final Function foreignKeyExtractor,
final ValueJoiner joiner,
final Materialized> materialized) {
return doJoinOnForeignKey(other, foreignKeyExtractor, joiner, NamedInternal.empty(), materialized, false);
}
@Override
public KTable join(final KTable other,
final Function foreignKeyExtractor,
final ValueJoiner joiner,
final Named named,
final Materialized> materialized) {
return doJoinOnForeignKey(other, foreignKeyExtractor, joiner, named, materialized, false);
}
@Override
public KTable leftJoin(final KTable other,
final Function foreignKeyExtractor,
final ValueJoiner joiner) {
return doJoinOnForeignKey(
other,
foreignKeyExtractor,
joiner,
NamedInternal.empty(),
Materialized.with(null, null),
true
);
}
@Override
public KTable leftJoin(final KTable other,
final Function foreignKeyExtractor,
final ValueJoiner joiner,
final Named named) {
return doJoinOnForeignKey(
other,
foreignKeyExtractor,
joiner,
named,
Materialized.with(null, null),
true
);
}
@Override
public KTable leftJoin(final KTable other,
final Function foreignKeyExtractor,
final ValueJoiner joiner,
final Named named,
final Materialized> materialized) {
return doJoinOnForeignKey(other, foreignKeyExtractor, joiner, named, materialized, true);
}
@Override
public KTable leftJoin(final KTable other,
final Function foreignKeyExtractor,
final ValueJoiner joiner,
final Materialized> materialized) {
return doJoinOnForeignKey(other, foreignKeyExtractor, joiner, NamedInternal.empty(), materialized, true);
}
@SuppressWarnings("unchecked")
private KTable doJoinOnForeignKey(final KTable foreignKeyTable,
final Function foreignKeyExtractor,
final ValueJoiner joiner,
final Named joinName,
final Materialized> materialized,
final boolean leftJoin) {
Objects.requireNonNull(foreignKeyTable, "foreignKeyTable can't be null");
Objects.requireNonNull(foreignKeyExtractor, "foreignKeyExtractor can't be null");
Objects.requireNonNull(joiner, "joiner can't be null");
Objects.requireNonNull(joinName, "joinName can't be null");
Objects.requireNonNull(materialized, "materialized can't be null");
//Old values are a useful optimization. The old values from the foreignKeyTable table are compared to the new values,
//such that identical values do not cause a prefixScan. PrefixScan and propagation can be expensive and should
//not be done needlessly.
((KTableImpl) foreignKeyTable).enableSendingOldValues(true);
//Old values must be sent such that the ForeignJoinSubscriptionSendProcessorSupplier can propagate deletions to the correct node.
//This occurs whenever the extracted foreignKey changes values.
enableSendingOldValues(true);
final NamedInternal renamed = new NamedInternal(joinName);
final String subscriptionTopicName = renamed.suffixWithOrElseGet(
"-subscription-registration",
builder,
SUBSCRIPTION_REGISTRATION
) + TOPIC_SUFFIX;
// the decoration can't be performed until we have the configuration available when the app runs,
// so we pass Suppliers into the components, which they can call at run time
final Supplier subscriptionPrimaryKeySerdePseudoTopic =
() -> internalTopologyBuilder().decoratePseudoTopic(subscriptionTopicName + "-pk");
final Supplier subscriptionForeignKeySerdePseudoTopic =
() -> internalTopologyBuilder().decoratePseudoTopic(subscriptionTopicName + "-fk");
final Supplier valueHashSerdePseudoTopic =
() -> internalTopologyBuilder().decoratePseudoTopic(subscriptionTopicName + "-vh");
builder.internalTopologyBuilder.addInternalTopic(subscriptionTopicName, InternalTopicProperties.empty());
final Serde foreignKeySerde = ((KTableImpl) foreignKeyTable).keySerde;
final Serde> subscriptionWrapperSerde = new SubscriptionWrapperSerde<>(subscriptionPrimaryKeySerdePseudoTopic, keySerde);
final SubscriptionResponseWrapperSerde responseWrapperSerde =
new SubscriptionResponseWrapperSerde<>(((KTableImpl) foreignKeyTable).valueSerde);
final CombinedKeySchema combinedKeySchema = new CombinedKeySchema<>(
subscriptionForeignKeySerdePseudoTopic,
foreignKeySerde,
subscriptionPrimaryKeySerdePseudoTopic,
keySerde
);
final ProcessorGraphNode
