org.apache.kafka.streams.kstream.internals.KTableImpl Maven / Gradle / Ivy
The newest version!
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kafka.streams.kstream.internals;
import org.apache.kafka.common.serialization.Serde;
import org.apache.kafka.streams.KeyValue;
import org.apache.kafka.streams.errors.TopologyBuilderException;
import org.apache.kafka.streams.kstream.ForeachAction;
import org.apache.kafka.streams.kstream.PrintForeachAction;
import org.apache.kafka.streams.kstream.KGroupedTable;
import org.apache.kafka.streams.kstream.KStream;
import org.apache.kafka.streams.kstream.KStreamBuilder;
import org.apache.kafka.streams.kstream.KTable;
import org.apache.kafka.streams.kstream.KeyValueMapper;
import org.apache.kafka.streams.kstream.Predicate;
import org.apache.kafka.streams.kstream.ValueJoiner;
import org.apache.kafka.streams.kstream.ValueMapper;
import org.apache.kafka.streams.processor.ProcessorSupplier;
import org.apache.kafka.streams.processor.StateStoreSupplier;
import org.apache.kafka.streams.processor.StreamPartitioner;
import org.apache.kafka.streams.state.KeyValueStore;
import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.util.Objects;
import java.util.Set;
/**
* The implementation class of {@link KTable}.
*
* @param the key type
* @param the source's (parent's) value type
* @param the value type
*/
public class KTableImpl extends AbstractStream implements KTable {
private static final String FILTER_NAME = "KTABLE-FILTER-";
private static final String FOREACH_NAME = "KTABLE-FOREACH-";
public static final String JOINTHIS_NAME = "KTABLE-JOINTHIS-";
public static final String JOINOTHER_NAME = "KTABLE-JOINOTHER-";
private static final String MAPVALUES_NAME = "KTABLE-MAPVALUES-";
public static final String MERGE_NAME = "KTABLE-MERGE-";
private static final String PRINTING_NAME = "KSTREAM-PRINTER-";
private static final String SELECT_NAME = "KTABLE-SELECT-";
public static final String SOURCE_NAME = "KTABLE-SOURCE-";
private static final String TOSTREAM_NAME = "KTABLE-TOSTREAM-";
public static final String STATE_STORE_NAME = "STATE-STORE-";
private final ProcessorSupplier processorSupplier;
private final String queryableStoreName;
private final boolean isQueryable;
private boolean sendOldValues = false;
private final Serde keySerde;
private final Serde valSerde;
public KTableImpl(KStreamBuilder topology,
String name,
ProcessorSupplier processorSupplier,
Set sourceNodes,
final String queryableStoreName,
boolean isQueryable) {
super(topology, name, sourceNodes);
this.processorSupplier = processorSupplier;
this.queryableStoreName = queryableStoreName;
this.keySerde = null;
this.valSerde = null;
this.isQueryable = isQueryable;
}
public KTableImpl(KStreamBuilder topology,
String name,
ProcessorSupplier processorSupplier,
final Serde keySerde,
final Serde valSerde,
Set sourceNodes,
final String queryableStoreName,
boolean isQueryable) {
super(topology, name, sourceNodes);
this.processorSupplier = processorSupplier;
this.queryableStoreName = queryableStoreName;
this.keySerde = keySerde;
this.valSerde = valSerde;
this.isQueryable = isQueryable;
}
@Override
public String queryableStoreName() {
if (!isQueryable) {
return null;
}
return this.queryableStoreName;
}
String internalStoreName() {
return this.queryableStoreName;
}
private KTable doFilter(final Predicate predicate,
final StateStoreSupplier storeSupplier,
boolean isFilterNot) {
Objects.requireNonNull(predicate, "predicate can't be null");
String name = topology.newName(FILTER_NAME);
String internalStoreName = null;
if (storeSupplier != null) {
internalStoreName = storeSupplier.name();
}
KTableProcessorSupplier processorSupplier = new KTableFilter<>(this, predicate, isFilterNot, internalStoreName);
topology.addProcessor(name, processorSupplier, this.name);
if (storeSupplier != null) {
topology.addStateStore(storeSupplier, name);
}
return new KTableImpl<>(topology, name, processorSupplier, this.keySerde, this.valSerde, sourceNodes, internalStoreName, internalStoreName != null);
}
@Override
public KTable filter(final Predicate predicate) {
return filter(predicate, (String) null);
}
@Override
public KTable filter(final Predicate predicate, final String queryableStoreName) {
StateStoreSupplier storeSupplier = null;
if (queryableStoreName != null) {
storeSupplier = keyValueStore(this.keySerde, this.valSerde, queryableStoreName);
}
return doFilter(predicate, storeSupplier, false);
}
@Override
public KTable filter(final Predicate predicate, final StateStoreSupplier storeSupplier) {
Objects.requireNonNull(storeSupplier, "storeSupplier can't be null");
return doFilter(predicate, storeSupplier, false);
}
@Override
public KTable filterNot(final Predicate predicate) {
return filterNot(predicate, (String) null);
}
@Override
public KTable filterNot(final Predicate predicate, final String queryableStoreName) {
StateStoreSupplier storeSupplier = null;
if (queryableStoreName != null) {
storeSupplier = keyValueStore(this.keySerde, this.valSerde, queryableStoreName);
}
return doFilter(predicate, storeSupplier, true);
}
@Override
public KTable filterNot(final Predicate predicate, final StateStoreSupplier storeSupplier) {
Objects.requireNonNull(storeSupplier, "storeSupplier can't be null");
return doFilter(predicate, storeSupplier, true);
}
private KTable doMapValues(final ValueMapper mapper,
final Serde valueSerde,
final StateStoreSupplier storeSupplier) {
Objects.requireNonNull(mapper);
String name = topology.newName(MAPVALUES_NAME);
String internalStoreName = null;
if (storeSupplier != null) {
internalStoreName = storeSupplier.name();
}
KTableProcessorSupplier processorSupplier = new KTableMapValues<>(this, mapper, internalStoreName);
topology.addProcessor(name, processorSupplier, this.name);
if (storeSupplier != null) {
topology.addStateStore(storeSupplier, name);
return new KTableImpl<>(topology, name, processorSupplier, this.keySerde, valueSerde, sourceNodes, internalStoreName, true);
} else {
return new KTableImpl<>(topology, name, processorSupplier, sourceNodes, this.queryableStoreName, false);
}
}
@Override
public KTable mapValues(final ValueMapper mapper) {
return mapValues(mapper, null, (String) null);
}
@Override
public KTable mapValues(final ValueMapper mapper,
final Serde valueSerde,
final String queryableStoreName) {
StateStoreSupplier storeSupplier = null;
if (queryableStoreName != null) {
storeSupplier = keyValueStore(this.keySerde, valueSerde, queryableStoreName);
}
return doMapValues(mapper, valueSerde, storeSupplier);
}
@Override
public KTable mapValues(final ValueMapper mapper,
final Serde valueSerde,
final StateStoreSupplier storeSupplier) {
Objects.requireNonNull(storeSupplier, "storeSupplier can't be null");
return doMapValues(mapper, valueSerde, storeSupplier);
}
@Override
public void print() {
print(null, null, null);
}
@Override
public void print(String streamName) {
print(null, null, streamName);
}
@Override
public void print(Serde keySerde, Serde valSerde) {
print(keySerde, valSerde, null);
}
@Override
public void print(Serde keySerde, Serde valSerde, String streamName) {
String name = topology.newName(PRINTING_NAME);
streamName = (streamName == null) ? this.name : streamName;
topology.addProcessor(name, new KStreamPrint<>(new PrintForeachAction(null, streamName), keySerde, valSerde), this.name);
}
@Override
public void writeAsText(String filePath) {
writeAsText(filePath, null, null, null);
}
@Override
public void writeAsText(String filePath, String streamName) {
writeAsText(filePath, streamName, null, null);
}
@Override
public void writeAsText(String filePath, Serde keySerde, Serde valSerde) {
writeAsText(filePath, null, keySerde, valSerde);
}
/**
* @throws TopologyBuilderException if file is not found
*/
@Override
public void writeAsText(String filePath, String streamName, Serde keySerde, Serde valSerde) {
Objects.requireNonNull(filePath, "filePath can't be null");
if (filePath.trim().isEmpty()) {
throw new TopologyBuilderException("filePath can't be an empty string");
}
String name = topology.newName(PRINTING_NAME);
streamName = (streamName == null) ? this.name : streamName;
try {
PrintWriter printWriter = new PrintWriter(filePath, StandardCharsets.UTF_8.name());
topology.addProcessor(name, new KStreamPrint<>(new PrintForeachAction(printWriter, streamName), keySerde, valSerde), this.name);
} catch (FileNotFoundException | UnsupportedEncodingException e) {
String message = "Unable to write stream to file at [" + filePath + "] " + e.getMessage();
throw new TopologyBuilderException(message);
}
}
@Override
public void foreach(final ForeachAction action) {
Objects.requireNonNull(action, "action can't be null");
String name = topology.newName(FOREACH_NAME);
KStreamPeek> processorSupplier = new KStreamPeek<>(new ForeachAction>() {
@Override
public void apply(K key, Change value) {
action.apply(key, value.newValue);
}
}, false);
topology.addProcessor(name, processorSupplier, this.name);
}
@Override
public KTable through(final Serde keySerde,
final Serde valSerde,
final StreamPartitioner partitioner,
final String topic,
final String queryableStoreName) {
final String internalStoreName = queryableStoreName != null ? queryableStoreName : topology.newStoreName(KTableImpl.TOSTREAM_NAME);
to(keySerde, valSerde, partitioner, topic);
return topology.table(keySerde, valSerde, topic, internalStoreName);
}
@Override
public KTable through(final Serde keySerde,
final Serde valSerde,
final StreamPartitioner partitioner,
final String topic,
final StateStoreSupplier storeSupplier) {
Objects.requireNonNull(storeSupplier, "storeSupplier can't be null");
to(keySerde, valSerde, partitioner, topic);
return topology.table(keySerde, valSerde, topic, storeSupplier);
}
@Override
public KTable through(final Serde keySerde,
final Serde valSerde,
final StreamPartitioner partitioner,
final String topic) {
return through(keySerde, valSerde, partitioner, topic, (String) null);
}
@Override
public KTable through(final Serde keySerde,
final Serde valSerde,
final String topic,
final String queryableStoreName) {
return through(keySerde, valSerde, null, topic, queryableStoreName);
}
@Override
public KTable through(final Serde keySerde,
final Serde valSerde,
final String topic,
final StateStoreSupplier storeSupplier) {
Objects.requireNonNull(storeSupplier, "storeSupplier can't be null");
return through(keySerde, valSerde, null, topic, storeSupplier);
}
@Override
public KTable through(final Serde keySerde,
final Serde valSerde,
final String topic) {
return through(keySerde, valSerde, null, topic, (String) null);
}
@Override
public KTable through(final StreamPartitioner partitioner,
final String topic,
final String queryableStoreName) {
return through(null, null, partitioner, topic, queryableStoreName);
}
@Override
public KTable through(final StreamPartitioner partitioner,
final String topic,
final StateStoreSupplier storeSupplier) {
Objects.requireNonNull(storeSupplier, "storeSupplier can't be null");
return through(null, null, partitioner, topic, storeSupplier);
}
@Override
public KTable through(final StreamPartitioner partitioner,
final String topic) {
return through(null, null, partitioner, topic, (String) null);
}
@Override
public KTable through(final String topic,
final String queryableStoreName) {
return through(null, null, null, topic, queryableStoreName);
}
@Override
public KTable through(final String topic,
final StateStoreSupplier storeSupplier) {
Objects.requireNonNull(storeSupplier, "storeSupplier can't be null");
return through(null, null, null, topic, storeSupplier);
}
@Override
public KTable through(final String topic) {
return through(null, null, null, topic, (String) null);
}
@Override
public void to(String topic) {
to(null, null, null, topic);
}
@Override
public void to(StreamPartitioner partitioner, String topic) {
to(null, null, partitioner, topic);
}
@Override
public void to(Serde keySerde, Serde valSerde, String topic) {
this.toStream().to(keySerde, valSerde, null, topic);
}
@Override
public void to(Serde keySerde, Serde valSerde, StreamPartitioner partitioner, String topic) {
this.toStream().to(keySerde, valSerde, partitioner, topic);
}
@Override
public KStream toStream() {
String name = topology.newName(TOSTREAM_NAME);
topology.addProcessor(name, new KStreamMapValues, V>(new ValueMapper, V>() {
@Override
public V apply(Change change) {
return change.newValue;
}
}), this.name);
return new KStreamImpl<>(topology, name, sourceNodes, false);
}
@Override
public KStream toStream(KeyValueMapper mapper) {
return toStream().selectKey(mapper);
}
@Override
public KTable join(final KTable other,
final ValueJoiner joiner) {
return doJoin(other, joiner, false, false, null, (String) null);
}
@Override
public KTable join(final KTable other,
final ValueJoiner joiner,
final Serde joinSerde,
final String queryableStoreName) {
return doJoin(other, joiner, false, false, joinSerde, queryableStoreName);
}
@Override
public KTable join(final KTable other,
final ValueJoiner joiner,
final StateStoreSupplier storeSupplier) {
Objects.requireNonNull(storeSupplier, "storeSupplier can't be null");
return doJoin(other, joiner, false, false, storeSupplier);
}
@Override
public KTable outerJoin(final KTable other,
final ValueJoiner joiner) {
return doJoin(other, joiner, true, true, null, (String) null);
}
@Override
public KTable outerJoin(final KTable other,
final ValueJoiner joiner,
final Serde joinSerde,
final String queryableStoreName) {
return doJoin(other, joiner, true, true, joinSerde, queryableStoreName);
}
@Override
public KTable outerJoin(final KTable other,
final ValueJoiner joiner,
final StateStoreSupplier storeSupplier) {
Objects.requireNonNull(storeSupplier, "storeSupplier can't be null");
return doJoin(other, joiner, true, true, storeSupplier);
}
@Override
public KTable leftJoin(final KTable other,
final ValueJoiner joiner) {
return doJoin(other, joiner, true, false, null, (String) null);
}
@Override
public KTable leftJoin(final KTable other,
final ValueJoiner joiner,
final Serde joinSerde,
final String queryableStoreName) {
return doJoin(other, joiner, true, false, joinSerde, queryableStoreName);
}
@Override
public KTable leftJoin(final KTable other,
final ValueJoiner joiner,
final StateStoreSupplier storeSupplier) {
Objects.requireNonNull(storeSupplier, "storeSupplier can't be null");
return doJoin(other, joiner, true, false, storeSupplier);
}
@SuppressWarnings("unchecked")
private KTable doJoin(final KTable other,
ValueJoiner joiner,
final boolean leftOuter,
final boolean rightOuter,
final Serde joinSerde,
final String queryableStoreName) {
Objects.requireNonNull(other, "other can't be null");
Objects.requireNonNull(joiner, "joiner can't be null");
final StateStoreSupplier storeSupplier = queryableStoreName == null ? null : keyValueStore(this.keySerde, joinSerde, queryableStoreName);
return doJoin(other, joiner, leftOuter, rightOuter, storeSupplier);
}
private KTable doJoin(final KTable other,
ValueJoiner joiner,
final boolean leftOuter,
final boolean rightOuter,
final StateStoreSupplier storeSupplier) {
Objects.requireNonNull(other, "other can't be null");
Objects.requireNonNull(joiner, "joiner can't be null");
final String internalQueryableName = storeSupplier == null ? null : storeSupplier.name();
final Set allSourceNodes = ensureJoinableWith((AbstractStream) other);
if (leftOuter) {
enableSendingOldValues();
}
if (rightOuter) {
((KTableImpl) other).enableSendingOldValues();
}
final String joinThisName = topology.newName(JOINTHIS_NAME);
final String joinOtherName = topology.newName(JOINOTHER_NAME);
final String joinMergeName = topology.newName(MERGE_NAME);
final KTableKTableAbstractJoin joinThis;
final KTableKTableAbstractJoin joinOther;
if (!leftOuter) { // inner
joinThis = new KTableKTableJoin<>(this, (KTableImpl) other, joiner);
joinOther = new KTableKTableJoin<>((KTableImpl) other, this, reverseJoiner(joiner));
} else if (!rightOuter) { // left
joinThis = new KTableKTableLeftJoin<>(this, (KTableImpl) other, joiner);
joinOther = new KTableKTableRightJoin<>((KTableImpl) other, this, reverseJoiner(joiner));
} else { // outer
joinThis = new KTableKTableOuterJoin<>(this, (KTableImpl) other, joiner);
joinOther = new KTableKTableOuterJoin<>((KTableImpl) other, this, reverseJoiner(joiner));
}
final KTableKTableJoinMerger joinMerge = new KTableKTableJoinMerger<>(
new KTableImpl(topology, joinThisName, joinThis, sourceNodes, this.internalStoreName(), false),
new KTableImpl(topology, joinOtherName, joinOther, ((KTableImpl) other).sourceNodes,
((KTableImpl) other).internalStoreName(), false),
internalQueryableName
);
topology.addProcessor(joinThisName, joinThis, this.name);
topology.addProcessor(joinOtherName, joinOther, ((KTableImpl) other).name);
topology.addProcessor(joinMergeName, joinMerge, joinThisName, joinOtherName);
topology.connectProcessorAndStateStores(joinThisName, ((KTableImpl) other).valueGetterSupplier().storeNames());
topology.connectProcessorAndStateStores(joinOtherName, valueGetterSupplier().storeNames());
if (internalQueryableName != null) {
topology.addStateStore(storeSupplier, joinMergeName);
}
return new KTableImpl<>(topology, joinMergeName, joinMerge, allSourceNodes, internalQueryableName, internalQueryableName != null);
}
@Override
public KGroupedTable groupBy(KeyValueMapper> selector,
Serde keySerde,
Serde valueSerde) {
Objects.requireNonNull(selector, "selector can't be null");
String selectName = topology.newName(SELECT_NAME);
KTableProcessorSupplier> selectSupplier = new KTableRepartitionMap(this, selector);
// select the aggregate key and values (old and new), it would require parent to send old values
topology.addProcessor(selectName, selectSupplier, this.name);
this.enableSendingOldValues();
return new KGroupedTableImpl<>(topology, selectName, this.name, keySerde, valueSerde);
}
@Override
public KGroupedTable groupBy(KeyValueMapper> selector) {
return this.groupBy(selector, null, null);
}
@SuppressWarnings("unchecked")
KTableValueGetterSupplier valueGetterSupplier() {
if (processorSupplier instanceof KTableSource) {
KTableSource source = (KTableSource) processorSupplier;
return new KTableSourceValueGetterSupplier<>(source.storeName);
} else if (processorSupplier instanceof KStreamAggProcessorSupplier) {
return ((KStreamAggProcessorSupplier) processorSupplier).view();
} else {
return ((KTableProcessorSupplier) processorSupplier).view();
}
}
@SuppressWarnings("unchecked")
void enableSendingOldValues() {
if (!sendOldValues) {
if (processorSupplier instanceof KTableSource) {
KTableSource source = (KTableSource) processorSupplier;
source.enableSendingOldValues();
} else if (processorSupplier instanceof KStreamAggProcessorSupplier) {
((KStreamAggProcessorSupplier) processorSupplier).enableSendingOldValues();
} else {
((KTableProcessorSupplier) processorSupplier).enableSendingOldValues();
}
sendOldValues = true;
}
}
boolean sendingOldValueEnabled() {
return sendOldValues;
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy