io.dstream.DStream Maven / Gradle / Ivy
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.dstream;
import java.util.stream.Stream;
import io.dstream.DStream.DStream2.DStream2WithPredicate;
import io.dstream.DStream.DStream3.DStream3WithPredicate;
import io.dstream.DStream.DStream4.DStream4WithPredicate;
import io.dstream.DStream.DStream5.DStream5WithPredicate;
import io.dstream.DStream.DStream6.DStream6WithPredicate;
import io.dstream.SerializableStreamAssets.SerPredicate;
import io.dstream.utils.Assert;
/**
* A type-safe {@link Stream}-like strategy that exposes distributable data as the sequence of
* elements that support sequential and parallel aggregate operations.
*
* Below is the example of rudimentary Word Count written using the API:
*
* DStream.ofType(String.class, "wc")
* .flatMap(line -> Stream.of(line.split("\\s+")))
* .reduceValues(word -> word, word -> 1, Integer::sum)
* .executeAs("WordCount");
*
*
* It defines operations with the following classifications:
*
* -
* intermediate - similar to intermediate operations
* of Java {@link Stream}.
*
* -
* terminal - similar to terminal operations
* of Java {@link Stream}.
*
* -
* shuffle - an operation which typically results in synchronization (typically data is written to shuffle output).
*
* -
* composable-shuffle - an operation which typically results in synchronization if previous operation is composable-transformation.
* However, if previous operation is composable-shuffle, each subsequent operation that is also composable-shuffle could be composed
* into a single operation.
*
* -
* composable-transformation - an operation that defines a simple transformation (e.g., map, flatMap, filter etc.) and could be composed
* with subsequent operations of the same type.
*
*
*
* @param the type of the stream element
*/
public interface DStream extends BaseDStream> {
/**
* Factory method which creates an instance of the {@code DStream} of type A.
*
* @param sourceElementType the type of the elements of this stream
* @param name the value which represents the name of this stream and is also used
* in configuration to resolve stream specific configuration values.
* For example, one such configuration properties is the source of this stream
* (e.g., dstream.source.foo=file://foo.txt where 'foo' is the name).
* See {@link DStreamConstants#SOURCE} for more details.
* Must be unique!. If you simply want to point to the same source, map it
* through configuration (e.g., dstream.source.foo=file://foo.txt, dstream.source.bar=file://foo.txt)
* @return the new {@link DStream} of type A
*
*/
@SuppressWarnings("unchecked")
public static DStream ofType(Class sourceElementType, String name) {
Assert.notNull(sourceElementType, "'sourceElementType' must not be null");
Assert.notEmpty(name, "'name' must not be null or empty");
return DStreamExecutionGraphsBuilder.as(sourceElementType, name, DStream.class);
}
/**
* Will join two {@link DStream}s returning an instance of {@link DStream2}.
*
* The actual instance of returned {@link DStream2} will be {@link DStream2WithPredicate}
* allowing predicate to be provided via {@link DStream2WithPredicate#on(SerPredicate)} operation.
*
* This is an intermediate operation.
*
* This is a composable-shuffle operation.
*
* @param ds stream that is joined with this stream
* @return new {@link DStream2} representing the result of the join.
*/
<_A> DStream2WithPredicate join(DStream<_A> ds);
/**
* Will join {@link DStream} with {@link DStream2} returning an instance of {@link DStream3}.
*
* The actual instance of returned {@link DStream3} will be {@link DStream3WithPredicate}
* allowing predicate to be provided via {@link DStream3WithPredicate#on(SerPredicate)} operation.
*
* This is an intermediate operation.
*
* This is a composable-shuffle operation.
*
* @param ds stream that is joined with this stream
* @return new {@link DStream3} representing the result of the join.
*/
<_A,_B> DStream3WithPredicate join(DStream2<_A,_B> ds);
/**
* Will join {@link DStream} with {@link DStream3} returning an instance of {@link DStream4}.
*
* The actual instance of returned {@link DStream4} will be {@link DStream4WithPredicate}
* allowing predicate to be provided via {@link DStream4WithPredicate#on(SerPredicate)} operation.
*
* This is an intermediate operation.
*
* This is a composable-shuffle operation.
*
* @param ds stream that is joined with this stream
* @return new {@link DStream4} representing the result of the join.
*/
<_A,_B,_C> DStream4WithPredicate join(DStream3<_A,_B,_C> ds);
/**
* Will join {@link DStream} with {@link DStream4} returning an instance of {@link DStream5}.
*
* The actual instance of returned {@link DStream5} will be {@link DStream5WithPredicate}
* allowing predicate to be provided via {@link DStream5WithPredicate#on(SerPredicate)} operation.
*
* This is an intermediate operation.
*
* This is a composable-shuffle operation.
*
* @param ds stream that is joined with this stream
* @return new {@link DStream5} representing the result of the join.
*/
<_A,_B,_C,_D> DStream5WithPredicate join(DStream4<_A,_B,_C,_D> ds);
/**
* Will join {@link DStream} with {@link DStream5} returning an instance of {@link DStream6}.
*
* The actual instance of returned {@link DStream6} will be {@link DStream6WithPredicate}
* allowing predicate to be provided via {@link DStream6WithPredicate#on(SerPredicate)} operation.
*
* This is an intermediate operation.
*
* This is a composable-shuffle operation.
*
* @param ds stream that is joined with this stream
* @return new {@link DStream6} representing the result of the join.
*/
<_A,_B,_C,_D,_E> DStream6WithPredicate join(DStream5<_A,_B,_C,_D,_E> ds);
/**
* Strategy which defines the type of {@link DStream} that contains two types of elements.
*
* @param - first element type
* @param - second element type
*/
interface DStream2 extends BaseDStream, DStream2> {
/**
* Strategy which defines the type of {@link DStream2} that allows for predicate to be applied.
*
* @param - first element type
* @param - second element type
*/
public interface DStream2WithPredicate extends DStream2{
DStream2 on(SerPredicate> predicate);
}
/**
* Will join {@link DStream2} with {@link DStream} returning an instance of {@link DStream3}.
*
* The actual instance of returned {@link DStream3} will be {@link DStream3WithPredicate}
* allowing predicate to be provided via {@link DStream3WithPredicate#on(SerPredicate)} operation.
*
* This is an intermediate operation.
*
* This is a composable-shuffle operation.
*
* @param ds stream that is joined with this stream
* @return new {@link DStream3} representing the result of the join.
*/
<_A> DStream3WithPredicate join(DStream<_A> ds);
/**
* Will join {@link DStream2} with {@link DStream2} returning an instance of {@link DStream4}.
*
* The actual instance of returned {@link DStream4} will be {@link DStream4WithPredicate}
* allowing predicate to be provided via {@link DStream4WithPredicate#on(SerPredicate)} operation.
*
* This is an intermediate operation.
*
* This is a composable-shuffle operation.
*
* @param ds stream that is joined with this stream
* @return new {@link DStream4} representing the result of the join.
*/
<_A,_B> DStream4WithPredicate join(DStream2<_A,_B> ds);
/**
* Will join {@link DStream2} with {@link DStream3} returning an instance of {@link DStream5}.
*
* The actual instance of returned {@link DStream5} will be {@link DStream5WithPredicate}
* allowing predicate to be provided via {@link DStream5WithPredicate#on(SerPredicate)} operation.
*
* This is an intermediate operation.
*
* This is a composable-shuffle operation.
*
* @param ds stream that is joined with this stream
* @return new {@link DStream5} representing the result of the join.
*/
<_A,_B,_C> DStream5WithPredicate join(DStream3<_A,_B,_C> ds);
/**
* Will join {@link DStream2} with {@link DStream4} returning an instance of {@link DStream6}.
*
* The actual instance of returned {@link DStream6} will be {@link DStream6WithPredicate}
* allowing predicate to be provided via {@link DStream6WithPredicate#on(SerPredicate)} operation.
*
* This is an intermediate operation.
*
* This is a composable-shuffle operation.
*
* @param ds stream that is joined with this stream
* @return new {@link DStream6} representing the result of the join.
*/
<_A,_B,_C,_D> DStream6WithPredicate join(DStream4<_A,_B,_C,_D> ds);
}
/**
* Strategy which defines the type of {@link DStream} that contains three types of elements.
*
* @param - first element type
* @param - second element type
* @param - third element type
*/
interface DStream3 extends BaseDStream, DStream3> {
/**
* Strategy which defines the type of {@link DStream3} that allows for predicate to be applied.
*
* @param - first element type
* @param - second element type
* @param - third element type
*/
interface DStream3WithPredicate extends DStream3{
DStream3 on(SerPredicate> predicate);
}
/**
* Will join {@link DStream3} with {@link DStream} returning an instance of {@link DStream4}.
*
* The actual instance of returned {@link DStream4} will be {@link DStream4WithPredicate}
* allowing predicate to be provided via {@link DStream4WithPredicate#on(SerPredicate)} operation.
*
* This is an intermediate operation.
*
* This is a composable-shuffle operation.
*
* @param ds stream that is joined with this stream
* @return new {@link DStream4} representing the result of the join.
*/
<_A> DStream4WithPredicate join(DStream<_A> ds);
/**
* Will join {@link DStream3} with {@link DStream2} returning an instance of {@link DStream5}.
*
* The actual instance of returned {@link DStream5} will be {@link DStream5WithPredicate}
* allowing predicate to be provided via {@link DStream5WithPredicate#on(SerPredicate)} operation.
*
* This is an intermediate operation.
*
* This is a composable-shuffle operation.
*
* @param ds stream that is joined with this stream
* @return new {@link DStream5} representing the result of the join.
*/
<_A,_B> DStream5WithPredicate join(DStream2<_A,_B> ds);
/**
* Will join {@link DStream3} with {@link DStream3} returning an instance of {@link DStream6}.
*
* The actual instance of returned {@link DStream6} will be {@link DStream6WithPredicate}
* allowing predicate to be provided via {@link DStream6WithPredicate#on(SerPredicate)} operation.
*
* This is an intermediate operation.
*
* This is a composable-shuffle operation.
*
* @param ds stream that is joined with this stream
* @return new {@link DStream6} representing the result of the join.
*/
<_A,_B,_C> DStream6WithPredicate join(DStream3<_A,_B,_C> ds);
}
/**
* Strategy which defines the type of {@link DStream} that contains four types of elements.
*
* @param - first element type
* @param - second element type
* @param - third element type
* @param - fourth element type
*/
interface DStream4 extends BaseDStream, DStream4> {
/**
* Strategy which defines the type of {@link DStream4} that allows for predicate to be applied.
*
* @param - first element type
* @param - second element type
* @param - third element type
* @param - fourth element type
*/
interface DStream4WithPredicate extends DStream4{
DStream4 on(SerPredicate> predicate);
}
/**
* Will join {@link DStream4} with {@link DStream} returning an instance of {@link DStream5}.
*
* The actual instance of returned {@link DStream5} will be {@link DStream5WithPredicate}
* allowing predicate to be provided via {@link DStream5WithPredicate#on(SerPredicate)} operation.
*
* This is an intermediate operation.
*
* This is a composable-shuffle operation.
*
* @param ds stream that is joined with this stream
* @return new {@link DStream5} representing the result of the join.
*/
<_A> DStream5WithPredicate join(DStream<_A> ds);
/**
* Will join {@link DStream4} with {@link DStream2} returning an instance of {@link DStream6}.
*
* The actual instance of returned {@link DStream6} will be {@link DStream6WithPredicate}
* allowing predicate to be provided via {@link DStream6WithPredicate#on(SerPredicate)} operation.
*
* This is an intermediate operation.
*
* This is a composable-shuffle operation.
*
* @param ds stream that is joined with this stream
* @return new {@link DStream6} representing the result of the join.
*/
<_A,_B> DStream6WithPredicate join(DStream2<_A,_B> ds);
}
/**
* Strategy which defines the type of {@link DStream} that contains five types of elements.
*
* @param - first element type
* @param - second element type
* @param - third element type
* @param - fourth element type
* @param - fifth element type
*/
interface DStream5 extends BaseDStream, DStream5> {
/**
* Strategy which defines the type of {@link DStream5} that allows for predicate to be applied.
*
* @param - first element type
* @param - second element type
* @param - third element type
* @param - fourth element type
* @param - fifth element type
*/
interface DStream5WithPredicate extends DStream5{
DStream5 on(SerPredicate> predicate);
}
/**
* Will join {@link DStream5} with {@link DStream} returning an instance of {@link DStream6}.
*
* The actual instance of returned {@link DStream6} will be {@link DStream6WithPredicate}
* allowing predicate to be provided via {@link DStream6WithPredicate#on(SerPredicate)} operation.
*
* This is an intermediate operation.
*
* This is a composable-shuffle operation.
*
* @param ds stream that is joined with this stream
* @return new {@link DStream6} representing the result of the join.
*/
<_A> DStream6WithPredicate join(DStream<_A> ds);
}
/**
* Strategy which defines the type of {@link DStream} that contains six
* types of elements.
*
* @param - first element type
* @param - second element type
* @param - third element type
* @param - fourth element type
* @param - fifth element type
* @param - sixth element type
*/
interface DStream6 extends BaseDStream, DStream6> {
/**
* Strategy which defines the type of {@link DStream5} that allows for predicate to be applied.
*
* @param - first element type
* @param - second element type
* @param - third element type
* @param - fourth element type
* @param - fifth element type
* @param - sixth element type
*/
interface DStream6WithPredicate extends DStream6{
DStream6 on(SerPredicate> predicate);
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy