Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Copyright (c) 2008-2018, Hazelcast, Inc. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hazelcast.jet.impl.pipeline;
import com.hazelcast.jet.core.DAG;
import com.hazelcast.jet.core.Edge;
import com.hazelcast.jet.core.Processor;
import com.hazelcast.jet.core.ProcessorMetaSupplier;
import com.hazelcast.jet.core.ProcessorSupplier;
import com.hazelcast.jet.core.Vertex;
import com.hazelcast.jet.core.WatermarkEmissionPolicy;
import com.hazelcast.jet.function.DistributedSupplier;
import com.hazelcast.jet.impl.pipeline.transform.SinkTransform;
import com.hazelcast.jet.impl.pipeline.transform.StreamSourceTransform;
import com.hazelcast.jet.impl.pipeline.transform.TimestampTransform;
import com.hazelcast.jet.impl.pipeline.transform.Transform;
import com.hazelcast.jet.impl.util.Util;
import javax.annotation.Nonnull;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import static com.hazelcast.jet.core.Edge.from;
import static com.hazelcast.jet.core.SlidingWindowPolicy.tumblingWinPolicy;
import static com.hazelcast.jet.core.WatermarkEmissionPolicy.emitByFrame;
import static com.hazelcast.jet.core.WatermarkEmissionPolicy.noThrottling;
import static com.hazelcast.jet.impl.TopologicalSorter.topologicalSort;
import static java.util.stream.Collectors.toList;
@SuppressWarnings("unchecked")
public class Planner {
public final DAG dag = new DAG();
public final Map xform2vertex = new HashMap<>();
private final PipelineImpl pipeline;
private final Set vertexNames = new HashSet<>();
Planner(PipelineImpl pipeline) {
this.pipeline = pipeline;
}
DAG createDag() {
Map> adjacencyMap = pipeline.adjacencyMap();
validateNoLeakage(adjacencyMap);
// Calculate greatest common denominator of frame lengths from all transforms in the pipeline
long frameSizeGcd = Util.gcd(adjacencyMap.keySet().stream()
.map(Transform::watermarkFrameSize)
.filter(frameSize -> frameSize > 0)
.mapToLong(i -> i)
.toArray());
WatermarkEmissionPolicy emitPolicy = frameSizeGcd > 0
? emitByFrame(tumblingWinPolicy(frameSizeGcd))
: noThrottling();
// Replace emission policy
for (Transform transform : adjacencyMap.keySet()) {
if (transform instanceof StreamSourceTransform) {
StreamSourceTransform t = (StreamSourceTransform) transform;
if (t.getWmParams() != null) {
t.setWmGenerationParams(t.getWmParams().withEmitPolicy(emitPolicy));
}
} else if (transform instanceof TimestampTransform) {
TimestampTransform t = (TimestampTransform) transform;
t.setWmGenerationParams(t.getWmGenParams().withEmitPolicy(emitPolicy));
}
}
Iterable sorted = topologicalSort(adjacencyMap, Object::toString);
for (Transform transform : sorted) {
transform.addToDag(this);
}
return dag;
}
private static void validateNoLeakage(Map> adjacencyMap) {
List leakages = adjacencyMap
.entrySet().stream()
.filter(e -> !(e.getKey() instanceof SinkTransform))
.filter(e -> e.getValue().isEmpty())
.map(Entry::getKey)
.collect(toList());
if (!leakages.isEmpty()) {
throw new IllegalArgumentException("These transforms have nothing attached to them: " + leakages);
}
}
public PlannerVertex addVertex(Transform transform, String name, int localParallelism,
DistributedSupplier procSupplier) {
return addVertex(transform, name, localParallelism, ProcessorMetaSupplier.of(procSupplier));
}
public PlannerVertex addVertex(Transform transform, String name, int localParallelism,
ProcessorSupplier procSupplier) {
return addVertex(transform, name, localParallelism, ProcessorMetaSupplier.of(procSupplier));
}
public PlannerVertex addVertex(Transform transform, String name, int localParallelism,
ProcessorMetaSupplier metaSupplier) {
PlannerVertex pv = new PlannerVertex(dag.newVertex(name, metaSupplier));
pv.v.localParallelism(localParallelism);
xform2vertex.put(transform, pv);
return pv;
}
public void addEdges(Transform transform, Vertex toVertex, BiConsumer configureEdgeFn) {
int destOrdinal = 0;
for (Transform fromTransform : transform.upstream()) {
PlannerVertex fromPv = xform2vertex.get(fromTransform);
Edge edge = from(fromPv.v, fromPv.nextAvailableOrdinal()).to(toVertex, destOrdinal);
dag.edge(edge);
configureEdgeFn.accept(edge, destOrdinal);
destOrdinal++;
}
}
public void addEdges(Transform transform, Vertex toVertex, Consumer configureEdgeFn) {
addEdges(transform, toVertex, (e, ord) -> configureEdgeFn.accept(e));
}
public void addEdges(Transform transform, Vertex toVertex) {
addEdges(transform, toVertex, e -> { });
}
/**
* Makes the proposed name unique in the DAG by adding optional "-N"
* between the name and the suffix.
*
* @return unique name to be used for the vertex
*/
public String uniqueVertexName(@Nonnull String proposedName, @Nonnull String proposedNameSuffix) {
for (int index = 1; ; index++) {
String candidate = proposedName
+ (index == 1 ? "" : "-" + index)
+ proposedNameSuffix;
if (vertexNames.add(candidate)) {
return candidate;
}
}
}
public static List tailList(List list) {
return list.subList(1, list.size());
}
public static class PlannerVertex {
public final Vertex v;
private int availableOrdinal;
PlannerVertex(Vertex v) {
this.v = v;
}
@Override
public String toString() {
return v.toString();
}
public int nextAvailableOrdinal() {
return availableOrdinal++;
}
}
}
