org.apache.hadoop.hive.ql.lib.TaskGraphWalker Maven / Gradle / Ivy
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.ql.lib;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.Stack;
import org.apache.hadoop.hive.ql.exec.ConditionalTask;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.parse.SemanticException;
/**
* base class for operator graph walker this class takes list of starting ops
* and walks them one by one. it maintains list of walked operators
* (dispatchedList) and a list of operators that are discovered but not yet
* dispatched
*/
public class TaskGraphWalker implements GraphWalker {
public class TaskGraphWalkerContext{
private final HashMap reMap;
public TaskGraphWalkerContext(HashMap reMap){
this.reMap = reMap;
}
public void addToDispatchList(Node dispatchedObj){
if(dispatchedObj != null) {
retMap.put(dispatchedObj, null);
}
}
}
protected Stack opStack;
private final List toWalk = new ArrayList();
private final HashMap retMap = new HashMap();
private final Dispatcher dispatcher;
private final TaskGraphWalkerContext walkerCtx;
/**
* Constructor.
*
* @param disp
* dispatcher to call for each op encountered
*/
public TaskGraphWalker(Dispatcher disp) {
dispatcher = disp;
opStack = new Stack();
walkerCtx = new TaskGraphWalkerContext(retMap);
}
/**
* @return the toWalk
*/
public List getToWalk() {
return toWalk;
}
/**
* @return the doneList
*/
public Set getDispatchedList() {
return retMap.keySet();
}
/**
* Dispatch the current operator.
*
* @param nd
* node being walked
* @param ndStack
* stack of nodes encountered
* @throws SemanticException
*/
public void dispatch(Node nd, Stack ndStack,TaskGraphWalkerContext walkerCtx) throws SemanticException {
Object[] nodeOutputs = null;
if (nd.getChildren() != null) {
nodeOutputs = new Object[nd.getChildren().size()+1];
nodeOutputs[0] = walkerCtx;
int i = 1;
for (Node child : nd.getChildren()) {
nodeOutputs[i++] = retMap.get(child);
}
}else{
nodeOutputs = new Object[1];
nodeOutputs[0] = walkerCtx;
}
Object retVal = dispatcher.dispatch(nd, ndStack, nodeOutputs);
retMap.put(nd, retVal);
}
/**
* starting point for walking.
*
* @throws SemanticException
*/
public void startWalking(Collection startNodes,
HashMap nodeOutput) throws SemanticException {
toWalk.addAll(startNodes);
while (toWalk.size() > 0) {
Node nd = toWalk.remove(0);
walk(nd);
if (nodeOutput != null) {
nodeOutput.put(nd, retMap.get(nd));
}
}
}
/**
* walk the current operator and its descendants.
*
* @param nd
* current operator in the graph
* @throws SemanticException
*/
public void walk(Node nd) throws SemanticException {
if(!(nd instanceof Task)){
throw new SemanticException("Task Graph Walker only walks for Task Graph");
}
if (getDispatchedList().contains(nd)) {
return;
}
if (opStack.empty() || nd != opStack.peek()) {
opStack.push(nd);
}
List> nextTaskList = null;
Set> nextTaskSet = new HashSet>();
List> taskListInConditionalTask = null;
if(nd instanceof ConditionalTask ){
//for conditional task, next task list should return the children tasks of each task, which
//is contained in the conditional task.
taskListInConditionalTask = ((ConditionalTask) nd).getListTasks();
for(Task tsk: taskListInConditionalTask){
List> childTask = tsk.getChildTasks();
if(childTask != null){
nextTaskSet.addAll(tsk.getChildTasks());
}
}
//convert the set into list
if(nextTaskSet.size()>0){
nextTaskList = new ArrayList>();
for(Task tsk:nextTaskSet ){
nextTaskList.add(tsk);
}
}
}else{
//for other tasks, just return its children tasks
nextTaskList = ((Task)nd).getChildTasks();
}
if ((nextTaskList == null)
|| getDispatchedList().containsAll(nextTaskList)) {
dispatch(nd, opStack,this.walkerCtx);
opStack.pop();
return;
}
// add children, self to the front of the queue in that order
getToWalk().add(0, nd);
getToWalk().removeAll(nextTaskList);
getToWalk().addAll(0, nextTaskList);
}
}