org.apache.pig.newplan.logical.Util Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.pig.newplan.logical;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.newplan.Operator;
import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
import org.apache.pig.newplan.logical.expression.ProjectExpression;
import org.apache.pig.newplan.logical.relational.LOForEach;
import org.apache.pig.newplan.logical.relational.LOGenerate;
import org.apache.pig.newplan.logical.relational.LOInnerLoad;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
import org.apache.pig.newplan.logical.relational.LogicalSchema;
public class Util {
public static LogicalSchema translateSchema(Schema schema) {
if (schema == null) {
return null;
}
LogicalSchema s2 = new LogicalSchema();
List ll = schema.getFields();
for (Schema.FieldSchema f: ll) {
LogicalSchema.LogicalFieldSchema f2 =
new LogicalSchema.LogicalFieldSchema(f.alias, translateSchema(f.schema), f.type);
s2.addField(f2);
}
return s2;
}
public static LogicalSchema.LogicalFieldSchema translateFieldSchema(Schema.FieldSchema fs) {
LogicalSchema newSchema = null;
if (fs.schema!=null) {
newSchema = translateSchema(fs.schema);
}
LogicalSchema.LogicalFieldSchema newFs = new LogicalSchema.LogicalFieldSchema(fs.alias, newSchema, fs.type);
return newFs;
}
/**
* This function translates the new LogicalSchema into old Schema format required
* by PhysicalOperators
* @param schema LogicalSchema to be converted to Schema
* @return Schema that is converted from LogicalSchema
* @throws FrontendException
*/
public static Schema translateSchema(LogicalSchema schema) {
if (schema == null) {
return null;
}
Schema s2 = new Schema();
List ll = schema.getFields();
for (LogicalSchema.LogicalFieldSchema f: ll) {
Schema.FieldSchema f2 = null;
try {
f2 = new Schema.FieldSchema(f.alias, translateSchema(f.schema), f.type);
f2.canonicalName = ((Long)f.uid).toString();
s2.add(f2);
} catch (FrontendException e) {
}
}
return s2;
}
/**
* If schema argument has fields where a bag does not contain a tuple schema,
* it inserts a tuple schema. It does so for all inner levels.
* eg bag({int}) => bag({(int)})
* @param sch
* @return modified schema
* @throws FrontendException
*/
public static Schema fixSchemaAddTupleInBag(Schema sch) throws FrontendException{
LogicalSchema logSch = translateSchema(sch);
logSch.normalize();
return translateSchema(logSch);
}
public static Schema.FieldSchema translateFieldSchema(LogicalSchema.LogicalFieldSchema fs) {
if(fs == null)
return null;
Schema newSchema = null;
if (fs.schema!=null) {
newSchema = translateSchema(fs.schema);
}
Schema.FieldSchema newFs = null;
try {
newFs = new Schema.FieldSchema(null, newSchema, fs.type);
} catch (FrontendException e) {
}
return newFs;
}
public static LOForEach addForEachAfter(LogicalPlan plan, LogicalRelationalOperator op, int branch,
Set columnsToDrop) throws FrontendException {
LOForEach foreach = new LOForEach(plan);
plan.add(foreach);
List next = plan.getSuccessors(op);
if (next != null) {
LogicalRelationalOperator nextOp = (LogicalRelationalOperator)next.get(branch);
plan.insertBetween(op, foreach, nextOp);
}
else {
plan.connect(op, foreach);
}
LogicalPlan innerPlan = new LogicalPlan();
foreach.setInnerPlan(innerPlan);
LogicalSchema schema = op.getSchema();
// build foreach inner plan
List exps = new ArrayList();
LOGenerate gen = new LOGenerate(innerPlan, exps, new boolean[schema.size()-columnsToDrop.size()]);
innerPlan.add(gen);
for (int i=0, j=0; i