org.apache.pig.newplan.logical.relational.LOGenerate Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.pig.newplan.logical.relational;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.pig.data.DataType;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.newplan.Operator;
import org.apache.pig.newplan.OperatorPlan;
import org.apache.pig.newplan.PlanVisitor;
import org.apache.pig.newplan.logical.expression.LogicalExpression;
import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
import org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema;
public class LOGenerate extends LogicalRelationalOperator {
private List outputPlans;
private boolean[] flattenFlags;
// mUserDefinedSchema is the original input from the user, we don't suppose
// to store uid in mUserDefinedSchema
private List mUserDefinedSchema = null;
private List outputPlanSchemas = null;
// If LOGenerate generate new uid, cache it here.
// This happens when expression plan does not have complete schema, however,
// user give complete schema in ForEach statement in script
private List uidOnlySchemas = null;
public LOGenerate(OperatorPlan plan, List ps, boolean[] flatten) {
this( plan );
outputPlans = ps;
flattenFlags = flatten;
}
public void setOutputPlans(List plans) {
this.outputPlans = plans;
}
public LOGenerate(OperatorPlan plan) {
super( "LOGenerate", plan );
}
@Override
public LogicalSchema getSchema() throws FrontendException {
if (schema != null) {
return schema;
}
if (uidOnlySchemas == null) {
uidOnlySchemas = new ArrayList();
for (int i=0;i();
for(int i=0; i innerFieldSchemas = new ArrayList();
if (flattenFlags[i]) {
if (fieldSchema.type == DataType.BAG) {
// if it is bag, get the schema of tuples
if (fieldSchema.schema!=null) {
if (fieldSchema.schema.getField(0).schema!=null)
innerFieldSchemas = fieldSchema.schema.getField(0).schema.getFields();
for (LogicalSchema.LogicalFieldSchema fs : innerFieldSchemas) {
fs.alias = fs.alias == null ? null : fieldSchema.alias + "::" + fs.alias;
}
}
} else { // DataType.TUPLE
innerFieldSchemas = fieldSchema.schema.getFields();
for (LogicalSchema.LogicalFieldSchema fs : innerFieldSchemas) {
fs.alias = fs.alias == null ? null : fieldSchema.alias + "::" + fs.alias;
}
}
for (LogicalSchema.LogicalFieldSchema fs : innerFieldSchemas)
expSchema.addField(fs);
}
else
expSchema.addField(fieldSchema);
}
}
}
// Merge with user defined schema
if (expSchema!=null && expSchema.size()==0)
expSchema = null;
LogicalSchema planSchema = new LogicalSchema();
if (mUserDefinedSchemaCopy!=null) {
LogicalSchema mergedSchema = new LogicalSchema();
// merge with userDefinedSchema
if (expSchema==null) {
// Use user defined schema
for (LogicalFieldSchema fs : mUserDefinedSchemaCopy.getFields()) {
fs.stampFieldSchema();
mergedSchema.addField(new LogicalFieldSchema(fs));
}
for (LogicalFieldSchema fs : mergedSchema.getFields()) {
if (fs.type == DataType.NULL){
//this is the use case where a new alias has been specified by user
fs.type = DataType.BYTEARRAY;
}
}
} else {
// Merge uid with the exp field schema
mergedSchema = LogicalSchema.merge(mUserDefinedSchemaCopy, expSchema, LogicalSchema.MergeMode.LoadForEach);
if (mergedSchema==null) {
throw new FrontendException(this, "Cannot merge (" + expSchema.toString(false) +
") with user defined schema (" + mUserDefinedSchemaCopy.toString(false) + ")", 1117);
}
mergedSchema.mergeUid(expSchema);
}
for (LogicalFieldSchema fs : mergedSchema.getFields())
planSchema.addField(fs);
} else {
// if any plan do not have schema, the whole LOGenerate do not have schema
if (expSchema==null) {
planSchema = null;
}
else {
// Merge schema for the plan
for (LogicalFieldSchema fs : expSchema.getFields())
planSchema.addField(fs);
}
}
if (planSchema==null) {
schema = null;
break;
}
for (LogicalFieldSchema fs : planSchema.getFields())
schema.addField(fs);
// If the schema is generated by user defined schema, keep uid
if (expSchema==null) {
LogicalSchema uidOnlySchema = planSchema.mergeUid(uidOnlySchemas.get(i));
uidOnlySchemas.set(i, uidOnlySchema);
}
outputPlanSchemas.add(planSchema);
}
if (schema==null || schema.size()==0) {
schema = null;
outputPlanSchemas = null;
}
return schema;
}
public List getOutputPlans() {
return outputPlans;
}
public boolean[] getFlattenFlags() {
return flattenFlags;
}
public void setFlattenFlags(boolean[] flatten) {
flattenFlags = flatten;
}
@Override
public boolean isEqual(Operator other) throws FrontendException {
if (!(other instanceof LOGenerate)) {
return false;
}
List otherPlan = ((LOGenerate)other).getOutputPlans();
boolean[] fs = ((LOGenerate)other).getFlattenFlags();
if (outputPlans.size() != otherPlan.size()) {
return false;
}
for(int i=0; i entry : annotations.entrySet()) {
msg.append(entry);
}
}
return msg.toString();
}
public List getUserDefinedSchema() {
return mUserDefinedSchema;
}
public void setUserDefinedSchema(List userDefinedSchema) {
mUserDefinedSchema = userDefinedSchema;
}
/**
* Get the output schema corresponding to each input expression plan
* @return list of output schemas
*/
public List getOutputPlanSchemas() {
return outputPlanSchemas;
}
public void setOutputPlanSchemas(List outputPlanSchemas) {
this.outputPlanSchemas = outputPlanSchemas;
}
public List getUidOnlySchemas() {
return uidOnlySchemas;
}
public void setUidOnlySchemas(List uidOnlySchemas) {
this.uidOnlySchemas = uidOnlySchemas;
}
@Override
public void resetUid() {
this.uidOnlySchemas = null;
}
@Override
public void resetSchema(){
super.resetSchema();
outputPlanSchemas = null;
}
}