org.apache.hadoop.hive.ql.plan.CreateTableDesc Maven / Gradle / Ivy
The newest version!
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.ParseUtils;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.mapred.OutputFormat;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
/**
* CreateTableDesc.
*
*/
@Explain(displayName = "Create Table", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class CreateTableDesc extends DDLDesc implements Serializable {
private static final long serialVersionUID = 1L;
private static Log LOG = LogFactory.getLog(CreateTableDesc.class);
String databaseName;
String tableName;
boolean isExternal;
List cols;
List partCols;
List bucketCols;
List sortCols;
int numBuckets;
String fieldDelim;
String fieldEscape;
String collItemDelim;
String mapKeyDelim;
String lineDelim;
String nullFormat;
String comment;
String inputFormat;
String outputFormat;
String location;
String serName;
String storageHandler;
Map serdeProps;
Map tblProps;
boolean ifNotExists;
List skewedColNames;
List> skewedColValues;
boolean isStoredAsSubDirectories = false;
boolean isTemporary = false;
private boolean replaceMode = false;
public CreateTableDesc() {
}
public CreateTableDesc(String databaseName, String tableName, boolean isExternal, boolean isTemporary,
List cols, List partCols,
List bucketCols, List sortCols, int numBuckets,
String fieldDelim, String fieldEscape, String collItemDelim,
String mapKeyDelim, String lineDelim, String comment, String inputFormat,
String outputFormat, String location, String serName,
String storageHandler,
Map serdeProps,
Map tblProps,
boolean ifNotExists, List skewedColNames, List> skewedColValues) {
this(tableName, isExternal, isTemporary, cols, partCols,
bucketCols, sortCols, numBuckets, fieldDelim, fieldEscape,
collItemDelim, mapKeyDelim, lineDelim, comment, inputFormat,
outputFormat, location, serName, storageHandler, serdeProps,
tblProps, ifNotExists, skewedColNames, skewedColValues);
this.databaseName = databaseName;
}
public CreateTableDesc(String tableName, boolean isExternal, boolean isTemporary,
List cols, List partCols,
List bucketCols, List sortCols, int numBuckets,
String fieldDelim, String fieldEscape, String collItemDelim,
String mapKeyDelim, String lineDelim, String comment, String inputFormat,
String outputFormat, String location, String serName,
String storageHandler,
Map serdeProps,
Map tblProps,
boolean ifNotExists, List skewedColNames, List> skewedColValues) {
this.tableName = tableName;
this.isExternal = isExternal;
this.isTemporary = isTemporary;
this.bucketCols = new ArrayList(bucketCols);
this.sortCols = new ArrayList(sortCols);
this.collItemDelim = collItemDelim;
this.cols = new ArrayList(cols);
this.comment = comment;
this.fieldDelim = fieldDelim;
this.fieldEscape = fieldEscape;
this.inputFormat = inputFormat;
this.outputFormat = outputFormat;
this.lineDelim = lineDelim;
this.location = location;
this.mapKeyDelim = mapKeyDelim;
this.numBuckets = numBuckets;
this.partCols = new ArrayList(partCols);
this.serName = serName;
this.storageHandler = storageHandler;
this.serdeProps = serdeProps;
this.tblProps = tblProps;
this.ifNotExists = ifNotExists;
this.skewedColNames = copyList(skewedColNames);
this.skewedColValues = copyList(skewedColValues);
}
private static List copyList(List copy) {
return copy == null ? null : new ArrayList(copy);
}
@Explain(displayName = "columns", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public List getColsString() {
return Utilities.getFieldSchemaString(getCols());
}
@Explain(displayName = "partition columns", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public List getPartColsString() {
return Utilities.getFieldSchemaString(getPartCols());
}
@Explain(displayName = "if not exists", displayOnlyOnTrue = true)
public boolean getIfNotExists() {
return ifNotExists;
}
public void setIfNotExists(boolean ifNotExists) {
this.ifNotExists = ifNotExists;
}
@Explain(displayName = "name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getTableName() {
return tableName;
}
public String getDatabaseName(){
return databaseName;
}
public void setTableName(String tableName) {
this.tableName = tableName;
}
public List getCols() {
return cols;
}
public void setCols(ArrayList cols) {
this.cols = cols;
}
public List getPartCols() {
return partCols;
}
public void setPartCols(ArrayList partCols) {
this.partCols = partCols;
}
@Explain(displayName = "bucket columns", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public List getBucketCols() {
return bucketCols;
}
public void setBucketCols(ArrayList bucketCols) {
this.bucketCols = bucketCols;
}
@Explain(displayName = "# buckets", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public Integer getNumBucketsExplain() {
if (numBuckets == -1) {
return null;
} else {
return numBuckets;
}
}
public int getNumBuckets() {
return numBuckets;
}
public void setNumBuckets(int numBuckets) {
this.numBuckets = numBuckets;
}
@Explain(displayName = "field delimiter")
public String getFieldDelim() {
return fieldDelim;
}
public void setFieldDelim(String fieldDelim) {
this.fieldDelim = fieldDelim;
}
@Explain(displayName = "field escape")
public String getFieldEscape() {
return fieldEscape;
}
public void setFieldEscape(String fieldEscape) {
this.fieldEscape = fieldEscape;
}
@Explain(displayName = "collection delimiter")
public String getCollItemDelim() {
return collItemDelim;
}
public void setCollItemDelim(String collItemDelim) {
this.collItemDelim = collItemDelim;
}
@Explain(displayName = "map key delimiter")
public String getMapKeyDelim() {
return mapKeyDelim;
}
public void setMapKeyDelim(String mapKeyDelim) {
this.mapKeyDelim = mapKeyDelim;
}
@Explain(displayName = "line delimiter")
public String getLineDelim() {
return lineDelim;
}
public void setLineDelim(String lineDelim) {
this.lineDelim = lineDelim;
}
@Explain(displayName = "comment")
public String getComment() {
return comment;
}
public void setComment(String comment) {
this.comment = comment;
}
@Explain(displayName = "input format", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getInputFormat() {
return inputFormat;
}
public void setInputFormat(String inputFormat) {
this.inputFormat = inputFormat;
}
@Explain(displayName = "output format", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getOutputFormat() {
return outputFormat;
}
public void setOutputFormat(String outputFormat) {
this.outputFormat = outputFormat;
}
@Explain(displayName = "storage handler", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public String getStorageHandler() {
return storageHandler;
}
public void setStorageHandler(String storageHandler) {
this.storageHandler = storageHandler;
}
@Explain(displayName = "location")
public String getLocation() {
return location;
}
public void setLocation(String location) {
this.location = location;
}
@Explain(displayName = "isExternal", displayOnlyOnTrue = true)
public boolean isExternal() {
return isExternal;
}
public void setExternal(boolean isExternal) {
this.isExternal = isExternal;
}
/**
* @return the sortCols
*/
@Explain(displayName = "sort columns")
public List getSortCols() {
return sortCols;
}
/**
* @param sortCols
* the sortCols to set
*/
public void setSortCols(ArrayList sortCols) {
this.sortCols = sortCols;
}
/**
* @return the serDeName
*/
@Explain(displayName = "serde name")
public String getSerName() {
return serName;
}
/**
* @param serName
* the serName to set
*/
public void setSerName(String serName) {
this.serName = serName;
}
/**
* @return the serDe properties
*/
@Explain(displayName = "serde properties")
public Map getSerdeProps() {
return serdeProps;
}
/**
* @param serdeProps
* the serde properties to set
*/
public void setSerdeProps(Map serdeProps) {
this.serdeProps = serdeProps;
}
/**
* @return the table properties
*/
@Explain(displayName = "table properties")
public Map getTblProps() {
return tblProps;
}
/**
* @param tblProps
* the table properties to set
*/
public void setTblProps(Map tblProps) {
this.tblProps = tblProps;
}
/**
* @return the skewedColNames
*/
public List getSkewedColNames() {
return skewedColNames;
}
/**
* @param skewedColNames the skewedColNames to set
*/
public void setSkewedColNames(ArrayList skewedColNames) {
this.skewedColNames = skewedColNames;
}
/**
* @return the skewedColValues
*/
public List> getSkewedColValues() {
return skewedColValues;
}
/**
* @param skewedColValues the skewedColValues to set
*/
public void setSkewedColValues(ArrayList> skewedColValues) {
this.skewedColValues = skewedColValues;
}
public void validate(HiveConf conf)
throws SemanticException {
if ((this.getCols() == null) || (this.getCols().size() == 0)) {
// for now make sure that serde exists
if (Table.hasMetastoreBasedSchema(conf, getSerName())) {
throw new SemanticException(ErrorMsg.INVALID_TBL_DDL_SERDE.getMsg());
}
return;
}
if (this.getStorageHandler() == null) {
try {
Class origin = Class.forName(this.getOutputFormat(), true,
Utilities.getSessionSpecifiedClassLoader());
Class replaced = HiveFileFormatUtils
.getOutputFormatSubstitute(origin);
if (!HiveOutputFormat.class.isAssignableFrom(replaced)) {
throw new SemanticException(ErrorMsg.INVALID_OUTPUT_FORMAT_TYPE
.getMsg());
}
} catch (ClassNotFoundException e) {
throw new SemanticException(ErrorMsg.GENERIC_ERROR.getMsg(), e);
}
}
List colNames = ParseUtils.validateColumnNameUniqueness(this.getCols());
if (this.getBucketCols() != null) {
// all columns in cluster and sort are valid columns
Iterator bucketCols = this.getBucketCols().iterator();
while (bucketCols.hasNext()) {
String bucketCol = bucketCols.next();
boolean found = false;
Iterator colNamesIter = colNames.iterator();
while (colNamesIter.hasNext()) {
String colName = colNamesIter.next();
if (bucketCol.equalsIgnoreCase(colName)) {
found = true;
break;
}
}
if (!found) {
throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg());
}
}
}
if (this.getSortCols() != null) {
// all columns in cluster and sort are valid columns
Iterator sortCols = this.getSortCols().iterator();
while (sortCols.hasNext()) {
String sortCol = sortCols.next().getCol();
boolean found = false;
Iterator colNamesIter = colNames.iterator();
while (colNamesIter.hasNext()) {
String colName = colNamesIter.next();
if (sortCol.equalsIgnoreCase(colName)) {
found = true;
break;
}
}
if (!found) {
throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg());
}
}
}
if (this.getPartCols() != null) {
// there is no overlap between columns and partitioning columns
Iterator partColsIter = this.getPartCols().iterator();
while (partColsIter.hasNext()) {
FieldSchema fs = partColsIter.next();
String partCol = fs.getName();
TypeInfo pti = null;
try {
pti = TypeInfoFactory.getPrimitiveTypeInfo(fs.getType());
} catch (Exception err) {
LOG.error(err);
}
if(null == pti){
throw new SemanticException(ErrorMsg.PARTITION_COLUMN_NON_PRIMITIVE.getMsg() + " Found "
+ partCol + " of type: " + fs.getType());
}
Iterator colNamesIter = colNames.iterator();
while (colNamesIter.hasNext()) {
String colName = BaseSemanticAnalyzer.unescapeIdentifier(colNamesIter.next());
if (partCol.equalsIgnoreCase(colName)) {
throw new SemanticException(
ErrorMsg.COLUMN_REPEATED_IN_PARTITIONING_COLS.getMsg());
}
}
}
}
/* Validate skewed information. */
ValidationUtility.validateSkewedInformation(colNames, this.getSkewedColNames(),
this.getSkewedColValues());
}
/**
* @return the isStoredAsSubDirectories
*/
public boolean isStoredAsSubDirectories() {
return isStoredAsSubDirectories;
}
/**
* @param isStoredAsSubDirectories the isStoredAsSubDirectories to set
*/
public void setStoredAsSubDirectories(boolean isStoredAsSubDirectories) {
this.isStoredAsSubDirectories = isStoredAsSubDirectories;
}
/**
* @return the nullFormat
*/
public String getNullFormat() {
return nullFormat;
}
/**
* Set null format string
* @param nullFormat
*/
public void setNullFormat(String nullFormat) {
this.nullFormat = nullFormat;
}
/**
* @return the isTemporary
*/
@Explain(displayName = "isTemporary", displayOnlyOnTrue = true)
public boolean isTemporary() {
return isTemporary;
}
/**
* @param isTemporary table is Temporary or not.
*/
public void setTemporary(boolean isTemporary) {
this.isTemporary = isTemporary;
}
/**
* @param replaceMode Determine if this CreateTable should behave like a replace-into alter instead
*/
public void setReplaceMode(boolean replaceMode) {
this.replaceMode = replaceMode;
}
/**
* @return true if this CreateTable should behave like a replace-into alter instead
*/
public boolean getReplaceMode() {
return replaceMode;
}
}