org.apache.lens.cube.metadata.MetastoreUtil Maven / Gradle / Ivy
The newest version!
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.lens.cube.metadata;
import static org.apache.lens.cube.error.LensCubeErrorCode.EXPRESSION_NOT_PARSABLE;
import static org.apache.lens.cube.metadata.MetastoreConstants.*;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.hadoop.hive.ql.parse.ParseDriver;
import org.apache.hadoop.hive.ql.parse.ParseUtils;
import org.antlr.runtime.CommonToken;
import com.google.common.collect.Sets;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class MetastoreUtil {
private MetastoreUtil() {
}
public static String getFactOrDimtableStorageTableName(String factName, String storageName) {
return getStorageTableName(factName, Storage.getPrefix(storageName));
}
public static String getStorageTableName(String cubeTableName, String storagePrefix) {
return (storagePrefix + cubeTableName).toLowerCase();
}
public static String getStorageClassKey(String name) {
return getStorageEntityPrefix(name) + CLASS_SFX;
}
public static String getStorageEntityPrefix(String storageName) {
return STORAGE_ENTITY_PFX + storageName.toLowerCase();
}
// //////////////////////////
// Dimension properties ///
// /////////////////////////
public static String getDimPrefix(String dimName) {
return DIMENSION_PFX + dimName.toLowerCase();
}
public static String getDimAttributeListKey(String dimName) {
return getDimPrefix(dimName) + ATTRIBUTES_LIST_SFX;
}
public static String getDimTablePartsKey(String dimtableName) {
return DIM_TABLE_PFX + dimtableName + PARTCOLS_SFX;
}
public static String getDimTimedDimensionKey(String dimName) {
return getDimPrefix(dimName) + TIMED_DIMENSION_SFX;
}
// ///////////////////////
// Dimension attribute properties//
// ///////////////////////
public static String getDimensionKeyPrefix(String dimName) {
return DIM_KEY_PFX + dimName.toLowerCase();
}
public static String getDimensionClassPropertyKey(String dimName) {
return getDimensionKeyPrefix(dimName) + CLASS_SFX;
}
public static String getInlineDimensionValuesKey(String name) {
return getDimensionKeyPrefix(name) + INLINE_VALUES_SFX;
}
public static String getDimTypePropertyKey(String dimName) {
return getDimensionKeyPrefix(dimName) + TYPE_SFX;
}
public static String getDimNumOfDistinctValuesPropertyKey(String dimName) {
return getDimensionKeyPrefix(dimName) + NUM_DISTINCT_VALUES;
}
public static String getHierachyElementKeyPFX(String dimName) {
return getDimensionKeyPrefix(dimName) + HIERARCHY_SFX;
}
public static String getHierachyElementKeyName(String dimName, int index) {
return getHierachyElementKeyPFX(dimName) + index;
}
public static Integer getHierachyElementIndex(String dimName, String param) {
return Integer.parseInt(param.substring(getHierachyElementKeyPFX(dimName).length()));
}
public static String getDimensionSrcReferenceKey(String dimName) {
return getDimensionKeyPrefix(dimName) + DIM_REFERS_SFX;
}
public static String getDimRefChainNameKey(String dimName) {
return getDimensionKeyPrefix(dimName) + CHAIN_NAME_SFX;
}
public static String getDimRefChainColumnKey(String dimName) {
return getDimensionKeyPrefix(dimName) + CHAIN_REF_COLUMN_SFX;
}
public static String getDimUseAsJoinKey(String dimName) {
return getDimensionKeyPrefix(dimName) + IS_JOIN_KEY_SFX;
}
public static String getReferencesString(List references) {
String[] toks = new String[references.size()];
for (int i = 0; i < references.size(); i++) {
TableReference reference = references.get(i);
toks[i] = reference.getDestTable() + TABLE_COLUMN_SEPERATOR + reference.getDestColumn();
toks[i] += TABLE_COLUMN_SEPERATOR + reference.isMapsToMany();
}
return StringUtils.join(toks, ',');
}
// ////////////////////////
// Column properties //
// ////////////////////////
public static String getColumnKeyPrefix(String colName) {
return COL_PFX + colName.toLowerCase();
}
public static String getCubeColStartTimePropertyKey(String colName) {
return getColumnKeyPrefix(colName) + START_TIME_SFX;
}
public static String getCubeColEndTimePropertyKey(String colName) {
return getColumnKeyPrefix(colName) + END_TIME_SFX;
}
public static String getStoragetableStartTimesKey() {
return STORAGE_PFX + "start.times";
}
public static String getStoragetableEndTimesKey() {
return STORAGE_PFX + "end.times";
}
public static String getCubeColCostPropertyKey(String colName) {
return getColumnKeyPrefix(colName) + COST_SFX;
}
public static String getCubeColDescriptionKey(String colName) {
return getColumnKeyPrefix(colName) + DESC_SFX;
}
public static String getCubeColDisplayKey(String colName) {
return getColumnKeyPrefix(colName) + DISPLAY_SFX;
}
public static String getCubeColTagKey(String colName) {
return getColumnKeyPrefix(colName) + TAGS_PFX;
}
public static String getExprColumnKey(String colName) {
return getColumnKeyPrefix(colName) + EXPR_SFX;
}
public static String getExprTypePropertyKey(String colName) {
return getColumnKeyPrefix(colName) + TYPE_SFX;
}
public static String getExprEncodingPropertyKey(String colName) {
return getExprColumnKey(colName) + BASE64_SFX;
}
////////////////////////////
// Join chain properties //
////////////////////////////
public static String getCubeJoinChainKey(String colName) {
return CUBE_KEY_PFX + JOIN_CHAIN_KEY + colName.toLowerCase();
}
public static String getCubeJoinChainNumChainsKey(String colName) {
return getCubeJoinChainKey(colName) + NUM_CHAINS_SFX;
}
public static String getCubeJoinChainFullChainKey(String colName, int index) {
return getCubeJoinChainKey(colName) + FULL_CHAIN_KEY + index;
}
public static String getCubeJoinChainDescriptionKey(String colName) {
return getCubeJoinChainKey(colName) + DESC_SFX;
}
public static String getCubeJoinChainDisplayKey(String colName) {
return getCubeJoinChainKey(colName) + DISPLAY_SFX;
}
public static String getDimensionJoinChainKey(String colName) {
return DIMENSION_PFX + JOIN_CHAIN_KEY + colName.toLowerCase();
}
public static String getDimensionJoinChainNumChainsKey(String colName) {
return getDimensionJoinChainKey(colName) + NUM_CHAINS_SFX;
}
public static String getDimensionJoinChainFullChainKey(String colName, int index) {
return getDimensionJoinChainKey(colName) + FULL_CHAIN_KEY + index;
}
public static String getDimensionJoinChainDescriptionKey(String colName) {
return getDimensionJoinChainKey(colName) + DESC_SFX;
}
public static String getDimensionJoinChainDisplayKey(String colName) {
return getDimensionJoinChainKey(colName) + DISPLAY_SFX;
}
// ////////////////////////
// Dimension table properties //
// ////////////////////////
public static String getDimensionTablePrefix(String dimTblName) {
return DIM_TBL_PFX + dimTblName.toLowerCase();
}
public static String getDimensionDumpPeriodKey(String dimTblName, String storage) {
return getDimensionTablePrefix(dimTblName) + "." + storage.toLowerCase() + DUMP_PERIOD_SFX;
}
public static String getDimensionStorageListKey(String dimTblName) {
return getDimensionTablePrefix(dimTblName) + STORAGE_LIST_SFX;
}
public static String getDimNameKey(String dimTblName) {
return getDimensionTablePrefix(dimTblName) + DIM_NAME_SFX;
}
// //////////////////////////
// Measure properties ///
// /////////////////////////
public static String getMeasurePrefix(String measureName) {
return MEASURE_KEY_PFX + measureName.toLowerCase();
}
public static String getMeasureClassPropertyKey(String measureName) {
return getMeasurePrefix(measureName) + CLASS_SFX;
}
public static String getMeasureUnitPropertyKey(String measureName) {
return getMeasurePrefix(measureName) + UNIT_SFX;
}
public static String getMeasureTypePropertyKey(String measureName) {
return getMeasurePrefix(measureName) + TYPE_SFX;
}
public static String getMeasureFormatPropertyKey(String measureName) {
return getMeasurePrefix(measureName) + FORMATSTRING_SFX;
}
public static String getMeasureAggrPropertyKey(String measureName) {
return getMeasurePrefix(measureName) + AGGR_SFX;
}
public static String getMeasureMinPropertyKey(String measureName) {
return getMeasurePrefix(measureName) + MIN_SFX;
}
public static String getMeasureMaxPropertyKey(String measureName) {
return getMeasurePrefix(measureName) + MAX_SFX;
}
public static String getExpressionListKey(String name) {
return getBasePrefix(name) + EXPRESSIONS_LIST_SFX;
}
// //////////////////////////
// Cube properties ///
// /////////////////////////
public static String getBasePrefix(String base) {
return BASE_KEY_PFX + base.toLowerCase();
}
public static String getCubePrefix(String cubeName) {
return CUBE_KEY_PFX + cubeName.toLowerCase();
}
public static String getCubeMeasureListKey(String cubeName) {
return getCubePrefix(cubeName) + MEASURES_LIST_SFX;
}
public static String getCubeDimensionListKey(String cubeName) {
return getCubePrefix(cubeName) + DIMENSIONS_LIST_SFX;
}
public static String getCubeTimedDimensionListKey(String cubeName) {
return getCubePrefix(cubeName) + TIMED_DIMENSIONS_LIST_SFX;
}
public static String getCubeJoinChainListKey(String cubeName) {
return getCubePrefix(cubeName) + JOIN_CHAIN_LIST_SFX;
}
public static String getDimensionJoinChainListKey(String dimName) {
return getDimPrefix(dimName) + JOIN_CHAIN_LIST_SFX;
}
public static String getParentCubeNameKey(String cubeName) {
return getCubePrefix(cubeName) + PARENT_CUBE_SFX;
}
public static String getCubeTableKeyPrefix(String tableName) {
return CUBE_TABLE_PFX + tableName.toLowerCase();
}
// //////////////////////////
// Fact properties ///
// /////////////////////////
public static String getFactStorageListKey(String name) {
return getFactKeyPrefix(name) + STORAGE_LIST_SFX;
}
public static String getFactKeyPrefix(String factName) {
return FACT_KEY_PFX + factName.toLowerCase();
}
public static String getFactUpdatePeriodKey(String name, String storage) {
return getFactKeyPrefix(name) + "." + storage.toLowerCase() + UPDATE_PERIOD_SFX;
}
public static String getFactCubeNameKey(String name) {
return getFactKeyPrefix(name) + CUBE_NAME_SFX;
}
public static String getSourceFactNameKey(String name) {
return getFactKeyPrefix(name) + SOURCE_NAME_SFX;
}
public static String getValidColumnsKey(String name) {
return getFactKeyPrefix(name) + VALID_COLUMNS_SFX;
}
public static String getCubeTableWeightKey(String name) {
return getCubeTableKeyPrefix(name) + WEIGHT_KEY_SFX;
}
public static String getLatestPartTimestampKey(String partCol) {
return STORAGE_PFX + partCol + LATEST_PART_TIMESTAMP_SFX;
}
// //////////////////////////
// Segmentation propertes ///
// /////////////////////////
public static String getSegmentationKeyPrefix(String segName) {
return SEGMENTATION_KEY_PFX + segName.toLowerCase();
}
public static String getSegmentationCubeNameKey(String name) {
return getSegmentationKeyPrefix(name) + CUBE_NAME_SFX;
}
public static String getSegmentsListKey(String name) {
return getSegmentationKeyPrefix(name) + SEGMENTATION_CUBE_SEGMENT_SFX;
}
public static String getSegmentPropertyKey(String segName) {
return getSegmentationKeyPrefix(segName) + SEGMENT_PROP_SFX;
}
// //////////////////////////
// Utils ///
// /////////////////////////
public static Date getDateFromProperty(String prop, boolean relative, boolean start) {
try {
if (StringUtils.isNotBlank(prop)) {
if (relative) {
return DateUtil.resolveRelativeDate(prop, new Date());
} else {
return DateUtil.resolveAbsoluteDate(prop);
}
}
} catch (LensException e) {
log.error("unable to parse {} {} date: {}", relative ? "relative" : "absolute", start ? "start" : "end", prop);
}
return start ? DateUtil.MIN_DATE : DateUtil.MAX_DATE;
}
public static String getNamedStr(Collection set) {
if (set == null) {
return "";
}
String sep = "";
StringBuilder valueStr = new StringBuilder();
for (E aSet : set) {
valueStr.append(sep).append(aSet.getName());
sep = ",";
}
return valueStr.toString();
}
static List getNamedStrs(Collection set, int maxLength) {
List namedStrings = new ArrayList<>();
if (set == null || set.isEmpty()) {
return namedStrings;
}
StringBuilder valueStr = new StringBuilder();
Iterator it = set.iterator();
for (int i = 0; i < (set.size() - 1); i++) {
E next = it.next();
if (valueStr.length() + next.getName().length() >= maxLength) {
namedStrings.add(valueStr.toString());
valueStr.setLength(0);
}
valueStr.append(next.getName());
valueStr.append(",");
}
E next = it.next();
if (valueStr.length() + next.getName().length() >= maxLength) {
namedStrings.add(valueStr.toString());
valueStr.setLength(0);
}
valueStr.append(next.getName());
namedStrings.add(valueStr.toString());
return namedStrings;
}
private static final int MAX_PARAM_LENGTH = 3999;
public static Set getNamedSetFromStringSet(Set strings) {
Set nameds = Sets.newHashSet();
for (final String s : strings) {
nameds.add(new Named() {
@Override
public String getName() {
return s;
}
});
}
return nameds;
}
public static void addNameStrings(Map props, String key, Collection set) {
addNameStrings(props, key, set, MAX_PARAM_LENGTH);
}
public static void addNameStrings(Map props, String key,
Collection set, int maxLength) {
List namedStrings = getNamedStrs(set, maxLength);
props.put(key + ".size", String.valueOf(namedStrings.size()));
for (int i = 0; i < namedStrings.size(); i++) {
props.put(key + i, namedStrings.get(i));
}
}
public static String getNamedStringValue(Map props, String key) {
if (props.containsKey(key + ".size")) {
int size = Integer.parseInt(props.get(key + ".size"));
StringBuilder valueStr = new StringBuilder();
for (int i = 0; i < size; i++) {
valueStr.append(props.get(key + i));
}
return valueStr.toString();
} else if (props.containsKey(key)) {
return props.get(key);
} else {
return null;
}
}
public static String getObjectStr(Collection set) {
if (set == null || set.isEmpty()) {
return "";
}
StringBuilder valueStr = new StringBuilder();
Iterator it = set.iterator();
for (int i = 0; i < (set.size() - 1); i++) {
valueStr.append(it.next().toString());
valueStr.append(",");
}
valueStr.append(it.next().toString());
return valueStr.toString();
}
public static String getStr(Collection set) {
if (set == null || set.isEmpty()) {
return "";
}
StringBuilder valueStr = new StringBuilder();
Iterator it = set.iterator();
for (int i = 0; i < (set.size() - 1); i++) {
valueStr.append(it.next());
valueStr.append(",");
}
valueStr.append(it.next());
return valueStr.toString();
}
public static void addColumnNames(CubeDimAttribute dim, Set cols) {
if (dim instanceof HierarchicalDimAttribute) {
HierarchicalDimAttribute h = (HierarchicalDimAttribute) dim;
for (CubeDimAttribute d : h.getHierarchy()) {
addColumnNames(d, cols);
}
} else {
cols.add(dim.getName().toLowerCase());
}
}
public static String getPartitionInfoKeyPrefix(UpdatePeriod updatePeriod, String partCol) {
return STORAGE_PFX + PARTITION_TIMELINE_CACHE + updatePeriod.getName() + "." + partCol + ".";
}
public static String getPartitionTimelineStorageClassKey(UpdatePeriod updatePeriod, String partCol) {
return getPartitionInfoKeyPrefix(updatePeriod, partCol) + STORAGE_CLASS;
}
public static String getPartitionTimelineCachePresenceKey() {
return STORAGE_PFX + PARTITION_TIMELINE_CACHE + "present";
}
public static void filterPartitionsByUpdatePeriod(List partitions, UpdatePeriod updatePeriod) {
Iterator iter = partitions.iterator();
while (iter.hasNext()) {
Partition part = iter.next();
if (!UpdatePeriod.valueOf(part.getParameters().get(PARTITION_UPDATE_PERIOD)).equals(updatePeriod)) {
iter.remove();
}
}
}
public static List filterPartitionsByNonTimeParts(List partitions,
Map nonTimePartSpec,
String latestPartCol) {
ListIterator iter = partitions.listIterator();
while (iter.hasNext()) {
Partition part = iter.next();
boolean ignore = false;
for (Map.Entry entry1 : part.getSpec().entrySet()) {
if ((nonTimePartSpec == null || !nonTimePartSpec.containsKey(entry1.getKey()))
&& !entry1.getKey().equals(latestPartCol)) {
try {
UpdatePeriod.valueOf(part.getParameters().get(PARTITION_UPDATE_PERIOD))
.format()
.parse(entry1.getValue());
} catch (ParseException e) {
ignore = true;
}
}
}
if (ignore) {
iter.remove();
}
}
return partitions;
}
public static Date getLatestTimeStampFromPartition(Partition part, String partCol) throws HiveException {
if (part != null) {
String latestTimeStampStr = part.getParameters().get(MetastoreUtil.getLatestPartTimestampKey(partCol));
String latestPartUpdatePeriod = part.getParameters().get(PARTITION_UPDATE_PERIOD);
UpdatePeriod latestUpdatePeriod = UpdatePeriod.valueOf(latestPartUpdatePeriod.toUpperCase());
try {
return latestUpdatePeriod.parse(latestTimeStampStr);
} catch (ParseException e) {
throw new HiveException(e);
}
}
return null;
}
static ASTNode parseExpr(String expr) throws LensException {
ParseDriver driver = new ParseDriver();
ASTNode tree;
try {
tree = driver.parseExpression(expr);
} catch (org.apache.hadoop.hive.ql.parse.ParseException e) {
throw new LensException(EXPRESSION_NOT_PARSABLE.getLensErrorInfo(), e, e.getMessage(), expr);
}
return ParseUtils.findRootNonNullToken(tree);
}
public static ASTNode copyAST(ASTNode original) {
return copyAST(original, x -> Pair.of(new ASTNode(x), true));
}
public static ASTNode copyAST(ASTNode original,
Function> overrideCopyFunction) {
Pair copy1 = overrideCopyFunction.apply(original);
ASTNode copy = copy1.getLeft();
if (copy1.getRight()) {
if (original.getChildren() != null) {
for (Node o : original.getChildren()) {
copy.addChild(copyAST((ASTNode) o, overrideCopyFunction));
}
}
}
return copy;
}
public static String getUpdatePeriodStoragePrefixKey(String factTableName, String storageName, String updatePeriod) {
return MetastoreUtil.getFactKeyPrefix(factTableName) + "." + storageName + "." + updatePeriod;
}
public static ASTNode getStringLiteralAST(String literal) {
return new ASTNode(new CommonToken(HiveParser.StringLiteral, literal));
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy