
io.prestosql.plugin.tpch.TpchMetadata Maven / Gradle / Ivy
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.plugin.tpch;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import io.airlift.slice.Slice;
import io.airlift.slice.Slices;
import io.prestosql.plugin.tpch.statistics.ColumnStatisticsData;
import io.prestosql.plugin.tpch.statistics.StatisticsEstimator;
import io.prestosql.plugin.tpch.statistics.TableStatisticsData;
import io.prestosql.plugin.tpch.statistics.TableStatisticsDataRepository;
import io.prestosql.spi.connector.ColumnHandle;
import io.prestosql.spi.connector.ColumnMetadata;
import io.prestosql.spi.connector.ConnectorMetadata;
import io.prestosql.spi.connector.ConnectorSession;
import io.prestosql.spi.connector.ConnectorTableHandle;
import io.prestosql.spi.connector.ConnectorTableMetadata;
import io.prestosql.spi.connector.ConnectorTablePartitioning;
import io.prestosql.spi.connector.ConnectorTableProperties;
import io.prestosql.spi.connector.Constraint;
import io.prestosql.spi.connector.ConstraintApplicationResult;
import io.prestosql.spi.connector.LocalProperty;
import io.prestosql.spi.connector.SchemaTableName;
import io.prestosql.spi.connector.SchemaTablePrefix;
import io.prestosql.spi.connector.SortOrder;
import io.prestosql.spi.connector.SortingProperty;
import io.prestosql.spi.predicate.Domain;
import io.prestosql.spi.predicate.NullableValue;
import io.prestosql.spi.predicate.TupleDomain;
import io.prestosql.spi.statistics.ColumnStatistics;
import io.prestosql.spi.statistics.ComputedStatistics;
import io.prestosql.spi.statistics.DoubleRange;
import io.prestosql.spi.statistics.Estimate;
import io.prestosql.spi.statistics.TableStatistics;
import io.prestosql.spi.statistics.TableStatisticsMetadata;
import io.prestosql.spi.type.Type;
import io.prestosql.spi.type.VarcharType;
import io.prestosql.tpch.Distributions;
import io.prestosql.tpch.LineItemColumn;
import io.prestosql.tpch.OrderColumn;
import io.prestosql.tpch.OrderGenerator;
import io.prestosql.tpch.PartColumn;
import io.prestosql.tpch.TpchColumn;
import io.prestosql.tpch.TpchColumnType;
import io.prestosql.tpch.TpchEntity;
import io.prestosql.tpch.TpchTable;
import java.time.LocalDate;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static com.google.common.collect.Maps.asMap;
import static io.prestosql.plugin.tpch.util.PredicateUtils.convertToPredicate;
import static io.prestosql.plugin.tpch.util.PredicateUtils.filterOutColumnFromPredicate;
import static io.prestosql.spi.statistics.TableStatisticType.ROW_COUNT;
import static io.prestosql.spi.type.BigintType.BIGINT;
import static io.prestosql.spi.type.DateType.DATE;
import static io.prestosql.spi.type.DoubleType.DOUBLE;
import static io.prestosql.spi.type.IntegerType.INTEGER;
import static io.prestosql.spi.type.VarcharType.createVarcharType;
import static io.prestosql.tpch.OrderColumn.ORDER_STATUS;
import static java.lang.String.format;
import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
public class TpchMetadata
implements ConnectorMetadata
{
public static final String TINY_SCHEMA_NAME = "tiny";
public static final double TINY_SCALE_FACTOR = 0.01;
public static final List SCHEMA_NAMES = ImmutableList.of(
TINY_SCHEMA_NAME, "sf1", "sf100", "sf300", "sf1000", "sf3000", "sf10000", "sf30000", "sf100000");
public static final String ROW_NUMBER_COLUMN_NAME = "row_number";
private static final Set ORDER_STATUS_VALUES = ImmutableSet.of("F", "O", "P").stream()
.map(Slices::utf8Slice)
.collect(toImmutableSet());
private static final Set ORDER_STATUS_NULLABLE_VALUES = ORDER_STATUS_VALUES.stream()
.map(value -> new NullableValue(getPrestoType(OrderColumn.ORDER_STATUS), value))
.collect(toSet());
private static final Set PART_TYPE_VALUES = Distributions.getDefaultDistributions().getPartTypes().getValues().stream()
.map(Slices::utf8Slice)
.collect(toImmutableSet());
private static final Set PART_TYPE_NULLABLE_VALUES = PART_TYPE_VALUES.stream()
.map(value -> new NullableValue(getPrestoType(PartColumn.TYPE), value))
.collect(toSet());
private static final Set PART_CONTAINER_VALUES = Distributions.getDefaultDistributions().getPartContainers().getValues().stream()
.map(Slices::utf8Slice)
.collect(toImmutableSet());
private static final Set PART_CONTAINER_NULLABLE_VALUES = PART_CONTAINER_VALUES.stream()
.map(value -> new NullableValue(getPrestoType(PartColumn.CONTAINER), value))
.collect(toSet());
private final Set tableNames;
private final ColumnNaming columnNaming;
private final StatisticsEstimator statisticsEstimator;
private final boolean predicatePushdownEnabled;
private final boolean partitioningEnabled;
public TpchMetadata()
{
this(ColumnNaming.SIMPLIFIED, true, true);
}
public TpchMetadata(ColumnNaming columnNaming, boolean predicatePushdownEnabled, boolean partitioningEnabled)
{
ImmutableSet.Builder tableNames = ImmutableSet.builder();
for (TpchTable> tpchTable : TpchTable.getTables()) {
tableNames.add(tpchTable.getTableName());
}
this.tableNames = tableNames.build();
this.columnNaming = columnNaming;
this.predicatePushdownEnabled = predicatePushdownEnabled;
this.partitioningEnabled = partitioningEnabled;
this.statisticsEstimator = createStatisticsEstimator();
}
private static StatisticsEstimator createStatisticsEstimator()
{
ObjectMapper objectMapper = new ObjectMapper()
.registerModule(new Jdk8Module());
TableStatisticsDataRepository tableStatisticsDataRepository = new TableStatisticsDataRepository(objectMapper);
return new StatisticsEstimator(tableStatisticsDataRepository);
}
@Override
public boolean schemaExists(ConnectorSession session, String schemaName)
{
return schemaNameToScaleFactor(schemaName) > 0;
}
@Override
public List listSchemaNames(ConnectorSession session)
{
return SCHEMA_NAMES;
}
@Override
public TpchTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName)
{
requireNonNull(tableName, "tableName is null");
if (!tableNames.contains(tableName.getTableName())) {
return null;
}
// parse the scale factor
double scaleFactor = schemaNameToScaleFactor(tableName.getSchemaName());
if (scaleFactor <= 0) {
return null;
}
return new TpchTableHandle(tableName.getTableName(), scaleFactor);
}
@Override
public ConnectorTableHandle getTableHandleForStatisticsCollection(ConnectorSession session, SchemaTableName tableName, Map analyzeProperties)
{
return getTableHandle(session, tableName);
}
private Set filterValues(Set nullableValues, TpchColumn> column, Constraint constraint)
{
return nullableValues.stream()
.filter(convertToPredicate(constraint.getSummary(), toColumnHandle(column)))
.filter(value -> constraint.predicate().isEmpty() || constraint.predicate().get().test(ImmutableMap.of(toColumnHandle(column), value)))
.collect(toSet());
}
@Override
public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle tableHandle)
{
TpchTableHandle tpchTableHandle = (TpchTableHandle) tableHandle;
TpchTable> tpchTable = TpchTable.getTable(tpchTableHandle.getTableName());
String schemaName = scaleFactorSchemaName(tpchTableHandle.getScaleFactor());
return getTableMetadata(schemaName, tpchTable, columnNaming);
}
private static ConnectorTableMetadata getTableMetadata(String schemaName, TpchTable> tpchTable, ColumnNaming columnNaming)
{
ImmutableList.Builder columns = ImmutableList.builder();
for (TpchColumn extends TpchEntity> column : tpchTable.getColumns()) {
columns.add(ColumnMetadata.builder()
.setName(columnNaming.getName(column))
.setType(getPrestoType(column))
.setNullable(false)
.build());
}
columns.add(ColumnMetadata.builder()
.setName(ROW_NUMBER_COLUMN_NAME)
.setType(BIGINT)
.setHidden(true)
.build());
SchemaTableName tableName = new SchemaTableName(schemaName, tpchTable.getTableName());
return new ConnectorTableMetadata(tableName, columns.build());
}
@Override
public Map getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle)
{
ImmutableMap.Builder builder = ImmutableMap.builder();
for (ColumnMetadata columnMetadata : getTableMetadata(session, tableHandle).getColumns()) {
builder.put(columnMetadata.getName(), new TpchColumnHandle(columnMetadata.getName(), columnMetadata.getType()));
}
return builder.build();
}
@Override
public Map> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix)
{
ImmutableMap.Builder> tableColumns = ImmutableMap.builder();
for (String schemaName : getSchemaNames(session, prefix.getSchema())) {
for (TpchTable> tpchTable : TpchTable.getTables()) {
if (prefix.getTable().map(tpchTable.getTableName()::equals).orElse(true)) {
ConnectorTableMetadata tableMetadata = getTableMetadata(schemaName, tpchTable, columnNaming);
tableColumns.put(new SchemaTableName(schemaName, tpchTable.getTableName()), tableMetadata.getColumns());
}
}
}
return tableColumns.build();
}
@Override
public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Constraint constraint)
{
TpchTableHandle tpchTableHandle = (TpchTableHandle) tableHandle;
String tableName = tpchTableHandle.getTableName();
TpchTable> tpchTable = TpchTable.getTable(tableName);
Map, List
© 2015 - 2025 Weber Informatics LLC | Privacy Policy