org.dinky.shaded.paimon.catalog.AbstractCatalog Maven / Gradle / Ivy
The newest version!
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.dinky.shaded.paimon.catalog;
import org.dinky.shaded.paimon.annotation.VisibleForTesting;
import org.dinky.shaded.paimon.factories.FactoryUtil;
import org.dinky.shaded.paimon.fs.FileIO;
import org.dinky.shaded.paimon.fs.Path;
import org.dinky.shaded.paimon.lineage.LineageMetaFactory;
import org.dinky.shaded.paimon.operation.FileStoreCommit;
import org.dinky.shaded.paimon.operation.Lock;
import org.dinky.shaded.paimon.options.Options;
import org.dinky.shaded.paimon.schema.Schema;
import org.dinky.shaded.paimon.schema.SchemaChange;
import org.dinky.shaded.paimon.schema.TableSchema;
import org.dinky.shaded.paimon.table.AbstractFileStoreTable;
import org.dinky.shaded.paimon.table.CatalogEnvironment;
import org.dinky.shaded.paimon.table.FileStoreTable;
import org.dinky.shaded.paimon.table.FileStoreTableFactory;
import org.dinky.shaded.paimon.table.Table;
import org.dinky.shaded.paimon.table.sink.BatchWriteBuilder;
import org.dinky.shaded.paimon.table.system.SystemTableLoader;
import org.dinky.shaded.paimon.utils.StringUtils;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import static org.dinky.shaded.paimon.options.CatalogOptions.LINEAGE_META;
import static org.dinky.shaded.paimon.options.OptionsUtils.convertToPropertiesPrefixKey;
import static org.dinky.shaded.paimon.utils.Preconditions.checkArgument;
/** Common implementation of {@link Catalog}. */
public abstract class AbstractCatalog implements Catalog {
public static final String DB_SUFFIX = ".db";
protected static final String TABLE_DEFAULT_OPTION_PREFIX = "table-default.";
protected final FileIO fileIO;
protected final Map tableDefaultOptions;
protected final Options catalogOptions;
@Nullable protected final LineageMetaFactory lineageMetaFactory;
protected AbstractCatalog(FileIO fileIO) {
this.fileIO = fileIO;
this.lineageMetaFactory = null;
this.tableDefaultOptions = new HashMap<>();
this.catalogOptions = new Options();
}
protected AbstractCatalog(FileIO fileIO, Options options) {
this.fileIO = fileIO;
this.lineageMetaFactory =
findAndCreateLineageMeta(options, AbstractCatalog.class.getClassLoader());
this.tableDefaultOptions =
convertToPropertiesPrefixKey(options.toMap(), TABLE_DEFAULT_OPTION_PREFIX);
this.catalogOptions = options;
}
@Override
public boolean databaseExists(String databaseName) {
if (isSystemDatabase(databaseName)) {
return true;
}
return databaseExistsImpl(databaseName);
}
protected abstract boolean databaseExistsImpl(String databaseName);
@Override
public void createDatabase(String name, boolean ignoreIfExists)
throws DatabaseAlreadyExistException {
if (isSystemDatabase(name)) {
throw new ProcessSystemDatabaseException();
}
if (databaseExists(name)) {
if (ignoreIfExists) {
return;
}
throw new DatabaseAlreadyExistException(name);
}
createDatabaseImpl(name);
}
@Override
public void dropPartition(Identifier identifier, Map partitionSpec)
throws TableNotExistException {
Table table = getTable(identifier);
AbstractFileStoreTable fileStoreTable = (AbstractFileStoreTable) table;
FileStoreCommit commit = fileStoreTable.store().newCommit(UUID.randomUUID().toString());
commit.dropPartitions(
Collections.singletonList(partitionSpec), BatchWriteBuilder.COMMIT_IDENTIFIER);
}
protected abstract void createDatabaseImpl(String name);
@Override
public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade)
throws DatabaseNotExistException, DatabaseNotEmptyException {
if (isSystemDatabase(name)) {
throw new ProcessSystemDatabaseException();
}
if (!databaseExists(name)) {
if (ignoreIfNotExists) {
return;
}
throw new DatabaseNotExistException(name);
}
if (!cascade && listTables(name).size() > 0) {
throw new DatabaseNotEmptyException(name);
}
dropDatabaseImpl(name);
}
protected abstract void dropDatabaseImpl(String name);
@Override
public List listTables(String databaseName) throws DatabaseNotExistException {
if (isSystemDatabase(databaseName)) {
return SystemTableLoader.loadGlobalTableNames();
}
if (!databaseExists(databaseName)) {
throw new DatabaseNotExistException(databaseName);
}
return listTablesImpl(databaseName).stream().sorted().collect(Collectors.toList());
}
protected abstract List listTablesImpl(String databaseName);
@Override
public void dropTable(Identifier identifier, boolean ignoreIfNotExists)
throws TableNotExistException {
checkNotSystemTable(identifier, "dropTable");
if (!tableExists(identifier)) {
if (ignoreIfNotExists) {
return;
}
throw new TableNotExistException(identifier);
}
dropTableImpl(identifier);
}
protected abstract void dropTableImpl(Identifier identifier);
@Override
public void createTable(Identifier identifier, Schema schema, boolean ignoreIfExists)
throws TableAlreadyExistException, DatabaseNotExistException {
checkNotSystemTable(identifier, "createTable");
validateIdentifierNameCaseInsensitive(identifier);
validateFieldNameCaseInsensitive(schema.rowType().getFieldNames());
if (!databaseExists(identifier.getDatabaseName())) {
throw new DatabaseNotExistException(identifier.getDatabaseName());
}
if (tableExists(identifier)) {
if (ignoreIfExists) {
return;
}
throw new TableAlreadyExistException(identifier);
}
copyTableDefaultOptions(schema.options());
createTableImpl(identifier, schema);
}
protected abstract void createTableImpl(Identifier identifier, Schema schema);
@Override
public void renameTable(Identifier fromTable, Identifier toTable, boolean ignoreIfNotExists)
throws TableNotExistException, TableAlreadyExistException {
checkNotSystemTable(fromTable, "renameTable");
checkNotSystemTable(toTable, "renameTable");
validateIdentifierNameCaseInsensitive(toTable);
if (!tableExists(fromTable)) {
if (ignoreIfNotExists) {
return;
}
throw new TableNotExistException(fromTable);
}
if (tableExists(toTable)) {
throw new TableAlreadyExistException(toTable);
}
renameTableImpl(fromTable, toTable);
}
protected abstract void renameTableImpl(Identifier fromTable, Identifier toTable);
@Override
public void alterTable(
Identifier identifier, List changes, boolean ignoreIfNotExists)
throws TableNotExistException, ColumnAlreadyExistException, ColumnNotExistException {
checkNotSystemTable(identifier, "alterTable");
validateIdentifierNameCaseInsensitive(identifier);
validateFieldNameCaseInsensitiveInSchemaChange(changes);
if (!tableExists(identifier)) {
if (ignoreIfNotExists) {
return;
}
throw new TableNotExistException(identifier);
}
alterTableImpl(identifier, changes);
}
protected abstract void alterTableImpl(Identifier identifier, List changes)
throws TableNotExistException, ColumnAlreadyExistException, ColumnNotExistException;
@Nullable
private LineageMetaFactory findAndCreateLineageMeta(Options options, ClassLoader classLoader) {
return options.getOptional(LINEAGE_META)
.map(
meta ->
FactoryUtil.discoverFactory(
classLoader, LineageMetaFactory.class, meta))
.orElse(null);
}
@Override
public Table getTable(Identifier identifier) throws TableNotExistException {
if (isSystemDatabase(identifier.getDatabaseName())) {
String tableName = identifier.getObjectName();
Table table =
SystemTableLoader.loadGlobal(
tableName,
fileIO,
this::allTablePaths,
catalogOptions,
lineageMetaFactory);
if (table == null) {
throw new TableNotExistException(identifier);
}
return table;
} else if (isSpecifiedSystemTable(identifier)) {
String[] splits = tableAndSystemName(identifier);
String tableName = splits[0];
String type = splits[1];
FileStoreTable originTable =
getDataTable(new Identifier(identifier.getDatabaseName(), tableName));
Table table = SystemTableLoader.load(type, fileIO, originTable);
if (table == null) {
throw new TableNotExistException(identifier);
}
return table;
} else {
return getDataTable(identifier);
}
}
private FileStoreTable getDataTable(Identifier identifier) throws TableNotExistException {
TableSchema tableSchema = getDataTableSchema(identifier);
return FileStoreTableFactory.create(
fileIO,
getDataTableLocation(identifier),
tableSchema,
new CatalogEnvironment(
Lock.factory(lockFactory().orElse(null), identifier),
metastoreClientFactory(identifier).orElse(null),
lineageMetaFactory));
}
@VisibleForTesting
public Path newDatabasePath(String database) {
return newDatabasePath(warehouse(), database);
}
Map> allTablePaths() {
try {
Map> allPaths = new HashMap<>();
for (String database : listDatabases()) {
Map tableMap =
allPaths.computeIfAbsent(database, d -> new HashMap<>());
for (String table : listTables(database)) {
tableMap.put(table, getDataTableLocation(Identifier.create(database, table)));
}
}
return allPaths;
} catch (DatabaseNotExistException e) {
throw new RuntimeException("Database is deleted while listing", e);
}
}
public abstract String warehouse();
public Map options() {
return catalogOptions.toMap();
}
protected abstract TableSchema getDataTableSchema(Identifier identifier)
throws TableNotExistException;
@VisibleForTesting
public Path getDataTableLocation(Identifier identifier) {
return newTableLocation(warehouse(), identifier);
}
private static boolean isSpecifiedSystemTable(Identifier identifier) {
return identifier.getObjectName().contains(SYSTEM_TABLE_SPLITTER);
}
protected boolean isSystemTable(Identifier identifier) {
return isSystemDatabase(identifier.getDatabaseName()) || isSpecifiedSystemTable(identifier);
}
private void checkNotSystemTable(Identifier identifier, String method) {
if (isSystemTable(identifier)) {
throw new IllegalArgumentException(
String.format(
"Cannot '%s' for system table '%s', please use data table.",
method, identifier));
}
}
public void copyTableDefaultOptions(Map options) {
tableDefaultOptions.forEach(options::putIfAbsent);
}
public FileIO fileIO() {
return fileIO;
}
private String[] tableAndSystemName(Identifier identifier) {
String[] splits = StringUtils.split(identifier.getObjectName(), SYSTEM_TABLE_SPLITTER);
if (splits.length != 2) {
throw new IllegalArgumentException(
"System table can only contain one '$' separator, but this is: "
+ identifier.getObjectName());
}
return splits;
}
public static Path newTableLocation(String warehouse, Identifier identifier) {
if (isSpecifiedSystemTable(identifier)) {
throw new IllegalArgumentException(
String.format(
"Table name[%s] cannot contain '%s' separator",
identifier.getObjectName(), SYSTEM_TABLE_SPLITTER));
}
return new Path(
newDatabasePath(warehouse, identifier.getDatabaseName()),
identifier.getObjectName());
}
public static Path newDatabasePath(String warehouse, String database) {
return new Path(warehouse, database + DB_SUFFIX);
}
private boolean isSystemDatabase(String database) {
return SYSTEM_DATABASE_NAME.equals(database);
}
/** Validate database, table and field names must be lowercase when not case-sensitive. */
public static void validateCaseInsensitive(
boolean caseSensitive, String type, String... names) {
validateCaseInsensitive(caseSensitive, type, Arrays.asList(names));
}
/** Validate database, table and field names must be lowercase when not case-sensitive. */
public static void validateCaseInsensitive(
boolean caseSensitive, String type, List names) {
if (caseSensitive) {
return;
}
List illegalNames =
names.stream().filter(f -> !f.equals(f.toLowerCase())).collect(Collectors.toList());
checkArgument(
illegalNames.isEmpty(),
String.format(
"%s name %s cannot contain upper case in the catalog.",
type, illegalNames));
}
private void validateIdentifierNameCaseInsensitive(Identifier identifier) {
validateCaseInsensitive(caseSensitive(), "Database", identifier.getDatabaseName());
validateCaseInsensitive(caseSensitive(), "Table", identifier.getObjectName());
}
private void validateFieldNameCaseInsensitiveInSchemaChange(List changes) {
List fieldNames = new ArrayList<>();
for (SchemaChange change : changes) {
if (change instanceof SchemaChange.AddColumn) {
SchemaChange.AddColumn addColumn = (SchemaChange.AddColumn) change;
fieldNames.add(addColumn.fieldName());
} else if (change instanceof SchemaChange.RenameColumn) {
SchemaChange.RenameColumn rename = (SchemaChange.RenameColumn) change;
fieldNames.add(rename.newName());
} else {
// do nothing
}
}
validateFieldNameCaseInsensitive(fieldNames);
}
private void validateFieldNameCaseInsensitive(List fieldNames) {
validateCaseInsensitive(caseSensitive(), "Field", fieldNames);
}
}