com.clickzetta.platform.client.schemachange.EvolutionManagerImpl Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of clickzetta-java Show documentation
Show all versions of clickzetta-java Show documentation
The java SDK for clickzetta's Lakehouse
package com.clickzetta.platform.client.schemachange;
import com.clickzetta.platform.arrow.ArrowTable;
import com.clickzetta.platform.client.Table;
import com.clickzetta.platform.client.api.ClientContext;
import com.clickzetta.platform.client.api.Options;
import com.clickzetta.platform.client.proxy.RpcProxy;
import com.clickzetta.platform.common.Constant;
import com.clickzetta.platform.util.AsyncUtil;
import com.google.common.base.Preconditions;
import cz.proto.DataType;
import cz.proto.DataTypeCategory;
import cz.proto.FieldSchema;
import cz.proto.access.Ddl;
import cz.proto.ingestion.v2.IngestionV2;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Tuple2;
import scala.Tuple3;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
public class EvolutionManagerImpl implements EvolutionManager {
private static final Logger LOG = LoggerFactory.getLogger(EvolutionManager.class);
private static final List EMPTY_LIST = Collections.emptyList();
private final boolean evolutionSupport;
private RpcProxy rpcProxy; // only init when evolution support.
public EvolutionManagerImpl(Options options, RpcProxy rpcProxy) {
Object obj = options.getProperties().getOrDefault(Constant.SCHEMA_EVOLUTION_SUPPORT, false);
this.evolutionSupport = obj instanceof String ? Boolean.parseBoolean((String) obj) : (boolean) obj;
if (this.evolutionSupport) {
this.rpcProxy = rpcProxy;
}
}
@Override
public boolean schemaEvolutionSupport() {
return this.evolutionSupport;
}
@Override
public List extractSchemaChanges(Table table, List updatedDataFields) {
if (evolutionSupport) {
Preconditions.checkArgument(table instanceof ArrowTable, "only v2 support schema evolution.");
// collect all data fields & find new add column.
ArrowTable arrowTable = (ArrowTable) table;
List dataFields = arrowTable.getStreamSchema().getDataFieldsList();
Map originalFieldMaps = new HashMap<>();
for (IngestionV2.DataField oldField : dataFields) {
originalFieldMaps.put(oldField.getName(), oldField);
}
// to all lower case compare with table field schema & updatedDataFields.
updatedDataFields = updatedDataFields.stream().map(dataField -> {
if (StringUtils.isAllLowerCase(dataField.getName())) {
return dataField;
} else {
return dataField.toBuilder().setName(dataField.getName().toLowerCase(Locale.ROOT)).build();
}
}).collect(Collectors.toList());
// only support add column.
List result = new ArrayList<>();
for (IngestionV2.DataField newField : updatedDataFields) {
if (!originalFieldMaps.containsKey(newField.getName())) {
result.add(SchemaChange.addColumn(newField.getName(), newField.getType()));
}
}
if (!result.isEmpty()) {
LOG.info("extract schema changes with {}", result);
}
return result;
}
return EMPTY_LIST;
}
@Override
public Future applySchemaChange(String schemaName, String tableName,
Table table, List schemaChanges,
SchemaChange.Callback callback, String serverToken) throws IOException {
if (evolutionSupport && !schemaChanges.isEmpty()) {
// send rpc to controller to do schema change.
CompletableFuture finalFuture = new CompletableFuture<>();
// trigger async call to apply schema change.
CompletableFuture triggerFuture = new CompletableFuture<>();
triggerFuture.whenCompleteAsync((aBoolean, throwable) -> {
try {
rpcProxy.schemaChange(schemaName, tableName, schemaChanges, this::buildSchemaChange, serverToken);
// wait for partition loaded. try to get tablet workers with retry.
List> workers;
ClientContext clientContext = rpcProxy.getClientContext();
boolean waitForLoaded;
int retryTime = 0;
int GET_WORKER_MAX_TIMES = 1000;
Properties properties = clientContext.getProperties();
if (properties.get(Constant.GET_WORKER_MAX_RETRY_TIMES) != null) {
GET_WORKER_MAX_TIMES = properties.get(Constant.GET_WORKER_MAX_RETRY_TIMES) instanceof Integer ?
(Integer) properties.get(Constant.GET_WORKER_MAX_RETRY_TIMES) :
Integer.parseInt(properties.getProperty(Constant.GET_WORKER_MAX_RETRY_TIMES));
}
do {
workers = rpcProxy.getRouteWorkers(clientContext.instanceId(), clientContext.workspace(), schemaName, tableName,
clientContext.igsConnectMode());
waitForLoaded = workers.stream().anyMatch(w -> w._1 == null || w._1.length() == 0);
if (waitForLoaded) {
LOG.info("sleep & retry to get all tablet loaded for table {}.{}. times {}", schemaName, tableName, ++retryTime);
try {
Thread.sleep(2 * 1000);
} catch (InterruptedException ite) {
throw new IOException(ite);
}
}
} while (waitForLoaded && retryTime <= GET_WORKER_MAX_TIMES);
if (waitForLoaded) {
throw new IOException(String.format("tablet loaded failed & getRouteWorkers failed reach max retry %s times.", GET_WORKER_MAX_TIMES));
}
Boolean result = callback.call(schemaName, tableName, table, schemaChanges).get();
finalFuture.complete(result);
} catch (Throwable t) {
finalFuture.completeExceptionally(t);
}
}, AsyncUtil.getForkJoinPool());
triggerFuture.complete(true);
return finalFuture;
}
// default.
CompletableFuture future = new CompletableFuture<>();
future.complete(true);
return future;
}
private Tuple2 buildSchemaChange(Tuple3 tuple3) {
String schemaName = tuple3._1();
String tableName = tuple3._2();
SchemaChange schemaChange = tuple3._3();
Ddl.TableChange.Builder builder = Ddl.TableChange.newBuilder();
String ddlSql;
if (schemaChange instanceof SchemaChange.AddColumn) {
SchemaChange.AddColumn addColumn = (SchemaChange.AddColumn) schemaChange;
builder.setColumnAdd(Ddl.ColumnAdd.newBuilder()
.setColumn(FieldSchema.newBuilder()
.setName(addColumn.fieldName())
.setType(addColumn.dataType())
.build())
.setIfNotExists(true).build());
ddlSql = buildDdlSql(schemaName, tableName, schemaChange);
// } else if (schemaChange instanceof SchemaChange.DropColumn) {
// } else if (schemaChange instanceof SchemaChange.RenameColumn) {
// } else if (schemaChange instanceof SchemaChange.UpdateColumnType) {
// } else if (schemaChange instanceof SchemaChange.UpdateColumnPosition) {
// } else if (schemaChange instanceof SchemaChange.UpdateColumnNullability) {
} else {
throw new UnsupportedOperationException(String.format("not support schema change: %s", schemaChange));
}
if (StringUtils.isEmpty(ddlSql)) {
throw new UnsupportedOperationException(String.format("schema change ddlSql is empty with: %s", schemaChange));
} else {
LOG.info("generator ddl sql: {}", ddlSql);
}
return new Tuple2<>(builder.build(), ddlSql);
}
private String buildDdlSql(String schemaName, String tableName, SchemaChange schemaChange) {
String ddlSql = "";
if (schemaChange instanceof SchemaChange.AddColumn) {
SchemaChange.AddColumn addColumn = (SchemaChange.AddColumn) schemaChange;
String name = addColumn.fieldName();
DataType dataType = addColumn.dataType();
ddlSql += String.format("ALTER TABLE %s.%s ADD COLUMN IF NOT EXISTS %s ", schemaName, tableName, name);
ddlSql += buildCondition(dataType);
} else {
throw new UnsupportedOperationException(
String.format("not support buildDdlSql with schema change: %s", schemaChange.getClass().getName()));
}
return ddlSql.replaceAll("\\s+", " ").toLowerCase(Locale.ROOT);
}
private String buildCondition(DataType dataType) {
DataTypeCategory fieldType = dataType.getCategory();
boolean nullable = dataType.getNullable();
String condition = "";
switch (fieldType) {
case BOOLEAN:
condition = " BOOLEAN ";
break;
case INT8:
condition = " TINYINT ";
break;
case INT16:
condition = " SMALLINT ";
break;
case INT32:
condition = " INT ";
break;
case INT64:
condition = " BIGINT ";
break;
case FLOAT32:
condition = " FLOAT ";
break;
case FLOAT64:
condition = " DOUBLE ";
break;
case DECIMAL:
final int precision = Long.valueOf(dataType.getDecimalTypeInfo().getPrecision()).intValue();
final int scale = Long.valueOf(dataType.getDecimalTypeInfo().getScale()).intValue();
condition = String.format(" DECIMAL(%s, %s) ", precision, scale);
break;
case VARCHAR:
final int varCharLength = Long.valueOf(dataType.getVarCharTypeInfo().getLength()).intValue();
condition = String.format(" VARCHAR(%s) ", varCharLength);
break;
case CHAR:
final int charLength = Long.valueOf(dataType.getCharTypeInfo().getLength()).intValue();
condition = String.format(" VARCHAR(%s) ", charLength);
break;
case STRING:
condition = " STRING ";
break;
case BINARY:
condition = " BINARY ";
break;
case DATE:
condition = " DATE ";
break;
case TIMESTAMP_LTZ:
condition = " TIMESTAMP_LTZ ";
break;
case ARRAY:
case MAP:
case STRUCT:
default:
throw new UnsupportedOperationException("not support fieldType: " + fieldType);
}
if (!nullable) {
condition += " NOT NULL ";
}
return condition;
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy