org.apache.iotdb.db.service.DataNode Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.iotdb.db.service;
import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TEndPoint;
import org.apache.iotdb.common.rpc.thrift.TNodeResource;
import org.apache.iotdb.commons.client.exception.ClientManagerException;
import org.apache.iotdb.commons.concurrent.IoTDBDefaultThreadExceptionHandler;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.consensus.ConsensusGroupId;
import org.apache.iotdb.commons.exception.StartupException;
import org.apache.iotdb.commons.file.SystemFileFactory;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.plugin.meta.PipePluginMeta;
import org.apache.iotdb.commons.service.JMXService;
import org.apache.iotdb.commons.service.RegisterManager;
import org.apache.iotdb.commons.service.ServiceType;
import org.apache.iotdb.commons.service.metric.MetricService;
import org.apache.iotdb.commons.trigger.TriggerInformation;
import org.apache.iotdb.commons.trigger.exception.TriggerManagementException;
import org.apache.iotdb.commons.trigger.service.TriggerExecutableManager;
import org.apache.iotdb.commons.udf.UDFInformation;
import org.apache.iotdb.commons.udf.service.UDFClassLoaderManager;
import org.apache.iotdb.commons.udf.service.UDFExecutableManager;
import org.apache.iotdb.commons.udf.service.UDFManagementService;
import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRegisterReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRegisterResp;
import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRestartReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRestartResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetJarInListReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetJarInListResp;
import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo;
import org.apache.iotdb.confignode.rpc.thrift.TRuntimeConfiguration;
import org.apache.iotdb.confignode.rpc.thrift.TSystemConfigurationResp;
import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.db.conf.DataNodeStartupCheck;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.IoTDBStartCheck;
import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor;
import org.apache.iotdb.db.consensus.DataRegionConsensusImpl;
import org.apache.iotdb.db.consensus.SchemaRegionConsensusImpl;
import org.apache.iotdb.db.pipe.agent.PipeAgent;
import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager;
import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
import org.apache.iotdb.db.protocol.thrift.impl.ClientRPCServiceImpl;
import org.apache.iotdb.db.protocol.thrift.impl.DataNodeRegionManager;
import org.apache.iotdb.db.qp.sql.IoTDBSqlParser;
import org.apache.iotdb.db.qp.sql.SqlLexer;
import org.apache.iotdb.db.queryengine.execution.exchange.MPPDataExchangeService;
import org.apache.iotdb.db.queryengine.execution.schedule.DriverScheduler;
import org.apache.iotdb.db.queryengine.plan.parser.ASTVisitor;
import org.apache.iotdb.db.queryengine.plan.parser.StatementGenerator;
import org.apache.iotdb.db.queryengine.plan.planner.LogicalPlanVisitor;
import org.apache.iotdb.db.queryengine.plan.planner.distribution.DistributionPlanContext;
import org.apache.iotdb.db.queryengine.plan.planner.distribution.SourceRewriter;
import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan;
import org.apache.iotdb.db.schemaengine.SchemaEngine;
import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager;
import org.apache.iotdb.db.service.metrics.DataNodeMetricsHelper;
import org.apache.iotdb.db.service.metrics.IoTDBInternalLocalReporter;
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.buffer.CacheHitRatioMonitor;
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduleTaskManager;
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager;
import org.apache.iotdb.db.storageengine.dataregion.flush.FlushManager;
import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor;
import org.apache.iotdb.db.storageengine.dataregion.wal.WALManager;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALMode;
import org.apache.iotdb.db.storageengine.rescon.disk.TierManager;
import org.apache.iotdb.db.subscription.agent.SubscriptionAgent;
import org.apache.iotdb.db.trigger.executor.TriggerExecutor;
import org.apache.iotdb.db.trigger.service.TriggerInformationUpdater;
import org.apache.iotdb.db.trigger.service.TriggerManagementService;
import org.apache.iotdb.metrics.config.MetricConfigDescriptor;
import org.apache.iotdb.metrics.utils.InternalReporterType;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.udf.api.exception.UDFManagementException;
import org.antlr.v4.runtime.CommonTokenStream;
import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static org.apache.iotdb.commons.conf.IoTDBConstant.DEFAULT_CLUSTER_NAME;
public class DataNode implements DataNodeMBean {
private static final Logger logger = LoggerFactory.getLogger(DataNode.class);
private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
private final String mbeanName =
String.format(
"%s:%s=%s",
IoTDBConstant.IOTDB_SERVICE_JMX_NAME,
IoTDBConstant.JMX_TYPE,
ServiceType.DATA_NODE.getJmxName());
private static File SYSTEM_PROPERTIES =
SystemFileFactory.INSTANCE.getFile(
config.getSystemDir() + File.separator + IoTDBStartCheck.PROPERTIES_FILE_NAME);
/**
* When joining a cluster or getting configuration this node will retry at most "DEFAULT_RETRY"
* times before returning a failure to the client.
*/
private static final int DEFAULT_RETRY = 50;
private static final long DEFAULT_RETRY_INTERVAL_IN_MS = config.getJoinClusterRetryIntervalMs();
private final TEndPoint thisNode = new TEndPoint();
/** Hold the information of trigger, udf...... */
private final ResourcesInformationHolder resourcesInformationHolder =
new ResourcesInformationHolder();
/** Responsible for keeping trigger information up to date. */
private final TriggerInformationUpdater triggerInformationUpdater =
new TriggerInformationUpdater();
private static final String REGISTER_INTERRUPTION =
"Unexpected interruption when waiting to register to the cluster";
private boolean schemaRegionConsensusStarted = false;
private boolean dataRegionConsensusStarted = false;
private DataNode() {
// We do not init anything here, so that we can re-initialize the instance in IT.
}
// TODO: This needs removal of statics ...
public static void reinitializeStatics() {
SYSTEM_PROPERTIES =
SystemFileFactory.INSTANCE.getFile(
config.getSystemDir() + File.separator + IoTDBStartCheck.PROPERTIES_FILE_NAME);
registerManager = new RegisterManager();
}
private static RegisterManager registerManager = new RegisterManager();
public static DataNode getInstance() {
return DataNodeHolder.INSTANCE;
}
public static void main(String[] args) {
logger.info("IoTDB-DataNode environment variables: {}", IoTDBConfig.getEnvironmentVariables());
logger.info("IoTDB-DataNode default charset is: {}", Charset.defaultCharset().displayName());
new DataNodeServerCommandLine().doMain(args);
}
protected void doAddNode() {
boolean isFirstStart = false;
try {
// Check if this DataNode is start for the first time and do other pre-checks
isFirstStart = prepareDataNode();
if (isFirstStart) {
// Set target ConfigNodeList from iotdb-datanode.properties file
ConfigNodeInfo.getInstance()
.updateConfigNodeList(Collections.singletonList(config.getSeedConfigNode()));
} else {
// Load registered ConfigNodes from system.properties
ConfigNodeInfo.getInstance().loadConfigNodeList();
}
// Pull and check system configurations from ConfigNode-leader
pullAndCheckSystemConfigurations();
if (isFirstStart) {
// Register this DataNode to the cluster when first start
sendRegisterRequestToConfigNode();
} else {
// Send restart request of this DataNode
sendRestartRequestToConfigNode();
}
// TierManager need DataNodeId to do some operations so the reset method need to be invoked
// after DataNode adding
TierManager.getInstance().resetFolders();
// Active DataNode
active();
// Setup metric service
setUpMetricService();
// Setup rpc service
setUpRPCService();
// Serialize mutable system properties
IoTDBStartCheck.getInstance().serializeMutableSystemPropertiesIfNecessary();
logger.info("IoTDB configuration: {}", config.getConfigMessage());
logger.info("Congratulations, IoTDB DataNode is set up successfully. Now, enjoy yourself!");
} catch (StartupException | IOException e) {
logger.error("Fail to start server", e);
if (isFirstStart) {
// Delete the system.properties file when first start failed.
// Therefore, the next time this DataNode is start will still be seen as the first time.
SYSTEM_PROPERTIES.deleteOnExit();
}
stop();
System.exit(-1);
}
}
/** Prepare cluster IoTDB-DataNode */
private boolean prepareDataNode() throws StartupException, IOException {
long startTime = System.currentTimeMillis();
// Notice: Consider this DataNode as first start if the system.properties file doesn't exist
IoTDBStartCheck.getInstance().checkOldSystemConfig();
boolean isFirstStart = IoTDBStartCheck.getInstance().checkIsFirstStart();
// Set this node
thisNode.setIp(config.getInternalAddress());
thisNode.setPort(config.getInternalPort());
// Startup checks
DataNodeStartupCheck checks = new DataNodeStartupCheck(IoTDBConstant.DN_ROLE, config);
checks.startUpCheck();
long endTime = System.currentTimeMillis();
logger.info("The DataNode is prepared successfully, which takes {} ms", (endTime - startTime));
return isFirstStart;
}
/**
* Pull and check the following system configurations:
*
* 1. GlobalConfig
*
*
2. RatisConfig
*
*
3. CQConfig
*
* @throws StartupException When failed connect to ConfigNode-leader
*/
private void pullAndCheckSystemConfigurations() throws StartupException {
logger.info("Pulling system configurations from the ConfigNode-leader...");
long startTime = System.currentTimeMillis();
/* Pull system configurations */
int retry = DEFAULT_RETRY;
TSystemConfigurationResp configurationResp = null;
while (retry > 0) {
try (ConfigNodeClient configNodeClient =
ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
configurationResp = configNodeClient.getSystemConfiguration();
break;
} catch (TException | ClientManagerException e) {
logger.warn(
"Cannot pull system configurations from ConfigNode-leader, because: {}",
e.getMessage());
retry--;
}
try {
// wait to start the next try
Thread.sleep(DEFAULT_RETRY_INTERVAL_IN_MS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
logger.warn(REGISTER_INTERRUPTION, e);
retry = -1;
}
}
if (configurationResp == null
|| !configurationResp.isSetStatus()
|| configurationResp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
// All tries failed
logger.error(
"Cannot pull system configurations from ConfigNode-leader after {} retries.",
DEFAULT_RETRY);
throw new StartupException(
"Cannot pull system configurations from ConfigNode-leader. "
+ "Please check whether the dn_seed_config_node in iotdb-datanode.properties is correct or alive.");
}
/* Load system configurations */
IoTDBDescriptor.getInstance().loadGlobalConfig(configurationResp.globalConfig);
IoTDBDescriptor.getInstance().loadRatisConfig(configurationResp.ratisConfig);
IoTDBDescriptor.getInstance().loadCQConfig(configurationResp.cqConfig);
CommonDescriptor.getInstance().loadGlobalConfig(configurationResp.globalConfig);
/* Set cluster consensus protocol class */
if (!IoTDBStartCheck.getInstance()
.checkConsensusProtocolExists(TConsensusGroupType.DataRegion)) {
config.setDataRegionConsensusProtocolClass(
configurationResp.globalConfig.getDataRegionConsensusProtocolClass());
}
if (!IoTDBStartCheck.getInstance()
.checkConsensusProtocolExists(TConsensusGroupType.SchemaRegion)) {
config.setSchemaRegionConsensusProtocolClass(
configurationResp.globalConfig.getSchemaRegionConsensusProtocolClass());
}
/* Check system configurations */
try {
IoTDBStartCheck.getInstance().checkSystemConfig();
IoTDBStartCheck.getInstance().checkDirectory();
if (!config.getDataRegionConsensusProtocolClass().equals(ConsensusFactory.IOT_CONSENSUS)) {
// In current implementation, only IoTConsensus need separated memory from Consensus
IoTDBDescriptor.getInstance().reclaimConsensusMemory();
}
} catch (Exception e) {
throw new StartupException(e.getMessage());
}
long endTime = System.currentTimeMillis();
logger.info(
"Successfully pull system configurations from ConfigNode-leader, which takes {} ms",
(endTime - startTime));
}
/**
* Store runtime configurations, which includes:
*
*
1. All ConfigNodes in cluster
*
*
2. All template information
*
*
3. All UDF information
*
*
4. All trigger information
*
*
5. All Pipe information
*
*
6. All TTL information
*/
private void storeRuntimeConfigurations(
List configNodeLocations, TRuntimeConfiguration runtimeConfiguration) {
/* Store ConfigNodeList */
List configNodeList = new ArrayList<>();
for (TConfigNodeLocation configNodeLocation : configNodeLocations) {
configNodeList.add(configNodeLocation.getInternalEndPoint());
}
ConfigNodeInfo.getInstance().updateConfigNodeList(configNodeList);
/* Store templateSetInfo */
ClusterTemplateManager.getInstance()
.updateTemplateSetInfo(runtimeConfiguration.getTemplateInfo());
/* Store udfInformationList */
getUDFInformationList(runtimeConfiguration.getAllUDFInformation());
/* Store triggerInformationList */
getTriggerInformationList(runtimeConfiguration.getAllTriggerInformation());
/* Store pipeInformationList */
getPipeInformationList(runtimeConfiguration.getAllPipeInformation());
/* Store ttl information */
StorageEngine.getInstance().updateTTLInfo(runtimeConfiguration.getAllTTLInformation());
/* Store cluster ID */
IoTDBDescriptor.getInstance().getConfig().setClusterId(runtimeConfiguration.getClusterId());
}
/**
* Register this DataNode into cluster.
*
* @throws StartupException if register failed.
* @throws IOException if serialize cluster name and dataNode Id failed.
*/
private void sendRegisterRequestToConfigNode() throws StartupException, IOException {
logger.info("Sending register request to ConfigNode-leader...");
long startTime = System.currentTimeMillis();
/* Send register request */
int retry = DEFAULT_RETRY;
TDataNodeRegisterReq req = new TDataNodeRegisterReq();
req.setDataNodeConfiguration(generateDataNodeConfiguration());
req.setClusterName(config.getClusterName());
req.setVersionInfo(new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO));
TDataNodeRegisterResp dataNodeRegisterResp = null;
while (retry > 0) {
try (ConfigNodeClient configNodeClient =
ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
dataNodeRegisterResp = configNodeClient.registerDataNode(req);
break;
} catch (TException | ClientManagerException e) {
logger.warn("Cannot register to the cluster, because: {}", e.getMessage());
retry--;
}
try {
// Wait to start the next try
Thread.sleep(DEFAULT_RETRY_INTERVAL_IN_MS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
logger.warn(REGISTER_INTERRUPTION, e);
retry = -1;
}
}
if (dataNodeRegisterResp == null) {
// All tries failed
logger.error("Cannot register into cluster after {} retries.", DEFAULT_RETRY);
throw new StartupException(
"Cannot register into the cluster. "
+ "Please check whether the dn_seed_config_node in iotdb-datanode.properties is correct or alive.");
}
if (dataNodeRegisterResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
/* Store runtime configurations when register success */
int dataNodeID = dataNodeRegisterResp.getDataNodeId();
config.setDataNodeId(dataNodeID);
IoTDBStartCheck.getInstance()
.serializeClusterNameAndDataNodeId(config.getClusterName(), dataNodeID);
storeRuntimeConfigurations(
dataNodeRegisterResp.getConfigNodeList(), dataNodeRegisterResp.getRuntimeConfiguration());
long endTime = System.currentTimeMillis();
logger.info(
"Successfully register to the cluster: {} , which takes {} ms.",
config.getClusterName(),
(endTime - startTime));
} else {
/* Throw exception when register failed */
logger.error(dataNodeRegisterResp.getStatus().getMessage());
throw new StartupException("Cannot register to the cluster.");
}
}
private void removeInvalidRegions(List dataNodeConsensusGroupIds) {
List invalidConsensusGroupIds =
DataRegionConsensusImpl.getInstance().getAllConsensusGroupIdsWithoutStarting().stream()
.filter(consensusGroupId -> !dataNodeConsensusGroupIds.contains(consensusGroupId))
.collect(Collectors.toList());
if (!invalidConsensusGroupIds.isEmpty()) {
logger.info("Remove invalid region directories... {}", invalidConsensusGroupIds);
for (ConsensusGroupId consensusGroupId : invalidConsensusGroupIds) {
File oldDir =
new File(
DataRegionConsensusImpl.getInstance()
.getRegionDirFromConsensusGroupId(consensusGroupId));
if (oldDir.exists()) {
try {
FileUtils.recursivelyDeleteFolder(oldDir.getPath());
logger.info("delete {} succeed.", oldDir.getAbsolutePath());
} catch (IOException e) {
logger.error("delete {} failed.", oldDir.getAbsolutePath());
}
} else {
logger.info("delete {} failed, because it does not exist.", oldDir.getAbsolutePath());
}
}
}
}
private void sendRestartRequestToConfigNode() throws StartupException {
logger.info("Sending restart request to ConfigNode-leader...");
long startTime = System.currentTimeMillis();
/* Send restart request */
int retry = DEFAULT_RETRY;
TDataNodeRestartReq req = new TDataNodeRestartReq();
req.setClusterName(
config.getClusterName() == null ? DEFAULT_CLUSTER_NAME : config.getClusterName());
req.setDataNodeConfiguration(generateDataNodeConfiguration());
req.setVersionInfo(new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO));
TDataNodeRestartResp dataNodeRestartResp = null;
while (retry > 0) {
try (ConfigNodeClient configNodeClient =
ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
dataNodeRestartResp = configNodeClient.restartDataNode(req);
break;
} catch (TException | ClientManagerException e) {
logger.warn(
"Cannot send restart request to the ConfigNode-leader, because: {}", e.getMessage());
retry--;
}
try {
// Wait to start the next try
Thread.sleep(DEFAULT_RETRY_INTERVAL_IN_MS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
logger.warn(REGISTER_INTERRUPTION, e);
retry = -1;
}
}
if (dataNodeRestartResp == null) {
// All tries failed
logger.error(
"Cannot send restart DataNode request to ConfigNode-leader after {} retries.",
DEFAULT_RETRY);
throw new StartupException(
"Cannot send restart DataNode request to ConfigNode-leader. "
+ "Please check whether the dn_seed_config_node in iotdb-datanode.properties is correct or alive.");
}
if (dataNodeRestartResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
/* Store runtime configurations when restart request is accepted */
storeRuntimeConfigurations(
dataNodeRestartResp.getConfigNodeList(), dataNodeRestartResp.getRuntimeConfiguration());
long endTime = System.currentTimeMillis();
logger.info(
"Restart request to cluster: {} is accepted, which takes {} ms.",
config.getClusterName(),
(endTime - startTime));
List consensusGroupIds = dataNodeRestartResp.getConsensusGroupIds();
List dataNodeConsensusGroupIds =
consensusGroupIds.stream()
.map(ConsensusGroupId.Factory::createFromTConsensusGroupId)
.collect(Collectors.toList());
removeInvalidRegions(dataNodeConsensusGroupIds);
} else {
/* Throw exception when restart is rejected */
throw new StartupException(dataNodeRestartResp.getStatus().getMessage());
}
}
private void prepareResources() throws StartupException {
prepareUDFResources();
prepareTriggerResources();
preparePipeResources();
}
/**
* Register services and set up DataNode.
*
* @throws StartupException if start up failed.
*/
private void active() throws StartupException {
try {
processPid();
setUp();
} catch (StartupException e) {
logger.error("Meet error while starting up.", e);
throw new StartupException("Error in activating IoTDB DataNode.");
}
logger.info("IoTDB DataNode has started.");
try {
long startTime = System.currentTimeMillis();
SchemaRegionConsensusImpl.getInstance().start();
long schemaRegionEndTime = System.currentTimeMillis();
logger.info(
"SchemaRegion consensus start successfully, which takes {} ms.",
(schemaRegionEndTime - startTime));
schemaRegionConsensusStarted = true;
DataRegionConsensusImpl.getInstance().start();
long dataRegionEndTime = System.currentTimeMillis();
logger.info(
"DataRegion consensus start successfully, which takes {} ms.",
(dataRegionEndTime - schemaRegionEndTime));
dataRegionConsensusStarted = true;
} catch (IOException e) {
throw new StartupException(e);
}
}
void processPid() {
String pidFile = System.getProperty(IoTDBConstant.IOTDB_PIDFILE);
if (pidFile != null) {
new File(pidFile).deleteOnExit();
}
}
private void setUp() throws StartupException {
logger.info("Setting up IoTDB DataNode...");
registerManager.register(new JMXService());
JMXService.registerMBean(getInstance(), mbeanName);
// Get resources for trigger,udf,pipe...
prepareResources();
Runtime.getRuntime().addShutdownHook(new IoTDBShutdownHook());
setUncaughtExceptionHandler();
logger.info("Recover the schema...");
initSchemaEngine();
classLoader();
registerManager.register(FlushManager.getInstance());
registerManager.register(CacheHitRatioMonitor.getInstance());
// Close wal when using ratis consensus
if (config.getDataRegionConsensusProtocolClass().equals(ConsensusFactory.RATIS_CONSENSUS)) {
config.setWalMode(WALMode.DISABLE);
}
registerManager.register(WALManager.getInstance());
// Must init before StorageEngine
registerManager.register(CompactionScheduleTaskManager.getInstance());
// In mpp mode we need to start some other services
registerManager.register(StorageEngine.getInstance());
registerManager.register(MPPDataExchangeService.getInstance());
registerManager.register(DriverScheduler.getInstance());
registerUdfServices();
logger.info(
"IoTDB DataNode is setting up, some databases may not be ready now, please wait several seconds...");
long startTime = System.currentTimeMillis();
while (!StorageEngine.getInstance().isAllSgReady()) {
try {
TimeUnit.MILLISECONDS.sleep(1000);
} catch (InterruptedException e) {
logger.warn("IoTDB DataNode failed to set up.", e);
Thread.currentThread().interrupt();
return;
}
}
long endTime = System.currentTimeMillis();
logger.info("Wait for all databases ready, which takes {} ms.", (endTime - startTime));
// Must init after SchemaEngine and StorageEngine prepared well
DataNodeRegionManager.getInstance().init();
// Start region migrate service
registerManager.register(RegionMigrateService.getInstance());
registerManager.register(CompactionTaskManager.getInstance());
// Register subscription agent before pipe agent
registerManager.register(SubscriptionAgent.runtime());
registerManager.register(PipeAgent.runtime());
}
/** Set up RPC and protocols after DataNode is available */
private void setUpRPCService() throws StartupException {
// Start InternalRPCService to indicate that the current DataNode can accept cluster scheduling
registerManager.register(DataNodeInternalRPCService.getInstance());
// Notice: During the period between starting the internal RPC service
// and starting the client RPC service , some requests may fail because
// DataNode is not marked as RUNNING by ConfigNode-leader yet.
// Start client RPCService to indicate that the current DataNode provide external services
IoTDBDescriptor.getInstance()
.getConfig()
.setRpcImplClassName(ClientRPCServiceImpl.class.getName());
if (config.isEnableRpcService()) {
registerManager.register(RPCService.getInstance());
}
// init service protocols
initProtocols();
}
private void setUpMetricService() throws StartupException {
MetricConfigDescriptor.getInstance().getMetricConfig().setNodeId(config.getDataNodeId());
registerManager.register(MetricService.getInstance());
// init metric service
if (MetricConfigDescriptor.getInstance()
.getMetricConfig()
.getInternalReportType()
.equals(InternalReporterType.IOTDB)) {
MetricService.getInstance().updateInternalReporter(new IoTDBInternalLocalReporter());
}
MetricService.getInstance().startInternalReporter();
// bind predefined metrics
DataNodeMetricsHelper.bind();
}
public static TDataNodeLocation generateDataNodeLocation() {
TDataNodeLocation location = new TDataNodeLocation();
location.setDataNodeId(config.getDataNodeId());
location.setClientRpcEndPoint(new TEndPoint(config.getRpcAddress(), config.getRpcPort()));
location.setInternalEndPoint(
new TEndPoint(config.getInternalAddress(), config.getInternalPort()));
location.setMPPDataExchangeEndPoint(
new TEndPoint(config.getInternalAddress(), config.getMppDataExchangePort()));
location.setDataRegionConsensusEndPoint(
new TEndPoint(config.getInternalAddress(), config.getDataRegionConsensusPort()));
location.setSchemaRegionConsensusEndPoint(
new TEndPoint(config.getInternalAddress(), config.getSchemaRegionConsensusPort()));
return location;
}
/**
* Generate dataNodeConfiguration. Warning: Don't private this method !!!
*
* @return TDataNodeConfiguration
*/
public TDataNodeConfiguration generateDataNodeConfiguration() {
// Set DataNodeLocation
TDataNodeLocation location = generateDataNodeLocation();
// Set NodeResource
TNodeResource resource = new TNodeResource();
resource.setCpuCoreNum(Runtime.getRuntime().availableProcessors());
resource.setMaxMemory(Runtime.getRuntime().totalMemory());
return new TDataNodeConfiguration(location, resource);
}
private void registerUdfServices() throws StartupException {
registerManager.register(TemporaryQueryDataFileService.getInstance());
registerManager.register(UDFClassLoaderManager.setupAndGetInstance(config.getUdfDir()));
}
private void initUDFRelatedInstance() throws StartupException {
try {
UDFExecutableManager.setupAndGetInstance(config.getUdfTemporaryLibDir(), config.getUdfDir());
UDFClassLoaderManager.setupAndGetInstance(config.getUdfDir());
} catch (IOException e) {
throw new StartupException(e);
}
}
private void prepareUDFResources() throws StartupException {
long startTime = System.currentTimeMillis();
initUDFRelatedInstance();
if (resourcesInformationHolder.getUDFInformationList() == null
|| resourcesInformationHolder.getUDFInformationList().isEmpty()) {
return;
}
// Get jars from config node
List udfNeedJarList = getJarListForUDF();
int index = 0;
while (index < udfNeedJarList.size()) {
List curList = new ArrayList<>();
int offset = 0;
while (offset < ResourcesInformationHolder.getJarNumOfOneRpc()
&& index + offset < udfNeedJarList.size()) {
curList.add(udfNeedJarList.get(index + offset));
offset++;
}
index += (offset + 1);
getJarOfUDFs(curList);
}
// Create instances of udf and do registration
try {
for (UDFInformation udfInformation : resourcesInformationHolder.getUDFInformationList()) {
UDFManagementService.getInstance().doRegister(udfInformation);
}
} catch (Exception e) {
throw new StartupException(e);
}
long endTime = System.currentTimeMillis();
logger.debug("successfully registered all the UDFs, which takes {} ms.", (endTime - startTime));
if (logger.isDebugEnabled()) {
for (UDFInformation udfInformation :
UDFManagementService.getInstance().getAllUDFInformation()) {
logger.debug("get udf: {}", udfInformation.getFunctionName());
}
}
}
private void getJarOfUDFs(List udfInformationList) throws StartupException {
try (ConfigNodeClient configNodeClient =
ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
List jarNameList =
udfInformationList.stream().map(UDFInformation::getJarName).collect(Collectors.toList());
TGetJarInListResp resp = configNodeClient.getUDFJar(new TGetJarInListReq(jarNameList));
if (resp.getStatus().getCode() == TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()) {
throw new StartupException("Failed to get UDF jar from config node.");
}
List jarList = resp.getJarList();
for (int i = 0; i < udfInformationList.size(); i++) {
UDFExecutableManager.getInstance()
.saveToInstallDir(jarList.get(i), udfInformationList.get(i).getJarName());
}
} catch (IOException | TException | ClientManagerException e) {
throw new StartupException(e);
}
}
/** Generate a list for UDFs that do not have jar on this node. */
private List getJarListForUDF() {
List res = new ArrayList<>();
for (UDFInformation udfInformation : resourcesInformationHolder.getUDFInformationList()) {
if (udfInformation.isUsingURI()) {
// Jar does not exist, add current udfInformation to list
if (!UDFExecutableManager.getInstance()
.hasFileUnderInstallDir(udfInformation.getJarName())) {
res.add(udfInformation);
} else {
try {
// Local jar has conflicts with jar on config node, add current triggerInformation to
// list
if (UDFManagementService.getInstance().isLocalJarConflicted(udfInformation)) {
res.add(udfInformation);
}
} catch (UDFManagementException e) {
res.add(udfInformation);
}
}
}
}
return res;
}
private void getUDFInformationList(List allUDFInformation) {
if (allUDFInformation != null && !allUDFInformation.isEmpty()) {
List list = new ArrayList<>();
for (ByteBuffer UDFInformationByteBuffer : allUDFInformation) {
list.add(UDFInformation.deserialize(UDFInformationByteBuffer));
}
resourcesInformationHolder.setUDFInformationList(list);
}
}
private void initTriggerRelatedInstance() throws StartupException {
try {
TriggerExecutableManager.setupAndGetInstance(
config.getTriggerTemporaryLibDir(), config.getTriggerDir());
} catch (IOException e) {
throw new StartupException(e);
}
}
private void prepareTriggerResources() throws StartupException {
long startTime = System.currentTimeMillis();
initTriggerRelatedInstance();
if (resourcesInformationHolder.getTriggerInformationList() == null
|| resourcesInformationHolder.getTriggerInformationList().isEmpty()) {
return;
}
// Get jars from config node
List triggerNeedJarList = getJarListForTrigger();
int index = 0;
while (index < triggerNeedJarList.size()) {
List curList = new ArrayList<>();
int offset = 0;
while (offset < ResourcesInformationHolder.getJarNumOfOneRpc()
&& index + offset < triggerNeedJarList.size()) {
curList.add(triggerNeedJarList.get(index + offset));
offset++;
}
index += (offset + 1);
getJarOfTriggers(curList);
}
// Create instances of triggers and do registration
try {
for (TriggerInformation triggerInformation :
resourcesInformationHolder.getTriggerInformationList()) {
TriggerManagementService.getInstance().doRegister(triggerInformation, true);
}
} catch (Exception e) {
throw new StartupException(e);
}
if (logger.isDebugEnabled()) {
for (TriggerInformation triggerInformation :
TriggerManagementService.getInstance().getAllTriggerInformationInTriggerTable()) {
logger.debug("get trigger: {}", triggerInformation.getTriggerName());
}
for (TriggerExecutor triggerExecutor :
TriggerManagementService.getInstance().getAllTriggerExecutors()) {
logger.debug(
"get trigger executor: {}", triggerExecutor.getTriggerInformation().getTriggerName());
}
}
// Start TriggerInformationUpdater
triggerInformationUpdater.startTriggerInformationUpdater();
long endTime = System.currentTimeMillis();
logger.info(
"successfully registered all the triggers, which takes {} ms.", (endTime - startTime));
}
private void getJarOfTriggers(List triggerInformationList)
throws StartupException {
try (ConfigNodeClient configNodeClient =
ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
List jarNameList =
triggerInformationList.stream()
.map(TriggerInformation::getJarName)
.collect(Collectors.toList());
TGetJarInListResp resp = configNodeClient.getTriggerJar(new TGetJarInListReq(jarNameList));
if (resp.getStatus().getCode() == TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()) {
throw new StartupException("Failed to get trigger jar from config node.");
}
List jarList = resp.getJarList();
for (int i = 0; i < triggerInformationList.size(); i++) {
TriggerExecutableManager.getInstance()
.saveToInstallDir(jarList.get(i), triggerInformationList.get(i).getJarName());
}
} catch (IOException | TException | ClientManagerException e) {
throw new StartupException(e);
}
}
/** Generate a list for triggers that do not have jar on this node. */
private List getJarListForTrigger() {
List res = new ArrayList<>();
for (TriggerInformation triggerInformation :
resourcesInformationHolder.getTriggerInformationList()) {
if (triggerInformation.isUsingURI()) {
// jar does not exist, add current triggerInformation to list
if (!TriggerExecutableManager.getInstance()
.hasFileUnderInstallDir(triggerInformation.getJarName())) {
res.add(triggerInformation);
} else {
try {
// local jar has conflicts with jar on config node, add current triggerInformation to
// list
if (TriggerManagementService.getInstance().isLocalJarConflicted(triggerInformation)) {
res.add(triggerInformation);
}
} catch (TriggerManagementException e) {
res.add(triggerInformation);
}
}
}
}
return res;
}
private void getTriggerInformationList(List allTriggerInformation) {
if (allTriggerInformation != null && !allTriggerInformation.isEmpty()) {
List list = new ArrayList<>();
for (ByteBuffer triggerInformationByteBuffer : allTriggerInformation) {
list.add(TriggerInformation.deserialize(triggerInformationByteBuffer));
}
resourcesInformationHolder.setTriggerInformationList(list);
}
}
private void preparePipeResources() throws StartupException {
long startTime = System.currentTimeMillis();
PipeAgent.runtime().preparePipeResources(resourcesInformationHolder);
long endTime = System.currentTimeMillis();
logger.info("Prepare pipe resources successfully, which takes {} ms.", (endTime - startTime));
}
private void getPipeInformationList(List allPipeInformation) {
final List list = new ArrayList<>();
if (allPipeInformation != null) {
for (ByteBuffer pipeInformationByteBuffer : allPipeInformation) {
list.add(PipePluginMeta.deserialize(pipeInformationByteBuffer));
}
}
resourcesInformationHolder.setPipePluginMetaList(list);
}
private void initSchemaEngine() {
long startTime = System.currentTimeMillis();
SchemaEngine.getInstance().init();
long endTime = System.currentTimeMillis();
logger.info("Recover schema successfully, which takes {} ms.", (endTime - startTime));
}
private void classLoader() {
try {
// StatementGenerator
Class.forName(StatementGenerator.class.getName());
Class.forName(ASTVisitor.class.getName());
Class.forName(SqlLexer.class.getName());
Class.forName(CommonTokenStream.class.getName());
Class.forName(IoTDBSqlParser.class.getName());
// SourceRewriter
Class.forName(SourceRewriter.class.getName());
Class.forName(DistributionPlanContext.class.getName());
// LogicalPlaner
Class.forName(LogicalPlanVisitor.class.getName());
Class.forName(LogicalQueryPlan.class.getName());
// TsFileProcessor
Class.forName(TsFileProcessor.class.getName());
} catch (ClassNotFoundException e) {
logger.error("load class error: ", e);
}
}
public void stop() {
deactivate();
SchemaEngine.getInstance().clear();
try {
MetricService.getInstance().stop();
if (schemaRegionConsensusStarted) {
SchemaRegionConsensusImpl.getInstance().stop();
}
if (dataRegionConsensusStarted) {
DataRegionConsensusImpl.getInstance().stop();
}
} catch (Exception e) {
logger.error("Stop data node error", e);
}
}
private void initProtocols() throws StartupException {
if (config.isEnableMQTTService()) {
registerManager.register(MQTTService.getInstance());
}
if (IoTDBRestServiceDescriptor.getInstance().getConfig().isEnableRestService()) {
registerManager.register(RestService.getInstance());
}
if (PipeConfig.getInstance().getPipeAirGapReceiverEnabled()) {
registerManager.register(PipeAgent.receiver().airGap());
}
}
private void deactivate() {
logger.info("Deactivating IoTDB DataNode...");
stopTriggerRelatedServices();
registerManager.deregisterAll();
JMXService.deregisterMBean(mbeanName);
logger.info("IoTDB DataNode is deactivated.");
}
private void stopTriggerRelatedServices() {
triggerInformationUpdater.stopTriggerInformationUpdater();
}
private void setUncaughtExceptionHandler() {
Thread.setDefaultUncaughtExceptionHandler(new IoTDBDefaultThreadExceptionHandler());
}
private static class DataNodeHolder {
private static final DataNode INSTANCE = new DataNode();
private DataNodeHolder() {
// Empty constructor
}
}
}