org.apache.kylin.rest.service.CubeService Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kylin.rest.service;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.EnumSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.lock.DistributedLock;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.CubeUpdate;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.cuboid.CuboidCLI;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.mr.CubingJob;
import org.apache.kylin.engine.mr.common.CubeJobLockUtil;
import org.apache.kylin.engine.mr.common.CuboidRecommenderUtil;
import org.apache.kylin.engine.spark.metadata.cube.PathManager;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.job.common.PatternedLogger;
import org.apache.kylin.job.constant.JobStatusEnum;
import org.apache.kylin.job.constant.JobTimeFilterEnum;
import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.measure.percentile.PercentileMeasureType;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.draft.Draft;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
//import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.metrics.MetricsManager;
import org.apache.kylin.metrics.property.QueryCubePropertyEnum;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.msg.Message;
import org.apache.kylin.rest.msg.MsgPicker;
import org.apache.kylin.rest.request.MetricsRequest;
import org.apache.kylin.rest.request.PrepareSqlRequest;
import org.apache.kylin.rest.response.CubeInstanceResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse.NodeInfo;
import org.apache.kylin.rest.response.StorageResponse;
import org.apache.kylin.rest.response.MetricsResponse;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.ValidateUtil;
import org.apache.kylin.storage.hybrid.HybridInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.security.access.AccessDeniedException;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;
import org.apache.kylin.shaded.com.google.common.base.Preconditions;
import org.apache.kylin.shaded.com.google.common.cache.Cache;
import org.apache.kylin.shaded.com.google.common.cache.CacheBuilder;
import org.apache.kylin.shaded.com.google.common.collect.Lists;
import org.apache.kylin.shaded.com.google.common.collect.Maps;
/**
* Stateless & lightweight service facade of cube management functions.
*
* @author yangli9
*/
@Component("cubeMgmtService")
public class CubeService extends BasicService implements InitializingBean {
private static final Logger logger = LoggerFactory.getLogger(CubeService.class);
protected Cache storageInfoCache = CacheBuilder.newBuilder().build();
@Autowired
@Qualifier("projectService")
private ProjectService projectService;
@Autowired
@Qualifier("jobService")
private JobService jobService;
@Autowired
@Qualifier("modelMgmtService")
private ModelService modelService;
@Autowired
@Qualifier("queryService")
private QueryService queryService;
@Autowired
private AclEvaluate aclEvaluate;
@Autowired
private HybridService hybridService;
public boolean isCubeNameVaildate(final String cubeName) {
if (StringUtils.isEmpty(cubeName) || !ValidateUtil.isAlphanumericUnderscore(cubeName)) {
return false;
}
for (CubeInstance cubeInstance : getCubeManager().listAllCubes()) {
if (cubeName.equalsIgnoreCase(cubeInstance.getName())) {
return false;
}
}
return true;
}
public List listAllCubes(final String cubeName, final String projectName, final String modelName,
boolean exactMatch) {
List cubeInstances = null;
if (null == projectName) {
cubeInstances = getCubeManager().listAllCubes();
aclEvaluate.checkIsGlobalAdmin();
} else {
cubeInstances = listAllCubes(projectName);
aclEvaluate.checkProjectReadPermission(projectName);
}
List filterModelCubes = new ArrayList();
if (modelName != null) {
for (CubeInstance cubeInstance : cubeInstances) {
boolean isModelMatch = cubeInstance.getDescriptor().getModelName().equalsIgnoreCase(modelName);
if (isModelMatch) {
filterModelCubes.add(0, cubeInstance);
}
}
} else {
filterModelCubes = cubeInstances;
}
List filterCubes = new ArrayList();
for (CubeInstance cubeInstance : filterModelCubes) {
boolean isCubeMatch = (null == cubeName)
|| (!exactMatch && cubeInstance.getName().toLowerCase(Locale.ROOT)
.contains(cubeName.toLowerCase(Locale.ROOT)))
|| (exactMatch && cubeInstance.getName().toLowerCase(Locale.ROOT)
.equals(cubeName.toLowerCase(Locale.ROOT)));
if (isCubeMatch) {
filterCubes.add(cubeInstance);
}
}
// sort the cube list by create time in descending order
filterCubes.sort((o1, o2) -> Long.compare(o2.getCreateTimeUTC(), o1.getCreateTimeUTC()));
return filterCubes;
}
public CubeInstance updateCubeCost(CubeInstance cube, int cost) throws IOException {
aclEvaluate.checkProjectWritePermission(cube);
if (cube.getCost() == cost) {
// Do nothing
return cube;
}
cube.setCost(cost);
String owner = SecurityContextHolder.getContext().getAuthentication().getName();
cube.setOwner(owner);
CubeUpdate update = new CubeUpdate(cube.latestCopyForWrite()).setOwner(owner).setCost(cost);
return getCubeManager().updateCube(update);
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+ " or hasPermission(#project, 'ADMINISTRATION') or hasPermission(#project, 'MANAGEMENT')")
public CubeInstance createCubeAndDesc(ProjectInstance project, CubeDesc desc) throws IOException {
Message msg = MsgPicker.getMsg();
String cubeName = desc.getName();
if (getCubeManager().getCube(cubeName) != null) {
throw new BadRequestException(String.format(Locale.ROOT, msg.getCUBE_ALREADY_EXIST(), cubeName));
}
if (getCubeDescManager().getCubeDesc(desc.getName()) != null) {
throw new BadRequestException(String.format(Locale.ROOT, msg.getCUBE_DESC_ALREADY_EXIST(), desc.getName()));
}
String owner = SecurityContextHolder.getContext().getAuthentication().getName();
CubeDesc createdDesc;
CubeInstance createdCube;
createdDesc = getCubeDescManager().createCubeDesc(desc);
if (createdDesc.isBroken()) {
throw new BadRequestException(createdDesc.getErrorsAsString());
}
int cuboidCount = CuboidCLI.simulateCuboidGeneration(createdDesc, false);
logger.info("New cube " + cubeName + " has " + cuboidCount + " cuboids");
createdCube = getCubeManager().createCube(cubeName, project.getName(), createdDesc, owner);
return createdCube;
}
public List listAllCubes(String projectName) {
ProjectManager projectManager = getProjectManager();
ProjectInstance project = projectManager.getProject(projectName);
if (project == null) {
return Collections.emptyList();
}
ArrayList result = new ArrayList();
for (RealizationEntry projectDataModel : project.getRealizationEntries()) {
if (projectDataModel.getType() == RealizationType.CUBE) {
CubeInstance cube = getCubeManager().getCube(projectDataModel.getRealization());
if (cube != null)
result.add(cube);
else
logger.error("Cube instance " + projectDataModel.getRealization() + " is failed to load");
}
}
return result;
}
protected boolean isCubeInProject(String projectName, CubeInstance target) {
ProjectManager projectManager = getProjectManager();
ProjectInstance project = projectManager.getProject(projectName);
if (project == null) {
return false;
}
for (RealizationEntry projectDataModel : project.getRealizationEntries()) {
if (projectDataModel.getType() == RealizationType.CUBE) {
CubeInstance cube = getCubeManager().getCube(projectDataModel.getRealization());
if (cube == null) {
logger.error("Project " + projectName + " contains realization " + projectDataModel.getRealization()
+ " which is not found by CubeManager");
continue;
}
if (cube.equals(target)) {
return true;
}
}
}
return false;
}
public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName, boolean forceUpdate)
throws IOException {
aclEvaluate.checkProjectWritePermission(cube);
Message msg = MsgPicker.getMsg();
final List cubingJobs = jobService.listJobsByRealizationName(cube.getName(), null,
EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
if (!cubingJobs.isEmpty()) {
throw new BadRequestException(String.format(Locale.ROOT, msg.getDISCARD_JOB_FIRST(), cube.getName()));
}
//double check again
if (!forceUpdate && !cube.getDescriptor().consistentWith(desc)) {
throw new BadRequestException(String.format(Locale.ROOT, msg.getINCONSISTENT_CUBE_DESC(), desc.getName()));
}
CubeDesc updatedCubeDesc = getCubeDescManager().updateCubeDesc(desc);
int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc, false);
logger.info("Updated cube " + cube.getName() + " has " + cuboidCount + " cuboids");
ProjectManager projectManager = getProjectManager();
if (!isCubeInProject(newProjectName, cube)) {
String owner = SecurityContextHolder.getContext().getAuthentication().getName();
ProjectInstance newProject = projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(),
newProjectName, owner);
}
return updatedCubeDesc;
}
public void deleteCube(CubeInstance cube) throws IOException {
aclEvaluate.checkProjectWritePermission(cube);
Message msg = MsgPicker.getMsg();
final List cubingJobs = jobService.listJobsByRealizationName(cube.getName(), null,
EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING, ExecutableState.ERROR));
if (!cubingJobs.isEmpty()) {
throw new BadRequestException(String.format(Locale.ROOT, msg.getDISCARD_JOB_FIRST(), cube.getName()));
}
try {
this.releaseAllJobs(cube);
} catch (Exception e) {
logger.error("error when releasing all jobs", e);
//ignore the exception
}
// remove from hybrid definition
ProjectInstance projectInstance = cube.getProjectInstance();
List hybridRealizationEntries = projectInstance.getRealizationEntries(RealizationType.HYBRID);
if (hybridRealizationEntries != null) {
for (RealizationEntry entry : hybridRealizationEntries) {
HybridInstance instance = getHybridManager().getHybridInstance(entry.getRealization());
List cubeRealizationEntries = instance.getRealizationEntries();
boolean needUpdateHybrid = false;
for (RealizationEntry cubeRealizationEntry : cubeRealizationEntries) {
if (cube.getName().equals(cubeRealizationEntry.getRealization())) {
needUpdateHybrid = true;
cubeRealizationEntries.remove(cubeRealizationEntry);
break;
}
}
if (needUpdateHybrid) {
String[] cubeNames = new String[cubeRealizationEntries.size()];
for (int i = 0; i < cubeRealizationEntries.size(); i++) {
cubeNames[i] = cubeRealizationEntries.get(i).getRealization();
}
hybridService.updateHybridCubeNoCheck(instance.getName(), projectInstance.getName(),
cube.getModel().getName(), cubeNames);
}
}
}
List toRemoveSegs = cube.getSegments();
int cubeNum = getCubeManager().getCubesByDesc(cube.getDescriptor().getName()).size();
getCubeManager().dropCube(cube.getName(), cubeNum == 1);//only delete cube desc when no other cube is using it
cleanSegmentStorage(cube, toRemoveSegs);
}
public void deleteCubeFast(CubeInstance cube) throws IOException {
aclEvaluate.checkProjectWritePermission(cube);
// user make sure no job running and no hybrid cube, so no check jobs status and hybrid definition
int cubeNum = getCubeManager().getCubesByDesc(cube.getDescriptor().getName()).size();
getCubeManager().dropCube(cube.getName(), cubeNum == 1);//only delete cube desc when no other cube is using it
}
/**
* Stop all jobs belonging to this cube and clean out all segments
*
* @param cube
* @return
* @throws IOException
* @throws JobException
*/
public CubeInstance purgeCube(CubeInstance cube) throws IOException {
aclEvaluate.checkProjectOperationPermission(cube);
Message msg = MsgPicker.getMsg();
String cubeName = cube.getName();
final List cubingJobs = jobService.listJobsByRealizationName(cubeName, null, EnumSet
.of(ExecutableState.READY, ExecutableState.RUNNING, ExecutableState.ERROR, ExecutableState.STOPPED));
if (!cubingJobs.isEmpty()) {
throw new BadRequestException(String.format(Locale.ROOT, msg.getDISCARD_JOB_FIRST(), cubeName));
}
RealizationStatusEnum ostatus = cube.getStatus();
if (null != ostatus && !RealizationStatusEnum.DISABLED.equals(ostatus)) {
throw new BadRequestException(
String.format(Locale.ROOT, msg.getPURGE_NOT_DISABLED_CUBE(), cubeName, ostatus));
}
this.releaseAllSegments(cube);
return cube;
}
/**
* Update a cube status from ready to disabled.
*
* @return
* @throws IOException
* @throws JobException
*/
public CubeInstance disableCube(CubeInstance cube) throws IOException {
aclEvaluate.checkProjectWritePermission(cube);
Message msg = MsgPicker.getMsg();
String cubeName = cube.getName();
RealizationStatusEnum ostatus = cube.getStatus();
if (null != ostatus && !RealizationStatusEnum.READY.equals(ostatus)) {
throw new BadRequestException(
String.format(Locale.ROOT, msg.getDISABLE_NOT_READY_CUBE(), cubeName, ostatus));
}
boolean isStreamingCube = cube.getDescriptor().isStreamingCube();
boolean cubeStatusUpdated = false;
try {
CubeInstance cubeInstance = getCubeManager().updateCubeStatus(cube, RealizationStatusEnum.DISABLED);
cubeStatusUpdated = true;
// for streaming cube.
// if (isStreamingCube) {
// //drop not ready segments
// CubeSegment[] buildingSegments = new CubeSegment[cubeInstance.getBuildingSegments().size()];
// Segments segments = cubeInstance.getBuildingSegments();
// if (!CollectionUtils.isEmpty(segments)) {
// for (int i = 0; i < segments.size(); i++) {
// buildingSegments[i] = (CubeSegment) segments.get(i);
// }
// getCubeManager().dropOptmizingSegments(cubeInstance, buildingSegments);
// }
// //unAssign cube
// getStreamingCoordinator().unAssignCube(cubeName);
//
// //discard jobs
// releaseAllJobs(cubeInstance);
//
// }
return cubeInstance;
} catch (Exception e) {
cube.setStatus(ostatus);
// roll back if cube status updated
if (cubeStatusUpdated) {
logger.info("roll back cube status to:{}", ostatus);
getCubeManager().updateCubeStatus(cube, ostatus);
}
throw e;
}
}
public void checkEnableCubeCondition(CubeInstance cube) {
aclEvaluate.checkProjectWritePermission(cube);
Message msg = MsgPicker.getMsg();
String cubeName = cube.getName();
RealizationStatusEnum ostatus = cube.getStatus();
if (!cube.getStatus().equals(RealizationStatusEnum.DISABLED)) {
throw new BadRequestException(
String.format(Locale.ROOT, msg.getENABLE_NOT_DISABLED_CUBE(), cubeName, ostatus));
}
if (cube.getSegments(SegmentStatusEnum.READY).size() == 0 && !cube.getDescriptor().isStreamingCube()) {
throw new BadRequestException(String.format(Locale.ROOT, msg.getNO_READY_SEGMENT(), cubeName));
}
if (!cube.getDescriptor().checkSignature()) {
throw new BadRequestException(
String.format(Locale.ROOT, msg.getINCONSISTENT_CUBE_DESC_SIGNATURE(), cube.getDescriptor()));
}
}
/**
* Update a cube status from disable to ready.
*
* @return
* @throws IOException
*/
public CubeInstance enableCube(CubeInstance cube) throws IOException {
boolean cubeStatusUpdated = false;
RealizationStatusEnum ostatus = cube.getStatus();
try {
CubeInstance cubeInstance = getCubeManager().updateCubeStatus(cube, RealizationStatusEnum.READY);
cubeStatusUpdated = true;
// for streaming cube.
// if (cube.getDescriptor().isStreamingCube()) {
// getStreamingCoordinator().assignCube(cube.getName());
// }
return cubeInstance;
} catch (Exception e) {
cube.setStatus(ostatus);
// roll back if cube status updated
if (cubeStatusUpdated) {
logger.info("roll back cube status to:{}", ostatus);
getCubeManager().updateCubeStatus(cube, ostatus);
}
throw e;
}
}
// private CoordinatorClient getStreamingCoordinator() {
// return CoordinatorClientFactory.createCoordinatorClient(StreamMetadataStoreFactory.getStreamMetaDataStore());
// }
public MetricsResponse calculateMetrics(MetricsRequest request) {
List cubes = this.getCubeManager().listAllCubes();
MetricsResponse metrics = new MetricsResponse();
Date startTime = (null == request.getStartTime()) ? new Date(-1) : request.getStartTime();
Date endTime = (null == request.getEndTime()) ? new Date() : request.getEndTime();
metrics.increase("totalCubes", (float) 0);
metrics.increase("totalStorage", (float) 0);
for (CubeInstance cube : cubes) {
Date createdDate = new Date(-1);
createdDate = (cube.getCreateTimeUTC() == 0) ? createdDate : new Date(cube.getCreateTimeUTC());
if (createdDate.getTime() > startTime.getTime() && createdDate.getTime() < endTime.getTime()) {
metrics.increase("totalCubes");
}
}
metrics.increase("aveStorage",
(metrics.get("totalCubes") == 0) ? 0 : metrics.get("totalStorage") / metrics.get("totalCubes"));
return metrics;
}
/**
* Calculate size of each region for given table and other info of the
* table.
*
* @param tableName The table name.
* @return The StorageResponse object contains table size, region count. null
* if error happens
* @throws IOException Exception when HTable resource is not closed correctly.
*/
public StorageResponse getHTableInfo(String cubeName, String tableName) throws IOException {
String key = cubeName + "/" + tableName;
StorageResponse sr = storageInfoCache.getIfPresent(key);
if (null != sr) {
return sr;
}
sr = new StorageResponse();
CubeInstance cube = CubeManager.getInstance(getConfig()).getCube(cubeName);
if (cube.getStorageType() == IStorageAware.ID_HBASE || cube.getStorageType() == IStorageAware.ID_SHARDED_HBASE
|| cube.getStorageType() == IStorageAware.ID_REALTIME_AND_HBASE) {
try {
logger.debug("Loading HTable info " + cubeName + ", " + tableName);
// use reflection to isolate NoClassDef errors when HBase is not available
sr = (StorageResponse)
Class.forName("org.apache.kylin.rest.service.HBaseInfoUtil")
.getMethod("getHBaseInfo", new Class[] { String.class, KylinConfig.class })
.invoke(null, tableName, this.getConfig());
sr.setStorageType("hbase");
} catch (Throwable e) {
throw new IOException(e);
}
}
storageInfoCache.put(key, sr);
return sr;
}
public void updateCubeNotifyList(CubeInstance cube, List notifyList) throws IOException {
aclEvaluate.checkProjectOperationPermission(cube);
CubeDesc desc = cube.getDescriptor();
desc.setNotifyList(notifyList);
getCubeDescManager().updateCubeDesc(desc);
}
public CubeInstance updateCubeOwner(CubeInstance cube, String owner) throws IOException {
aclEvaluate.checkProjectWritePermission(cube);
if (Objects.equals(cube.getOwner(), owner)) {
// Do nothing
return cube;
}
cube.setOwner(owner);
CubeUpdate update = new CubeUpdate(cube.latestCopyForWrite()).setOwner(owner);
return getCubeManager().updateCube(update);
}
public CubeInstance deleteSegmentById(CubeInstance cube, String uuid) throws IOException {
aclEvaluate.checkProjectWritePermission(cube);
Message msg = MsgPicker.getMsg();
CubeSegment toDelete = null;
toDelete = cube.getSegmentById(uuid);
if (toDelete == null) {
throw new BadRequestException(String.format(Locale.ROOT, msg.getSEG_NOT_FOUND(), uuid));
}
if (cube.getStatus() == RealizationStatusEnum.DISABLED || isOrphonSegment(cube, uuid)) {
CubeInstance cubeInstance = CubeManager.getInstance(getConfig()).updateCubeDropSegments(cube, toDelete);
cleanSegmentStorage(cubeInstance, Collections.singletonList(toDelete));
return cubeInstance;
} else {
throw new BadRequestException(
String.format(Locale.ROOT, msg.getDELETE_READY_SEG_BY_UUID(), uuid, cube.getName()));
}
}
public CubeInstance deleteSegment(CubeInstance cube, String segmentName) throws IOException {
aclEvaluate.checkProjectOperationPermission(cube);
Message msg = MsgPicker.getMsg();
if (cube.getStatus() == RealizationStatusEnum.READY) {
throw new BadRequestException(
String.format(Locale.ROOT, msg.getDELETE_SEG_FROM_READY_CUBE(), segmentName, cube.getName()));
}
CubeSegment toDelete = null;
for (CubeSegment seg : cube.getSegments()) {
if (seg.getName().equals(segmentName)) {
toDelete = seg;
break;
}
}
if (toDelete == null) {
throw new BadRequestException(String.format(Locale.ROOT, msg.getSEG_NOT_FOUND(), segmentName));
}
if (toDelete.getStatus() != SegmentStatusEnum.READY) {
if (toDelete.getStatus() == SegmentStatusEnum.NEW) {
if (!isOrphonSegment(cube, toDelete.getUuid())) {
throw new BadRequestException(
String.format(Locale.ROOT, msg.getDELETE_NOT_READY_SEG(), segmentName));
}
} else {
throw new BadRequestException(String.format(Locale.ROOT, msg.getDELETE_NOT_READY_SEG(), segmentName));
}
}
if (!segmentName.equals(cube.getSegments().get(0).getName())
&& !segmentName.equals(cube.getSegments().get(cube.getSegments().size() - 1).getName())) {
logger.warn(String.format(Locale.ROOT, msg.getDELETE_SEGMENT_CAUSE_GAPS(), cube.getName(), segmentName));
}
CubeInstance cubeInstance = CubeManager.getInstance(getConfig()).updateCubeDropSegments(cube, toDelete);
cleanSegmentStorage(cubeInstance, Collections.singletonList(toDelete));
return cubeInstance;
}
// clean segment data in hdfs
private void cleanSegmentStorage(CubeInstance cube, List toRemoveSegs) throws IOException {
if (!KylinConfig.getInstanceFromEnv().cleanStorageAfterDelOperation()) {
return;
}
if (toRemoveSegs != null && !toRemoveSegs.isEmpty()) {
for (CubeSegment segment : toRemoveSegs) {
PathManager.deleteSegmentParquetStoragePath(cube, segment.getName(), segment.getStorageLocationIdentifier());
}
}
}
public boolean isOrphonSegment(CubeInstance cube, String segId) {
List jobInstances = jobService.searchJobsByCubeName(
cube.getName(), cube.getProject(), Lists.newArrayList(JobStatusEnum.NEW, JobStatusEnum.PENDING,
JobStatusEnum.RUNNING, JobStatusEnum.ERROR, JobStatusEnum.STOPPED),
JobTimeFilterEnum.ALL, JobService.JobSearchMode.CUBING_ONLY);
for (JobInstance jobInstance : jobInstances) {
// if there are segment related jobs, can not delete this segment.
if (segId.equals(jobInstance.getRelatedSegment())) {
return false;
}
}
return true;
}
protected void releaseAllJobs(CubeInstance cube) {
final List cubingJobs = jobService.listJobsByRealizationName(cube.getName(), null);
for (CubingJob cubingJob : cubingJobs) {
final ExecutableState status = cubingJob.getStatus();
if (status != ExecutableState.SUCCEED && status != ExecutableState.DISCARDED) {
getExecutableManager().discardJob(cubingJob.getId());
//release global dict lock if exists
DistributedLock lock = KylinConfig.getInstanceFromEnv().getDistributedLockFactory()
.lockForCurrentThread();
if (lock.isLocked(CubeJobLockUtil.getLockPath(cube.getName(), cubingJob.getId()))) {//release cube job dict lock if exists
lock.purgeLocks(CubeJobLockUtil.getLockPath(cube.getName(), null));
logger.info("{} unlock cube job global lock path({}) success", cubingJob.getId(),
CubeJobLockUtil.getLockPath(cube.getName(), null));
if (lock.isLocked(CubeJobLockUtil.getEphemeralLockPath(cube.getName()))) {//release cube job Ephemeral lock if exists
lock.purgeLocks(CubeJobLockUtil.getEphemeralLockPath(cube.getName()));
logger.info("{} unlock cube job ephemeral lock path({}) success", cubingJob.getId(),
CubeJobLockUtil.getEphemeralLockPath(cube.getName()));
}
}
}
}
}
/**
* purge the cube
*
* @throws IOException
* @throws JobException
*/
private void releaseAllSegments(CubeInstance cube) throws IOException {
releaseAllJobs(cube);
List toRemoveSegs = cube.getSegments();
// remove from metadata
getCubeManager().clearSegments(cube);
cleanSegmentStorage(cube, toRemoveSegs);
}
public void updateOnNewSegmentReady(String cubeName) {
final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
String serverMode = kylinConfig.getServerMode();
if (Constant.SERVER_MODE_JOB.equals(serverMode.toLowerCase(Locale.ROOT))
|| Constant.SERVER_MODE_ALL.equals(serverMode.toLowerCase(Locale.ROOT))) {
CubeInstance cube = getCubeManager().getCube(cubeName);
if (cube != null) {
CubeSegment seg = cube.getLatestBuiltSegment();
if (seg != null && seg.getStatus() == SegmentStatusEnum.READY) {
keepCubeRetention(cubeName);
mergeCubeSegment(cubeName);
}
}
}
}
private void keepCubeRetention(String cubeName) {
logger.info("checking keepCubeRetention");
CubeInstance cube = getCubeManager().getCube(cubeName);
CubeDesc desc = cube.getDescriptor();
if (desc.getRetentionRange() <= 0)
return;
synchronized (CubeService.class) {
cube = getCubeManager().getCube(cubeName);
List readySegs = cube.getSegments(SegmentStatusEnum.READY);
if (readySegs.isEmpty())
return;
List toRemoveSegs = Lists.newArrayList();
long tail = readySegs.get(readySegs.size() - 1).getTSRange().end.v;
long head = tail - desc.getRetentionRange();
for (CubeSegment seg : readySegs) {
if (seg.getTSRange().end.v > 0) { // for streaming cube its initial value is 0
if (seg.getTSRange().end.v <= head) {
toRemoveSegs.add(seg);
}
}
}
if (toRemoveSegs.size() > 0) {
try {
getCubeManager().updateCubeDropSegments(cube, toRemoveSegs);
} catch (IOException e) {
logger.error("Failed to remove old segment from cube " + cubeName, e);
}
}
}
}
public String mergeCubeSegment(String cubeName) {
CubeInstance cube = getCubeManager().getCube(cubeName);
if (!cube.needAutoMerge())
return null;
if (!cube.isReady()) {
logger.info("The cube: {} is disabled", cubeName);
return null;
}
synchronized (CubeService.class) {
try {
cube = getCubeManager().getCube(cubeName);
SegmentRange offsets = cube.autoMergeCubeSegments();
if (offsets != null && !isMergingJobBeenDiscarded(cube, cubeName, cube.getProject(), offsets)) {
CubeSegment newSeg = getCubeManager().mergeSegments(cube, null, offsets, true);
logger.info("Will submit merge job on " + newSeg);
DefaultChainedExecutable job = EngineFactory.createBatchMergeJob(newSeg, "SYSTEM");
getExecutableManager().addJob(job);
return job.getId();
} else {
logger.info("Not ready for merge on cube " + cubeName);
}
} catch (IOException e) {
logger.error("Failed to auto merge cube " + cubeName, e);
}
}
return null;
}
//Don't merge the job that has been discarded manually before
private boolean isMergingJobBeenDiscarded(CubeInstance cubeInstance, String cubeName, String projectName,
SegmentRange offsets) {
SegmentRange.TSRange tsRange = new SegmentRange.TSRange((Long) offsets.start.v, (Long) offsets.end.v);
String segmentName = CubeSegment.makeSegmentName(tsRange, null, cubeInstance.getModel());
final List jobInstanceList = jobService.listJobsByRealizationName(cubeName, projectName,
EnumSet.of(ExecutableState.DISCARDED));
for (CubingJob cubingJob : jobInstanceList) {
String jobSegmentName = cubingJob.getSegmentName();
if (jobSegmentName != null && segmentName.equals(jobSegmentName)) {
logger.debug("Merge job {} has been discarded before, will not merge.", segmentName);
return true;
}
}
return false;
}
public void validateCubeDesc(CubeDesc desc, boolean isDraft) {
Message msg = MsgPicker.getMsg();
if (desc == null) {
throw new BadRequestException(msg.getINVALID_CUBE_DEFINITION());
}
String cubeName = desc.getName();
if (StringUtils.isEmpty(cubeName)) {
logger.info("Cube name should not be empty.");
throw new BadRequestException(msg.getEMPTY_CUBE_NAME());
}
if (!ValidateUtil.isAlphanumericUnderscore(cubeName)) {
logger.info("Invalid Cube name {}, only letters, numbers and underscore supported.", cubeName);
throw new BadRequestException(String.format(Locale.ROOT, msg.getINVALID_CUBE_NAME(), cubeName));
}
if (!isDraft) {
DataModelDesc modelDesc = modelService.getDataModelManager().getDataModelDesc(desc.getModelName());
if (modelDesc == null) {
throw new BadRequestException(
String.format(Locale.ROOT, msg.getMODEL_NOT_FOUND(), desc.getModelName()));
}
if (modelDesc.isDraft()) {
logger.info("Cannot use draft model.");
throw new BadRequestException(
String.format(Locale.ROOT, msg.getUSE_DRAFT_MODEL(), desc.getModelName()));
}
}
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+ " or hasPermission(#project, 'ADMINISTRATION') or hasPermission(#project, 'MANAGEMENT')")
public CubeDesc saveCube(CubeDesc desc, ProjectInstance project) throws IOException {
Message msg = MsgPicker.getMsg();
desc.setDraft(false);
if (desc.getUuid() == null)
desc.updateRandomUuid();
try {
createCubeAndDesc(project, desc);
} catch (AccessDeniedException accessDeniedException) {
throw new ForbiddenException(msg.getUPDATE_CUBE_NO_RIGHT());
}
if (desc.isBroken()) {
throw new BadRequestException(desc.getErrorsAsString());
}
return desc;
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+ " or hasPermission(#project, 'ADMINISTRATION') or hasPermission(#project, 'MANAGEMENT')")
public void saveDraft(ProjectInstance project, CubeInstance cube, String uuid, RootPersistentEntity... entities)
throws IOException {
Draft draft = new Draft();
draft.setProject(project.getName());
draft.setUuid(uuid);
draft.setEntities(entities);
getDraftManager().save(draft);
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+ " or hasPermission(#project, 'ADMINISTRATION') or hasPermission(#project, 'MANAGEMENT')")
public void saveDraft(ProjectInstance project, String uuid, RootPersistentEntity... entities) throws IOException {
Draft draft = new Draft();
draft.setProject(project.getName());
draft.setUuid(uuid);
draft.setEntities(entities);
getDraftManager().save(draft);
}
public void deleteDraft(Draft draft) throws IOException {
aclEvaluate.checkProjectWritePermission(draft.getProject());
getDraftManager().delete(draft.getUuid());
}
public CubeDesc updateCube(CubeInstance cube, CubeDesc desc, ProjectInstance project) throws IOException {
aclEvaluate.checkProjectWritePermission(cube);
Message msg = MsgPicker.getMsg();
String projectName = project.getName();
desc.setDraft(false);
try {
if (cube.getSegments().size() != 0 && !cube.getDescriptor().consistentWith(desc)) {
throw new BadRequestException(
String.format(Locale.ROOT, msg.getINCONSISTENT_CUBE_DESC(), desc.getName()));
}
desc = updateCubeAndDesc(cube, desc, projectName, true);
} catch (AccessDeniedException accessDeniedException) {
throw new ForbiddenException(msg.getUPDATE_CUBE_NO_RIGHT());
}
if (desc.isBroken()) {
throw new BadRequestException(desc.getErrorsAsString());
}
return desc;
}
public Draft getCubeDraft(String cubeName, String projectName) throws IOException {
for (Draft d : listCubeDrafts(cubeName, null, projectName, true)) {
return d;
}
return null;
}
public List listCubeDrafts(String cubeName, String modelName, String project, boolean exactMatch)
throws IOException {
if (null == project) {
aclEvaluate.checkIsGlobalAdmin();
} else {
aclEvaluate.checkProjectReadPermission(project);
}
List result = new ArrayList<>();
for (Draft d : getDraftManager().list(project)) {
RootPersistentEntity e = d.getEntity();
if (e instanceof CubeDesc) {
CubeDesc c = (CubeDesc) e;
if ((cubeName == null
|| (exactMatch
&& cubeName.toLowerCase(Locale.ROOT).equals(c.getName().toLowerCase(Locale.ROOT)))
|| (!exactMatch
&& c.getName().toLowerCase(Locale.ROOT).contains(cubeName.toLowerCase(Locale.ROOT))))
&& (modelName == null || modelName.toLowerCase(Locale.ROOT)
.equals(c.getModelName().toLowerCase(Locale.ROOT)))) {
// backward compability for percentile
if (c.getMeasures() != null) {
for (MeasureDesc m : c.getMeasures()) {
FunctionDesc f = m.getFunction();
if (f.getExpression().equals(PercentileMeasureType.FUNC_PERCENTILE)) {
f.setExpression(PercentileMeasureType.FUNC_PERCENTILE_APPROX);
}
}
}
result.add(d);
}
}
}
return result;
}
@Override
public void afterPropertiesSet() throws Exception {
Broadcaster.getInstance(getConfig()).registerStaticListener(new HTableInfoSyncListener(), "cube");
}
public CubeInstanceResponse createCubeInstanceResponse(CubeInstance cube) {
return new CubeInstanceResponse(cube, projectService.getProjectOfCube(cube.getName()));
}
public CuboidTreeResponse getCuboidTreeResponse(CuboidScheduler cuboidScheduler, Map rowCountMap,
Map hitFrequencyMap, Map queryMatchMap, Set currentCuboidSet) {
long baseCuboidId = cuboidScheduler.getBaseCuboidId();
int dimensionCount = Long.bitCount(baseCuboidId);
// get cube query count total
long cubeQueryCount = 0L;
if (hitFrequencyMap != null) {
for (long queryCount : hitFrequencyMap.values()) {
cubeQueryCount += queryCount;
}
}
NodeInfo root = generateNodeInfo(baseCuboidId, dimensionCount, cubeQueryCount, rowCountMap, hitFrequencyMap,
queryMatchMap, currentCuboidSet);
List nodeQueue = Lists.newLinkedList();
nodeQueue.add(root);
while (!nodeQueue.isEmpty()) {
NodeInfo parentNode = nodeQueue.remove(0);
for (long childId : cuboidScheduler.getSpanningCuboid(parentNode.getId())) {
NodeInfo childNode = generateNodeInfo(childId, dimensionCount, cubeQueryCount, rowCountMap,
hitFrequencyMap, queryMatchMap, currentCuboidSet);
parentNode.addChild(childNode);
nodeQueue.add(childNode);
}
}
CuboidTreeResponse result = new CuboidTreeResponse();
result.setRoot(root);
return result;
}
private NodeInfo generateNodeInfo(long cuboidId, int dimensionCount, long cubeQueryCount,
Map rowCountMap, Map hitFrequencyMap, Map queryMatchMap,
Set currentCuboidSet) {
Long queryCount = hitFrequencyMap == null || hitFrequencyMap.get(cuboidId) == null ? 0L
: hitFrequencyMap.get(cuboidId);
float queryRate = cubeQueryCount <= 0 ? 0 : queryCount.floatValue() / cubeQueryCount;
long queryExactlyMatchCount = queryMatchMap == null || queryMatchMap.get(cuboidId) == null ? 0L
: queryMatchMap.get(cuboidId);
boolean ifExist = currentCuboidSet.contains(cuboidId);
long rowCount = (rowCountMap == null || rowCountMap.size() == 0) ? 0L : rowCountMap.get(cuboidId);
NodeInfo node = new NodeInfo();
node.setId(cuboidId);
node.setName(Cuboid.getDisplayName(cuboidId, dimensionCount));
node.setQueryCount(queryCount);
node.setQueryRate(queryRate);
node.setExactlyMatchCount(queryExactlyMatchCount);
node.setExisted(ifExist);
node.setRowCount(rowCount);
return node;
}
/** cube planner services */
public Map getRecommendCuboidStatistics(CubeInstance cube, Map hitFrequencyMap,
Map>> rollingUpCountSourceMap) throws IOException {
aclEvaluate.checkProjectAdminPermission(cube.getProject());
return CuboidRecommenderUtil.getRecommendCuboidList(cube, hitFrequencyMap, rollingUpCountSourceMap);
}
public Map formatQueryCount(List> orgQueryCount) {
Map formattedQueryCount = Maps.newLinkedHashMap();
for (List hit : orgQueryCount) {
formattedQueryCount.put(Long.parseLong(hit.get(0)), (long) Double.parseDouble(hit.get(1)));
}
return formattedQueryCount;
}
public Map>> formatRollingUpStats(List> orgRollingUpCount) {
Map>> formattedRollingUpStats = Maps.newLinkedHashMap();
for (List rollingUp : orgRollingUpCount) {
Map> childMap = Maps.newLinkedHashMap();
Long srcCuboid = Long.parseLong(rollingUp.get(0));
Long tgtCuboid = Long.parseLong(rollingUp.get(1));
Long rollupCount = (long) Double.parseDouble(rollingUp.get(2));
Long returnCount = (long) Double.parseDouble(rollingUp.get(3));
childMap.put(tgtCuboid, new Pair<>(rollupCount, returnCount));
formattedRollingUpStats.put(srcCuboid, childMap);
}
return formattedRollingUpStats;
}
public Map getCuboidHitFrequency(String cubeName, boolean isCuboidSource) {
String cuboidColumn = isCuboidSource ? QueryCubePropertyEnum.CUBOID_SOURCE.toString()
: QueryCubePropertyEnum.CUBOID_TARGET.toString();
String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
String table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQuerySparkJob());
String sql = "select " + cuboidColumn + ", sum(" + hitMeasure + ")" //
+ " from " + table//
+ " where " + QueryCubePropertyEnum.CUBE.toString() + " = ?" //
+ " group by " + cuboidColumn;
List> orgHitFrequency = getPrepareQueryResult(cubeName, sql);
return formatQueryCount(orgHitFrequency);
}
public Map>> getCuboidRollingUpStats(String cubeName) {
String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
String cuboidTgt = QueryCubePropertyEnum.CUBOID_TARGET.toString();
String aggCount = QueryCubePropertyEnum.AGGR_COUNT.toString();
String returnCount = QueryCubePropertyEnum.RETURN_COUNT.toString();
String table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQuerySparkJob());
String sql = "select " + cuboidSource + ", " + cuboidTgt + ", avg(" + aggCount + "), avg(" + returnCount + ")"//
+ " from " + table //
+ " where " + QueryCubePropertyEnum.CUBE.toString() + " = ?" //
+ " group by " + cuboidSource + ", " + cuboidTgt;
List> orgRollingUpCount = getPrepareQueryResult(cubeName, sql);
return formatRollingUpStats(orgRollingUpCount);
}
public Map getCuboidQueryMatchCount(String cubeName) {
String cuboidSource = QueryCubePropertyEnum.CUBOID_SOURCE.toString();
String hitMeasure = QueryCubePropertyEnum.WEIGHT_PER_HIT.toString();
String table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQuerySparkJob());
String sql = "select " + cuboidSource + ", sum(" + hitMeasure + ")" //
+ " from " + table //
+ " where " + QueryCubePropertyEnum.CUBE.toString() + " = ?" //
+ " and " + QueryCubePropertyEnum.IF_MATCH.toString() + " = true" //
+ " group by " + cuboidSource;
List> orgMatchHitFrequency = getPrepareQueryResult(cubeName, sql);
return formatQueryCount(orgMatchHitFrequency);
}
private List> getPrepareQueryResult(String cubeName, String sql) {
PrepareSqlRequest sqlRequest = new PrepareSqlRequest();
sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
PrepareSqlRequest.StateParam[] params = new PrepareSqlRequest.StateParam[1];
params[0] = new PrepareSqlRequest.StateParam();
params[0].setClassName("java.lang.String");
params[0].setValue(cubeName);
sqlRequest.setParams(params);
sqlRequest.setSql(sql);
return queryService.doQueryWithCache(sqlRequest, false).getResults();
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+ " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
public void migrateCube(CubeInstance cube, String projectName) {
KylinConfig config = KylinConfig.getInstanceFromEnv();
if (!config.isAllowAutoMigrateCube()) {
throw new InternalErrorException("One click migration is disabled, please contact your ADMIN");
}
for (CubeSegment segment : cube.getSegments()) {
if (segment.getStatus() != SegmentStatusEnum.READY) {
throw new InternalErrorException(
"At least one segment is not in READY state. Please check whether there are Running or Error jobs.");
}
}
String srcCfgUri = config.getAutoMigrateCubeSrcConfig();
String dstCfgUri = config.getAutoMigrateCubeDestConfig();
Preconditions.checkArgument(StringUtils.isNotEmpty(srcCfgUri), "Source configuration should not be empty.");
Preconditions.checkArgument(StringUtils.isNotEmpty(dstCfgUri),
"Destination configuration should not be empty.");
String stringBuilder = ("%s/bin/kylin.sh org.apache.kylin.tool.CubeMigrationCLI %s %s %s %s %s %s true true");
String cmd = String.format(Locale.ROOT,
stringBuilder,
KylinConfig.getKylinHome(),
CliCommandExecutor.checkParameterWhiteList(srcCfgUri),
CliCommandExecutor.checkParameterWhiteList(dstCfgUri),
cube.getName(),
CliCommandExecutor.checkParameterWhiteList(projectName),
config.isAutoMigrateCubeCopyAcl(),
config.isAutoMigrateCubePurge());
logger.info("One click migration cmd: " + cmd);
CliCommandExecutor exec = new CliCommandExecutor();
PatternedLogger patternedLogger = new PatternedLogger(logger);
try {
exec.execute(cmd, patternedLogger, null);
} catch (IOException e) {
throw new InternalErrorException("Failed to perform one-click migrating", e);
}
}
private class HTableInfoSyncListener extends Broadcaster.Listener {
@Override
public void onClearAll(Broadcaster broadcaster) throws IOException {
storageInfoCache.invalidateAll();
}
@Override
public void onEntityChange(Broadcaster broadcaster, String entity, Broadcaster.Event event, String cacheKey)
throws IOException {
String cubeName = cacheKey;
String keyPrefix = cubeName + "/";
for (String k : storageInfoCache.asMap().keySet()) {
if (k.startsWith(keyPrefix))
storageInfoCache.invalidate(k);
}
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy