All Downloads are FREE. Search and download functionalities are using the official Maven repository.
Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.apache.kylin.tool.HBaseUsageExtractor Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kylin.tool;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.List;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.OptionGroup;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationRegistry;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
public class HBaseUsageExtractor extends AbstractInfoExtractor {
private static final Logger logger = LoggerFactory.getLogger(HBaseUsageExtractor.class);
@SuppressWarnings("static-access")
private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("Specify which cube to extract").create("cube");
@SuppressWarnings("static-access")
private static final Option OPTION_PROJECT = OptionBuilder.withArgName("project").hasArg().isRequired(false).withDescription("Specify realizations in which project to extract").create("project");
private List htables = Lists.newArrayList();
private Configuration conf;
private CubeManager cubeManager;
private RealizationRegistry realizationRegistry;
private KylinConfig kylinConfig;
private ProjectManager projectManager;
public HBaseUsageExtractor() {
super();
packageType = "hbase";
OptionGroup realizationOrProject = new OptionGroup();
realizationOrProject.addOption(OPTION_CUBE);
realizationOrProject.addOption(OPTION_PROJECT);
realizationOrProject.setRequired(true);
options.addOptionGroup(realizationOrProject);
conf = HBaseConfiguration.create();
}
public static void main(String[] args) {
HBaseUsageExtractor extractor = new HBaseUsageExtractor();
extractor.execute(args);
}
private String getHBaseMasterUrl() throws IOException, KeeperException {
String host = conf.get("hbase.master.info.bindAddress");
if (host.equals("0.0.0.0")) {
host = MasterAddressTracker.getMasterAddress(new ZooKeeperWatcher(conf, null, null)).getHostname();
}
String port = conf.get("hbase.master.info.port");
return "http://" + host + ":" + port + "/";
}
@Override
protected void executeExtract(OptionsHelper optionsHelper, File exportDir) throws Exception {
kylinConfig = KylinConfig.getInstanceFromEnv();
cubeManager = CubeManager.getInstance(kylinConfig);
realizationRegistry = RealizationRegistry.getInstance(kylinConfig);
projectManager = ProjectManager.getInstance(kylinConfig);
if (optionsHelper.hasOption(OPTION_PROJECT)) {
String projectNames = optionsHelper.getOptionValue(OPTION_PROJECT);
for (String projectName: projectNames.split(",")) {
ProjectInstance projectInstance = projectManager.getProject(projectName);
if (projectInstance == null) {
throw new IllegalArgumentException("Project " + projectName + " does not exist");
}
List realizationEntries = projectInstance.getRealizationEntries();
for (RealizationEntry realizationEntry : realizationEntries) {
retrieveResourcePath(getRealization(realizationEntry));
}
}
} else if (optionsHelper.hasOption(OPTION_CUBE)) {
String cubeNames = optionsHelper.getOptionValue(OPTION_CUBE);
for (String cubeName : cubeNames.split(",")) {
IRealization realization = cubeManager.getRealization(cubeName);
if (realization != null) {
retrieveResourcePath(realization);
} else {
throw new IllegalArgumentException("No cube found with name of " + cubeName);
}
}
}
extractCommonInfo(exportDir);
extractHTables(exportDir);
}
private void extractHTables(File dest) throws IOException {
logger.info("These htables are going to be extracted:");
for (String htable : htables) {
logger.info(htable + "(required)");
}
File tableDir = new File(dest, "table");
FileUtils.forceMkdir(tableDir);
for (String htable : htables) {
try {
URL srcUrl = new URL(getHBaseMasterUrl() + "table.jsp?name=" + htable);
File destFile = new File(tableDir, htable + ".html");
FileUtils.copyURLToFile(srcUrl, destFile);
} catch (Exception e) {
logger.warn("HTable " + htable + "info fetch failed: ", e);
}
}
}
private void extractCommonInfo(File dest) throws IOException {
logger.info("The hbase master info/conf are going to be extracted...");
// hbase master page
try {
File masterDir = new File(dest, "master");
FileUtils.forceMkdir(masterDir);
URL srcMasterUrl = new URL(getHBaseMasterUrl() + "master-status");
File masterDestFile = new File(masterDir, "master-status.html");
FileUtils.copyURLToFile(srcMasterUrl, masterDestFile);
} catch (Exception e) {
logger.warn("HBase Master status fetch failed: ", e);
}
// hbase conf
try {
File confDir = new File(dest, "conf");
FileUtils.forceMkdir(confDir);
URL srcConfUrl = new URL(getHBaseMasterUrl() + "conf");
File destConfFile = new File(confDir, "hbase-conf.xml");
FileUtils.copyURLToFile(srcConfUrl, destConfFile);
} catch (Exception e) {
logger.warn("HBase conf fetch failed: ", e);
}
// hbase jmx
try {
File jmxDir = new File(dest, "jmx");
FileUtils.forceMkdir(jmxDir);
URL srcJmxUrl = new URL(getHBaseMasterUrl() + "jmx");
File jmxDestFile = new File(jmxDir, "jmx.html");
FileUtils.copyURLToFile(srcJmxUrl, jmxDestFile);
} catch (Exception e) {
logger.warn("HBase JMX fetch failed: ", e);
}
// hbase hdfs status
try {
File hdfsDir = new File(dest, "hdfs");
FileUtils.forceMkdir(hdfsDir);
CliCommandExecutor cliCommandExecutor = kylinConfig.getCliCommandExecutor();
String output = cliCommandExecutor.execute("hadoop fs -ls -R " + conf.get("hbase.rootdir") + "/data/default/KYLIN_*").getSecond();
FileUtils.writeStringToFile(new File(hdfsDir, "hdfs-files.list"), output, Charset.defaultCharset());
output = cliCommandExecutor.execute("hadoop fs -ls -R " + conf.get("hbase.rootdir") + "/data/default/kylin_*").getSecond();
FileUtils.writeStringToFile(new File(hdfsDir, "hdfs-files.list"), output, Charset.defaultCharset(), true);
} catch (Exception e) {
logger.warn("HBase hdfs status fetch failed: ", e);
}
}
private IRealization getRealization(RealizationEntry realizationEntry) {
return realizationRegistry.getRealization(realizationEntry.getType(), realizationEntry.getRealization());
}
private void retrieveResourcePath(IRealization realization) {
logger.info("Deal with realization {} of type {}", realization.getName(), realization.getType());
if (realization instanceof CubeInstance) {
CubeInstance cube = (CubeInstance) realization;
for (CubeSegment segment : cube.getSegments()) {
addHTable(segment.getStorageLocationIdentifier());
}
} else {
logger.warn("Unknown realization type: " + realization.getType());
}
}
private void addHTable(String record) {
logger.info("adding required resource {}", record);
htables.add(record);
}
}
