fr.jetoile.hadoopunit.component.OozieBootstrap Maven / Gradle / Ivy
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package fr.jetoile.hadoopunit.component;
import com.github.sakserv.minicluster.impl.MRLocalCluster;
import com.github.sakserv.minicluster.impl.OozieLocalServer;
import com.github.sakserv.minicluster.util.FileUtils;
import fr.jetoile.hadoopunit.Component;
import fr.jetoile.hadoopunit.Config;
import fr.jetoile.hadoopunit.HadoopBootstrap;
import fr.jetoile.hadoopunit.HadoopUtils;
import fr.jetoile.hadoopunit.exception.BootstrapException;
import fr.jetoile.hadoopunit.exception.NotFoundServiceException;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.oozie.client.OozieClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
public class OozieBootstrap implements Bootstrap {
final public static String NAME = Component.OOZIE.name();
final private Logger LOGGER = LoggerFactory.getLogger(OozieBootstrap.class);
public static final String OOZIE_PORT = "oozie.port";
public static final String OOZIE_HOST = "oozie.host";
private static final String OOZIE_SHARELIB_PATH_KEY = "oozie.sharelib.path";
private static final String OOZIE_SHARELIB_NAME_KEY = "oozie.sharelib.name";
private static final String SHARE_LIB_LOCAL_TEMP_PREFIX = "oozie_share_lib_tmp";
private static final String SHARE_LIB_PREFIX = "lib_";
private OozieLocalServer oozieLocalCluster;
private MRLocalCluster mrLocalCluster;
private State state = State.STOPPED;
private Configuration configuration;
private String oozieTestDir;
private String oozieHomeDir;
private String oozieUsername;
private String oozieGroupname;
private String oozieYarnResourceManagerAddress;
private org.apache.hadoop.conf.Configuration hadoopConf;
private String hdfsDefaultFs;
private String oozieHdfsShareLibDir;
private boolean oozieShareLibCreate;
private String oozieLocalShareLibCacheDir;
private boolean ooziePurgeLocalShareLibCache;
private int numNodeManagers;
private String jobHistoryAddress;
private String resourceManagerAddress;
private String resourceManagerHostname;
private String resourceManagerSchedulerAddress;
private String resourceManagerResourceTrackerAddress;
private String resourceManagerWebappAddress;
private boolean useInJvmContainerExecutor;
private String oozieShareLibPath;
private String oozieShareLibName;
private int ooziePort;
private String oozieHost;
private String fullOozieShareLibTarFilePath;
private String oozieShareLibExtractTempDir;
public OozieBootstrap() {
if (oozieLocalCluster == null) {
try {
loadConfig();
} catch (BootstrapException | NotFoundServiceException e) {
LOGGER.error("unable to load configuration", e);
}
}
}
@Override
public String getName() {
return NAME;
}
@Override
public String getProperties() {
return "[" +
"host:" + oozieHost +
", port:" + ooziePort +
"]";
}
private void init() {
}
private void build() throws NotFoundServiceException {
//is done lazy because hdfsBootstrap should be started
hdfsDefaultFs = HadoopBootstrap.INSTANCE.getService(Component.HDFS).getConfiguration().get("fs.defaultFS");
hadoopConf = HadoopBootstrap.INSTANCE.getService(Component.HDFS).getConfiguration();
mrLocalCluster = new MRLocalCluster.Builder()
.setNumNodeManagers(numNodeManagers)
.setJobHistoryAddress(jobHistoryAddress)
.setResourceManagerAddress(resourceManagerAddress)
.setResourceManagerHostname(resourceManagerHostname)
.setResourceManagerSchedulerAddress(resourceManagerSchedulerAddress)
.setResourceManagerResourceTrackerAddress(resourceManagerResourceTrackerAddress)
.setResourceManagerWebappAddress(resourceManagerWebappAddress)
.setUseInJvmContainerExecutor(useInJvmContainerExecutor)
.setHdfsDefaultFs(hdfsDefaultFs)
.setConfig(hadoopConf)
.build();
oozieLocalCluster = new OozieLocalServer.Builder()
.setOozieTestDir(oozieTestDir)
.setOozieHomeDir(oozieHomeDir)
.setOozieUsername(oozieUsername)
.setOozieGroupname(oozieGroupname)
.setOozieYarnResourceManagerAddress(oozieYarnResourceManagerAddress)
.setOozieHdfsDefaultFs(hdfsDefaultFs)
.setOozieConf(hadoopConf)
.setOozieHdfsShareLibDir(oozieHdfsShareLibDir)
.setOozieShareLibCreate(oozieShareLibCreate)
.setOozieLocalShareLibCacheDir(oozieLocalShareLibCacheDir)
.setOoziePurgeLocalShareLibCache(ooziePurgeLocalShareLibCache)
.setOoziePort(ooziePort)
.setOozieHost(oozieHost)
.build();
}
private void loadConfig() throws BootstrapException, NotFoundServiceException {
HadoopUtils.setHadoopHome();
try {
configuration = new PropertiesConfiguration(Config.DEFAULT_PROPS_FILE);
} catch (ConfigurationException e) {
throw new BootstrapException("bad config", e);
}
oozieTestDir = configuration.getString(Config.OOZIE_TEST_DIR_KEY);
oozieHomeDir = configuration.getString(Config.OOZIE_HOME_DIR_KEY);
oozieUsername = System.getProperty("user.name");
oozieGroupname = configuration.getString(Config.OOZIE_GROUPNAME_KEY);
oozieYarnResourceManagerAddress = configuration.getString(Config.YARN_RESOURCE_MANAGER_ADDRESS_KEY);
oozieHdfsShareLibDir = configuration.getString(Config.OOZIE_HDFS_SHARE_LIB_DIR_KEY);
oozieShareLibCreate = configuration.getBoolean(Config.OOZIE_SHARE_LIB_CREATE_KEY);
oozieLocalShareLibCacheDir = configuration.getString(Config.OOZIE_LOCAL_SHARE_LIB_CACHE_DIR_KEY);
ooziePurgeLocalShareLibCache = configuration.getBoolean(Config.OOZIE_PURGE_LOCAL_SHARE_LIB_CACHE_KEY);
numNodeManagers = Integer.parseInt(configuration.getString(Config.YARN_NUM_NODE_MANAGERS_KEY));
jobHistoryAddress = configuration.getString(Config.MR_JOB_HISTORY_ADDRESS_KEY);
resourceManagerAddress = configuration.getString(Config.YARN_RESOURCE_MANAGER_ADDRESS_KEY);
resourceManagerHostname = configuration.getString(Config.YARN_RESOURCE_MANAGER_HOSTNAME_KEY);
resourceManagerSchedulerAddress = configuration.getString(Config.YARN_RESOURCE_MANAGER_SCHEDULER_ADDRESS_KEY);
resourceManagerResourceTrackerAddress = configuration.getString(Config.YARN_RESOURCE_MANAGER_RESOURCE_TRACKER_ADDRESS_KEY);
resourceManagerWebappAddress = configuration.getString(Config.YARN_RESOURCE_MANAGER_WEBAPP_ADDRESS_KEY);
useInJvmContainerExecutor = configuration.getBoolean(Config.YARN_USE_IN_JVM_CONTAINER_EXECUTOR_KEY);
ooziePort = configuration.getInt(OOZIE_PORT);
oozieHost = configuration.getString(OOZIE_HOST);
oozieShareLibPath = configuration.getString(OOZIE_SHARELIB_PATH_KEY);
oozieShareLibName = configuration.getString(OOZIE_SHARELIB_NAME_KEY);
}
@Override
public Bootstrap start() {
if (state == State.STOPPED) {
state = State.STARTING;
LOGGER.info("{} is starting", this.getClass().getName());
init();
try {
build();
} catch (NotFoundServiceException e) {
LOGGER.error("unable to add oozie", e);
}
createShareLib();
try {
mrLocalCluster.start();
oozieLocalCluster.start();
} catch (Exception e) {
LOGGER.error("unable to add oozie", e);
}
state = State.STARTED;
LOGGER.info("{} is started", this.getClass().getName());
}
return this;
}
@Override
public Bootstrap stop() {
if (state == State.STARTED) {
state = State.STOPPING;
LOGGER.info("{} is stopping", this.getClass().getName());
try {
oozieLocalCluster.stop(true);
mrLocalCluster.stop(true);
cleanup();
} catch (Exception e) {
LOGGER.error("unable to stop hdfs", e);
}
state = State.STOPPED;
LOGGER.info("{} is stopped", this.getClass().getName());
}
return this;
}
private void cleanup() {
FileUtils.deleteFolder(fullOozieShareLibTarFilePath);
FileUtils.deleteFolder(oozieShareLibExtractTempDir);
}
@Override
public org.apache.hadoop.conf.Configuration getConfiguration() {
return oozieLocalCluster.getOozieConf();
}
public OozieClient getOozieClient() {
return oozieLocalCluster.getOozieClient();
}
// Main driver that downloads, extracts, and deploys the oozie sharelib
public void createShareLib() {
if (!oozieShareLibCreate) {
LOGGER.info("OOZIE: Share Lib Create Disabled... skipping");
} else {
try {
// Get and extract the oozie release
String oozieExtractTempDir = extractOozieTarFileToTempDir(new File(oozieShareLibPath + Path.SEPARATOR + oozieShareLibName));
// Extract the sharelib tarball to a temp dir
fullOozieShareLibTarFilePath = oozieExtractTempDir + Path.SEPARATOR + "oozie-" + getOozieVersionFromOozieTarFileName()
+ Path.SEPARATOR + "oozie-sharelib-" + getOozieVersionFromOozieTarFileName() + ".tar.gz";
;
oozieShareLibExtractTempDir = extractOozieShareLibTarFileToTempDir(new File(fullOozieShareLibTarFilePath));
// Copy the sharelib into HDFS
Path destPath = new Path(oozieHdfsShareLibDir + Path.SEPARATOR +
SHARE_LIB_PREFIX + getTimestampDirectory());
LOGGER.info("OOZIE: Writing share lib contents to: {}", destPath);
FileSystem hdfsFileSystem = null;
try {
hdfsFileSystem = ((HdfsBootstrap) HadoopBootstrap.INSTANCE.getService(Component.HDFS)).getHdfsFileSystemHandle();
} catch (Exception e) {
LOGGER.error("unable to get hdfs client");
}
hdfsFileSystem.copyFromLocalFile(false, new Path(new File(oozieShareLibExtractTempDir).toURI()), destPath);
// if (purgeLocalShareLibCache) {
// FileUtils.deleteDirectory(new File(shareLibCacheDir));
// }
} catch (IOException e) {
LOGGER.error("unable to copy oozie sharelib into hdfs");
}
}
}
public String extractOozieTarFileToTempDir(File fullOozieTarFilePath) throws IOException {
File tempDir = File.createTempFile(SHARE_LIB_LOCAL_TEMP_PREFIX, "");
tempDir.delete();
tempDir.mkdir();
tempDir.deleteOnExit();
FileUtil.unTar(fullOozieTarFilePath, tempDir);
return tempDir.getAbsolutePath();
}
public String extractOozieShareLibTarFileToTempDir(File fullOozieShareLibTarFilePath) throws IOException {
File tempDir = File.createTempFile(SHARE_LIB_LOCAL_TEMP_PREFIX, "");
tempDir.delete();
tempDir.mkdir();
tempDir.deleteOnExit();
FileUtil.unTar(fullOozieShareLibTarFilePath, tempDir);
return tempDir.getAbsolutePath();
}
public String getOozieVersionFromOozieTarFileName() {
return oozieShareLibName.replace("-distro.tar.gz", "").replace("oozie-", "");
}
public String getTimestampDirectory() {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss");
Date date = new Date();
return dateFormat.format(date).toString();
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy