All Downloads are FREE. Search and download functionalities are using the official Maven repository.

eu.stratosphere.client.CliFrontend Maven / Gradle / Ivy

/***********************************************************************************************************************
 * Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
 * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations under the License.
 **********************************************************************************************************************/

package eu.stratosphere.client;

import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.apache.commons.cli.UnrecognizedOptionException;

import eu.stratosphere.api.common.accumulators.AccumulatorHelper;
import eu.stratosphere.client.program.Client;
import eu.stratosphere.client.program.JobInstantiationException;
import eu.stratosphere.client.program.PackagedProgram;
import eu.stratosphere.client.program.ProgramInvocationException;
import eu.stratosphere.compiler.CompilerException;
import eu.stratosphere.configuration.ConfigConstants;
import eu.stratosphere.configuration.Configuration;
import eu.stratosphere.configuration.GlobalConfiguration;
import eu.stratosphere.nephele.client.JobExecutionResult;
import eu.stratosphere.nephele.event.job.RecentJobEvent;
import eu.stratosphere.nephele.ipc.RPC;
import eu.stratosphere.nephele.jobgraph.JobID;
import eu.stratosphere.nephele.jobgraph.JobStatus;
import eu.stratosphere.nephele.net.NetUtils;
import eu.stratosphere.nephele.protocols.ExtendedManagementProtocol;
import eu.stratosphere.util.StringUtils;

/**
 * Implementation of a simple command line fronted for executing PACT programs.
 */
public class CliFrontend {

	// actions
	private static final String ACTION_RUN = "run";
	private static final String ACTION_INFO = "info";
	private static final String ACTION_LIST = "list";
	private static final String ACTION_CANCEL = "cancel";
	
	private static final String GENERAL_OPTS = "general";
	
	// general options
	private static final Option HELP_OPTION = new Option("h", "help", false, "Show the help for the CLI Frontend.");
	private static final Option VERBOSE_OPTION = new Option("v", "verbose", false, "Print more detailed error messages.");
	
	// run options
	private static final Option JAR_OPTION = new Option("j", "jarfile", true, "Stratosphere program JAR file");
	private static final Option CLASS_OPTION = new Option("c", "class", true, "Program class");
	private static final Option ARGS_OPTION = new Option("a", "arguments", true, "Program arguments");
	private static final Option WAIT_OPTION = new Option("w", "wait", false, "Wait for program to finish");
	private static final Option ADDRESS_OPTION = new Option("m", "jobmanager", true, "Jobmanager to which the program is submitted");
	
	// info options
	private static final Option DESCR_OPTION = new Option("d", "description", false, "Show description of expected program arguments");
	private static final Option PLAN_OPTION = new Option("p", "plan", false, "Show optimized execution plan of the program (JSON)");
	
	// list options
	private static final Option RUNNING_OPTION = new Option("r", "running", false, "Show running programs and their JobIDs");
	private static final Option SCHEDULED_OPTION = new Option("s", "scheduled", false, "Show scheduled prorgrams and their JobIDs");
	
	// cancel options
	private static final Option ID_OPTION = new Option("i", "jobid", true, "JobID of program to cancel");
	
	// config dir parameters
	private static final String ENV_CONFIG_DIRECTORY = "STRATOSPHERE_CONF_DIR";
	private static final String CONFIG_DIRECTORY_FALLBACK_1 = "../conf";
	private static final String CONFIG_DIRECTORY_FALLBACK_2 = "conf";
	

	private CommandLineParser parser;
	private Map options;
	private boolean verbose;

	/**
	 * Initializes the class
	 */
	public CliFrontend() {

		parser = new PosixParser();
		
		// init options
		options = new HashMap();
		options.put(GENERAL_OPTS, getGeneralOptions());
		options.put(ACTION_RUN, getRunOptions());
		options.put(ACTION_INFO, getInfoOptions());
		options.put(ACTION_LIST, getListOptions());
		options.put(ACTION_CANCEL, getCancelOptions());
	}

	private Options getGeneralOptions() {
		
		Options options = new Options();
		
		// general options
		HELP_OPTION.setRequired(false);
		options.addOption(HELP_OPTION);
		VERBOSE_OPTION.setRequired(false);
		options.addOption(VERBOSE_OPTION);
		
		return options;
	}
	
	/**
	 * Builds command line options for the run action.
	 * 
	 * @return Command line options for the run action.
	 */
	private Options getRunOptions() {
		Options options = new Options();
		// run options
		JAR_OPTION.setRequired(false);
		JAR_OPTION.setArgName("jarfile");
		options.addOption(JAR_OPTION);
		CLASS_OPTION.setRequired(false);
		CLASS_OPTION.setArgName("classname");
		options.addOption(CLASS_OPTION);
		ARGS_OPTION.setRequired(false);
		ARGS_OPTION.setArgName("programArgs");
		ARGS_OPTION.setArgs(Option.UNLIMITED_VALUES);
		options.addOption(ARGS_OPTION);
		WAIT_OPTION.setRequired(false);
		options.addOption(WAIT_OPTION);
		ADDRESS_OPTION.setRequired(false);
		ADDRESS_OPTION.setArgName("host:port");
		options.addOption(ADDRESS_OPTION);
		
		return options;
	}
	
	/**
	 * Builds command line options for the info action.
	 * 
	 * @return Command line options for the info action.
	 */
	private Options getInfoOptions() {
		
		Options options = new Options();
		
		// info options
		DESCR_OPTION.setRequired(false);
		options.addOption(DESCR_OPTION);
		PLAN_OPTION.setRequired(false);
		options.addOption(PLAN_OPTION);
		JAR_OPTION.setRequired(false);
		JAR_OPTION.setArgName("jarfile");
		options.addOption(JAR_OPTION);
		CLASS_OPTION.setRequired(false);
		CLASS_OPTION.setArgName("classname");
		options.addOption(CLASS_OPTION);
		ARGS_OPTION.setRequired(false);
		ARGS_OPTION.setArgName("programArgs");
		ARGS_OPTION.setArgs(Option.UNLIMITED_VALUES);
		options.addOption(ARGS_OPTION);
		
		return options;
	}
	
	/**
	 * Builds command line options for the list action.
	 * 
	 * @return Command line options for the list action.
	 */
	private Options getListOptions() {
		
		Options options = new Options();
		
		// list options
		RUNNING_OPTION.setRequired(false);
		options.addOption(RUNNING_OPTION);
		SCHEDULED_OPTION.setRequired(false);
		options.addOption(SCHEDULED_OPTION);
		
		return options;
	}
	
	/**
	 * Builds command line options for the cancel action.
	 * 
	 * @return Command line options for the cancel action.
	 */
	private Options getCancelOptions() {
		
		Options options = new Options();
		
		// cancel options
		ID_OPTION.setRequired(false);
		ID_OPTION.setArgName("jobID");
		options.addOption(ID_OPTION);
		
		return options;
	}
	
	/**
	 * Executions the run action.
	 * 
	 * @param args Command line arguments for the run action.
	 */
	private void run(String[] args) {
		
		File jarFile = null;
		String assemblerClass = null;
		String[] programArgs = null;
		String address = null;
		boolean wait = false;
		
		// Parse command line options
		CommandLine line = null;
		try {
			line = parser.parse(this.options.get(ACTION_RUN), args, false);
		} catch (Exception e) {
			handleError(e);
		}
		
		if( line.hasOption(ADDRESS_OPTION.getOpt())) {
			address = line.getOptionValue(ADDRESS_OPTION.getOpt());
		} 
		
		// Get jar file
		if (line.hasOption(JAR_OPTION.getOpt())) {
			String jarFilePath = line.getOptionValue(JAR_OPTION.getOpt());
			jarFile = new File(jarFilePath);
			
			// Check if JAR file exists
			if(!jarFile.exists()) {
				System.err.println("Error: Jar file does not exist.");
				printHelp();
				System.exit(1);
			} else if(!jarFile.isFile()) {
				System.err.println("Error: Jar file is not a file.");
				printHelp();
				System.exit(1);
			}
		} else {
			System.err.println("Error: Jar file is not set.");	
			printHelp();
			System.exit(1);
		}
		
		// Get assembler class
		if(line.hasOption(CLASS_OPTION.getOpt())) {
			assemblerClass = line.getOptionValue(CLASS_OPTION.getOpt());
		}
		
		// get program arguments
		if(line.hasOption(ARGS_OPTION.getOpt())) {
			programArgs = line.getOptionValues(ARGS_OPTION.getOpt());
		}
		
		// get wait flag
		wait = line.hasOption(WAIT_OPTION.getOpt());
		
		// Try to get load plan
		PackagedProgram program = null;
		try {
			if (assemblerClass == null) {
				program = new PackagedProgram(jarFile, programArgs);
			} else {
				program = new PackagedProgram(jarFile, assemblerClass, programArgs);
			}
		} catch (ProgramInvocationException e) {
			handleError(e);
		}

		Configuration configuration = getConfiguration();
		Client client;
		InetSocketAddress socket = null;
		if(address != null && !address.isEmpty()) {
			socket = RemoteExecutor.getInetFromHostport(address);
			client = new Client(socket, configuration);
		} else {
			client = new Client(configuration);
		}
		client.setPrintStatusDuringExecution(true);
		
		JobExecutionResult execResult = null;
		try {
			execResult = client.run(program.getPlanWithJars(), wait);
		} catch (ProgramInvocationException e) {
			handleError(e);
		} catch (JobInstantiationException e) {
			handleError(e);
		} catch (IOException e) {
			handleError(e);
		} finally {
			program.deleteExtractedLibraries();
		}
		if(wait) {
			System.out.println("Job Runtime: "+execResult.getNetRuntime());
			Map accumulatorsResult = execResult.getAllAccumulatorResults();
			if(accumulatorsResult.size() > 0) {
				System.out.println("Accumulator Results: ");
				System.out.println(AccumulatorHelper.getResultsFormated(accumulatorsResult));
			}
		} else {
			if(address != null && !address.isEmpty()) {
				System.out.println("Job successfully submitted. Use -w (or --wait) option to track the progress here.\n"
						+ "JobManager web interface: http://"
						+ socket.getHostName()
						+":"+configuration.getInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, ConfigConstants.DEFAULT_JOB_MANAGER_WEB_FRONTEND_PORT));
			} else {
				System.out.println("Job successfully submitted. Use -w (or --wait) option to track the progress here.\n"
					+ "JobManager web interface: http://"
					+ configuration.getString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, null)
					+":"+configuration.getInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, ConfigConstants.DEFAULT_JOB_MANAGER_WEB_FRONTEND_PORT));
			}
		}
	}
	
	/**
	 * Executes the info action.
	 * 
	 * @param args Command line arguments for the info action. 
	 */
	private void info(String[] args) {
		
		File jarFile = null;
		String assemblerClass = null;
		String[] programArgs = null;
		
		boolean description;
		boolean plan;
		
		// Parse command line options
		CommandLine line = null;
		try {
			line = parser.parse(this.options.get(ACTION_INFO), args, false);
		} catch (Exception e) {
			handleError(e);
		}
		
		// Get jar file
		if (line.hasOption(JAR_OPTION.getOpt())) {
			jarFile = new File(line.getOptionValue(JAR_OPTION.getOpt()));
			
			// Check if JAR file exists
			if(!jarFile.exists()) {
				System.err.println("Error: Jar file does not exist.");
				printHelp();
				System.exit(1);
			} else if(!jarFile.isFile()) {
				System.err.println("Error: Jar file is not a file.");
				printHelp();
				System.exit(1);
			}
		} else {
			System.err.println("Error: Jar file is not set.");
			printHelp();
			System.exit(1);
		}
		
		// Get assembler class
		if(line.hasOption(CLASS_OPTION.getOpt())) {
			assemblerClass = line.getOptionValue(CLASS_OPTION.getOpt());
		}
		
		// get program arguments
		if(line.hasOption(ARGS_OPTION.getOpt())) {
			programArgs = line.getOptionValues(ARGS_OPTION.getOpt());
		}
		
		description = line.hasOption(DESCR_OPTION.getOpt());
		plan = line.hasOption(PLAN_OPTION.getOpt());
		
		if(!description && !plan) {
			System.err.println("ERROR: Specify the information to display.");
			printHelp();
			System.exit(1);
		}
		
		// Try to get load plan
		PackagedProgram program = null;
		try {
			if (assemblerClass == null) {
				program = new PackagedProgram(jarFile, programArgs);
			} else {
				program = new PackagedProgram(jarFile, assemblerClass, programArgs);
			}
		} catch (ProgramInvocationException e) {
			handleError(e);
		}
		
		// check for description request
		if (description) {
			String descr = null;
			try {
				descr = program.getDescription();
			} catch (Exception e) {
				handleError(e);
			}
			
			if (descr != null) {
				System.out.println("-------------------- Program Description ---------------------");
				System.out.println(descr);
				System.out.println("--------------------------------------------------------------");
			} else {
				System.err.println("No description available for this plan.");
			}
		}
		
		// check for json plan request
		if (plan) {
			String jsonPlan = null;
			
			Configuration configuration = getConfiguration();
			Client client = new Client(configuration);
			try {
				jsonPlan = client.getOptimizerPlanAsJSON(program.getPlanWithJars());
			} catch (ProgramInvocationException e) {
				handleError(e);
			} catch (JobInstantiationException e) {
				handleError(e);
			} catch (CompilerException e) {
				handleError(e);
			} catch (IOException e) {
				handleError(e);
			}
			
			if(jsonPlan != null) {
				System.out.println("----------------------- Execution Plan -----------------------");
				System.out.println(jsonPlan);
				System.out.println("--------------------------------------------------------------");
			} else {
				System.err.println("JSON plan could not be compiled.");
			}
		}
		program.deleteExtractedLibraries();
	}

	/**
	 * Executes the list action.
	 * 
	 * @param args Command line arguments for the list action.
	 */
	private void list(String[] args) {
		
		boolean running;
		boolean scheduled;
		
		// Parse command line options
		CommandLine line = null;
		try {
			line = parser.parse(this.options.get(ACTION_LIST), args, false);
		} catch (Exception e) {
			handleError(e);
		}
		
		// get list options
		running = line.hasOption(RUNNING_OPTION.getOpt());
		scheduled = line.hasOption(SCHEDULED_OPTION.getOpt());
		
		if(!running && !scheduled) {
			System.err.println("Error: Specify the status of the jobs to list.");
			printHelp();
			System.exit(1);
		}
		
		ExtendedManagementProtocol jmConn = null;
		try {
			
			jmConn = getJMConnection();
			List recentJobs = jmConn.getRecentJobs();
			
			ArrayList runningJobs = null;
			ArrayList scheduledJobs = null;
			if(running) {
				runningJobs = new ArrayList();
			}
			if(scheduled) {
				scheduledJobs = new ArrayList();
			}
			
			for(RecentJobEvent rje : recentJobs) {
				
				if(running && rje.getJobStatus().equals(JobStatus.RUNNING)) {
					runningJobs.add(rje);
				}
				if(scheduled && rje.getJobStatus().equals(JobStatus.SCHEDULED)) {
					scheduledJobs.add(rje);
				}
			}
			
			SimpleDateFormat df = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss");
			Comparator njec = new Comparator(){
				
				@Override
				public int compare(RecentJobEvent o1, RecentJobEvent o2) {
					return (int)(o1.getTimestamp()-o2.getTimestamp());
				}
			};
			
			if(running) {
				if(runningJobs.size() == 0) {
					System.out.println("No running jobs.");
				} else {
					Collections.sort(runningJobs, njec);
					
					System.out.println("------------------------ Running Jobs ------------------------");
					for(RecentJobEvent je : runningJobs) {
						System.out.println(df.format(new Date(je.getTimestamp()))+" : "+je.getJobID().toString()+" : "+je.getJobName());
					}
					System.out.println("--------------------------------------------------------------");
				}
			}
			if(scheduled) {
				if(scheduledJobs.size() == 0) {
					System.out.println("No scheduled jobs.");
				} else {
					Collections.sort(scheduledJobs, njec);
					
					System.out.println("----------------------- Scheduled Jobs -----------------------");
					for(RecentJobEvent je : scheduledJobs) {
						System.out.println(df.format(new Date(je.getTimestamp()))+" : "+je.getJobID().toString()+" : "+je.getJobName());
					}
					System.out.println("--------------------------------------------------------------");
				}
			}
			
		} catch (Throwable t) {
			handleError(t);
		} finally {
			if (jmConn != null) {
				try {
					RPC.stopProxy(jmConn);
				} catch (Throwable t) {
					System.err.println("Could not cleanly shut down connection from compiler to job manager");
				}
			}
			jmConn = null;
		}
		
	}
	
	/**
	 * Executes the cancel action.
	 * 
	 * @param args Command line arguments for the cancel action.
	 */
	private void cancel(String[] args) {
		
		String jobId = null;
		
		// Parse command line options
		CommandLine line = null;
		try {
			line = parser.parse(this.options.get(ACTION_CANCEL), args, false);
		} catch (Exception e) {
			handleError(e);
		}
		
		if(line.hasOption(ID_OPTION.getOpt())) {
			jobId = line.getOptionValue(ID_OPTION.getOpt());
		} else {
			System.err.println("Error: Specify a jobID to cancel a job.");
			printHelp();
			System.exit(1);
		}
		
		ExtendedManagementProtocol jmConn = null;
		try {
			
			jmConn = getJMConnection();
			jmConn.cancelJob(new JobID(StringUtils.hexStringToByte(jobId)));
		} catch (Throwable t) {
			handleError(t);
		} finally {
			if (jmConn != null) {
				try {
					RPC.stopProxy(jmConn);
				} catch (Throwable t) {
					System.err.println("Could not cleanly shut down connection from compiler to job manager");
				}
			}
			jmConn = null;
		}
		
	}
	
	/**
	 * Sets up a connection to the JobManager.
	 * 
	 * @return Connection to the JobManager.
	 * @throws IOException
	 */
	private ExtendedManagementProtocol getJMConnection() throws IOException {
		Configuration config = getConfiguration();
		String jmHost = config.getString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, null);
		String jmPort = config.getString(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, null);
		
		if(jmHost == null) {
			handleError(new Exception("JobManager address could not be determined."));
		}
		if(jmPort == null) {
			handleError(new Exception("JobManager port could not be determined."));
		}
		
		return RPC.getProxy(ExtendedManagementProtocol.class, 
			new InetSocketAddress(jmHost, Integer.parseInt(jmPort)), NetUtils.getSocketFactory());
	}
	

	/**
	 * Prints the help for the client.
	 * 
	 * @param options A map with options for actions. 
	 */
	private void printHelp() {
		HelpFormatter formatter = new HelpFormatter();
		formatter.setLeftPadding(5);
		
		System.out.println("./stratosphere [ACTION] [GENERAL_OPTIONS] [ACTION_ARGUMENTS]");
		
		formatter.setSyntaxPrefix("  general options:");
		formatter.printHelp(" ",this.options.get(GENERAL_OPTS));
		
		System.out.println("\nAction \"run\" compiles and submits a Stratosphere program.");
		formatter.setSyntaxPrefix("  \"run\" action arguments:");
		formatter.printHelp(" ", this.options.get(ACTION_RUN));
		
		System.out.println("\nAction \"info\" displays information about a Stratosphere program.");
		formatter.setSyntaxPrefix("  \"info\" action arguments:");
		formatter.printHelp(" ", this.options.get(ACTION_INFO));
		
		System.out.println("\nAction \"list\" lists submitted Stratosphere programs.");
		formatter.setSyntaxPrefix("  \"list\" action arguments:");
		formatter.printHelp(" ", this.options.get(ACTION_LIST));
		
		System.out.println("\nAction \"cancel\" cancels a submitted Stratosphere program.");
		formatter.setSyntaxPrefix("  \"cancel\" action arguments:");
		formatter.printHelp(" ", this.options.get(ACTION_CANCEL));
		
	}
	
	/**
	 * Displays exceptions.
	 * 
	 * @param e the exception to display.
	 */
	@SuppressWarnings({"unchecked"})
	private void handleError(Throwable t) {
		if(t instanceof UnrecognizedOptionException) {
			
			Options generalOptions = this.options.get(GENERAL_OPTS);
			boolean generalOption = false;
			for(Option o : (Collection




© 2015 - 2025 Weber Informatics LLC | Privacy Policy