All Downloads are FREE. Search and download functionalities are using the official Maven repository.

uk.ac.ebi.interpro.scan.jms.main.Run Maven / Gradle / Ivy

The newest version!
package uk.ac.ebi.interpro.scan.jms.main;

import org.apache.activemq.broker.BrokerService;
import org.apache.activemq.broker.TransportConnector;
import org.apache.commons.cli.*;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.springframework.context.support.AbstractApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.util.StringUtils;
import uk.ac.ebi.interpro.scan.io.ExternallySetLocationTemporaryDirectoryManager;
import uk.ac.ebi.interpro.scan.io.FileOutputFormat;
import uk.ac.ebi.interpro.scan.io.TemporaryDirectoryManager;
import uk.ac.ebi.interpro.scan.jms.converter.Converter;
import uk.ac.ebi.interpro.scan.jms.exception.InvalidInputException;
import uk.ac.ebi.interpro.scan.jms.master.*;
import uk.ac.ebi.interpro.scan.jms.monitoring.MasterControllerApplication;
import uk.ac.ebi.interpro.scan.jms.stats.SystemInfo;
import uk.ac.ebi.interpro.scan.util.Utilities;
import uk.ac.ebi.interpro.scan.jms.worker.WorkerImpl;
import uk.ac.ebi.interpro.scan.management.model.Job;
import uk.ac.ebi.interpro.scan.management.model.JobStatusWrapper;
import uk.ac.ebi.interpro.scan.management.model.Jobs;
import uk.ac.ebi.interpro.scan.management.model.implementations.panther.PantherNewBinaryStep;
import uk.ac.ebi.interpro.scan.model.SignatureLibrary;
import uk.ac.ebi.interpro.scan.model.SignatureLibraryRelease;

import java.io.File;
import java.io.FilePermission;
import java.io.IOException;
import java.net.DatagramSocket;
import java.net.InetAddress;
import java.net.ServerSocket;
import java.net.URI;
import java.security.AccessController;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * The main entry point for the the master and workers in a
 * Java Messaging configuration of InterProScan.
 * 

* Runs in mode 'master' by default. *

* Usage: * java -Dconfig=conf/myconfig.props -jar interproscan-5.jar master * java -Dconfig=conf/myconfig.props -jar interproscan-5.jar worker * java -Dconfig=conf/myconfig.props -jar interproscan-5.jar monitor */ public class Run extends AbstractI5Runner { private static final Logger LOGGER = Logger.getLogger(Run.class.getName()); /** * This is the REAL set of options that the Run class will accept */ private static final Options COMMAND_LINE_OPTIONS = new Options(); /** * Same contents as COMMAND_LINE_OPTIONS, however if the I5Option enum value * has includeInUsageMessage == false, the option is excluded. *

* This is to remove clutter from the help message that may confuse users. */ private static final Options COMMAND_LINE_OPTIONS_FOR_HELP = new Options(); private static final int MEGA = 1024 * 1024; String temporaryDirectory = null; private boolean deleteWorkingDirectoryOnCompletion = true; static { //Usual I5 options for (I5Option i5Option : I5Option.values()) { final Option.Builder builder; builder = (i5Option.getShortOpt() == null) ? Option.builder() : Option.builder(i5Option.getShortOpt()); builder.longOpt(i5Option.getLongOpt()); builder.desc(i5Option.getDescription()); if (i5Option.isRequired()) { builder.required(); } if (i5Option.getArgumentName() != null) { builder.argName(i5Option.getArgumentName()); if (i5Option.hasMultipleArgs()) { builder.hasArgs(); } else { builder.hasArg(); } } builder.valueSeparator(); final Option option = builder.build(); COMMAND_LINE_OPTIONS.addOption(option); } } public static void main(String[] args) { // create the command line parser CommandLineParser parser = new DefaultParser(); String modeArgument; Mode mode = null; String temporaryDirectory = null; boolean deleteWorkingDirectoryOnCompletion = true; try { //change Loglevel // changeLogLevel("DEBUG"); // parse the command line arguments CommandLine parsedCommandLine = parser.parse(COMMAND_LINE_OPTIONS, args); modeArgument = parsedCommandLine.getOptionValue(I5Option.MODE.getLongOpt()); try { mode = getMode(modeArgument); } catch (IllegalArgumentException iae) { LOGGER.fatal("The mode '" + modeArgument + "' is not handled. Should be one of: " + Mode.getCommaSepModeList()); System.exit(1); } for (Option option : COMMAND_LINE_OPTIONS.getOptions()) { final String shortOpt = option.getOpt(); if (I5Option.showOptInHelpMessage(shortOpt, mode)) { COMMAND_LINE_OPTIONS_FOR_HELP.addOption(option); } } ArrayList analysesHelpInformation = new ArrayList<>(); String i5Version = "5.36-75.0"; String i5BuildType = "64-Bit"; //32bitMessage:i5BuildType = "32-Bit"; //print version and exit if (parsedCommandLine.hasOption(I5Option.VERSION.getLongOpt())) { printVersion(i5Version, i5BuildType); System.exit(0); } System.out.println(Utilities.getTimeNow() + " Welcome to InterProScan-" + i5Version); //32bitMessage:System.out.println(Utilities.getTimeNow() + " You are running the 32-bit version"); //String config = System.getProperty("config"); if (LOGGER.isInfoEnabled()) { LOGGER.info("Memory free: " + Runtime.getRuntime().freeMemory() / MEGA + "MB total: " + Runtime.getRuntime().totalMemory() / MEGA + "MB max: " + Runtime.getRuntime().maxMemory() / MEGA + "MB"); LOGGER.info("Running in " + mode + " mode"); } //create the dot i5 dir/file //$USER_HOME/.interproscan-5/interproscan.properties if(System.getProperty("user.home") != null && ! System.getProperty("user.home").isEmpty()) { String dotInterproscan5Dir = System.getProperty("user.home") + "/.interproscan-5"; if (LOGGER.isDebugEnabled()) { LOGGER.debug("dotInterproscan5Dir : " + dotInterproscan5Dir); } String userInterproscan5Properties = dotInterproscan5Dir + "/interproscan.properties"; File userInterproscan5PropertiesFile = new File(userInterproscan5Properties); if (!checkPathExistence(dotInterproscan5Dir)) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Create dotInterproscan5Dir : " + dotInterproscan5Dir); } createDirectory(dotInterproscan5Dir); } else { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Directory $USER_HOME/.interproscan-5/interproscan.properties - " + dotInterproscan5Dir + " exists"); } } //Create file if it doesnot exists if (!userInterproscan5PropertiesFile.exists()) { if (LOGGER.isDebugEnabled()) { LOGGER.debug(" Creating the userInterproscan5Properties file : " + userInterproscan5Properties); } try { userInterproscan5PropertiesFile.createNewFile(); }catch (IOException e){ LOGGER.warn("Unable to access " + userInterproscan5Properties); //check the permisions in the directory of user.home try { String actions = "read,write"; AccessController.checkPermission(new FilePermission(System.getProperty("user.home"), actions)); // System.out.println("You have read/write permition to use : " + System.getProperty("user.home")); } catch (SecurityException se) { LOGGER.warn("You don't have read/write permition to use : " + System.getProperty("user.home")); } LOGGER.warn(e); } } //Deal with user supplied config file from the command line String systemInterproscanProperties = userInterproscan5Properties; if (System.getProperty("system.interproscan.properties") == null) { LOGGER.debug("USer has not supplied any properties file"); System.setProperty("system.interproscan.properties", systemInterproscanProperties); } }else{ //system and interproscan.properties are the same in case the user has not supplied any file if (System.getProperty("system.interproscan.properties") == null) { LOGGER.debug("USer has not supplied any properties file"); System.setProperty("system.interproscan.properties", "interproscan.properties"); } } final AbstractApplicationContext ctx = new ClassPathXmlApplicationContext(new String[]{mode.getContextXML()}); // The command-line distributed mode selects a random port number for communications. // This block selects the random port number and sets it on the broker. // Def. analysesToRun: List of analyses jobs which will be performed/submitted by I5 String[] analysesToRun = null; String[] depreactedAnalysesToRun = null; String[] excludedAnalyses = null; if (!mode.equals(Mode.INSTALLER) && !mode.equals(Mode.EMPTY_INSTALLER) && !mode.equals(Mode.CONVERT) && !mode.equals(Mode.MONITOR)) { Jobs jobs = (Jobs) ctx.getBean("jobs"); temporaryDirectory = jobs.getBaseDirectoryTemporaryFiles(); if (LOGGER.isDebugEnabled()) { LOGGER.debug("temporaryDirectory: jobs.getBaseDirectoryTemporaryFiles() - " + temporaryDirectory.toString()); } //Get deactivated jobs final Map deactivatedJobs = jobs.getDeactivatedJobs(); //Info about active and de-active jobs is shown in the manual instruction (help) as well final Map deprecatedJobs = jobs.getDeprecatedJobs(); if (isInvalid(mode, parsedCommandLine)) { printHelp(COMMAND_LINE_OPTIONS_FOR_HELP); analysesHelpInformation.add("Available analyses:\n"); // LEAVE as System.out for (Job job : jobs.getActiveAnalysisJobs().getJobList()) { // Print out available jobs SignatureLibraryRelease slr = job.getLibraryRelease(); if(! job.isDeprecated()) { analysesHelpInformation.add(String.format(" %25s (%s) : %s\n", slr.getLibrary().getName(), slr.getVersion(), job.getDescription())); // LEAVE as System.out } } if (deactivatedJobs.size() > 0) { analysesHelpInformation.add("\nDeactivated analyses:\n"); } for (Job deactivatedJob : deactivatedJobs.keySet()) { JobStatusWrapper jobStatusWrapper = deactivatedJobs.get(deactivatedJob); // Print out deactivated jobs SignatureLibraryRelease slr = deactivatedJob.getLibraryRelease(); analysesHelpInformation.add(String.format(" %25s (%s) : %s\n", slr.getLibrary().getName(), slr.getVersion(), jobStatusWrapper.getWarning())); } if (deprecatedJobs.size() > 0) { analysesHelpInformation.add("\nDeprecated analyses:\n"); } for (Job deprecatedJob : deprecatedJobs.keySet()) { JobStatusWrapper jobStatusWrapper = deprecatedJobs.get(deprecatedJob); // Print out deactivated jobs SignatureLibraryRelease slr = deprecatedJob.getLibraryRelease(); analysesHelpInformation.add(String.format(" %25s (%s) : %s\n", slr.getLibrary().getName(), slr.getVersion(), deprecatedJob.getDescription())); // LEAVE as System.out } printStringList(analysesHelpInformation); System.exit(1); } //print help and exit if (parsedCommandLine.hasOption(I5Option.HELP.getLongOpt())) { printHelp(COMMAND_LINE_OPTIONS_FOR_HELP); printStringList(analysesHelpInformation); System.exit(0); } try { depreactedAnalysesToRun = getDeprecatedApplications(parsedCommandLine, jobs); //System.out.println("depreacted Analyses To Run :" + depreactedAnalysesToRun.toString()); excludedAnalyses = getExcludedApplications(parsedCommandLine, jobs); //System.out.println("excludedAnalyses Analyses :" + excludedAnalyses.toString()); analysesToRun = getApplications(parsedCommandLine, jobs); if (LOGGER.isDebugEnabled()){ StringBuilder analysisItems = new StringBuilder(); for (String analysisItem : analysesToRun){ analysisItems.append(analysisItem).append(" "); } LOGGER.debug("analysesToRun :- " + analysisItems.toString()); } } catch (InvalidInputException e) { System.out.println("Invalid input specified for -appl/--applications parameter:\n" + e.getMessage()); System.exit(1); } } //Print help for the convert mode else if (mode.equals(Mode.CONVERT)) { if (isInvalid(mode, parsedCommandLine)) { exitI5(Mode.CONVERT, 1); } } // Validate the output formats supplied String[] parsedOutputFormats = null; if (parsedCommandLine.hasOption(I5Option.OUTPUT_FORMATS.getLongOpt())) { parsedOutputFormats = parsedCommandLine.getOptionValues(I5Option.OUTPUT_FORMATS.getLongOpt()); parsedOutputFormats = tidyOptionsArray(parsedOutputFormats); validateOutputFormatList(parsedOutputFormats, mode); } // Validate the sequence type String sequenceType = "p"; if (parsedCommandLine.hasOption(I5Option.SEQUENCE_TYPE.getLongOpt())) { sequenceType = parsedCommandLine.getOptionValue(I5Option.SEQUENCE_TYPE.getLongOpt()); // Check the sequence type is "n" or "p" Set sequenceTypes = (HashSet) ctx.getBean("sequenceTypes"); if (sequenceTypes != null && !sequenceTypes.contains(sequenceType)) { System.out.print("\n\nThe specified sequence type " + sequenceType + " was not recognised, expected: "); StringBuilder expectedSeqTypes = new StringBuilder(); for (String seqType : sequenceTypes) { if (expectedSeqTypes.length() > 0) { expectedSeqTypes.append(","); } expectedSeqTypes.append(seqType); } System.out.println(expectedSeqTypes + "\n\n"); System.exit(1); } } if (mode.getRunnableBean() != null) { final Runnable runnable = (Runnable) ctx.getBean(mode.getRunnableBean()); //Set up converter mode if (runnable instanceof Converter) { runConvertMode(runnable, parsedCommandLine, parsedOutputFormats); } else if (runnable instanceof MasterControllerApplication) { runMasterControllerApplicationMode(runnable, parsedCommandLine, ctx, mode); } else { checkIfMasterAndConfigure(runnable, analysesToRun, parsedCommandLine, parsedOutputFormats, ctx, mode, sequenceType); checkIfDistributedWorkerAndConfigure(runnable, parsedCommandLine, ctx, mode); } //checkIfDistributedWorkerAndConfigure(runnable, parsedCommandLine, ctx, mode); String workingTemporaryDirectory = ""; //get temp directory for cleanup if (! (mode.equals(Mode.INSTALLER) || mode.equals(Mode.WORKER) || mode.equals(Mode.DISTRIBUTED_WORKER) || mode.equals(Mode.CONVERT) || mode.equals(Mode.HIGHMEM_WORKER)) ) { final AbstractMaster master = (AbstractMaster) runnable; if (LOGGER.isDebugEnabled()) { LOGGER.debug(Utilities.getTimeNow() + " 1. Checking working Temporary Directory -master.getTemporaryDirectory() : " + master.getTemporaryDirectory()); } master.setupTemporaryDirectory(); if (LOGGER.isDebugEnabled()) { LOGGER.debug(Utilities.getTimeNow() + " 1. temporaryDirectory is " + temporaryDirectory); } try { temporaryDirectory = master.getWorkingTemporaryDirectoryPath(); } catch (IllegalStateException e) { final String tempDir = master.getTemporaryDirectory(); System.out.println("Could not write to temporary directory: " + tempDir.substring(0, tempDir.lastIndexOf(File.separator))); System.exit(1); } if (LOGGER.isDebugEnabled()) { LOGGER.debug(Utilities.getTimeNow() + " 1. BaseDirectoryTemporary is " + master.getJobs().getBaseDirectoryTemporaryFiles()); } workingTemporaryDirectory = master.getWorkingTemporaryDirectoryPath(); deleteWorkingDirectoryOnCompletion = master.isDeleteWorkingDirectoryOnCompletion(); if (LOGGER.isDebugEnabled()) { LOGGER.debug(Utilities.getTimeNow() + " 1. workingTemporaryDirectory is " + workingTemporaryDirectory); } } if (! (mode.equals(Mode.INSTALLER) || mode.equals(Mode.CONVERT)) ) { //deal with panther stepPantherHMM3RunPantherScore //this maynot be necessary anymore //TODO maybe remove //final PantherNewBinaryStep stepPantherRunBinary = (PantherNewBinaryStep) ctx.getBean("stepPantherRunBinary"); // final PantherScoreStep stepPantherRunBinary = (PantherScoreStep) ctx.getBean("stepPantherHMM3RunPantherScore"); //stepPantherRunBinary.setUserDir(parsedCommandLine.getOptionValue(I5Option.USER_DIR.getLongOpt()).trim()); } String operatingSystem = System.getProperty("os.name"); System.out.println(Utilities.getTimeNow() + " Running InterProScan v5 in " + mode + " mode... on " + operatingSystem ); runnable.run(); } //System.exit(0); } catch (UnrecognizedOptionException exp) { // bit of a hack - but it at least prints something informative for completely unknown options // if we've caught the exception above COMMAND_LINE_OPTIONS_FOR_HELP should be empty anyway if (COMMAND_LINE_OPTIONS_FOR_HELP.getOptions().size() == 0) { for (Option option : (Collection

* It tests the port is available and then attempts to use it (There is a very small chance * that another process will start using this port between checking and using, in which case this method * will barfe out and InterProScan will exit non-zero. This is very unlikely however, * as the two InterProScan instances would need to be running on the same host and have * randomly picked the same port number at the same moment!) *

* But never say never... * * @param ctx the Spring application context. * @return the URI as a String for this transport, e.g. tcp://myservername:1901 */ private static String configureTCPTransport(final AbstractApplicationContext ctx) { LOGGER.info(" configure TCP Transport, start Broker Service "); List portExclusionList = null; if (ctx.containsBean(PORT_EXCLUSION_LIST_BEAN_ID)) { final String exclusionString = (String) ctx.getBean(PORT_EXCLUSION_LIST_BEAN_ID); if (exclusionString != null && !exclusionString.isEmpty()) { final String[] exclusionStringArray = exclusionString.split(","); portExclusionList = new ArrayList(exclusionStringArray.length); for (String portString : exclusionStringArray) { final String trimmedPortString = portString.trim(); if (!trimmedPortString.isEmpty()) { try { portExclusionList.add(new Integer(trimmedPortString)); } catch (NumberFormatException nfe) { throw new IllegalStateException("Please check that the property 'tcp.port.exclusion.list' is a comma separated list of integer values (port numbers) or is empty. (White space is fine.)", nfe); } } } } } // Configure the Broker with a random TCP port number. //final BrokerService broker1 = (BrokerService) ctx.getBean("localhostJMSBroker"); final BrokerService broker = (BrokerService) ctx.getBean("jmsBroker"); try { // Get hostname //get canonical hostname as otherwise hostname may not be exactly how other machines see this host final String hostname = InetAddress.getLocalHost().getCanonicalHostName(); if (Utilities.verboseLogLevel >= 10){ Utilities.verboseLog("process hostname: " + hostname); } // Select a random port above 1024, excluding LSF ports and check availability. boolean portAssigned = false; int port = 0; final Random rand = new Random(); while (!portAssigned) { port = rand.nextInt(64510) + 1025; // > 1024 and < 65535 if (portExclusionList != null && portExclusionList.contains(port)) { continue; } // Test the port is available on this machine. portAssigned = available(port); } //if this is not a production master, set a random broker name, otherwise you get RMI protocol exception when workers running on the same machine //broker.setBrokerName(Utilities.createUniqueJobName(8)); //Setting transport connector final String uriString = new StringBuilder("tcp://").append(hostname).append(':').append(port).toString(); final TransportConnector tc = new TransportConnector(); tc.setUri(new URI(uriString)); broker.addConnector(tc); // broker.start(); if (LOGGER.isInfoEnabled()) { LOGGER.info("BrokerService running at uriString = " + uriString); } return uriString; } catch (Exception e) { throw new IllegalStateException("Unable to configure the TCPTransport on the Broker", e); } } /** * Checks to see if a specific port is available. * Checks for use both TCP and UDP use of ports. * * @param port number to check for availability * @return true if the port number is available. */ public static boolean available(int port) { ServerSocket ss = null; DatagramSocket ds = null; try { ss = new ServerSocket(port); ss.setReuseAddress(true); ds = new DatagramSocket(port); ds.setReuseAddress(true); // The port is currently not used for anything and is available for TCP / UDP return true; } catch (IOException e) { // Don't need to do anything here - if an Exception is thrown attempting to connect to // a port then this method returns false, indicating that the // specified port is not available. (Which is the desired behaviour). return false; } finally { if (ds != null) { ds.close(); } if (ss != null) { try { ss.close(); } catch (IOException e) { /* should not be thrown */ } } } } private static boolean isInvalid(final Mode mode, final CommandLine commandline) { Option[] options = commandline.getOptions(); if (options.length == 0) { return true; } else if (options.length == 1) { if (options[0].getLongOpt() == I5Option.USER_DIR.getLongOpt()) { return true; } } else if (!commandline.hasOption(I5Option.INPUT.getLongOpt())) { if (mode.equals(Mode.SINGLESEQ) || mode.equals(Mode.STANDALONE) || mode.equals(Mode.DISTRIBUTED_MASTER) || mode.equals(Mode.CLUSTER) || mode.equals(Mode.CONVERT)) { return true; } } return false; } public static void changeLogLevel(String logLevel){ Logger root = Logger.getLogger("uk.ac.ebi.interpro.scan"); //setting the logging level according to input if ("FATAL".equalsIgnoreCase(logLevel)) { root.setLevel(Level.FATAL); }else if ("ERROR".equalsIgnoreCase(logLevel)) { root.setLevel(Level.ERROR); }else if ("WARN".equalsIgnoreCase(logLevel)) { root.setLevel(Level.WARN); }else if ("DEBUG".equalsIgnoreCase(logLevel)) { root.setLevel(Level.DEBUG); } } /** * delete the temporaryWorkingFileDirectory * * @param dirPath * @throws IOException */ private static void deleteWorkingTemporaryDirectory(String dirPath) throws IOException { File dir = new File(dirPath); try { FileUtils.deleteDirectory(dir); }catch (IOException e) { LOGGER.warn("At run completion, unable to delete temporary directory " + dir.getAbsolutePath()); } } /** * if deleteWorkingDirectoryOnCompletion then clean up the temporaryFileDirectory * * @param deleteWorkingDirectoryOnCompletion * @param temporaryFileDirectory */ private static void cleanUpWorkingDirectory(boolean deleteWorkingDirectoryOnCompletion, String temporaryFileDirectory){ if(deleteWorkingDirectoryOnCompletion){ try { if(new File(temporaryFileDirectory).exists()) { LOGGER.debug("Cleaning up temporaryDirectoryName : " + temporaryFileDirectory); deleteWorkingTemporaryDirectory(temporaryFileDirectory); } } catch (IOException e) { LOGGER.warn("At run completion, unable to delete temporary working directory " + temporaryFileDirectory); e.printStackTrace(); } }else { LOGGER.warn("deleteWorkingDirectoryOnCompletion : " + deleteWorkingDirectoryOnCompletion); } } @Override public void finalize() throws Throwable{ try { if (temporaryDirectory != null) { //do some cleanup cleanUpWorkingDirectory(deleteWorkingDirectoryOnCompletion, temporaryDirectory); } }finally { super.finalize(); } } }





© 2015 - 2025 Weber Informatics LLC | Privacy Policy