All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.scalatest.tools.Runner.scala Maven / Gradle / Ivy

Go to download 

/*
 * Copyright 2001-2013 Artima, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.scalatest.tools

import org.scalatest._
import java.net.URL
import java.net.MalformedURLException
import java.net.URLClassLoader
import java.io.File
import java.io.IOException
import javax.swing.SwingUtilities
import java.util.concurrent.ArrayBlockingQueue
import java.util.regex.Pattern
import java.util.concurrent.Semaphore
import org.scalatest.events._
import java.util.concurrent.Executors
import java.util.concurrent.ExecutorService
import java.util.concurrent.ThreadFactory
import SuiteDiscoveryHelper._
import org.scalatest.time.Span
import org.scalatest.time.Seconds
import org.scalatest.time.Millis
import java.util.concurrent.atomic.AtomicInteger
import Suite.{mergeMap, CHOSEN_STYLES, SELECTED_TAG}
import ArgsParser._
import org.scalactic.Requirements._
// import org.scalatest.prop.Randomizer

/*
Command line args:

a - archive count for dashboard reporter     --archive
A - select previously failed and/or canceled tests to rerun --again
b - run a testNG test (b for Beust)          --testng
B - 
c - parallel execution (--parallel)          --parallel (deprecated, will be custom reporter)
C - custom reporter (temporarily)
d - dashboard reporter                       --dashboard
D - config map pair                          -D
e - standard error reporter (--stderr-reporter)    --stderr
E -
f - file reporter                            --file
F - Span scale factor
g - graphical reporter                       --graphical
G -
h - HTML Reporter                            --html
H -
i - this one is used for the Suite ID        --suiteId
I -
j - currently JUnit directly (can drop and use WrapWith)   --junit
J - 
k - socket reporter XML
K - socket reporter binary
l - tags to exclude                          --exclude
L -
m - members only path                        --members
M - memory reporter                          --memory (records failed tests in a file, so they can be rerun with A)
n - tags to include                          --include
N -
o - standard out reporter
O -
p - space-separated runpath (currently deprecated, will be parallel execution)
P - parallel execution (temporarily)
q - Suffix? -q Spec will only look at class files whose name ends in Spec
Q - equalivalent to -q Suite -q Spec
r - custom reporter (currently deprecated, will be runpath)
R - space-separated runpath (temporarily)
s - suite class name (to become a glob)
S - initial seed for Randomizer.default
t - test name
T - sorting timeout                        --sorting-timeout
u - JUnit XML reporter
U -
v - ScalaTest version number (also -version and --version)
V -
w - wildcard path
W - slowpoke detector (with two integral args, for timeout (delay) and interval (period), in seconds)
x - save for ScalaTest native XML
X -
y - sets org.scalatest.chosenstyle -y FunSpec or -y "FunSpec FunSuite" DEACTIVATED IN 3.1.0
-Y for after -h to set a custom style sheet
z - test name wildcard
Z -

StringReporter configuration params:
A - drop AlertProvided events
B - drop NoteProvided events
C - drop TestSucceeded events
D - show all durations
E - drop TestPending events
F - show full stack traces
*G - reminder with full stack traces
H - drop SuiteStarting events
*I - Reminder without stack traces
J
*K - exclude TestCanceled events from reminder
L - drop SuiteCompleted events
M - drop MarkupProvided events
N - drop TestStarting events
O - drop InfoProvided events
P - drop ScopeOpened events
Q - drop ScopeClosed events
R - drop ScopePending events 
S - show short stack traces
*T - reminder with short stack traces
U - unformatted mode
V
W - without color
X - drop TestIgnored events
Z
*/

private[tools] case class SuiteConfig(suite: Suite, dynaTags: DynaTags, requireSelectedTag: Boolean, excludeNestedSuites: Boolean)
private[scalatest] case class ConcurrentConfig(numThreads: Int, enableSuiteSortingReporter: Boolean)
private[tools] case class SlowpokeConfig(delayInMillis: Long, periodInMillis: Long)

/**
 * Application that runs a suite of tests.
 *
 * 

 * Note: this application offers the full range of ScalaTest features via command line arguments described below. If you just want
 * to run a suite of tests from the command line and see results on the standard output, you may prefer to use ScalaTest's simple runner.
 * 

 *
 * 

 * The basic form of a Runner invocation is:
 * 

 *
 * 
 * scala [-cp scalatest-<version>.jar:...] org.scalatest.tools.Runner [arguments]
 * 

 *
 * 

 * The arguments Runner accepts are described in the following table:
 * 

 *
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
 * 
argumentdescriptionexample
-Dkey=valuedefines a key/value pair for the config map-DmaxConnections=100
-R <runpath elements>the specifies the runpath from which tests classes will be
discovered and loaded (Note: only one -R allowed)		Unix: -R target/classes:target/generated/classes
Windows: -R target\classes;target\generated\classes
-n <tag name>specifies a tag to include (Note: only one tag name allowed per -n)-n UnitTests -n FastTests
-l <tag name>specifies a tag to exclude (Note: only one tag name allowed per -l)-l SlowTests -l PerfTests
-P[CS][integer thread count]specifies a parallel run, with optional cached thread pool, suite sorting and thread count
(Note: only one -P allowed)		-P, -PS, -PS8, -P8, -PC or -PCS.  Negative or zero number of thread count is not allowed.
-s <suite class name>specifies a suite class to run-s com.company.project.StackSpec
-m <members-only package>requests that suites that are direct members of the specified package
 be discovered and run		-m com.company.project
-w <wildcard package>requests that suites that are members of the specified package or its subpackages
be discovered and run		-w com.company.project
-q <suffixes>specify suffixes to discover-q Spec -q Suite
-Qdiscover only classes whose names end with Spec or Suite
(or other suffixes specified by -q)		-Q
-j <JUnit class name>instantiate and run a JUnit test class-j StackTestClass
-b <TestNG XML file>run TestNG tests using the specified TestNG XML file-b testng.xml
-F <span scale factor>a factor by which to scale time spans
(Note: only one -F is allowed)		-F 10 or -F 2.5
-T <sorting timeout>specifies a integer timeout (in seconds) for sorting the events of
parallel runs back into sequential order		-T 5
-i <suite ID>specifies a suite to run by ID (Note: must follow -s, 
and is intended to be used primarily by tools such as IDEs.)		-i com.company.project.FileSpec-file1.txt
-t <test name>select the test with the specified name-t "An empty Stack should complain when popped"
-z <test name substring>select tests whose names include the specified substring-z "popped"
-g[NCXEHLOPQMD]select the graphical reporter-g
-f[NCXEHLOPQMDWSFU] <filename>select the file reporter-f output.txt
-u <directory name>select the JUnit XML reporter-u target/junitxmldir
-h <directory name> [-Y <css file name>]select the HTML reporter, optionally including the specified CSS file-h target/htmldir -Y src/main/html/customStyles.css
-vprint the ScalaTest version-v or, also -version
-o[NCXEHLOPQMDWSFU]select the standard output reporter-o
-e[NCXEHLOPQMDWSFU]select the standard error reporter-e
-C[NCXEHLOPQMD] <reporter class>select a custom reporter-C com.company.project.BarReporter
-M <file name>memorize failed and canceled tests in a file, so they can be rerun with -A (again)-M rerun.txt
-A <file name>used in conjunction with -M (momento) to select previously failed
and canceled tests to rerun again		-A rerun.txt
-W <delay> <period>requests notifications of slowpoke tests, tests that have been running
longer than delay seconds, every period seconds.		-W 60 60

 *
 * 

 * The simplest way to start Runner is to specify the directory containing your compiled tests as the sole element of the runpath, for example:
 * 

 *
 * 
scala -classpath scalatest-<version>.jar org.scalatest.tools.Runner -R compiled_tests

 *
 * 

 * Given the previous command, Runner will discover and execute all Suites in the compiled_tests directory and its subdirectories,
 * and show results in graphical user interface (GUI).
 * 

 *
 * 
 * 
Executing suites

 *
 * 

 * Each -s argument must be followed by one and only one fully qualified class name. The class must either extend Suite and
 * have a public, no-arg constructor, or be annotated by a valid WrapWith annotation.
 * 

 *
 * 
 * 
Specifying the config map

 *
 * 

 * A config map contains pairs consisting of a string key and a value that may be of any type. (Keys that start with
 * "org.scalatest." are reserved for ScalaTest. Configuration values that are themselves strings may be specified on the
 * Runner command line.
 * Each configuration pair is denoted with a "-D", followed immediately by the key string, an "=", and the value string.
 * For example:
 * 

 *
 * 
-Ddbname=testdb -Dserver=192.168.1.188

 *
 * 
 * 
Specifying a runpath

 *
 * 

 * A runpath is the list of filenames, directory paths, and/or URLs that Runner
 * uses to load classes for the running test. If runpath is specified, Runner creates
 * a custom class loader to load classes available on the runpath.
 * The graphical user interface reloads the test classes anew for each run
 * by creating and using a new instance of the custom class loader for each run.
 * The classes that comprise the test may also be made available on
 * the classpath, in which case no runpath need be specified.
 * 

 *
 * 

 * The runpath is specified with the -R option. The -R must be followed by a space,
 * a double quote ("), a white-space-separated list of
 * paths and URLs, and a double quote. If specifying only one element in the runpath, you can leave off
 * the double quotes, which only serve to combine a white-space separated list of strings into one
 * command line argument. If you have path elements that themselves have a space in them, you must
 * place a backslash (\) in front of the space. Here's an example:
 * 

 *
 * 
-R "serviceuitest-1.1beta4.jar myjini http://myhost:9998/myfile.jar target/class\ files"

 *
 * 
 * 
Specifying reporters

 *
 * 

 * Reporters can be specified  on the command line in any of the following
 * ways:
 * 

 *
 * 
    
 * 
  •  -g[configs...] - causes display of a graphical user interface that allows
 *    tests to be run and results to be investigated
    • 
 * 
  •  -f[configs...] <filename> - causes test results to be written to
 *     the named file
    • 
 * 
  •  -u <directory> - causes test results to be written to
 *      junit-style xml files in the named directory
    • 
 * 
  •  -h <directory> [-Y <CSS file>] - causes test results to be written to
 *      HTML files in the named directory, optionally included the specified CSS file
    • 
 * 
  •  -a <number of files to archive> - causes specified number of old
 *      summary and durations files to be archived (in summaries/ and durations/ subdirectories)
 *      for dashboard reporter (default is two)
    • 
 * 
  •  -o[configs...] - causes test results to be written to
 *     the standard output
    • 
 * 
  •  -e[configs...] - causes test results to be written to
 *     the standard error
    • 
 * 
  •  -k <host> <port> - causes test results to be written to
 *      socket in the named host and port number, using XML format
    • 
 * 
  •  -K <host> <port> - causes test results to be written to
 *      socket in the named host and port number, using Java object binary format
    • 
 * 
  •  -C[configs...] <reporterclass> - causes test results to be reported to
 *     an instance of the specified fully qualified Reporter class name
    • 
 * 

 *
 * 

 * The [configs...] parameter, which is used to configure reporters, is described in the next section.
 * 

 *
 * 

 * The -C option causes the reporter specified in
 * <reporterclass> to be
 * instantiated.
 * Each reporter class specified with a -C option must be public, implement
 * org.scalatest.Reporter, and have a public no-arg constructor.
 * Reporter classes must be specified with fully qualified names. 
 * The specified reporter classes may be
 * deployed on the classpath. If a runpath is specified with the
 * -R option, the specified reporter classes may also be loaded from the runpath.
 * All specified reporter classes will be loaded and instantiated via their no-arg constructor.
 * 

 *
 * 

 * For example, to run a suite named MySuite from the mydir directory
 * using two reporters, the graphical reporter and a file reporter
 * writing to a file named "test.out", you would type:
 * 

 *
 * 
java -jar scalatest.jar -R mydir -g -f test.out -s MySuite

 *
 * 

 * The -g, -o, or -e options can
 * appear at most once each in any single command line.
 * Multiple appearances of -f and -C result in multiple reporters
 * unless the specified <filename> or <reporterclass> is
 * repeated. If any of -g, -o, -e,
 * <filename> or <reporterclass> are repeated on
 * the command line, the Runner will print an error message and not run the tests.
 * 

 *
 * 

 * Runner adds the reporters specified on the command line to a dispatch reporter,
 * which will dispatch each method invocation to each contained reporter. Runner will pass
 * the dispatch reporter to executed suites. As a result, every
 * specified reporter will receive every report generated by the running suite of tests.
 * If no reporters are specified, a graphical
 * runner will be displayed that provides a graphical report of
 * executed suites.
 * 

 *
 * 
 * 
Configuring reporters

 *
 * 

 * Each reporter option on the command line can include configuration characters. Configuration characters
 * are specified immediately following the -g, -o,
 * -e, -f, or -C. The following configuration
 * characters, which cause reports to be dropped, are valid for any reporter:
 * 

 *
 * 
    
 * 
  •  N - drop TestStarting events
    • 
 * 
  •  C - drop TestSucceeded events
    • 
 * 
  •  X - drop TestIgnored events
    • 
 * 
  •  E - drop TestPending events
    • 
 * 
  •  H - drop SuiteStarting events
    • 
 * 
  •  L - drop SuiteCompleted events
    • 
 * 
  •  O - drop InfoProvided events
    • 
 * 
  •  P - drop ScopeOpened events
    • 
 * 
  •  Q - drop ScopeClosed events
    • 
 * 
  •  R - drop ScopePending events
    • 
 * 
  •  M - drop MarkupProvided events
    • 
 * 

 *
 * 

 * A dropped event will not be delivered to the reporter at all. So the reporter will not know about it and therefore not
 * present information about the event in its report. For example, if you specify -oN, the standard output reporter
 * will never receive any TestStarting events and will therefore never report them. The purpose of these
 * configuration parameters is to allow users to selectively remove events they find add clutter to the report without
 * providing essential information.
 * 

 *
 * 

 * The following three reporter configuration parameters may additionally be used on standard output (-o), standard error (-e),
 * and file (-f) reporters: 
 * 

 *
 * 
    
 * 
  •  W - without color
    • 
 * 
  •  D - show all durations
    • 
 * 
  •  S - show short stack traces
    • 
 * 
  •  F - show full stack traces
    • 
 * 
  •  U - unformatted mode
    • 
 * 
  •  I - show reminder of failed and canceled tests without stack traces
    • 
 * 
  •  T - show reminder of failed and canceled tests with short stack traces
    • 
 * 
  •  G - show reminder of failed and canceled tests with full stack traces
    • 
 * 
  •  K - exclude TestCanceled events from reminder
    • 
 * 

 *
 * 

 * If you specify a W, D, S, F, U, R, T, G, or K for any reporter other than standard output, standard error, or file reporters, Runner
 * will complain with an error message and not perform the run.
 * 

 *
 * 

 * Configuring a standard output, error, or file reporter with D will cause that reporter to
 * print a duration for each test and suite.  When running in the default mode, a duration will only be printed for
 * the entire run.
 * 

 *
 * 

 * Configuring a standard output, error, or file reporter with F will cause that reporter to print full stack traces for all exceptions,
 * including TestFailedExceptions. Every TestFailedException contains a stack depth of the
 * line of test code that failed so that users won't need to search through a stack trace to find it. When running in the default,
 * mode, these reporters will only show full stack traces when other exceptions are thrown, such as an exception thrown
 * by production code. When a TestFailedException is thrown in default mode, only the source filename and
 * line number of the line of test code that caused the test to fail are printed along with the error message, not the full stack
 * trace. 
 * 

 *
 * 

 * The 'U' unformatted configuration removes some formatting from the output and adds verbosity.
 * The purpose of unformatted (or, "ugly") mode is to facilitate debugging of parallel runs. If you have
 * tests that fail or hang during parallel runs, but succeed when run sequentially, unformatted mode can help.
 * In unformatted mode, you can see exactly what is happening when it is happening. Rather than attempting to make the output
 * look as pretty and human-readable as possible, unformatted mode will just print out verbose information about each event
 * as it arrives, helping you track down the problem
 * you are trying to debug.
 * 

 *
 * 

 * By default, a standard output, error, or file reporter inserts ansi escape codes into the output printed to change and later reset
 * terminal colors. Information printed as a result of run starting, completed, and stopped events
 * is printed in cyan. Information printed as a result of ignored or pending test events is shown in yellow. Information printed
 * as a result of test failed, suite aborted, or run aborted events is printed in red. All other information is printed in green.
 * The purpose of these colors is to facilitate speedy reading of the output, especially the finding of failed tests, which can
 * get lost in a sea of passing tests. Configuring a standard output, error, or file reporter into without-color mode (W) will
 * turn off this behavior. No ansi codes will be inserted.
 * 

 *
 * 

 * The R, T, and G options enable "reminders" of failed and, optionally, canceled tests to be printed
 * at the end of the summary. This minimizes or eliminates the need to search and scroll backwards to find out what tests failed or were canceled.
 * For large test suites, the actual failure message could have scrolled off the top of the buffer, making it otherwise impossible
 * to see what failed. You can configure the detail level of the stack trace for regular reports of failed and canceled tests independently
 * from that of reminders. To set the detail level for regular reports, use S for short stack traces, F for
 * full stack traces, or nothing for the default of no stack trace. To set the detail level for reminder reports, use T for
 * reminders with short stack traces, G for reminders with full stack traces in reminders, or R for reminders
 * with no stack traces. If you wish to exclude reminders of canceled tests, i.e., only see reminders of failed tests, specify
 * K along with one of R, T, or G, as in "-oRK".
 * 

 *
 * 

 * For example, to run a suite using two reporters, the graphical reporter configured to present every reported event
 * and a standard error reporter configured to present everything but test starting, test succeeded, test ignored, test
 * pending, suite starting, suite completed, and info provided events, you would type:
 * 

 *
 * 

 * scala -classpath scalatest-<version>.jar -R mydir -g -eNDXEHLO -s MySuite
 * 

 *
 * 

 * Note that no white space is allowed between the reporter option and the initial configuration
 * parameters. So "-e NDXEHLO" will not work,
 * "-eNDXEHLO" will work.
 * 

 *
 * 
 * 
Specifying tags to include and exclude

 *
 * 

 * You can specify tag names of tests to include or exclude from a run. To specify tags to include,
 * use -n followed by a white-space-separated list of tag names to include, surrounded by
 * double quotes. (The double quotes are not needed if specifying just one tag.)  Similarly, to specify tags
 * to exclude, use -l followed by a white-space-separated
 * list of tag names to exclude, surrounded by double quotes. (As before, the double quotes are not needed
 * if specifying just one tag.) If tags to include is not specified, then all tests
 * except those mentioned in the tags to exclude (and in the org.scalatest.Ignore tag), will be executed.
 * (In other words, the absence of a -n option is like a wildcard, indicating all tests be included.)
 * If tags to include is specified, then only those tests whose tags are mentioned in the argument following -n
 * and not mentioned in the tags to exclude, will be executed. For more information on test tags, see
 * the documentation for Suite. Here are some examples:
 * 

 *
 * 

 * 
    
 * 
  • -n CheckinTests
    • 
 * 
  • -n FunctionalTests -l org.scalatest.tags.Slow
    • 
 * 
  • -n "CheckinTests FunctionalTests" -l "org.scalatest.tags.Slow org.scalatest.tags.Network"
    • 
 * 

 * 

 *
 * 
 * 
Specifying suffixes to discover

 *
 * 

 * You can specify suffixes of Suite names to discover. To specify suffixes to discover,
 * use -q followed by a vertical-bar-separated list of suffixes to discover, surrounded by
 * double quotes. (The double quotes are not needed if specifying just one suffix.)  Or you can specify
 * them individually using multiple -q's.
 * If suffixes to discover is not specified, then all suffixes are considered.
 * If suffixes is specified, then only those Suites whose class names end in one of the specified suffixes
 * will be considered during discovery. Here are some examples:
 * 

 *
 * 

 * 
    
 * 
  • -q Spec
    • 
 * 
  • -q "Spec|Suite"
    • 
 * 
  • -q Spec -q Suite
    • 
 * 

 * 

 *
 * 

 * Option -Q can be used to specify a default set of suffixes "Spec|Suite". If you specify both -Q and -q, you'll get Spec
 * and Suite in addition to the other suffix or suffixes you specify with -q.
 * 

 *
 * 

 * Specifying suffixes can speed up the discovery process because class files with names not ending the specified suffixes
 * can be immediately disqualified, without needing to load and inspect them to see if they either extend Suite
 * and declare a public, no-arg constructor, or are annotated with WrapWith. 
 * 

 *
 * 
 * 
Executing Suites in parallel

 *
 * 

 * With the proliferation of multi-core architectures, and the often parallelizable nature of tests, it is useful to be able to run
 * tests in parallel. If you include -P on the command line, Runner will pass a Distributor to
 * the Suites you specify with -s. Runner will set up a thread pool to execute any Suites
 * passed to the Distributor's put method in parallel. Trait Suite's implementation of
 * runNestedSuites will place any nested Suites into this Distributor. Thus, if you have a Suite
 * of tests that must be executed sequentially, you should override runNestedSuites as described in the documentation for Distributor.
 * 

 *
 * 

 * The -P option may optionally be appended with a number (e.g.
 * "-P10" -- no intervening space) to specify the number of
 * threads to be created in the thread pool.  If no number (or 0) is
 * specified, the number of threads will be decided based on the number of
 * processors available.
 * 

 *
 * 
 * 
Specifying Suites

 *
 * 

 * Suites are specified on the command line with a -s followed by the fully qualified
 * name of a Suite subclass, as in:
 * 

 *
 * 
-s com.artima.serviceuitest.ServiceUITestkit

 *
 * 

 * Each specified suite class must be public, a subclass of
 * org.scalatest.Suite, and contain a public no-arg constructor.
 * Suite classes must be specified with fully qualified names. 
 * The specified Suite classes may be
 * loaded from the classpath. If a runpath is specified with the
 * -R option, specified Suite classes may also be loaded from the runpath.
 * All specified Suite classes will be loaded and instantiated via their no-arg constructor.
 * 

 *
 * 

 * The runner will invoke execute on each instantiated org.scalatest.Suite,
 * passing in the dispatch reporter to each execute method.
 * 

 *
 * 

 * Runner is intended to be used from the command line. It is included in org.scalatest
 * package as a convenience for the user. If this package is incorporated into tools, such as IDEs, which take
 * over the role of runner, object org.scalatest.tools.Runner may be excluded from that implementation of the package.
 * All other public types declared in package org.scalatest.tools.Runner should be included in any such usage, however,
 * so client software can count on them being available.
 * 

 *
 * 
 * 
Specifying "members-only" and "wildcard" Suite paths

 *
 * 

 * If you specify Suite path names with -m or -w, Runner will automatically
 * discover and execute accessible Suites in the runpath that are either a member of (in the case of -m)
 * or enclosed by (in the case of -w) the specified path. As used in this context, a path is a portion of a fully qualified name.
 * For example, the fully qualifed name com.example.webapp.MySuite contains paths com, com.example, and com.example.webapp.
 * The fully qualifed name com.example.webapp.MyObject.NestedSuite contains paths com, com.example,
 * com.example.webapp, and com.example.webapp.MyObject.
 * An accessible Suite is a public class that extends org.scalatest.Suite
 * and defines a public no-arg constructor. Note that Suites defined inside classes and traits do not have no-arg constructors,
 * and therefore won't be discovered. Suites defined inside singleton objects, however, do get a no-arg constructor by default, thus
 * they can be discovered.
 * 

 *
 * 

 * For example, if you specify -m com.example.webapp
 * on the command line, and you've placed com.example.webapp.RedSuite and com.example.webapp.BlueSuite
 * on the runpath, then Runner will instantiate and execute both of those Suites. The difference
 * between -m and -w is that for -m, only Suites that are direct members of the named path
 * will be discovered. For -w, any Suites whose fully qualified
 * name begins with the specified path will be discovered. Thus, if com.example.webapp.controllers.GreenSuite
 * exists on the runpath, invoking Runner with -w com.example.webapp will cause GreenSuite
 * to be discovered, because its fully qualifed name begins with "com.example.webapp". But if you invoke Runner
 * with -m com.example.webapp, GreenSuite will not be discovered because it is directly
 * a member of com.example.webapp.controllers, not com.example.webapp.
 * 

 *
 * 

 * If you specify no -s, -m, or -w arguments on the command line to Runner, it will discover and execute all accessible Suites
 * in the runpath.
 * 

 *
 * 
 * 
Selecting suites and tests

 *
 * 

 * Runner accepts three arguments that facilitate selecting suites and tests: -i, -t, and -z.
 * The -i option enables a suite to be selected by suite ID. This argument is intended to allow tools such as IDEs or build tools to
 * rerun specific tests or suites from information included in the results of a previous run.  A -i must follow a -s
 * that specifies a class with a public, no-arg constructor. The -i parameter can be used, for example, to rerun a nested suite that
 * declares no zero-arg constructor, which was created by containing suite that does declare a no-arg constructor. In this case, -s would be
 * used to specify the class ScalaTest can instantiate directly, the containing suite that has a public, no-arg constructor, and -i would be
 * used to select the desired nested suite. One important use case for -i is to enable such a nested suite that aborted during the previous run
 * to be rerun. 
 * 

 *
 * 

 * The -t argument allows a test to be selected by its (complete) test name. Like -i, the -t argument is primarily intented
 * to be used by tools such as IDEs or build tools, to rerun selected tests based on information obtained from the results of a previous run.
 * For example, -t could be used to rerun a test that failed in the previous run.
 * The -t argument can be used directly by users, but because descriptive test names are usually rather long, the -z argument (described next), will
 * usually be a more practical choice for users. If a -t follows either -s or -i, then it only applies to the suite
 * identified.  If it is specified independent of a -s or -i, then discovery is performed to find all Suites containing the test name.
 * 

 *
 * 

 * The -z option allows tests to be selected by a simplified wildcard: any test whose name includes the substring specified after -z
 * will be selected. For example, -z popped would select tests named "An empty stack should complain when popped" and "A non-empty stack
 * should return the last-pushed value when popped, but not "An empty stack should be empty". In short, -z popped would select any
 * tests whose name includes the substring "popped", and not select any tests whose names don't include "popped". This simplified
 * approach to test name wildcards, which was suggested by Mathias Doenitz, works around the difficulty of finding an actual wildcard character that will work
 * reliably on different operating systems.  Like -t, if -z follows -s or -i, then it only applies to the Suite specified.  Otherwise discovery is performed to find all Suites containing test names that include the substring.
 * 

 *
 * 
 * 
Specifying a span scale factor

 *
 * 

 * If you specify a integer or floating point span scale factor with -F, trait ScaledTimeSpans
 * trait will  return the specified value from its implementation of spanScaleFactor. This allows you to tune the "patience" of a run (how long to wait
 * for asynchronous operations) from the command line. For more information, see the documentation for trait ScaledTimeSpans.
 * 

 *
 * 
 * 
Specifying TestNG XML config file paths

 *
 * 

 * If you specify one or more file paths with -b (b for Beust, the last name of TestNG's creator), Runner will create a org.scalatest.testng.TestNGWrapperSuite,
 * passing in a List of the specified paths. When executed, the TestNGWrapperSuite will create one TestNG instance
 * and pass each specified file path to it for running. If you include -b arguments, you must include TestNG's jar file on the class path or runpath.
 * The -b argument will enable you to run existing TestNG tests, including tests written in Java, as part of a ScalaTest run.
 * You need not use -b to run suites written in Scala that extend TestNGSuite. You can simply run such suites with 
 * -s, -m, or -w parameters.
 * 

 *
 * 
 * 
Specifying JUnit tests

 *
 * 

 * JUnit tests, including ones written in Java, may be run by specifying
 * -j classname, where the classname is a valid JUnit class
 * such as a TestCase, TestSuite, or a class implementing a static suite()
 * method returning a TestSuite. 

 * 

 * To use this option you must include a JUnit jar file on your classpath.
 * 

 *
 *  
 * 
Memorizing and rerunning failed and canceled tests

 *
 * 

 * You can memorize failed and canceled tests using -M:
 * 

 *
 * 
 * -M failed-canceled.txt
 * 

 *
 * All failed and canceled tests will be memorized in failed-canceled.txt, to rerun them again, you use -A:
 *
 * 
 * -A failed-canceled.txt
 * 

 *
 *  
 * 
Slowpoke notifications

 *
 * 

 * You can request to recieve periodic notifications of slowpokes, tests that have been running longer than a given amount of time, specified in
 * seconds by the first integer after -W, the delay.
 * You specify the period between slowpoke notifications in seconds with the second integer after -W, the period. Thus to receive
 * notifications very minute of tests that have been running longer than two minutes, you'd use:
 * 

 * 
 * 
 * -W 120 60
 * 

 *
 * 

 * Slowpoke notifications will be sent via AlertProvided events. The standard out reporter, for example, 
 * will report such notifications like:
 * 

 *
 * 
 * *** Test still running after 2 minutes, 13 seconds: suite name: ExampleSpec, test name: An egg timer should take 10 minutes.
 * 

 *
 * @author Bill Venners
 * @author George Berger
 * @author Josh Cough
 * @author Chee Seng
 */
object Runner {

  private val RUNNER_JFRAME_START_X: Int = 150
  private val RUNNER_JFRAME_START_Y: Int = 100
  
  @volatile private[scalatest] var spanScaleFactor: Double = 1.0

  private final val DefaultNumFilesToArchive = 2
  
  //                     TO
  // We always include a PassFailReporter on runs in order to determine
  // whether or not all tests passed.
  //
  // The thread that calls Runner.run() will either start a GUI, if a graphic
  // reporter was requested, or just run the tests itself. If a GUI is started,
  // an event handler thread will get going, and it will start a RunnerThread,
  // which will actually do the running. The GUI can repeatedly start RunnerThreads
  // and RerunnerThreads, until the GUI is closed. If -P is specified, that means
  // the tests should be run concurrently, which in turn means a Distributor will
  // be passed to the execute method of the Suites, which will in turn populate
  // it with its nested suites instead of executing them directly in the same
  // thread. The Distributor works in conjunction with a pool of threads that
  // will take suites out of the distributor queue and execute them. The DispatchReporter
  // will serialize all reports via an actor, which because that actor uses receive
  // not react, will have its own thread. So the DispatchReporter actor's thread will
  // be the one that actually invokes TestFailed, RunAborted, etc., on this PassFailReporter.
  // The thread that invoked Runner.run(), will be the one that calls allTestsPassed.
  //
  // The thread that invoked Runner.run() will be the one to instantiate the PassFailReporter
  // and in its primary constructor acquire the single semaphore permit. This permit will
  // only be released by the DispatchReporter's actor thread when a runAborted, runStopped,
  // or runCompleted is invoked. allTestsPassed will block until it can reacquire the lone
  // semaphore permit. Thus, a PassFailReporter can just be used for one run, then it is
  // spent. A new PassFailReporter is therefore created each time the Runner.run() method is invoked.
  //
  private class PassFailReporter extends Reporter {

    @volatile private var failedAbortedOrStopped = false
    private val runDoneSemaphore = new Semaphore(1)
    runDoneSemaphore.acquire()

    override def apply(event: Event): Unit = {
      event match {
        case _: TestFailed =>
          failedAbortedOrStopped = true

        case _: RunAborted =>
          failedAbortedOrStopped = true
          runDoneSemaphore.release()

        case _: SuiteAborted =>
          failedAbortedOrStopped = true

        case _: RunStopped =>
          failedAbortedOrStopped = true
          runDoneSemaphore.release() 

        case _: RunCompleted =>
          runDoneSemaphore.release()

        case _ =>
      }
    }

    def allTestsPassed = {
      runDoneSemaphore.acquire()
      !failedAbortedOrStopped
    }
  }

  // TODO: I don't think I'm enforcing that properties can't start with "org.scalatest"
  // TODO: I don't think I'm handling rejecting multiple -f/-C with the same arg. -f fred.txt -f fred.txt should
  // fail, as should -C MyReporter -C MyReporter. I'm failing on -o -o, -g -g, and -e -e, but the error messages
  // could indeed be nicer.
  /**
   * Runs a suite of tests, with optional GUI. See the main documentation for this singleton object for the details.
   */
  def main(args: Array[String]): Unit = {
    // println("FOR DEBUGGING, THESE ARE THE ARGS PASSED TO main(): " + args.mkString(" "))
    Thread.currentThread.setName("ScalaTest-main")
    val result = 
      if (args.contains("-v") || args.contains("--version")) {
        val version = org.scalatest.ScalaTestVersions.ScalaTestVersion
        val scalaVersion = org.scalatest.ScalaTestVersions.BuiltForScalaVersion
        println("ScalaTest " + version + " (Built for Scala " + scalaVersion + ")")
        runOptionallyWithPassFailReporter(args.filter(arg => arg != "-v" && arg != "--version"), true)
      }
      else
        runOptionallyWithPassFailReporter(args, true)

    if (result)
      System.exit(0)
    else
      System.exit(1)
  }

  /**
   * Runs a suite of tests, with optional GUI. See the main documentation for this singleton object for the details.
   * The difference between this method and main is simply that this method will block until the run
   * has completed, aborted, or been stopped, and return true if all tests executed and passed. In other
   * words, if any test fails, or if any suite aborts, or if the run aborts or is stopped, this method will
   * return false. This value is used, for example, by the ScalaTest ant task to determine whether
   * to continue the build if haltOnFailure is set to true.
   *
   * @return true if all tests were executed and passed.
   */
  def run(args: Array[String]): Boolean = {
    // println("FOR DEBUGGING, THESE ARE THE ARGS PASSED TO run(): " + args.mkString(" "))
    val originalThreadName = Thread.currentThread.getName
    try {
      Thread.currentThread.setName("ScalaTest-run")
      runOptionallyWithPassFailReporter(args, true)
    }
    finally Thread.currentThread.setName(originalThreadName)
  }

  private def runOptionallyWithPassFailReporter(args: Array[String], runWithPassFailReporter: Boolean): Boolean = {

    checkArgsForValidity(args) match {
      case Some(s) => {
        println(s)
        System.exit(1) // TODO: Shouldn't this be returning false?
      }
      case None =>
    }

    val ParsedArgs(
      runpathArgs,
      reporterArgs,
      suiteArgs,
      againArgs,
      junitArgs,
      propertiesArgs,
      tagsToIncludeArgs,
      tagsToExcludeArgs,
      concurrentArgs,
      membersOnlyArgs,
      wildcardArgs,
      testNGArgs,
      suffixes, 
      chosenStyles, 
      spanScaleFactors, 
      testSortingReporterTimeouts,
      slowpokeArgs,
      seedArgs
    ) = parseArgs(args)

    val fullReporterConfigurations: ReporterConfigurations =
      if (reporterArgs.isEmpty)
        // If no reporters specified, just give them a graphic reporter
        new ReporterConfigurations(Some(GraphicReporterConfiguration(Set())), Nil, Nil, Nil, /*Nil, Nil, */None, None, Nil, Nil, Nil, Nil)
      else
        parseReporterArgsIntoConfigurations(reporterArgs)

    val (suitesList: List[SuiteParam], testSpecs: List[TestSpec]) =
      parseSuiteArgs(suiteArgs)
    val agains: List[String] = parseAgainArgs(againArgs)
    val junitsList: List[String] = parseSuiteArgsIntoNameStrings(junitArgs, "-j")
    val runpathList: List[String] = parseRunpathArgIntoList(runpathArgs)
    val propertiesMap: ConfigMap = parsePropertiesArgsIntoMap(propertiesArgs)
    val tagsToInclude: Set[String] = parseCompoundArgIntoSet(tagsToIncludeArgs, "-n")
    val tagsToExclude: Set[String] = parseCompoundArgIntoSet(tagsToExcludeArgs, "-l")
    val concurrent: Boolean = !concurrentArgs.isEmpty
    val concurrentConfig: ConcurrentConfig = parseConcurrentConfig(concurrentArgs)
    val membersOnlyList: List[String] = parseSuiteArgsIntoNameStrings(membersOnlyArgs, "-m")
    val wildcardList: List[String] = parseSuiteArgsIntoNameStrings(wildcardArgs, "-w")
    val testNGList: List[String] = parseSuiteArgsIntoNameStrings(testNGArgs, "-b")
    val chosenStyleSet: Set[String] = parseChosenStylesIntoChosenStyleSet(chosenStyles, "-y")
    val slowpokeConfig: Option[SlowpokeConfig] = parseSlowpokeConfig(slowpokeArgs)
    val seedList: Option[Long] = parseLongArgument(seedArgs, "-S")

    spanScaleFactor = parseDoubleArgument(spanScaleFactors, "-F", 1.0)
    val testSortingReporterTimeout = Span(parseDoubleArgument(testSortingReporterTimeouts, "-T", Suite.defaultTestSortingReporterTimeoutInSeconds), Seconds)

    seedList match {
      case Some(seed) => // Randomizer.defaultSeed.getAndSet(Some(seed))
        println("Note: -S for setting the Randomizer seed is not yet supported.")
      case None => // do nothing
    }

    // If there's a graphic reporter, we need to leave it out of
    // reporterSpecs, because we want to pass all reporterSpecs except
    // the graphic reporter's to the RunnerJFrame (because RunnerJFrame *is*
    // the graphic reporter).
    val reporterConfigs: ReporterConfigurations =
      fullReporterConfigurations.graphicReporterConfiguration match {
        case None => fullReporterConfigurations
        case Some(grs) => {
          new ReporterConfigurations(
            None,
            fullReporterConfigurations.fileReporterConfigurationList,
            fullReporterConfigurations.memoryReporterConfigurationList,
            fullReporterConfigurations.junitXmlReporterConfigurationList,
            //fullReporterConfigurations.dashboardReporterConfigurationList,
            //fullReporterConfigurations.xmlReporterConfigurationList,
            fullReporterConfigurations.standardOutReporterConfiguration,
            fullReporterConfigurations.standardErrReporterConfiguration,
            fullReporterConfigurations.htmlReporterConfigurationList,
            fullReporterConfigurations.customReporterConfigurationList, 
            fullReporterConfigurations.xmlSocketReporterConfigurationList, 
            fullReporterConfigurations.socketReporterConfigurationList
          )
        }
      }

    val passFailReporter = if (runWithPassFailReporter) Some(new PassFailReporter) else None

    if (propertiesMap.isDefinedAt(CHOSEN_STYLES))
      throw new IllegalArgumentException("Property name '" + CHOSEN_STYLES + "' is used by ScalaTest, please choose other property name.")
    val configMap: ConfigMap = 
      if (chosenStyleSet.isEmpty)
        propertiesMap
      else
        propertiesMap + (CHOSEN_STYLES -> chosenStyleSet)

    if (chosenStyleSet.nonEmpty)
      println(Resources.deprecatedChosenStyleWarning)

    val (detectSlowpokes: Boolean, slowpokeDetectionDelay: Long, slowpokeDetectionPeriod: Long) =
      slowpokeConfig match {
        case Some(SlowpokeConfig(delayInMillis, periodInMillis)) => (true, delayInMillis, periodInMillis)
        case _ => (false, 60000L, 60000L)
      }
    fullReporterConfigurations.graphicReporterConfiguration match {
      case Some(GraphicReporterConfiguration(configSet)) => {
        val graphicEventsToPresent: Set[EventToPresent] = EventToPresent.allEventsToPresent filter
          (if (configSet.contains(FilterTestStarting)) {_ != PresentTestStarting} else etp => true) filter
          (if (configSet.contains(FilterTestSucceeded)) {_ != PresentTestSucceeded} else etp => true) filter
          (if (configSet.contains(FilterTestIgnored)) {_ != PresentTestIgnored} else etp => true) filter
          (if (configSet.contains(FilterTestPending)) {_ != PresentTestPending} else etp => true) filter
          (if (configSet.contains(FilterScopeOpened)) {_ != PresentScopeOpened} else etp => true) filter
          (if (configSet.contains(FilterScopeClosed)) {_ != PresentScopeClosed} else etp => true) filter
          (if (configSet.contains(FilterScopePending)) {_ != PresentScopePending} else etp => true) filter
          (if (configSet.contains(FilterSuiteStarting)) {_ != PresentSuiteStarting} else etp => true) filter
          (if (configSet.contains(FilterSuiteCompleted)) {_ != PresentSuiteCompleted} else etp => true) filter
          (if (configSet.contains(FilterInfoProvided)) {_ != PresentInfoProvided} else etp => true) filter 
          (if (configSet.contains(FilterMarkupProvided)) {_ != PresentMarkupProvided} else etp => true)

        val abq = new ArrayBlockingQueue[RunnerJFrame](1)
        usingEventDispatchThread {
          val rjf = new RunnerJFrame(
            graphicEventsToPresent,
            reporterConfigs,
            suitesList,
            agains,
            testSpecs,
            junitsList,
            runpathList,
            tagsToInclude,
            tagsToExclude,
            configMap,
            concurrent,
            membersOnlyList,
            wildcardList,
            testNGList,
            passFailReporter,
            concurrentConfig,
            suffixes,
            chosenStyleSet,
            detectSlowpokes,
            slowpokeDetectionDelay,
            slowpokeDetectionPeriod,
            testSortingReporterTimeout
          )
          rjf.setLocation(RUNNER_JFRAME_START_X, RUNNER_JFRAME_START_Y)
          rjf.setVisible(true)
          rjf.prepUIForRunning()
          rjf.runFromGUI()
          abq.put(rjf)
        }
        // To get the Ant task to work, the main thread needs to block until
        // The GUI window exits.
        val rjf = abq.take()
        rjf.blockUntilWindowClosed()
      }
      case None => { // Run the test without a GUI
        withClassLoaderAndDispatchReporter(
          runpathList,
          reporterConfigs,
          None,
          passFailReporter,
          detectSlowpokes,
          slowpokeDetectionDelay,
          slowpokeDetectionPeriod
       ) { (loader, dispatchReporter) =>
          doRunRunRunDaDoRunRun(
            dispatchReporter,
            suitesList,
            agains,
            testSpecs,
            junitsList,
            Stopper.default,
            tagsToInclude,
            tagsToExclude,
            configMap,
            concurrent,
            membersOnlyList,
            wildcardList,
            testNGList,
            runpathList,
            loader,
            new RunDoneListener {},
            1,
            concurrentConfig,
            suffixes,
            chosenStyleSet,
            testSortingReporterTimeout
          )
        }
      }
    }
    
    passFailReporter match {
      case Some(pfr) => pfr.allTestsPassed
      case None => false
    }
  }

  // For debugging.
/*
  private[scalatest] def printOpts(opt: EventToPresent.Set32) {
    if (opt.contains(EventToPresent.PresentRunStarting))
      println("PresentRunStarting")
    if (opt.contains(EventToPresent.PresentTestStarting))
      println("PresentTestStarting")
    if (opt.contains(EventToPresent.PresentTestSucceeded))
      println("PresentTestSucceeded")
    if (opt.contains(EventToPresent.PresentTestFailed))
      println("PresentTestFailed")
    if (opt.contains(EventToPresent.PresentTestIgnored))
      println("PresentTestIgnored")
    if (opt.contains(EventToPresent.PresentSuiteStarting))
      println("PresentSuiteStarting")
    if (opt.contains(EventToPresent.PresentSuiteCompleted))
      println("PresentSuiteCompleted")
    if (opt.contains(EventToPresent.PresentSuiteAborted))
      println("PresentSuiteAborted")
    if (opt.contains(EventToPresent.PresentInfoProvided))
      println("PresentInfoProvided")
    if (opt.contains(EventToPresent.PresentRunStopped))
      println("PresentRunStopped")
    if (opt.contains(EventToPresent.PresentRunCompleted))
      println("PresentRunCompleted")
    if (opt.contains(EventToPresent.PresentRunAborted))
      println("PresentRunAborted")
  }
*/
  
  // We number our named threads so that people can keep track
  // of it as it goes through different suites. But in case the
  // multiple doRunRunRunDaDoRunRun's get called, we want to
  // use different numbers. So this is a "global" count in Runner.
  private val atomicThreadCounter = new AtomicInteger

  private[scalatest] def doRunRunRunDaDoRunRun(
    dispatch: DispatchReporter,
    suitesList: List[SuiteParam],
    agains: List[String],
    testSpecs: List[TestSpec],
    junitsList: List[String],
    stopper: Stopper,
    tagsToIncludeSet: Set[String], 
    tagsToExcludeSet: Set[String], 
    configMap: ConfigMap,
    concurrent: Boolean,
    membersOnlyList: List[String],
    wildcardList: List[String],
    testNGList: List[String],
    runpath: List[String],
    loader: ClassLoader,
    doneListener: RunDoneListener,
    runStamp: Int,
    concurrentConfig: ConcurrentConfig,
    suffixes: Option[Pattern],
    chosenStyleSet: Set[String],
    testSortingReporterTimeout: Span
  ): Unit = {

    // TODO: add more, and to RunnerThread too
    requireNonNull(dispatch,
                   suitesList,
                   agains,
                   testSpecs,
                   junitsList,
                   stopper,
                   tagsToIncludeSet,
                   tagsToExcludeSet,
                   configMap,
                   membersOnlyList,
                   wildcardList,
                   testNGList,
                   runpath,
                   loader,
                   doneListener,
                   chosenStyleSet)

    val (globSuites, nonGlobSuites) = suitesList.partition(_.isGlob)

    var tracker = new Tracker(new Ordinal(runStamp))

    //
    // Generates SuiteConfigs for Suites found via discovery.
    //
    def genDiscoSuites: List[SuiteConfig] = {
      val emptyDynaTags = DynaTags(Map.empty[String, Set[String]], Map.empty[String, Map[String, Set[String]]])

      //
      // If user specified any -t or -z arguments independent of
      // a Suite name, or any members-only or wildcard args, or
      // suite args with glob characters, then we need to do
      // discovery to find the Suites associated with those
      // args.
      //
      val discoArgsArePresent =
        !membersOnlyList.isEmpty || !wildcardList.isEmpty ||
        !testSpecs.isEmpty || !globSuites.isEmpty

      //
      // If user specified any specific Suites to run, then we don't
      // have to do any discovery unless discoArgsArePresent.
      //
      val suiteArgsArePresent =
        !nonGlobSuites.isEmpty || !junitsList.isEmpty || !testNGList.isEmpty ||
        !agains.isEmpty

      if (suiteArgsArePresent && !discoArgsArePresent) {
        Nil // No DiscoverySuites in this case. Just run Suites
            // named with -s or -j or -b
      }
      else {
        val discoveryStartTime = System.currentTimeMillis
        dispatch(DiscoveryStarting(tracker.nextOrdinal(), configMap))

        val accessibleSuites: Set[String] =
          discoverSuiteNames(runpath, loader, suffixes)

        val discoSuites =
          if (!discoArgsArePresent && !suiteArgsArePresent) {
            // In this case, they didn't specify any -w, -m, -s,
            // -j or -b on the command line, so the default is to
            // run any accessible Suites discovered on the runpath
            List(SuiteConfig(new DiscoverySuite("", accessibleSuites, true, loader), emptyDynaTags, false, false))
          }
          else {
            val membersOnlyInstances =
              for (membersOnlyName <- membersOnlyList)
                yield SuiteConfig(new DiscoverySuite(membersOnlyName, accessibleSuites, false, loader), emptyDynaTags, false, false)
  
            val wildcardInstances =
              for (wildcardName <- wildcardList)
                yield SuiteConfig(new DiscoverySuite(wildcardName, accessibleSuites, true, loader), emptyDynaTags, false, false)
  
            val testSpecSuiteParams =
              SuiteDiscoveryHelper.discoverTests(
                testSpecs, accessibleSuites, loader)
  
            val testSpecInstances = 
              for (suiteParam <- testSpecSuiteParams)
                yield genSuiteConfig(suiteParam, loader)
  
            val deglobbedSuiteParams: List[SuiteParam] =
              deglobSuiteParams(globSuites, accessibleSuites)
  
            val globInstances = 
              for (suiteParam <- deglobbedSuiteParams)
                yield genSuiteConfig(suiteParam, loader)
  
            membersOnlyInstances ::: wildcardInstances ::: testSpecInstances ::: globInstances
          }

        val discoveryDuration = System.currentTimeMillis - discoveryStartTime
        dispatch(
          DiscoveryCompleted(tracker.nextOrdinal(), Some(discoveryDuration)))

        discoSuites
      }
    }

    // suites specified by name, either directly via -s or in a file via -A
    val againSuites = readMemoryFiles(agains, dispatch, tracker)
    val specificSuites = nonGlobSuites ::: againSuites

    val runStartTime = System.currentTimeMillis
    
    try {
      val loadProblemsExist =
        try {
          val unrunnableList = specificSuites.filter{ suiteParam => 
            val className = suiteParam.className
            loader.loadClass(className) // Check if the class exist, so if not we get the nice cannot load suite error message.
            !isAccessibleSuite(className, loader) && !isRunnable(className, loader)
          }
          if (!unrunnableList.isEmpty) {
            val names = for (suiteParam <- unrunnableList) yield " " + suiteParam.className
            dispatch(RunAborted(tracker.nextOrdinal(), Resources.nonSuite + names.mkString(", "), None))
            true
          }
          else {
            false
          }
        }
        catch {
          case e: ClassNotFoundException => {
            dispatch(RunAborted(tracker.nextOrdinal(), Resources.cannotLoadSuite(e.getMessage), Some(e)))
            true
          }
        }

      if (!loadProblemsExist) {
        try {
          val namedSuiteInstances: List[SuiteConfig] =
            for (suiteParam <- specificSuites)
              yield genSuiteConfig(suiteParam, loader)
          
          val emptyDynaTags = DynaTags(Map.empty[String, Set[String]], Map.empty[String, Map[String, Set[String]]])

          val junitSuiteInstances: List[SuiteConfig] =
            if (junitsList.isEmpty)
              List.empty
            else {
              // TODO: should change the class name to org.scalatestplus.junit.JUnitWrapperSuite after we move junit out.
              val junitWrapperClass = loader.loadClass("org.scalatest.junit.JUnitWrapperSuite")
              val junitWrapperClassConstructor = junitWrapperClass.getDeclaredConstructor(classOf[String], classOf[ClassLoader])
              for (junitClassName <- junitsList)
                yield SuiteConfig(junitWrapperClassConstructor.newInstance(junitClassName, loader).asInstanceOf[Suite], emptyDynaTags, false, true) // JUnit suite should exclude nested suites
            }

          val testNGWrapperSuiteList: List[SuiteConfig] =
            if (!testNGList.isEmpty) {
              // TODO: should change the class name to org.scalatestplus.testng.TestNGWrapperSuite after we move junit out.
              val testngWrapperClass = loader.loadClass("org.scalatest.testng.TestNGWrapperSuite")
              val testngWrapperClassConstructor = testngWrapperClass.getDeclaredConstructor(classOf[List[String]])
              List(SuiteConfig(testngWrapperClassConstructor.newInstance(testNGList).asInstanceOf[Suite], emptyDynaTags, false, true)) // TestNG suite should exclude nested suites
            }
            else
              Nil

          val discoSuiteInstances = genDiscoSuites

          val suiteInstances: List[SuiteConfig] = namedSuiteInstances ::: junitSuiteInstances ::: discoSuiteInstances ::: testNGWrapperSuiteList

          val testCountList =
            for (suiteConfig <- suiteInstances)
              yield { 
              val tagsToInclude = if (suiteConfig.requireSelectedTag) tagsToIncludeSet ++ Set(SELECTED_TAG) else tagsToIncludeSet
              val filter = Filter(if (tagsToInclude.isEmpty) None else Some(tagsToInclude), tagsToExcludeSet, suiteConfig.excludeNestedSuites, suiteConfig.dynaTags)
              suiteConfig.suite.expectedTestCount(filter)
            }
  
          def sumInts(list: List[Int]): Int =
            list match {
              case Nil => 0
              case x :: xs => x + sumInts(xs)
            }

          val expectedTestCount = sumInts(testCountList)

          dispatch(RunStarting(tracker.nextOrdinal(), expectedTestCount, configMap))
          
          if (concurrent) {

            // Because some tests may do IO, will create a pool of 2 times the number of processors reported
            // by the Runtime's availableProcessors method.
            val poolSize =
            if (concurrentConfig.numThreads == 0)
              Runtime.getRuntime.availableProcessors * 2
            else
              concurrentConfig.numThreads

            val distributedSuiteSorter = 
              if (concurrentConfig.enableSuiteSortingReporter)
                Some(new SuiteSortingReporter(dispatch, Span(testSortingReporterTimeout.millisPart, Millis), System.err))
              else
                None
              
            val concurrentDispatch = 
              distributedSuiteSorter match {
                case Some(dss) => dss
                case None => dispatch
              }
                
            val threadFactory =
              new ThreadFactory {
                val defaultThreadFactory = Executors.defaultThreadFactory
                def newThread(runnable: Runnable): Thread = {
                  val thread = defaultThreadFactory.newThread(runnable)
                  thread.setName("ScalaTest-" + atomicThreadCounter.incrementAndGet())
                  thread
                }
              }
            val execSvc: ExecutorService =
              if (poolSize > 0)
                Executors.newFixedThreadPool(poolSize, threadFactory)
              else
                Executors.newCachedThreadPool(threadFactory)
            try {

              val distributor = new ConcurrentDistributor(Args(dispatch, stopper, Filter(if (tagsToIncludeSet.isEmpty) None else Some(tagsToIncludeSet), tagsToExcludeSet), configMap, None, tracker, chosenStyleSet, false, None, None), execSvc)
              if (System.getProperty("org.scalatest.tools.Runner.forever", "false") == "true") {

                while (true) {
                  val statuses = for (suiteConfig <- suiteInstances) yield {
                    val tagsToInclude = if (suiteConfig.requireSelectedTag) tagsToIncludeSet ++ Set(SELECTED_TAG) else tagsToIncludeSet
                    val filter = Filter(if (tagsToInclude.isEmpty) None else Some(tagsToInclude), tagsToExcludeSet, suiteConfig.excludeNestedSuites, suiteConfig.dynaTags)
                    val runArgs = Args(concurrentDispatch, stopper, filter, configMap, Some(distributor), tracker.nextTracker(), chosenStyleSet, false, None, distributedSuiteSorter)
                    distributor.apply(suiteConfig.suite, runArgs)
                  }
                  distributor.waitUntilDone()
                  (new CompositeStatus(statuses.toSet)).waitUntilCompleted()
                }
              }
              else {
                val statuses = for (suiteConfig <- suiteInstances) yield {
                  val tagsToInclude = if (suiteConfig.requireSelectedTag) tagsToIncludeSet ++ Set(SELECTED_TAG) else tagsToIncludeSet
                  val filter = Filter(if (tagsToInclude.isEmpty) None else Some(tagsToInclude), tagsToExcludeSet, suiteConfig.excludeNestedSuites, suiteConfig.dynaTags)
                  val runArgs = Args(concurrentDispatch, stopper, filter, configMap, Some(distributor), tracker.nextTracker(), chosenStyleSet, false, None, distributedSuiteSorter)
                  distributor.apply(suiteConfig.suite, runArgs)
                }
                distributor.waitUntilDone()
                (new CompositeStatus(statuses.toSet)).waitUntilCompleted()
              }
            }
            finally {
              execSvc.shutdown()
            }
          }
          else {
            for (suiteConfig <- suiteInstances) {
              val tagsToInclude = if (suiteConfig.requireSelectedTag) tagsToIncludeSet ++ Set(SELECTED_TAG) else tagsToIncludeSet
              val filter = Filter(if (tagsToInclude.isEmpty) None else Some(tagsToInclude), tagsToExcludeSet, suiteConfig.excludeNestedSuites, suiteConfig.dynaTags)
              val runArgs = Args(dispatch, stopper, filter, configMap, None, tracker, chosenStyleSet, false, None, None)
              val status = new ScalaTestStatefulStatus()
              val suiteRunner = new SuiteRunner(suiteConfig.suite, runArgs, status)
              suiteRunner.run()
              status.waitUntilCompleted()
            }
          }

          val duration = System.currentTimeMillis - runStartTime
          if (stopper.stopRequested) {
            dispatch(RunStopped(tracker.nextOrdinal(), Some(duration)))
          }
          else {
            dispatch(RunCompleted(tracker.nextOrdinal(), Some(duration)))
          }
        }
        catch {
          case e: InstantiationException =>
            dispatch(RunAborted(tracker.nextOrdinal(), Resources.cannotInstantiateSuite(e.getMessage), Some(e), Some(System.currentTimeMillis - runStartTime)))
          case e: IllegalAccessException =>
            dispatch(RunAborted(tracker.nextOrdinal(), Resources.cannotInstantiateSuite(e.getMessage), Some(e), Some(System.currentTimeMillis - runStartTime)))
          case e: NoClassDefFoundError =>
            dispatch(RunAborted(tracker.nextOrdinal(), Resources.cannotLoadClass(e.getMessage), Some(e), Some(System.currentTimeMillis - runStartTime)))
          case e: Throwable =>
            dispatch(RunAborted(tracker.nextOrdinal(), Resources.bigProblems(e), Some(e), Some(System.currentTimeMillis - runStartTime)))
        }
      }
    }
    finally {
      dispatch.dispatchDisposeAndWaitUntilDone()
      doneListener.done()
    }
  }

  //
  // Creates a SuiteParam for each Suite found that matches one
  // of the patterns in a list of SuiteParams containing globs.
  //
  private[tools] def deglobSuiteParams(globsList: List[SuiteParam],
                                       accessibleSuites: Set[String]):
  List[SuiteParam] =
    for {
      suiteParam <- globsList
      name <- accessibleSuites
      if suiteParam.matches(name)
    }
    yield suiteParam.copy(className = name)

  //
  // Reads each specified file and generates a SuiteParam object
  // for each entry in file.  Files are of the format created when
  // -M option is specified to record failed/canceled/aborted
  // tests so they can be run again later.
  //
  private[tools] def readMemoryFiles(fileNames: List[String],
                                     reporter: Reporter,
                                     tracker: Tracker):
  List[SuiteParam] =
  {
    val mementos =
      for {
        fileName <- fileNames
        memento <- Memento.readFromFile(fileName)
      }
      yield memento

    val (unrerunnables, rerunnables) = mementos.partition(_.className == None)

    for (memento <- unrerunnables)
      reporter.apply(
        AlertProvided(
          tracker.nextOrdinal(),
          Resources.cannotRerun(memento.eventName, memento.suiteId,
                    memento.testName),
          None))

    rerunnables.map(_.toSuiteParam)
  }

  private[tools] def genSuiteConfig(suiteParam: SuiteParam, loader: ClassLoader): SuiteConfig = {
    val suiteClassName = suiteParam.className
    val clazz = loader.loadClass(suiteClassName)
    val wrapWithAnnotation = clazz.getAnnotation(classOf[WrapWith])
    val suiteInstance = 
      if (wrapWithAnnotation == null) 
        clazz.newInstance.asInstanceOf[Suite]
      else {
        val suiteClazz = wrapWithAnnotation.value
        val constructorList = suiteClazz.getDeclaredConstructors()
        val constructor = constructorList.find { c => 
          val types = c.getParameterTypes
          types.length == 1 && types(0) == classOf[java.lang.Class[_]]
        }
        constructor.get.newInstance(clazz).asInstanceOf[Suite]
      }
    
    if (suiteParam.testNames.length == 0 && suiteParam.wildcardTestNames.length == 0 && suiteParam.nestedSuites.length == 0)
      SuiteConfig(suiteInstance, new DynaTags(Map.empty, Map.empty), false, false) // -s suiteClass, no dynamic tagging required.
    else {
      val nestedSuites = suiteParam.nestedSuites
      
      val (selectSuiteList, selectTestList) = nestedSuites.partition(ns => ns.testNames.length == 0 || ns.wildcardTestNames.length == 0)
      val suiteDynaTags: Map[String, Set[String]] = Map() ++ selectSuiteList.map(ns => (ns.suiteId -> Set(SELECTED_TAG)))
      
      val suiteExactTestDynaTags: Map[String, Map[String, Set[String]]] = 
        if (suiteParam.testNames.length > 0) 
          Map(suiteInstance.suiteId -> (Map() ++ suiteParam.testNames.map(tn => (tn -> Set(SELECTED_TAG)))))
        else 
          Map.empty
      
      val suiteWildcardTestDynaTags: Map[String, Map[String, Set[String]]] = 
        if (suiteParam.wildcardTestNames.length > 0) {
          val wildcardTestNames = suiteParam.wildcardTestNames
          val allTestNames = suiteInstance.testNames
          val wildcardTestTags = Map() ++ allTestNames.filter(tn => wildcardTestNames.find(wc => tn.contains(wc)).isDefined)
                                              .map(tn => (tn -> Set(SELECTED_TAG)))
          Map(suiteInstance.suiteId -> wildcardTestTags)
        }
        else
          Map.empty
          
      def getNestedSuiteSelectedTestNames(nestedSuite: NestedSuiteParam): Array[String] = {
        if (nestedSuite.wildcardTestNames.length == 0)
          nestedSuite.testNames
        else {
          val wildcardTestNames = nestedSuite.wildcardTestNames
          val allTestNames = suiteInstance.testNames
          nestedSuite.testNames ++ allTestNames.filter(tn => wildcardTestNames.find(wc => tn.contains(wc)).isDefined)
        }
      }
      
      val nestedSuitesTestDynaTags: Map[String, Map[String, Set[String]]] 
        = Map() ++ selectTestList.map(ns => (ns.suiteId -> (Map() ++ getNestedSuiteSelectedTestNames(ns).map(tn => (tn, Set(SELECTED_TAG))))))
        
      val testDynaTags = mergeMap[String, Map[String, Set[String]]](List(suiteExactTestDynaTags, suiteWildcardTestDynaTags, nestedSuitesTestDynaTags)) { (suiteTestMap1, suiteTestMap2) => 
                           mergeMap[String, Set[String]](List(suiteTestMap1, suiteTestMap2)) { (tagSet1, tagSet2) =>
                             tagSet1 ++ tagSet2
                           }
                         }
      // Only exclude nested suites when using -s XXX -t XXXX, or -s XXX -z XXX
      val excludeNestedSuites = suiteParam.testNames.length > 0 && nestedSuites.length == 0
      SuiteConfig(suiteInstance, new DynaTags(suiteDynaTags, testDynaTags), true, excludeNestedSuites)
    }
  }

  private[scalatest] def excludesWithIgnore(excludes: Set[String]) = excludes + "org.scalatest.Ignore"

  private[scalatest] def withClassLoaderAndDispatchReporter(
    runpathList: List[String],
    reporterSpecs: ReporterConfigurations,
    graphicReporter: Option[Reporter],
    passFailReporter: Option[Reporter],
    detectSlowpokes: Boolean,
    slowpokeDetectionDelay: Long,
    slowpokeDetectionPeriod: Long
  )(f: (ClassLoader, DispatchReporter) => Unit): Unit = {

    val loader: ClassLoader = getRunpathClassLoader(runpathList)
    try {
      Thread.currentThread.setContextClassLoader(loader)
      try {
        val dispatchReporter = ReporterFactory.getDispatchReporter(reporterSpecs, graphicReporter, passFailReporter, loader, None, detectSlowpokes, slowpokeDetectionDelay, slowpokeDetectionPeriod)
        try {
          f(loader, dispatchReporter)
        }
        finally {
          dispatchReporter.dispatchDisposeAndWaitUntilDone()
        }
      }
      catch {
        // getDispatchReporter may complete abruptly with an exception, if there is an problem trying to load
        // or instantiate a custom reporter class.
        case ex: Throwable => {
          System.err.println(Resources.bigProblemsMaybeCustomReporter)
          ex.printStackTrace(System.err)
        }
      }
    }
    finally {
      // eventually call close on the RunpathClassLoader
    }
  }

  private[scalatest] def getRunpathClassLoader(runpathList: List[String]): ClassLoader = {

    requireNonNull(runpathList)
    if (runpathList.isEmpty) {
      classOf[Suite].getClassLoader // Could this be null technically?
    }
    else {
      val urlsList: List[URL] =
        for (raw <- runpathList) yield {
          try {
            new URL(raw)
          }
          catch {
            case murle: MalformedURLException => {
  
              // Assume they tried to just pass in a file name
              val file: File = new File(raw)
  
              // file.toURL may throw MalformedURLException too, but for now
              // just let that propagate up.
              file.toURI.toURL // If a dir, comes back terminated by a slash
            }
          }
        }
  
      // Here is where the Jini preferred class loader stuff went.

      // Tell the URLConnections to not use caching, so that repeated runs and reruns actually work
      // on the latest binaries.
      for (url <- urlsList) {
        try {
          url.openConnection.setDefaultUseCaches(false)
        }
        catch {
          case e: IOException => // just ignore these
        }
      }

      new URLClassLoader(urlsList.toArray, classOf[Suite].getClassLoader)
    }
  }

  private[scalatest] def usingEventDispatchThread(f: => Unit): Unit = {
    SwingUtilities.invokeLater(
      new Runnable() {
        def run(): Unit = {
          f
        }
      }
    )
  }
}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy