org.scalatest.tools.Runner.scala Maven / Gradle / Ivy
/* * Copyright 2001-2013 Artima, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.scalatest.tools import org.scalatest._ import java.net.URL import java.net.MalformedURLException import java.net.URLClassLoader import java.io.File import java.io.IOException import javax.swing.SwingUtilities import java.util.concurrent.ArrayBlockingQueue import java.util.regex.Pattern import java.util.concurrent.Semaphore import org.scalatest.events._ import java.util.concurrent.Executors import java.util.concurrent.ExecutorService import java.util.concurrent.ThreadFactory import SuiteDiscoveryHelper._ import org.scalatest.time.Span import org.scalatest.time.Seconds import org.scalatest.time.Millis import java.util.concurrent.atomic.AtomicInteger import Suite.{mergeMap, CHOSEN_STYLES, SELECTED_TAG} import ArgsParser._ import org.scalactic.Requirements._ import org.scalatest.prop.Seed /* Command line args: a - archive count for dashboard reporter --archive A - select previously failed and/or canceled tests to rerun --again b - run a testNG test (b for Beust) --testng B - c - parallel execution (--parallel) --parallel (deprecated, will be custom reporter) C - custom reporter (temporarily) d - dashboard reporter --dashboard D - config map pair -D e - standard error reporter (--stderr-reporter) --stderr E - f - file reporter --file F - Span scale factor g - graphical reporter --graphical G - h - HTML Reporter --html H - i - this one is used for the Suite ID --suiteId I - j - currently JUnit directly (can drop and use WrapWith) --junit J - k - socket reporter XML K - socket reporter binary l - tags to exclude --exclude L - m - members only path --members M - memory reporter --memory (records failed tests in a file, so they can be rerun with A) n - tags to include --include N - o - standard out reporter O - p - space-separated runpath (currently deprecated, will be parallel execution) P - parallel execution (temporarily) q - Suffix? -q Spec will only look at class files whose name ends in Spec Q - equalivalent to -q Suite -q Spec r - custom reporter (currently deprecated, will be runpath) R - space-separated runpath (temporarily) s - suite class name (to become a glob) S - initial seed for Randomizer.default t - test name T - sorting timeout --sorting-timeout u - JUnit XML reporter U - v - ScalaTest version number (also -version and --version) V - w - wildcard path W - slowpoke detector (with two integral args, for timeout (delay) and interval (period), in seconds) x - save for ScalaTest native XML X - y - sets org.scalatest.chosenstyle -y FunSpec or -y "FunSpec FunSuite" DEACTIVATED IN 3.1.0 -Y for after -h to set a custom style sheet z - test name wildcard Z - StringReporter configuration params: A - drop AlertProvided events B - drop NoteProvided events C - drop TestSucceeded events D - show all durations E - drop TestPending events F - show full stack traces *G - reminder with full stack traces H - drop SuiteStarting events *I - Reminder without stack traces J *K - exclude TestCanceled events from reminder L - drop SuiteCompleted events M - drop MarkupProvided events N - drop TestStarting events O - drop InfoProvided events P - drop ScopeOpened events Q - drop ScopeClosed events R - drop ScopePending events S - show short stack traces *T - reminder with short stack traces U - unformatted mode V W - without color X - drop TestIgnored events Z */ private[tools] case class SuiteConfig(suite: Suite, dynaTags: DynaTags, requireSelectedTag: Boolean, excludeNestedSuites: Boolean) private[scalatest] case class ConcurrentConfig(numThreads: Int, enableSuiteSortingReporter: Boolean) private[tools] case class SlowpokeConfig(delayInMillis: Long, periodInMillis: Long) /** * Application that runs a suite of tests. * *
-z. * The* Note: this application offers the full range of ScalaTest features via command line arguments described below. If you just want * to run a suite of tests from the command line and see results on the standard output, you may prefer to use ScalaTest's simple runner. *
* ** The basic form of a
* *Runner
invocation is: ** scala [-cp scalatest-<version>.jar:...] org.scalatest.tools.Runner [arguments] ** ** The arguments
* *Runner
accepts are described in the following table: **
* ** argument description example * -Dkey=value
defines a key/value pair for the config map -DmaxConnections=100
* -R <runpath elements>
the specifies the runpath from which tests classes will be
discovered and loaded (Note: only one-R
allowed)Unix: -R target/classes:target/generated/classes
Windows:-R target\classes;target\generated\classes
* -n <tag name>
specifies a tag to include (Note: only one tag name allowed per -n
)-n UnitTests -n FastTests
* -l <tag name>
specifies a tag to exclude (Note: only one tag name allowed per -l
)-l SlowTests -l PerfTests
* -P[CS][integer thread count]
specifies a parallel run, with optional cached thread pool, suite sorting and thread count
(Note: only one-P
allowed)-P
,-PS
,-PS8
,-P8
,-PC
or-PCS
. Negative or zero number of thread count is not allowed.* -s <suite class name>
specifies a suite class to run -s com.company.project.StackSpec
* -m <members-only package>
requests that suites that are direct members of the specified package
be discovered and run-m com.company.project
* -w <wildcard package>
requests that suites that are members of the specified package or its subpackages
be discovered and run-w com.company.project
* -q <suffixes>
specify suffixes to discover -q Spec -q Suite
* -Q
discover only classes whose names end with Spec
orSuite
(or other suffixes specified by-q
)-Q
* -j <JUnit class name>
instantiate and run a JUnit test class -j StackTestClass
* -b <TestNG XML file>
run TestNG tests using the specified TestNG XML file -b testng.xml
* -F <span scale factor>
a factor by which to scale time spans
(Note: only one-F
is allowed)-F 10
or-F 2.5
* -T <sorting timeout>
specifies a integer timeout (in seconds) for sorting the events of
parallel runs back into sequential order-T 5
* -i <suite ID>
specifies a suite to run by ID (Note: must follow -s
,
and is intended to be used primarily by tools such as IDEs.)-i com.company.project.FileSpec-file1.txt
* -t <test name>
select the test with the specified name -t "An empty Stack should complain when popped"
* -z <test name substring>
select tests whose names include the specified substring -z "popped"
* -g[NCXEHLOPQMD]
select the graphical reporter -g
* -f[NCXEHLOPQMDWSFU] <filename>
select the file reporter -f output.txt
* -u <directory name>
select the JUnit XML reporter -u target/junitxmldir
* -h <directory name> [-Y <css file name>]
select the HTML reporter, optionally including the specified CSS file -h target/htmldir -Y src/main/html/customStyles.css
* -v
print the ScalaTest version -v
or, also-version
* -o[NCXEHLOPQMDWSFU]
select the standard output reporter -o
* -e[NCXEHLOPQMDWSFU]
select the standard error reporter -e
* -C[NCXEHLOPQMD] <reporter class>
select a custom reporter -C com.company.project.BarReporter
* -M <file name>
memorize failed and canceled tests in a file, so they can be rerun with -A (again) -M rerun.txt
* -A <file name>
used in conjunction with -M (momento) to select previously failed
and canceled tests to rerun again-A rerun.txt
* -W <delay> <period>
requests notifications of slowpoke tests, tests that have been running
longer than delay seconds, every period seconds.-W 60 60
* The simplest way to start
* *Runner
is to specify the directory containing your compiled tests as the sole element of the runpath, for example: *scala -classpath scalatest-<version>.jar org.scalatest.tools.Runner -R compiled_tests* ** Given the previous command,
* * *Runner
will discover and execute allSuite
s in thecompiled_tests
directory and its subdirectories, * and show results in graphical user interface (GUI). *Executing suites
* ** Each
* * *-s
argument must be followed by one and only one fully qualified class name. The class must either extendSuite
and * have a public, no-arg constructor, or be annotated by a validWrapWith
annotation. *Specifying the config map
* ** A config map contains pairs consisting of a string key and a value that may be of any type. (Keys that start with * "org.scalatest." are reserved for ScalaTest. Configuration values that are themselves strings may be specified on the *
* *Runner
command line. * Each configuration pair is denoted with a "-D", followed immediately by the key string, an "=", and the value string. * For example: *-Ddbname=testdb -Dserver=192.168.1.188* * *Specifying a runpath
* ** A runpath is the list of filenames, directory paths, and/or URLs that
* *Runner
* uses to load classes for the running test. If runpath is specified,Runner
creates * a custom class loader to load classes available on the runpath. * The graphical user interface reloads the test classes anew for each run * by creating and using a new instance of the custom class loader for each run. * The classes that comprise the test may also be made available on * the classpath, in which case no runpath need be specified. ** The runpath is specified with the -R option. The -R must be followed by a space, * a double quote (
* *"
), a white-space-separated list of * paths and URLs, and a double quote. If specifying only one element in the runpath, you can leave off * the double quotes, which only serve to combine a white-space separated list of strings into one * command line argument. If you have path elements that themselves have a space in them, you must * place a backslash (\) in front of the space. Here's an example: *-R "serviceuitest-1.1beta4.jar myjini http://myhost:9998/myfile.jar target/class\ files"* * *Specifying reporters
* ** Reporters can be specified on the command line in any of the following * ways: *
* **
* *- *
-g[configs...]
- causes display of a graphical user interface that allows * tests to be run and results to be investigated- *
-f[configs...] <filename>
- causes test results to be written to * the named file- *
-u <directory>
- causes test results to be written to * junit-style xml files in the named directory- *
-h <directory> [-Y <CSS file>]
- causes test results to be written to * HTML files in the named directory, optionally included the specified CSS file- *
-a <number of files to archive>
- causes specified number of old * summary and durations files to be archived (in summaries/ and durations/ subdirectories) * for dashboard reporter (default is two)- *
-o[configs...]
- causes test results to be written to * the standard output- *
-e[configs...]
- causes test results to be written to * the standard error- *
-k <host> <port>
- causes test results to be written to * socket in the named host and port number, using XML format- *
-K <host> <port>
- causes test results to be written to * socket in the named host and port number, using Java object binary format- *
-C[configs...] <reporterclass>
- causes test results to be reported to * an instance of the specified fully qualifiedReporter
class name* The
* *[configs...]
parameter, which is used to configure reporters, is described in the next section. ** The
* *-C
option causes the reporter specified in *<reporterclass>
to be * instantiated. * Each reporter class specified with a -C option must be public, implement *org.scalatest.Reporter
, and have a public no-arg constructor. * Reporter classes must be specified with fully qualified names. * The specified reporter classes may be * deployed on the classpath. If a runpath is specified with the *-R
option, the specified reporter classes may also be loaded from the runpath. * All specified reporter classes will be loaded and instantiated via their no-arg constructor. ** For example, to run a suite named
* *MySuite
from themydir
directory * using two reporters, the graphical reporter and a file reporter * writing to a file named"test.out"
, you would type: *java -jar scalatest.jar -R mydir -g -f test.out -s MySuite* ** The
* *-g
,-o
, or-e
options can * appear at most once each in any single command line. * Multiple appearances of-f
and-C
result in multiple reporters * unless the specified<filename>
or<reporterclass>
is * repeated. If any of-g
,-o
,-e
, *<filename>
or<reporterclass>
are repeated on * the command line, theRunner
will print an error message and not run the tests. **
* * *Runner
adds the reporters specified on the command line to a dispatch reporter, * which will dispatch each method invocation to each contained reporter.Runner
will pass * the dispatch reporter to executed suites. As a result, every * specified reporter will receive every report generated by the running suite of tests. * If no reporters are specified, a graphical * runner will be displayed that provides a graphical report of * executed suites. *Configuring reporters
* ** Each reporter option on the command line can include configuration characters. Configuration characters * are specified immediately following the
* *-g
,-o
, *-e
,-f
, or-C
. The following configuration * characters, which cause reports to be dropped, are valid for any reporter: **
* *- *
N
- dropTestStarting
events- *
C
- dropTestSucceeded
events- *
X
- dropTestIgnored
events- *
E
- dropTestPending
events- *
H
- dropSuiteStarting
events- *
L
- dropSuiteCompleted
events- *
O
- dropInfoProvided
events- *
P
- dropScopeOpened
events- *
Q
- dropScopeClosed
events- *
R
- dropScopePending
events- *
M
- dropMarkupProvided
events* A dropped event will not be delivered to the reporter at all. So the reporter will not know about it and therefore not * present information about the event in its report. For example, if you specify
* *-oN
, the standard output reporter * will never receive anyTestStarting
events and will therefore never report them. The purpose of these * configuration parameters is to allow users to selectively remove events they find add clutter to the report without * providing essential information. ** The following three reporter configuration parameters may additionally be used on standard output (-o), standard error (-e), * and file (-f) reporters: *
* **
* *- *
W
- without color- *
D
- show all durations- *
S
- show short stack traces- *
F
- show full stack traces- *
U
- unformatted mode- *
I
- show reminder of failed and canceled tests without stack traces- *
T
- show reminder of failed and canceled tests with short stack traces- *
G
- show reminder of failed and canceled tests with full stack traces- *
K
- excludeTestCanceled
events from reminder* If you specify a W, D, S, F, U, R, T, G, or K for any reporter other than standard output, standard error, or file reporters,
* *Runner
* will complain with an error message and not perform the run. ** Configuring a standard output, error, or file reporter with
* *D
will cause that reporter to * print a duration for each test and suite. When running in the default mode, a duration will only be printed for * the entire run. ** Configuring a standard output, error, or file reporter with
* *F
will cause that reporter to print full stack traces for all exceptions, * includingTestFailedExceptions
. EveryTestFailedException
contains a stack depth of the * line of test code that failed so that users won't need to search through a stack trace to find it. When running in the default, * mode, these reporters will only show full stack traces when other exceptions are thrown, such as an exception thrown * by production code. When aTestFailedException
is thrown in default mode, only the source filename and * line number of the line of test code that caused the test to fail are printed along with the error message, not the full stack * trace. ** The 'U' unformatted configuration removes some formatting from the output and adds verbosity. * The purpose of unformatted (or, "ugly") mode is to facilitate debugging of parallel runs. If you have * tests that fail or hang during parallel runs, but succeed when run sequentially, unformatted mode can help. * In unformatted mode, you can see exactly what is happening when it is happening. Rather than attempting to make the output * look as pretty and human-readable as possible, unformatted mode will just print out verbose information about each event * as it arrives, helping you track down the problem * you are trying to debug. *
* ** By default, a standard output, error, or file reporter inserts ansi escape codes into the output printed to change and later reset * terminal colors. Information printed as a result of run starting, completed, and stopped events * is printed in cyan. Information printed as a result of ignored or pending test events is shown in yellow. Information printed * as a result of test failed, suite aborted, or run aborted events is printed in red. All other information is printed in green. * The purpose of these colors is to facilitate speedy reading of the output, especially the finding of failed tests, which can * get lost in a sea of passing tests. Configuring a standard output, error, or file reporter into without-color mode (
* *W
) will * turn off this behavior. No ansi codes will be inserted. ** The
* *R
,T
, andG
options enable "reminders" of failed and, optionally, canceled tests to be printed * at the end of the summary. This minimizes or eliminates the need to search and scroll backwards to find out what tests failed or were canceled. * For large test suites, the actual failure message could have scrolled off the top of the buffer, making it otherwise impossible * to see what failed. You can configure the detail level of the stack trace for regular reports of failed and canceled tests independently * from that of reminders. To set the detail level for regular reports, useS
for short stack traces,F
for * full stack traces, or nothing for the default of no stack trace. To set the detail level for reminder reports, useT
for * reminders with short stack traces,G
for reminders with full stack traces in reminders, orR
for reminders * with no stack traces. If you wish to exclude reminders of canceled tests, i.e., only see reminders of failed tests, specify *K
along with one ofR
,T
, orG
, as in"-oRK"
. ** For example, to run a suite using two reporters, the graphical reporter configured to present every reported event * and a standard error reporter configured to present everything but test starting, test succeeded, test ignored, test * pending, suite starting, suite completed, and info provided events, you would type: *
* **
* *scala -classpath scalatest-<version>.jar -R mydir -g -eNDXEHLO -s MySuite
** Note that no white space is allowed between the reporter option and the initial configuration * parameters. So
* * *"-e NDXEHLO"
will not work, *"-eNDXEHLO"
will work. *Specifying tags to include and exclude
* ** You can specify tag names of tests to include or exclude from a run. To specify tags to include, * use
* *-n
followed by a white-space-separated list of tag names to include, surrounded by * double quotes. (The double quotes are not needed if specifying just one tag.) Similarly, to specify tags * to exclude, use-l
followed by a white-space-separated * list of tag names to exclude, surrounded by double quotes. (As before, the double quotes are not needed * if specifying just one tag.) If tags to include is not specified, then all tests * except those mentioned in the tags to exclude (and in theorg.scalatest.Ignore
tag), will be executed. * (In other words, the absence of a-n
option is like a wildcard, indicating all tests be included.) * If tags to include is specified, then only those tests whose tags are mentioned in the argument following-n
* and not mentioned in the tags to exclude, will be executed. For more information on test tags, see * the documentation forSuite
. Here are some examples: **
*
* * * *- *
-n CheckinTests
- *
-n FunctionalTests -l org.scalatest.tags.Slow
- *
-n "CheckinTests FunctionalTests" -l "org.scalatest.tags.Slow org.scalatest.tags.Network"
Specifying suffixes to discover
* ** You can specify suffixes of
* *Suite
names to discover. To specify suffixes to discover, * use-q
followed by a vertical-bar-separated list of suffixes to discover, surrounded by * double quotes. (The double quotes are not needed if specifying just one suffix.) Or you can specify * them individually using multiple -q's. * If suffixes to discover is not specified, then all suffixes are considered. * If suffixes is specified, then only those Suites whose class names end in one of the specified suffixes * will be considered during discovery. Here are some examples: **
*
* * *- *
-q Spec
- *
-q "Spec|Suite"
- *
-q Spec -q Suite
* Option -Q can be used to specify a default set of suffixes "Spec|Suite". If you specify both -Q and -q, you'll get Spec * and Suite in addition to the other suffix or suffixes you specify with -q. *
* ** Specifying suffixes can speed up the discovery process because class files with names not ending the specified suffixes * can be immediately disqualified, without needing to load and inspect them to see if they either extend
* * *Suite
* and declare a public, no-arg constructor, or are annotated withWrapWith
. *Executing
* *Suite
s in parallel* With the proliferation of multi-core architectures, and the often parallelizable nature of tests, it is useful to be able to run * tests in parallel. If you include
* *-P
on the command line,Runner
will pass aDistributor
to * theSuite
s you specify with-s
.Runner
will set up a thread pool to execute anySuite
s * passed to theDistributor
'sput
method in parallel. TraitSuite
's implementation of *runNestedSuites
will place any nestedSuite
s into thisDistributor
. Thus, if you have aSuite
* of tests that must be executed sequentially, you should overriderunNestedSuites
as described in the documentation forDistributor
. ** The
* * *-P
option may optionally be appended with a number (e.g. * "-P10
" -- no intervening space) to specify the number of * threads to be created in the thread pool. If no number (or 0) is * specified, the number of threads will be decided based on the number of * processors available. *Specifying
* *Suite
s* Suites are specified on the command line with a -s followed by the fully qualified * name of a
* *Suite
subclass, as in: *-s com.artima.serviceuitest.ServiceUITestkit* ** Each specified suite class must be public, a subclass of *
* *org.scalatest.Suite
, and contain a public no-arg constructor. *Suite
classes must be specified with fully qualified names. * The specifiedSuite
classes may be * loaded from the classpath. If a runpath is specified with the *-R
option, specifiedSuite
classes may also be loaded from the runpath. * All specifiedSuite
classes will be loaded and instantiated via their no-arg constructor. ** The runner will invoke
* *execute
on each instantiatedorg.scalatest.Suite
, * passing in the dispatch reporter to eachexecute
method. **
* * *Runner
is intended to be used from the command line. It is included inorg.scalatest
* package as a convenience for the user. If this package is incorporated into tools, such as IDEs, which take * over the role of runner, objectorg.scalatest.tools.Runner
may be excluded from that implementation of the package. * All other public types declared in packageorg.scalatest.tools.Runner
should be included in any such usage, however, * so client software can count on them being available. *Specifying "members-only" and "wildcard"
* *Suite
paths* If you specify
* *Suite
path names with-m
or-w
,Runner
will automatically * discover and execute accessibleSuite
s in the runpath that are either a member of (in the case of-m
) * or enclosed by (in the case of-w
) the specified path. As used in this context, a path is a portion of a fully qualified name. * For example, the fully qualifed namecom.example.webapp.MySuite
contains pathscom
,com.example
, andcom.example.webapp
. * The fully qualifed namecom.example.webapp.MyObject.NestedSuite
contains pathscom
,com.example
, *com.example.webapp
, andcom.example.webapp.MyObject
. * An accessibleSuite
is a public class that extendsorg.scalatest.Suite
* and defines a public no-arg constructor. Note thatSuite
s defined inside classes and traits do not have no-arg constructors, * and therefore won't be discovered.Suite
s defined inside singleton objects, however, do get a no-arg constructor by default, thus * they can be discovered. ** For example, if you specify
* *-m com.example.webapp
* on the command line, and you've placedcom.example.webapp.RedSuite
andcom.example.webapp.BlueSuite
* on the runpath, thenRunner
will instantiate and execute both of thoseSuite
s. The difference * between-m
and-w
is that for-m
, onlySuite
s that are direct members of the named path * will be discovered. For-w
, anySuite
s whose fully qualified * name begins with the specified path will be discovered. Thus, ifcom.example.webapp.controllers.GreenSuite
* exists on the runpath, invokingRunner
with-w com.example.webapp
will causeGreenSuite
* to be discovered, because its fully qualifed name begins with"com.example.webapp"
. But if you invokeRunner
* with-m com.example.webapp
,GreenSuite
will not be discovered because it is directly * a member ofcom.example.webapp.controllers
, notcom.example.webapp
. ** If you specify no
* * *-s
,-m
, or-w
arguments on the command line toRunner
, it will discover and execute all accessibleSuite
s * in the runpath. *Selecting suites and tests
* **
Runner
accepts three arguments that facilitate selecting suites and tests:-i
,-t
, and-i
option enables a suite to be selected by suite ID. This argument is intended to allow tools such as IDEs or build tools to * rerun specific tests or suites from information included in the results of a previous run. A-i
must follow a-s
* that specifies a class with a public, no-arg constructor. The-i
parameter can be used, for example, to rerun a nested suite that * declares no zero-arg constructor, which was created by containing suite that does declare a no-arg constructor. In this case,-s
would be * used to specify the class ScalaTest can instantiate directly, the containing suite that has a public, no-arg constructor, and-i
would be * used to select the desired nested suite. One important use case for-i
is to enable such a nested suite that aborted during the previous run * to be rerun. * * ** The
* *-t
argument allows a test to be selected by its (complete) test name. Like-i
, the-t
argument is primarily intented * to be used by tools such as IDEs or build tools, to rerun selected tests based on information obtained from the results of a previous run. * For example,-t
could be used to rerun a test that failed in the previous run. * The-t
argument can be used directly by users, but because descriptive test names are usually rather long, the-z
argument (described next), will * usually be a more practical choice for users. If a-t
follows either-s
or-i
, then it only applies to the suite * identified. If it is specified independent of a-s
or-i
, then discovery is performed to find all Suites containing the test name. ** The
* * *-z
option allows tests to be selected by a simplified wildcard: any test whose name includes the substring specified after-z
* will be selected. For example,-z popped
would select tests named"An empty stack should complain when popped"
and"A non-empty stack * should return the last-pushed value when popped
, but not"An empty stack should be empty"
. In short,-z popped
would select any * tests whose name includes the substring"popped"
, and not select any tests whose names don't include"popped"
. This simplified * approach to test name wildcards, which was suggested by Mathias Doenitz, works around the difficulty of finding an actual wildcard character that will work * reliably on different operating systems. Like-t
, if-z
follows-s
or-i
, then it only applies to the Suite specified. Otherwise discovery is performed to find all Suites containing test names that include the substring. *Specifying a span scale factor
* ** If you specify a integer or floating point span scale factor with
* * *-F
, traitScaledTimeSpans
* trait will return the specified value from its implementation ofspanScaleFactor
. This allows you to tune the "patience" of a run (how long to wait * for asynchronous operations) from the command line. For more information, see the documentation for traitScaledTimeSpans
. *Specifying TestNG XML config file paths
* ** If you specify one or more file paths with
* * *-b
(b for Beust, the last name of TestNG's creator),Runner
will create aorg.scalatest.testng.TestNGWrapperSuite
, * passing in aList
of the specified paths. When executed, theTestNGWrapperSuite
will create oneTestNG
instance * and pass each specified file path to it for running. If you include-b
arguments, you must include TestNG's jar file on the class path or runpath. * The-b
argument will enable you to run existingTestNG
tests, including tests written in Java, as part of a ScalaTest run. * You need not use-b
to run suites written in Scala that extendTestNGSuite
. You can simply run such suites with *-s
,-m
, or -w parameters. *Specifying JUnit tests
* ** JUnit tests, including ones written in Java, may be run by specifying *
*-j classname
, where the classname is a valid JUnit class * such as a TestCase, TestSuite, or a class implementing a static suite() * method returning a TestSuite.* To use this option you must include a JUnit jar file on your classpath. *
* * *Memorizing and rerunning failed and canceled tests
* ** You can memorize failed and canceled tests using
* *-M
: ** -M failed-canceled.txt ** * All failed and canceled tests will be memorized infailed-canceled.txt
, to rerun them again, you use-A
: * ** -A failed-canceled.txt ** * *Slowpoke notifications
* ** You can request to recieve periodic notifications of slowpokes, tests that have been running longer than a given amount of time, specified in * seconds by the first integer after
* *-W
, the delay. * You specify the period between slowpoke notifications in seconds with the second integer after-W
, the period. Thus to receive * notifications very minute of tests that have been running longer than two minutes, you'd use: *** *-W 120 60
** Slowpoke notifications will be sent via
* *AlertProvided
events. The standard out reporter, for example, * will report such notifications like: ** *** Test still running after 2 minutes, 13 seconds: suite name: ExampleSpec, test name: An egg timer should take 10 minutes. *
* * @author Bill Venners * @author George Berger * @author Josh Cough * @author Chee Seng */ object Runner { private val RUNNER_JFRAME_START_X: Int = 150 private val RUNNER_JFRAME_START_Y: Int = 100 @volatile private[scalatest] var spanScaleFactor: Double = 1.0 private final val DefaultNumFilesToArchive = 2 // TO // We always include a PassFailReporter on runs in order to determine // whether or not all tests passed. // // The thread that calls Runner.run() will either start a GUI, if a graphic // reporter was requested, or just run the tests itself. If a GUI is started, // an event handler thread will get going, and it will start a RunnerThread, // which will actually do the running. The GUI can repeatedly start RunnerThreads // and RerunnerThreads, until the GUI is closed. If -P is specified, that means // the tests should be run concurrently, which in turn means a Distributor will // be passed to the execute method of the Suites, which will in turn populate // it with its nested suites instead of executing them directly in the same // thread. The Distributor works in conjunction with a pool of threads that // will take suites out of the distributor queue and execute them. The DispatchReporter // will serialize all reports via an actor, which because that actor uses receive // not react, will have its own thread. So the DispatchReporter actor's thread will // be the one that actually invokes TestFailed, RunAborted, etc., on this PassFailReporter. // The thread that invoked Runner.run(), will be the one that calls allTestsPassed. // // The thread that invoked Runner.run() will be the one to instantiate the PassFailReporter // and in its primary constructor acquire the single semaphore permit. This permit will // only be released by the DispatchReporter's actor thread when a runAborted, runStopped, // or runCompleted is invoked. allTestsPassed will block until it can reacquire the lone // semaphore permit. Thus, a PassFailReporter can just be used for one run, then it is // spent. A new PassFailReporter is therefore created each time the Runner.run() method is invoked. // private class PassFailReporter extends Reporter { @volatile private var failedAbortedOrStopped = false private val runDoneSemaphore = new Semaphore(1) runDoneSemaphore.acquire() override def apply(event: Event): Unit = { event match { case _: TestFailed => failedAbortedOrStopped = true case _: RunAborted => failedAbortedOrStopped = true runDoneSemaphore.release() case _: SuiteAborted => failedAbortedOrStopped = true case _: RunStopped => failedAbortedOrStopped = true runDoneSemaphore.release() case _: RunCompleted => runDoneSemaphore.release() case _ => } } def allTestsPassed = { runDoneSemaphore.acquire() !failedAbortedOrStopped } } // TODO: I don't think I'm enforcing that properties can't start with "org.scalatest" // TODO: I don't think I'm handling rejecting multiple -f/-C with the same arg. -f fred.txt -f fred.txt should // fail, as should -C MyReporter -C MyReporter. I'm failing on -o -o, -g -g, and -e -e, but the error messages // could indeed be nicer. /** * Runs a suite of tests, with optional GUI. See the main documentation for this singleton object for the details. */ def main(args: Array[String]): Unit = { // println("FOR DEBUGGING, THESE ARE THE ARGS PASSED TO main(): " + args.mkString(" ")) Thread.currentThread.setName("ScalaTest-main") val result = if (args.contains("-v") || args.contains("--version")) { val version = org.scalatest.ScalaTestVersions.ScalaTestVersion val scalaVersion = org.scalatest.ScalaTestVersions.BuiltForScalaVersion println("ScalaTest " + version + " (Built for Scala " + scalaVersion + ")") runOptionallyWithPassFailReporter(args.filter(arg => arg != "-v" && arg != "--version"), true) } else runOptionallyWithPassFailReporter(args, true) if (result) System.exit(0) else System.exit(1) } /** * Runs a suite of tests, with optional GUI. See the main documentation for this singleton object for the details. * The difference between this method andmain
is simply that this method will block until the run * has completed, aborted, or been stopped, and returntrue
if all tests executed and passed. In other * words, if any test fails, or if any suite aborts, or if the run aborts or is stopped, this method will * returnfalse
. This value is used, for example, by the ScalaTest ant task to determine whether * to continue the build ifhaltOnFailure
is set totrue
. * * @return true if all tests were executed and passed. */ def run(args: Array[String]): Boolean = { // println("FOR DEBUGGING, THESE ARE THE ARGS PASSED TO run(): " + args.mkString(" ")) val originalThreadName = Thread.currentThread.getName try { Thread.currentThread.setName("ScalaTest-run") runOptionallyWithPassFailReporter(args, true) } finally Thread.currentThread.setName(originalThreadName) } private def runOptionallyWithPassFailReporter(args: Array[String], runWithPassFailReporter: Boolean): Boolean = { checkArgsForValidity(args) match { case Some(s) => { println(s) System.exit(1) // TODO: Shouldn't this be returning false? } case None => } val ParsedArgs( runpathArgs, reporterArgs, suiteArgs, againArgs, junitArgs, propertiesArgs, tagsToIncludeArgs, tagsToExcludeArgs, concurrentArgs, membersOnlyArgs, wildcardArgs, testNGArgs, suffixes, chosenStyles, spanScaleFactors, testSortingReporterTimeouts, slowpokeArgs, seedArgs ) = parseArgs(args) val fullReporterConfigurations: ReporterConfigurations = if (reporterArgs.isEmpty) // If no reporters specified, just give them a graphic reporter new ReporterConfigurations(Some(GraphicReporterConfiguration(Set())), Nil, Nil, Nil, /*Nil, Nil, */None, None, Nil, Nil, Nil, Nil) else parseReporterArgsIntoConfigurations(reporterArgs) val (suitesList: List[SuiteParam], testSpecs: List[TestSpec]) = parseSuiteArgs(suiteArgs) val agains: List[String] = parseAgainArgs(againArgs) val junitsList: List[String] = parseSuiteArgsIntoNameStrings(junitArgs, "-j") val runpathList: List[String] = parseRunpathArgIntoList(runpathArgs) val propertiesMap: ConfigMap = parsePropertiesArgsIntoMap(propertiesArgs) val tagsToInclude: Set[String] = parseCompoundArgIntoSet(tagsToIncludeArgs, "-n") val tagsToExclude: Set[String] = parseCompoundArgIntoSet(tagsToExcludeArgs, "-l") val concurrent: Boolean = !concurrentArgs.isEmpty val concurrentConfig: ConcurrentConfig = parseConcurrentConfig(concurrentArgs) val membersOnlyList: List[String] = parseSuiteArgsIntoNameStrings(membersOnlyArgs, "-m") val wildcardList: List[String] = parseSuiteArgsIntoNameStrings(wildcardArgs, "-w") val testNGList: List[String] = parseSuiteArgsIntoNameStrings(testNGArgs, "-b") val chosenStyleSet: Set[String] = parseChosenStylesIntoChosenStyleSet(chosenStyles, "-y") val slowpokeConfig: Option[SlowpokeConfig] = parseSlowpokeConfig(slowpokeArgs) val seedList: Option[Long] = parseLongArgument(seedArgs, "-S") spanScaleFactor = parseDoubleArgument(spanScaleFactors, "-F", 1.0) val testSortingReporterTimeout = Span(parseDoubleArgument(testSortingReporterTimeouts, "-T", Suite.defaultTestSortingReporterTimeoutInSeconds), Seconds) seedList match { case Some(seed) => Seed.configuredRef.getAndSet(Some(seed)) case None => // do nothing } // If there's a graphic reporter, we need to leave it out of // reporterSpecs, because we want to pass all reporterSpecs except // the graphic reporter's to the RunnerJFrame (because RunnerJFrame *is* // the graphic reporter). val reporterConfigs: ReporterConfigurations = fullReporterConfigurations.graphicReporterConfiguration match { case None => fullReporterConfigurations case Some(grs) => { new ReporterConfigurations( None, fullReporterConfigurations.fileReporterConfigurationList, fullReporterConfigurations.memoryReporterConfigurationList, fullReporterConfigurations.junitXmlReporterConfigurationList, //fullReporterConfigurations.dashboardReporterConfigurationList, //fullReporterConfigurations.xmlReporterConfigurationList, fullReporterConfigurations.standardOutReporterConfiguration, fullReporterConfigurations.standardErrReporterConfiguration, fullReporterConfigurations.htmlReporterConfigurationList, fullReporterConfigurations.customReporterConfigurationList, fullReporterConfigurations.xmlSocketReporterConfigurationList, fullReporterConfigurations.socketReporterConfigurationList ) } } val passFailReporter = if (runWithPassFailReporter) Some(new PassFailReporter) else None if (propertiesMap.isDefinedAt(CHOSEN_STYLES)) throw new IllegalArgumentException("Property name '" + CHOSEN_STYLES + "' is used by ScalaTest, please choose other property name.") val configMap: ConfigMap = if (chosenStyleSet.isEmpty) propertiesMap else propertiesMap + (CHOSEN_STYLES -> chosenStyleSet) if (chosenStyleSet.nonEmpty) println(Resources.deprecatedChosenStyleWarning) val (detectSlowpokes: Boolean, slowpokeDetectionDelay: Long, slowpokeDetectionPeriod: Long) = slowpokeConfig match { case Some(SlowpokeConfig(delayInMillis, periodInMillis)) => (true, delayInMillis, periodInMillis) case _ => (false, 60000L, 60000L) } fullReporterConfigurations.graphicReporterConfiguration match { case Some(GraphicReporterConfiguration(configSet)) => { val graphicEventsToPresent: Set[EventToPresent] = EventToPresent.allEventsToPresent filter (if (configSet.contains(FilterTestStarting)) {_ != PresentTestStarting} else etp => true) filter (if (configSet.contains(FilterTestSucceeded)) {_ != PresentTestSucceeded} else etp => true) filter (if (configSet.contains(FilterTestIgnored)) {_ != PresentTestIgnored} else etp => true) filter (if (configSet.contains(FilterTestPending)) {_ != PresentTestPending} else etp => true) filter (if (configSet.contains(FilterScopeOpened)) {_ != PresentScopeOpened} else etp => true) filter (if (configSet.contains(FilterScopeClosed)) {_ != PresentScopeClosed} else etp => true) filter (if (configSet.contains(FilterScopePending)) {_ != PresentScopePending} else etp => true) filter (if (configSet.contains(FilterSuiteStarting)) {_ != PresentSuiteStarting} else etp => true) filter (if (configSet.contains(FilterSuiteCompleted)) {_ != PresentSuiteCompleted} else etp => true) filter (if (configSet.contains(FilterInfoProvided)) {_ != PresentInfoProvided} else etp => true) filter (if (configSet.contains(FilterMarkupProvided)) {_ != PresentMarkupProvided} else etp => true) val abq = new ArrayBlockingQueue[RunnerJFrame](1) usingEventDispatchThread { val rjf = new RunnerJFrame( graphicEventsToPresent, reporterConfigs, suitesList, agains, testSpecs, junitsList, runpathList, tagsToInclude, tagsToExclude, configMap, concurrent, membersOnlyList, wildcardList, testNGList, passFailReporter, concurrentConfig, suffixes, chosenStyleSet, detectSlowpokes, slowpokeDetectionDelay, slowpokeDetectionPeriod, testSortingReporterTimeout ) rjf.setLocation(RUNNER_JFRAME_START_X, RUNNER_JFRAME_START_Y) rjf.setVisible(true) rjf.prepUIForRunning() rjf.runFromGUI() abq.put(rjf) } // To get the Ant task to work, the main thread needs to block until // The GUI window exits. val rjf = abq.take() rjf.blockUntilWindowClosed() } case None => { // Run the test without a GUI withClassLoaderAndDispatchReporter( runpathList, reporterConfigs, None, passFailReporter, detectSlowpokes, slowpokeDetectionDelay, slowpokeDetectionPeriod ) { (loader, dispatchReporter) => doRunRunRunDaDoRunRun( dispatchReporter, suitesList, agains, testSpecs, junitsList, Stopper.default, tagsToInclude, tagsToExclude, configMap, concurrent, membersOnlyList, wildcardList, testNGList, runpathList, loader, new RunDoneListener {}, 1, concurrentConfig, suffixes, chosenStyleSet, testSortingReporterTimeout ) } } } passFailReporter match { case Some(pfr) => pfr.allTestsPassed case None => false } } // For debugging. /* private[scalatest] def printOpts(opt: EventToPresent.Set32) { if (opt.contains(EventToPresent.PresentRunStarting)) println("PresentRunStarting") if (opt.contains(EventToPresent.PresentTestStarting)) println("PresentTestStarting") if (opt.contains(EventToPresent.PresentTestSucceeded)) println("PresentTestSucceeded") if (opt.contains(EventToPresent.PresentTestFailed)) println("PresentTestFailed") if (opt.contains(EventToPresent.PresentTestIgnored)) println("PresentTestIgnored") if (opt.contains(EventToPresent.PresentSuiteStarting)) println("PresentSuiteStarting") if (opt.contains(EventToPresent.PresentSuiteCompleted)) println("PresentSuiteCompleted") if (opt.contains(EventToPresent.PresentSuiteAborted)) println("PresentSuiteAborted") if (opt.contains(EventToPresent.PresentInfoProvided)) println("PresentInfoProvided") if (opt.contains(EventToPresent.PresentRunStopped)) println("PresentRunStopped") if (opt.contains(EventToPresent.PresentRunCompleted)) println("PresentRunCompleted") if (opt.contains(EventToPresent.PresentRunAborted)) println("PresentRunAborted") } */ // We number our named threads so that people can keep track // of it as it goes through different suites. But in case the // multiple doRunRunRunDaDoRunRun's get called, we want to // use different numbers. So this is a "global" count in Runner. private val atomicThreadCounter = new AtomicInteger private[scalatest] def loadJUnitWrapperClass(loader: ClassLoader) = loader.loadClass("org.scalatestplus.junit.JUnitWrapperSuite") private[scalatest] def loadTestNGWrapperClass(loader: ClassLoader) = loader.loadClass("org.scalatestplus.testng.TestNGWrapperSuite") private[scalatest] def doRunRunRunDaDoRunRun( dispatch: DispatchReporter, suitesList: List[SuiteParam], agains: List[String], testSpecs: List[TestSpec], junitsList: List[String], stopper: Stopper, tagsToIncludeSet: Set[String], tagsToExcludeSet: Set[String], configMap: ConfigMap, concurrent: Boolean, membersOnlyList: List[String], wildcardList: List[String], testNGList: List[String], runpath: List[String], loader: ClassLoader, doneListener: RunDoneListener, runStamp: Int, concurrentConfig: ConcurrentConfig, suffixes: Option[Pattern], chosenStyleSet: Set[String], testSortingReporterTimeout: Span ): Unit = { // TODO: add more, and to RunnerThread too requireNonNull(dispatch, suitesList, agains, testSpecs, junitsList, stopper, tagsToIncludeSet, tagsToExcludeSet, configMap, membersOnlyList, wildcardList, testNGList, runpath, loader, doneListener, chosenStyleSet) val (globSuites, nonGlobSuites) = suitesList.partition(_.isGlob) var tracker = new Tracker(new Ordinal(runStamp)) // // Generates SuiteConfigs for Suites found via discovery. // def genDiscoSuites: List[SuiteConfig] = { val emptyDynaTags = DynaTags(Map.empty[String, Set[String]], Map.empty[String, Map[String, Set[String]]]) // // If user specified any -t or -z arguments independent of // a Suite name, or any members-only or wildcard args, or // suite args with glob characters, then we need to do // discovery to find the Suites associated with those // args. // val discoArgsArePresent = !membersOnlyList.isEmpty || !wildcardList.isEmpty || !testSpecs.isEmpty || !globSuites.isEmpty // // If user specified any specific Suites to run, then we don't // have to do any discovery unless discoArgsArePresent. // val suiteArgsArePresent = !nonGlobSuites.isEmpty || !junitsList.isEmpty || !testNGList.isEmpty || !agains.isEmpty if (suiteArgsArePresent && !discoArgsArePresent) { Nil // No DiscoverySuites in this case. Just run Suites // named with -s or -j or -b } else { val discoveryStartTime = System.currentTimeMillis dispatch(DiscoveryStarting(tracker.nextOrdinal(), configMap)) val accessibleSuites: Set[String] = discoverSuiteNames(runpath, loader, suffixes) val discoSuites = if (!discoArgsArePresent && !suiteArgsArePresent) { // In this case, they didn't specify any -w, -m, -s, // -j or -b on the command line, so the default is to // run any accessible Suites discovered on the runpath List(SuiteConfig(new DiscoverySuite("", accessibleSuites, true, loader), emptyDynaTags, false, false)) } else { val membersOnlyInstances = for (membersOnlyName <- membersOnlyList) yield SuiteConfig(new DiscoverySuite(membersOnlyName, accessibleSuites, false, loader), emptyDynaTags, false, false) val wildcardInstances = for (wildcardName <- wildcardList) yield SuiteConfig(new DiscoverySuite(wildcardName, accessibleSuites, true, loader), emptyDynaTags, false, false) val testSpecSuiteParams = SuiteDiscoveryHelper.discoverTests( testSpecs, accessibleSuites, loader) val testSpecInstances = for (suiteParam <- testSpecSuiteParams) yield genSuiteConfig(suiteParam, loader) val deglobbedSuiteParams: List[SuiteParam] = deglobSuiteParams(globSuites, accessibleSuites) val globInstances = for (suiteParam <- deglobbedSuiteParams) yield genSuiteConfig(suiteParam, loader) membersOnlyInstances ::: wildcardInstances ::: testSpecInstances ::: globInstances } val discoveryDuration = System.currentTimeMillis - discoveryStartTime dispatch( DiscoveryCompleted(tracker.nextOrdinal(), Some(discoveryDuration))) discoSuites } } // suites specified by name, either directly via -s or in a file via -A val againSuites = readMemoryFiles(agains, dispatch, tracker) val specificSuites = nonGlobSuites ::: againSuites val runStartTime = System.currentTimeMillis try { val loadProblemsExist = try { val unrunnableList = specificSuites.filter{ suiteParam => val className = suiteParam.className loader.loadClass(className) // Check if the class exist, so if not we get the nice cannot load suite error message. !isAccessibleSuite(className, loader) && !isRunnable(className, loader) } if (!unrunnableList.isEmpty) { val names = for (suiteParam <- unrunnableList) yield " " + suiteParam.className dispatch(RunAborted(tracker.nextOrdinal(), Resources.nonSuite + names.mkString(", "), None)) true } else { false } } catch { case e: ClassNotFoundException => { dispatch(RunAborted(tracker.nextOrdinal(), Resources.cannotLoadSuite(e.getMessage), Some(e))) true } } if (!loadProblemsExist) { try { val namedSuiteInstances: List[SuiteConfig] = for (suiteParam <- specificSuites) yield genSuiteConfig(suiteParam, loader) val emptyDynaTags = DynaTags(Map.empty[String, Set[String]], Map.empty[String, Map[String, Set[String]]]) val junitSuiteInstances: List[SuiteConfig] = if (junitsList.isEmpty) List.empty else { val junitWrapperClass = loadJUnitWrapperClass(loader) val junitWrapperClassConstructor = junitWrapperClass.getDeclaredConstructor(classOf[String], classOf[ClassLoader]) for (junitClassName <- junitsList) yield SuiteConfig(junitWrapperClassConstructor.newInstance(junitClassName, loader).asInstanceOf[Suite], emptyDynaTags, false, true) // JUnit suite should exclude nested suites } val testNGWrapperSuiteList: List[SuiteConfig] = if (!testNGList.isEmpty) { val testngWrapperClass = loadTestNGWrapperClass(loader) val testngWrapperClassConstructor = testngWrapperClass.getDeclaredConstructor(classOf[List[String]]) List(SuiteConfig(testngWrapperClassConstructor.newInstance(testNGList).asInstanceOf[Suite], emptyDynaTags, false, true)) // TestNG suite should exclude nested suites } else Nil val discoSuiteInstances = genDiscoSuites val suiteInstances: List[SuiteConfig] = namedSuiteInstances ::: junitSuiteInstances ::: discoSuiteInstances ::: testNGWrapperSuiteList val testCountList = for (suiteConfig <- suiteInstances) yield { val tagsToInclude = if (suiteConfig.requireSelectedTag) tagsToIncludeSet ++ Set(SELECTED_TAG) else tagsToIncludeSet val filter = Filter(if (tagsToInclude.isEmpty) None else Some(tagsToInclude), tagsToExcludeSet, suiteConfig.excludeNestedSuites, suiteConfig.dynaTags) suiteConfig.suite.expectedTestCount(filter) } def sumInts(list: List[Int]): Int = list match { case Nil => 0 case x :: xs => x + sumInts(xs) } val expectedTestCount = sumInts(testCountList) dispatch(RunStarting(tracker.nextOrdinal(), expectedTestCount, configMap)) if (concurrent) { // Because some tests may do IO, will create a pool of 2 times the number of processors reported // by the Runtime's availableProcessors method. val poolSize = if (concurrentConfig.numThreads == 0) Runtime.getRuntime.availableProcessors * 2 else concurrentConfig.numThreads val distributedSuiteSorter = if (concurrentConfig.enableSuiteSortingReporter) Some(new SuiteSortingReporter(dispatch, Span(testSortingReporterTimeout.millisPart, Millis), System.err)) else None val concurrentDispatch = distributedSuiteSorter match { case Some(dss) => dss case None => dispatch } val threadFactory = new ThreadFactory { val defaultThreadFactory = Executors.defaultThreadFactory def newThread(runnable: Runnable): Thread = { val thread = defaultThreadFactory.newThread(runnable) thread.setName("ScalaTest-" + atomicThreadCounter.incrementAndGet()) thread } } val execSvc: ExecutorService = if (poolSize > 0) Executors.newFixedThreadPool(poolSize, threadFactory) else Executors.newCachedThreadPool(threadFactory) try { val distributor = new ConcurrentDistributor(Args(dispatch, stopper, Filter(if (tagsToIncludeSet.isEmpty) None else Some(tagsToIncludeSet), tagsToExcludeSet), configMap, None, tracker, chosenStyleSet, false, None, None), execSvc) if (System.getProperty("org.scalatest.tools.Runner.forever", "false") == "true") { while (true) { val statuses = for (suiteConfig <- suiteInstances) yield { val tagsToInclude = if (suiteConfig.requireSelectedTag) tagsToIncludeSet ++ Set(SELECTED_TAG) else tagsToIncludeSet val filter = Filter(if (tagsToInclude.isEmpty) None else Some(tagsToInclude), tagsToExcludeSet, suiteConfig.excludeNestedSuites, suiteConfig.dynaTags) val runArgs = Args(concurrentDispatch, stopper, filter, configMap, Some(distributor), tracker.nextTracker(), chosenStyleSet, false, None, distributedSuiteSorter) distributor.apply(suiteConfig.suite, runArgs) } distributor.waitUntilDone() (new CompositeStatus(statuses.toSet)).waitUntilCompleted() } } else { val statuses = for (suiteConfig <- suiteInstances) yield { val tagsToInclude = if (suiteConfig.requireSelectedTag) tagsToIncludeSet ++ Set(SELECTED_TAG) else tagsToIncludeSet val filter = Filter(if (tagsToInclude.isEmpty) None else Some(tagsToInclude), tagsToExcludeSet, suiteConfig.excludeNestedSuites, suiteConfig.dynaTags) val runArgs = Args(concurrentDispatch, stopper, filter, configMap, Some(distributor), tracker.nextTracker(), chosenStyleSet, false, None, distributedSuiteSorter) distributor.apply(suiteConfig.suite, runArgs) } distributor.waitUntilDone() (new CompositeStatus(statuses.toSet)).waitUntilCompleted() } } finally { execSvc.shutdown() } } else { for (suiteConfig <- suiteInstances) { val tagsToInclude = if (suiteConfig.requireSelectedTag) tagsToIncludeSet ++ Set(SELECTED_TAG) else tagsToIncludeSet val filter = Filter(if (tagsToInclude.isEmpty) None else Some(tagsToInclude), tagsToExcludeSet, suiteConfig.excludeNestedSuites, suiteConfig.dynaTags) val runArgs = Args(dispatch, stopper, filter, configMap, None, tracker, chosenStyleSet, false, None, None) val status = new ScalaTestStatefulStatus() val suiteRunner = new SuiteRunner(suiteConfig.suite, runArgs, status) suiteRunner.run() status.waitUntilCompleted() } } val duration = System.currentTimeMillis - runStartTime if (stopper.stopRequested) { dispatch(RunStopped(tracker.nextOrdinal(), Some(duration))) } else { dispatch(RunCompleted(tracker.nextOrdinal(), Some(duration))) } } catch { case e: InstantiationException => dispatch(RunAborted(tracker.nextOrdinal(), Resources.cannotInstantiateSuite(e.getMessage), Some(e), Some(System.currentTimeMillis - runStartTime))) case e: IllegalAccessException => dispatch(RunAborted(tracker.nextOrdinal(), Resources.cannotInstantiateSuite(e.getMessage), Some(e), Some(System.currentTimeMillis - runStartTime))) case e: NoClassDefFoundError => dispatch(RunAborted(tracker.nextOrdinal(), Resources.cannotLoadClass(e.getMessage), Some(e), Some(System.currentTimeMillis - runStartTime))) case e: Throwable => dispatch(RunAborted(tracker.nextOrdinal(), Resources.bigProblems(e), Some(e), Some(System.currentTimeMillis - runStartTime))) } } } finally { dispatch.dispatchDisposeAndWaitUntilDone() doneListener.done() } } // // Creates a SuiteParam for each Suite found that matches one // of the patterns in a list of SuiteParams containing globs. // private[tools] def deglobSuiteParams(globsList: List[SuiteParam], accessibleSuites: Set[String]): List[SuiteParam] = for { suiteParam <- globsList name <- accessibleSuites if suiteParam.matches(name) } yield suiteParam.copy(className = name) // // Reads each specified file and generates a SuiteParam object // for each entry in file. Files are of the format created when // -M option is specified to record failed/canceled/aborted // tests so they can be run again later. // private[tools] def readMemoryFiles(fileNames: List[String], reporter: Reporter, tracker: Tracker): List[SuiteParam] = { val mementos = for { fileName <- fileNames memento <- Memento.readFromFile(fileName) } yield memento val (unrerunnables, rerunnables) = mementos.partition(_.className == None) for (memento <- unrerunnables) reporter.apply( AlertProvided( tracker.nextOrdinal(), Resources.cannotRerun(memento.eventName, memento.suiteId, memento.testName), None)) rerunnables.map(_.toSuiteParam) } private[tools] def genSuiteConfig(suiteParam: SuiteParam, loader: ClassLoader): SuiteConfig = { val suiteClassName = suiteParam.className val clazz = loader.loadClass(suiteClassName) val wrapWithAnnotation = clazz.getAnnotation(classOf[WrapWith]) val suiteInstance = if (wrapWithAnnotation == null) clazz.newInstance.asInstanceOf[Suite] else { val suiteClazz = wrapWithAnnotation.value val constructorList = suiteClazz.getDeclaredConstructors() val constructor = constructorList.find { c => val types = c.getParameterTypes types.length == 1 && types(0) == classOf[java.lang.Class[_]] } constructor.get.newInstance(clazz).asInstanceOf[Suite] } if (suiteParam.testNames.length == 0 && suiteParam.wildcardTestNames.length == 0 && suiteParam.nestedSuites.length == 0) SuiteConfig(suiteInstance, new DynaTags(Map.empty, Map.empty), false, false) // -s suiteClass, no dynamic tagging required. else { val nestedSuites = suiteParam.nestedSuites val (selectSuiteList, selectTestList) = nestedSuites.partition(ns => ns.testNames.length == 0 || ns.wildcardTestNames.length == 0) val suiteDynaTags: Map[String, Set[String]] = Map() ++ selectSuiteList.map(ns => (ns.suiteId -> Set(SELECTED_TAG))) val suiteExactTestDynaTags: Map[String, Map[String, Set[String]]] = if (suiteParam.testNames.length > 0) Map(suiteInstance.suiteId -> (Map() ++ suiteParam.testNames.map(tn => (tn -> Set(SELECTED_TAG))))) else Map.empty val suiteWildcardTestDynaTags: Map[String, Map[String, Set[String]]] = if (suiteParam.wildcardTestNames.length > 0) { val wildcardTestNames = suiteParam.wildcardTestNames val allTestNames = suiteInstance.testNames val wildcardTestTags = Map() ++ allTestNames.filter(tn => wildcardTestNames.find(wc => tn.contains(wc)).isDefined) .map(tn => (tn -> Set(SELECTED_TAG))) Map(suiteInstance.suiteId -> wildcardTestTags) } else Map.empty def getNestedSuiteSelectedTestNames(nestedSuite: NestedSuiteParam): Array[String] = { if (nestedSuite.wildcardTestNames.length == 0) nestedSuite.testNames else { val wildcardTestNames = nestedSuite.wildcardTestNames val allTestNames = suiteInstance.testNames nestedSuite.testNames ++ allTestNames.filter(tn => wildcardTestNames.find(wc => tn.contains(wc)).isDefined) } } val nestedSuitesTestDynaTags: Map[String, Map[String, Set[String]]] = Map() ++ selectTestList.map(ns => (ns.suiteId -> (Map() ++ getNestedSuiteSelectedTestNames(ns).map(tn => (tn, Set(SELECTED_TAG)))))) val testDynaTags = mergeMap[String, Map[String, Set[String]]](List(suiteExactTestDynaTags, suiteWildcardTestDynaTags, nestedSuitesTestDynaTags)) { (suiteTestMap1, suiteTestMap2) => mergeMap[String, Set[String]](List(suiteTestMap1, suiteTestMap2)) { (tagSet1, tagSet2) => tagSet1 ++ tagSet2 } } // Only exclude nested suites when using -s XXX -t XXXX, or -s XXX -z XXX val excludeNestedSuites = suiteParam.testNames.length > 0 && nestedSuites.length == 0 SuiteConfig(suiteInstance, new DynaTags(suiteDynaTags, testDynaTags), true, excludeNestedSuites) } } private[scalatest] def excludesWithIgnore(excludes: Set[String]) = excludes + "org.scalatest.Ignore" private[scalatest] def withClassLoaderAndDispatchReporter( runpathList: List[String], reporterSpecs: ReporterConfigurations, graphicReporter: Option[Reporter], passFailReporter: Option[Reporter], detectSlowpokes: Boolean, slowpokeDetectionDelay: Long, slowpokeDetectionPeriod: Long )(f: (ClassLoader, DispatchReporter) => Unit): Unit = { val loader: ClassLoader = getRunpathClassLoader(runpathList) try { Thread.currentThread.setContextClassLoader(loader) try { val dispatchReporter = ReporterFactory.getDispatchReporter(reporterSpecs, graphicReporter, passFailReporter, loader, None, detectSlowpokes, slowpokeDetectionDelay, slowpokeDetectionPeriod) try { f(loader, dispatchReporter) } finally { dispatchReporter.dispatchDisposeAndWaitUntilDone() } } catch { // getDispatchReporter may complete abruptly with an exception, if there is an problem trying to load // or instantiate a custom reporter class. case ex: Throwable => { val msg = Resources.bigProblemsMaybeCustomReporter System.err.println(msg) ex.printStackTrace(System.err) val tracker = new Tracker(new Ordinal(1)) val runAborted = RunAborted(tracker.nextOrdinal(), msg, None) graphicReporter.foreach { rep => rep(runAborted) } passFailReporter.foreach { rep => rep(runAborted) } } } } finally { // eventually call close on the RunpathClassLoader } } private[scalatest] def getRunpathClassLoader(runpathList: List[String]): ClassLoader = { requireNonNull(runpathList) if (runpathList.isEmpty) { classOf[Suite].getClassLoader // Could this be null technically? } else { val urlsList: List[URL] = for (raw <- runpathList) yield { try { new URL(raw) } catch { case murle: MalformedURLException => { // Assume they tried to just pass in a file name val file: File = new File(raw) // file.toURL may throw MalformedURLException too, but for now // just let that propagate up. file.toURI.toURL // If a dir, comes back terminated by a slash } } } // Here is where the Jini preferred class loader stuff went. // Tell the URLConnections to not use caching, so that repeated runs and reruns actually work // on the latest binaries. for (url <- urlsList) { try { url.openConnection.setDefaultUseCaches(false) } catch { case e: IOException => // just ignore these } } new URLClassLoader(urlsList.toArray, classOf[Suite].getClassLoader) } } private[scalatest] def usingEventDispatchThread(f: => Unit): Unit = { SwingUtilities.invokeLater( new Runnable() { def run(): Unit = { f } } ) } }