io.cloudslang.lang.tools.build.tester.SlangTestRunner Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of cloudslang-content-verifier Show documentation
There is a newer version: 2.0.94
Show newest version
/*******************************************************************************
 * (c) Copyright 2016 Hewlett-Packard Development Company, L.P.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Apache License v2.0 which accompany this distribution.
 *
 * The Apache License is available at
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 *******************************************************************************/
package io.cloudslang.lang.tools.build.tester;

import io.cloudslang.lang.api.Slang;
import io.cloudslang.lang.compiler.SlangSource;
import io.cloudslang.lang.compiler.modeller.DependenciesHelper;
import io.cloudslang.lang.compiler.modeller.model.Executable;
import io.cloudslang.lang.entities.CompilationArtifact;
import io.cloudslang.lang.entities.ScoreLangConstants;
import io.cloudslang.lang.entities.SystemProperty;
import io.cloudslang.lang.entities.bindings.values.Value;
import io.cloudslang.lang.tools.build.SlangBuildMain;
import io.cloudslang.lang.tools.build.SlangBuildMain.BulkRunMode;
import io.cloudslang.lang.tools.build.SlangBuildMain.TestCaseRunMode;
import io.cloudslang.lang.logging.LoggingService;
import io.cloudslang.lang.tools.build.tester.parallel.MultiTriggerTestCaseEventListener;
import io.cloudslang.lang.tools.build.tester.parallel.report.LoggingSlangTestCaseEventListener;
import io.cloudslang.lang.tools.build.tester.parallel.report.ThreadSafeRunTestResults;
import io.cloudslang.lang.tools.build.tester.parallel.services.ParallelTestCaseExecutorService;
import io.cloudslang.lang.tools.build.tester.parallel.services.TestCaseEventDispatchService;
import io.cloudslang.lang.tools.build.tester.parallel.testcaseevents.FailedSlangTestCaseEvent;
import io.cloudslang.lang.tools.build.tester.parse.SlangTestCase;
import io.cloudslang.lang.tools.build.tester.parse.TestCasesYamlParser;
import io.cloudslang.lang.tools.build.tester.runconfiguration.BuildModeConfig;
import io.cloudslang.lang.tools.build.tester.runconfiguration.TestRunInfoService;
import io.cloudslang.lang.tools.build.tester.runconfiguration.strategy.RunMultipleTestSuiteConflictResolutionStrategy;
import io.cloudslang.lang.tools.build.tester.runconfiguration.strategy.SequentialRunTestSuiteResolutionStrategy;
import io.cloudslang.score.events.EventConstants;
import java.io.File;
import java.io.Serializable;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Future;
import java.util.concurrent.TimeoutException;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.apache.logging.log4j.Level;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import static java.lang.Long.parseLong;
import static java.lang.String.valueOf;
import static java.lang.System.getProperty;
import static java.util.concurrent.TimeUnit.MINUTES;
import static org.apache.commons.collections4.CollectionUtils.containsAny;
import static org.apache.commons.collections4.CollectionUtils.isEmpty;


@Component
public class SlangTestRunner {

    private static final String PROJECT_PATH_TOKEN = "${project_path}";
    public static final long MAX_TIME_PER_TESTCASE_IN_MINUTES = 10;
    public static final String TEST_CASE_TIMEOUT_IN_MINUTES_KEY = "test.case.timeout.in.minutes";
    public static final String PREFIX_DASH = "    - ";

    @Autowired
    private TestCasesYamlParser parser;

    @Autowired
    private Slang slang;

    @Autowired
    private ParallelTestCaseExecutorService parallelTestCaseExecutorService;

    @Autowired
    private TestCaseEventDispatchService testCaseEventDispatchService;

    @Autowired
    private TestRunInfoService testRunInfoService;

    @Autowired
    private DependenciesHelper dependenciesHelper;

    @Autowired
    private LoggingService loggingService;

    @Autowired
    private LoggingSlangTestCaseEventListener loggingSlangTestCaseEventListener;

    private String[] testCaseFileExtensions = {"yaml", "yml"};
    private static final String TEST_CASE_PASSED = "Test case passed: ";
    private static final String TEST_CASE_FAILED = "Test case failed: ";

    private static final String UNAVAILABLE_NAME = "N/A";

    public enum TestCaseRunState {
        PARALLEL,
        SEQUENTIAL,
        INACTIVE
    }

    public Map createTestCases(String testPath, Set allAvailableExecutables) {
        Validate.notEmpty(testPath, "You must specify a path for tests");
        File testPathDir = new File(testPath);
        Validate.isTrue(testPathDir.isDirectory(),
                "Directory path argument \'" + testPath + "\' does not lead to a directory");
        Collection testCasesFiles = FileUtils.listFiles(testPathDir, testCaseFileExtensions, true);

        loggingService.logEvent(Level.INFO, "");
        loggingService.logEvent(Level.INFO, "--- parsing test cases ---");
        loggingService.logEvent(Level.INFO, "Start parsing all test cases files under: " + testPath);
        loggingService.logEvent(Level.INFO, testCasesFiles.size() + " test cases files were found");

        Map testCases = new HashMap<>();
        Set testCasesWithMissingReference = new HashSet<>();
        for (File testCaseFile : testCasesFiles) {
            Validate.isTrue(testCaseFile.isFile(),
                    "file path \'" + testCaseFile.getAbsolutePath() + "\' must lead to a file");

            Map testCasesFromCurrentFile =
                    parser.parseTestCases(SlangSource.fromFile(testCaseFile));
            for (Map.Entry currentTestCaseEntry : testCasesFromCurrentFile.entrySet()) {
                SlangTestCase currentTestCase = currentTestCaseEntry.getValue();
                String currentTestCaseName = currentTestCaseEntry.getKey();
                String testFlowPath = currentTestCase.getTestFlowPath();
                currentTestCase.setName(currentTestCaseName);
                if (StringUtils.isBlank(currentTestCase.getResult())) {
                    currentTestCase.setResult(getResultFromFileName(testFlowPath));
                }
                if (currentTestCase.getThrowsException() == null) {
                    currentTestCase.setThrowsException(false);
                }
                // Make sure the new test cases names are unique
                if (testCases.containsKey(currentTestCaseName)) {
                    throw new RuntimeException("Test case with the name: " + currentTestCaseName +
                            " already exists. Test case name should be unique across the project"
                    );
                }
                if (!allAvailableExecutables.contains(testFlowPath)) {
                    testCasesWithMissingReference.add(currentTestCase);
                }
                testCases.put(currentTestCaseName, currentTestCase);
            }
        }
        printTestCasesWithMissingReference(testCasesWithMissingReference);
        return testCases;
    }

    /**
     *
     * @param projectPath the project path
     * @param testCases the test cases
     * @param compiledFlows the compiled flows
     * @param runTestsResults is updated to reflect skipped, failed passes test cases.
     */
    public void runTestsSequential(String projectPath, Map testCases,
                                   Map compiledFlows, IRunTestResults runTestsResults) {

        if (MapUtils.isEmpty(testCases)) {
            return;
        }
        printTestForActualRunSummary(TestCaseRunMode.SEQUENTIAL, testCases);

        for (Map.Entry testCaseEntry : testCases.entrySet()) {
            SlangTestCase testCase = testCaseEntry.getValue();

            loggingService.logEvent(Level.INFO, "Running test: " +
                    SlangTestCase.generateTestCaseReference(testCase) + " - " + testCase.getDescription());
            try {
                CompilationArtifact compiledTestFlow = getCompiledTestFlow(compiledFlows, testCase);
                runTest(testCase, compiledTestFlow, projectPath);
                runTestsResults.addPassedTest(testCase.getName(), new TestRun(testCase, null));
            } catch (RuntimeException e) {
                runTestsResults.addFailedTest(testCase.getName(), new TestRun(testCase, e.getMessage()));
            }
        }
    }

    public void runTestsParallel(String projectPath, Map testCases,
                                 Map compiledFlows,
                                 ThreadSafeRunTestResults runTestsResults) {
        if (MapUtils.isEmpty(testCases)) {
            return;
        }
        printTestForActualRunSummary(TestCaseRunMode.PARALLEL, testCases);

        testCaseEventDispatchService.unregisterAllListeners();
        testCaseEventDispatchService.registerListener(runTestsResults); // for gathering of report data
        testCaseEventDispatchService.registerListener(loggingSlangTestCaseEventListener); // for logging purpose

        MultiTriggerTestCaseEventListener multiTriggerTestCaseEventListener = new MultiTriggerTestCaseEventListener();
        slang.subscribeOnEvents(multiTriggerTestCaseEventListener, createListenerEventTypesSet());
        try {
            Map> testCaseFutures = new LinkedHashMap<>();
            for (Map.Entry testCaseEntry : testCases.entrySet()) {
                SlangTestCase testCase = testCaseEntry.getValue();
                SlangTestCaseRunnable slangTestCaseRunnable =
                        new SlangTestCaseRunnable(testCase, compiledFlows, projectPath, this,
                                testCaseEventDispatchService, multiTriggerTestCaseEventListener);
                testCaseFutures.put(testCase, parallelTestCaseExecutorService.submitTestCase(slangTestCaseRunnable));
            }

            final long testCaseTimeoutMinutes = getTestCaseTimeoutInMinutes();
            for (Map.Entry> slangTestCaseFutureEntry : testCaseFutures.entrySet()) {
                SlangTestCase testCase = slangTestCaseFutureEntry.getKey();
                Future testCaseFuture = slangTestCaseFutureEntry.getValue();
                try {
                    testCaseFuture.get(testCaseTimeoutMinutes, MINUTES);
                } catch (InterruptedException e) {
                    loggingService.logEvent(Level.ERROR, "Interrupted while waiting for result: ", e);
                } catch (TimeoutException e) {
                    testCaseEventDispatchService.notifyListeners(new FailedSlangTestCaseEvent(testCase,
                            "Timeout reached for test case " + testCase.getName(), e));
                } catch (Exception e) {
                    testCaseEventDispatchService.notifyListeners(
                            new FailedSlangTestCaseEvent(testCase, e.getMessage(), e));
                }
            }
        } finally {
            testCaseEventDispatchService.unregisterAllListeners();
            slang.unSubscribeOnEvents(multiTriggerTestCaseEventListener);
        }
    }

    private void printTestForActualRunSummary(TestCaseRunMode runMode, Map testCases) {
        if (!MapUtils.isEmpty(testCases)) {
            loggingService.logEvent(Level.INFO, "Running " + testCases.size() + " test(s) in " +
                    runMode.toString().toLowerCase(Locale.ENGLISH) + ": ");
            for (Map.Entry stringSlangTestCaseEntry : testCases.entrySet()) {
                final SlangTestCase slangTestCase = stringSlangTestCaseEntry.getValue();
                loggingService.logEvent(Level.INFO, PREFIX_DASH +
                        SlangTestCase.generateTestCaseReference(slangTestCase));
            }
        }
    }

    /**
     * Processes skipped tests and also handles null testcase failures
     * @param bulkRunMode the bulk run mode
     * @param testCases the test cases
     * @param testSuites active test suites
     * @param runTestsResults is updated to reflect skipped and fail fast scenarios
     * @param buildModeConfig the build mode config
     * @return a map with split test cases
     */
    public Map> splitTestCasesByRunState(
            final BulkRunMode bulkRunMode,
            final Map testCases,
            final List testSuites,
            final IRunTestResults runTestsResults,
            final BuildModeConfig buildModeConfig) {
        Map> resultMap = new HashMap<>();

        // Prepare the 3 categories inactive, parallel, sequential
        for (TestCaseRunState testCaseRunState : TestCaseRunState.values()) {
            resultMap.put(testCaseRunState, new LinkedHashMap());
        }

        for (Map.Entry testCaseEntry : testCases.entrySet()) {
            final SlangTestCase testCase = testCaseEntry.getValue();
            if (testCase == null) {
                processQuickFailTest(runTestsResults);
                continue;
            }

            if (isTestCaseInActiveSuite(testCase, testSuites) &&
                    isEnabledByBuildMode(buildModeConfig.getBuildMode(), testCase,
                            buildModeConfig.getChangedFiles(), buildModeConfig.getAllTestedFlowModels())) {
                processActiveTest(bulkRunMode, resultMap, testCaseEntry, testCase);
            } else {
                processSkippedTest(runTestsResults, testCaseEntry, testCase, resultMap);
            }
        }

        return resultMap;
    }

    private boolean isEnabledByBuildMode(
            SlangBuildMain.BuildMode buildMode,
            SlangTestCase slangTestCase,
            Set changedExecutables,
            Map allTestedFlowModels) {
        return (buildMode == SlangBuildMain.BuildMode.BASIC) ||
                (buildMode == SlangBuildMain.BuildMode.CHANGED &&
                        isAffectedTestCase(slangTestCase, changedExecutables, allTestedFlowModels));
    }

    private boolean isAffectedTestCase(SlangTestCase slangTestCase, Set changedExecutables,
                                       Map allTestedFlowModels) {
        String testFlowPath = slangTestCase.getTestFlowPath();
        Executable testCaseReference = allTestedFlowModels.get(testFlowPath);
        if (testCaseReference == null) {
            throw new RuntimeException("Test case reference[" + testFlowPath + "] not found in compiled models.");
        }
        Set testCaseDependencies = dependenciesHelper.fetchDependencies(testCaseReference, allTestedFlowModels);
        testCaseDependencies.add(testFlowPath);
        return containsAny(testCaseDependencies, changedExecutables);
    }

    private void processQuickFailTest(final IRunTestResults runTestsResults) {
        runTestsResults.addFailedTest(UNAVAILABLE_NAME, new TestRun(null, "Test case cannot be null"));
    }

    private void processActiveTest(final BulkRunMode bulkRunMode, final Map> resultMap, Map.Entry testCaseEntry,
                                   SlangTestCase testCase) {
        if (bulkRunMode == BulkRunMode.POSSIBLY_MIXED) {
            TestCaseRunMode runModeForTestCase = testRunInfoService
                    .getRunModeForTestCase(testCase, new RunMultipleTestSuiteConflictResolutionStrategy(),
                    new SequentialRunTestSuiteResolutionStrategy());
            if (runModeForTestCase == TestCaseRunMode.SEQUENTIAL) {
                resultMap.get(TestCaseRunState.SEQUENTIAL).put(testCaseEntry.getKey(), testCase);
            } else if (runModeForTestCase == TestCaseRunMode.PARALLEL) {
                resultMap.get(TestCaseRunState.PARALLEL).put(testCaseEntry.getKey(), testCase);
            }
        } else if (bulkRunMode == BulkRunMode.ALL_SEQUENTIAL) {
            resultMap.get(TestCaseRunState.SEQUENTIAL).put(testCaseEntry.getKey(), testCase);
        } else if (bulkRunMode == BulkRunMode.ALL_PARALLEL) {
            resultMap.get(TestCaseRunState.PARALLEL).put(testCaseEntry.getKey(), testCase);
        }
    }

    private void processSkippedTest(final IRunTestResults runTestsResults,
                                    Map.Entry testCaseEntry, SlangTestCase testCase,
                                    final Map> resultMap) {
        String message = "Skipping test: " + SlangTestCase.generateTestCaseReference(testCase) +
                " because it is not in active test suites";
        loggingService.logEvent(Level.INFO, message);

        runTestsResults.addSkippedTest(testCase.getName(), new TestRun(testCase, message));
        resultMap.get(TestCaseRunState.INACTIVE).put(testCaseEntry.getKey(), testCaseEntry.getValue());
    }

    private long getTestCaseTimeoutInMinutes() {
        try {
            return parseLong(getProperty(TEST_CASE_TIMEOUT_IN_MINUTES_KEY, valueOf(MAX_TIME_PER_TESTCASE_IN_MINUTES)));
        } catch (NumberFormatException nfEx) {
            loggingService.logEvent(Level.WARN,
                    String.format("Misconfigured test case timeout '%s'. Using default timeout %d.",
                            getProperty(TEST_CASE_TIMEOUT_IN_MINUTES_KEY), MAX_TIME_PER_TESTCASE_IN_MINUTES));
            return MAX_TIME_PER_TESTCASE_IN_MINUTES;
        }
    }

    public boolean isTestCaseInActiveSuite(SlangTestCase testCase, List testSuites) {
        return (isEmpty(testCase.getTestSuites()) && testSuites.contains(SlangBuildMain.DEFAULT_TESTS)) ||
                containsAny(testSuites, testCase.getTestSuites());
    }

    private void printTestCasesWithMissingReference(Set testCasesWithMissingReference) {
        int testCasesWithMissingReferenceSize = testCasesWithMissingReference.size();
        if (testCasesWithMissingReferenceSize > 0) {
            loggingService.logEvent(Level.INFO, "");
            loggingService.logEvent(Level.INFO, testCasesWithMissingReferenceSize +
                    " test cases have missing test flow references:");
            for (SlangTestCase slangTestCase : testCasesWithMissingReference) {
                loggingService.logEvent(Level.INFO,
                        "For test case: " + SlangTestCase.generateTestCaseReference(slangTestCase) +
                                " testFlowPath reference not found: " +
                                slangTestCase.getTestFlowPath()
                );
            }
        }
    }

    public CompilationArtifact getCompiledTestFlow(Map compiledFlows,
                                                   SlangTestCase testCase) {
        String testFlowPath = testCase.getTestFlowPath();
        if (StringUtils.isEmpty(testFlowPath)) {
            throw new RuntimeException("For test case: " + SlangTestCase.generateTestCaseReference(testCase) +
                    " testFlowPath property is mandatory");
        }
        String testFlowPathTransformed = testFlowPath.replace(File.separatorChar, '.');
        CompilationArtifact compiledTestFlow = compiledFlows.get(testFlowPathTransformed);
        if (compiledTestFlow == null) {
            throw new RuntimeException("Test flow: " + testFlowPath + " is missing. Referenced in test case: " +
                    SlangTestCase.generateTestCaseReference(testCase));
        }
        return compiledTestFlow;
    }

    public void runTest(SlangTestCase testCase, CompilationArtifact compiledTestFlow, String projectPath) {

        Map convertedInputs = getTestCaseInputsMap(testCase);
        Set systemProperties = getTestSystemProperties(testCase, projectPath);

        runTestCaseSequentiallyToCompletion(testCase, compiledTestFlow, convertedInputs, systemProperties);
    }

    public void runTestCaseParallel(SlangTestCase testCase, CompilationArtifact compiledTestFlow,
                                    String projectPath,
                                    MultiTriggerTestCaseEventListener multiTriggerTestCaseEventListener) {

        Map convertedInputs = getTestCaseInputsMap(testCase);
        Set systemProperties = getTestSystemProperties(testCase, projectPath);

        runTestCaseInParallelToCompletion(testCase, compiledTestFlow, convertedInputs,
                systemProperties, multiTriggerTestCaseEventListener);
    }

    private Set getTestSystemProperties(SlangTestCase testCase, String projectPath) {
        String systemPropertiesFile = testCase.getSystemPropertiesFile();
        if (StringUtils.isEmpty(systemPropertiesFile)) {
            return new HashSet<>();
        }
        systemPropertiesFile = StringUtils.replace(systemPropertiesFile, PROJECT_PATH_TOKEN, projectPath);
        return parser.parseProperties(systemPropertiesFile);
    }

    private Map getTestCaseInputsMap(SlangTestCase testCase) {
        List inputs = testCase.getInputs();
        Map convertedInputs = new HashMap<>();
        convertedInputs = convertMapParams(inputs, convertedInputs);
        return io.cloudslang.lang.entities.utils.MapUtils.convertMapNonSensitiveValues(convertedInputs);
    }

    private Map convertMapParams(List params, Map convertedInputs) {
        if (CollectionUtils.isNotEmpty(params)) {
            for (Map param : params) {
                convertedInputs.put(
                        (String) param.keySet().iterator().next(),
                        (Serializable) param.values().iterator().next());
            }
        }
        return convertedInputs;
    }

    private Map getTestCaseOutputsMap(SlangTestCase testCase) {
        List outputs = testCase.getOutputs();
        Map convertedOutputs = new HashMap<>();
        return convertMapParams(outputs, convertedOutputs);
    }

    /**
     * This method will trigger the flow in a synchronize matter, meaning only one flow can run at a time.
     *
     * @param testCase the test Case
     * @param compilationArtifact the artifact to trigger
     * @param inputs flow inputs
     * @param systemProperties the system properties
     * @return executionId
     */
    public Long runTestCaseSequentiallyToCompletion(SlangTestCase testCase, CompilationArtifact compilationArtifact,
                                                    Map inputs,
                                                    Set systemProperties) {

        final String result = testCase.getResult();
        final Map outputs = getTestCaseOutputsMap(testCase);
        String flowName = testCase.getTestFlowPath();

        TriggerTestCaseEventListener testsEventListener = new TriggerTestCaseEventListener();
        slang.subscribeOnEvents(testsEventListener, createListenerEventTypesSet());

        Long executionId = slang.run(compilationArtifact, inputs, systemProperties);

        while (!testsEventListener.isFlowFinished()) {
            poll();
        }
        slang.unSubscribeOnEvents(testsEventListener);

        String errorMessageFlowExecution = testsEventListener.getErrorMessage();

        String message;
        String testCaseReference = SlangTestCase.generateTestCaseReference(testCase);

        if (BooleanUtils.isTrue(testCase.getThrowsException())) {
            return handleExpectedExceptionCase(testCase, compilationArtifact, flowName, executionId,
                    errorMessageFlowExecution, testCaseReference);
        }

        if (StringUtils.isNotBlank(errorMessageFlowExecution)) {
            // unexpected exception occurred during flow execution
            message = "Error occurred while running test: " + testCaseReference + " - " +
                    testCase.getDescription() + "\n\t" + errorMessageFlowExecution;
            loggingService.logEvent(Level.INFO, message);
            throw new RuntimeException(message);
        }

        String executionResult = testsEventListener.getResult();
        if (result != null && !result.equals(executionResult)) {
            message = TEST_CASE_FAILED + testCaseReference + " - " + testCase.getDescription() +
                    "\n\tExpected result: " + result + "\n\tActual result: " + executionResult;
            loggingService.logEvent(Level.ERROR, message);
            throw new RuntimeException(message);
        }

        Map executionOutputs = testsEventListener.getOutputs();
        handleTestCaseFailuresFromOutputs(testCase, testCaseReference, outputs, executionOutputs);

        loggingService.logEvent(Level.INFO, TEST_CASE_PASSED + testCaseReference +
                ". Finished running: " + flowName + " with result: " + executionResult);
        return executionId;
    }

    public Long runTestCaseInParallelToCompletion(SlangTestCase testCase, CompilationArtifact compilationArtifact,
                                                  Map inputs,
                                                  Set systemProperties,
                                                  MultiTriggerTestCaseEventListener globalListener) {

        final String result = testCase.getResult();
        final Map outputs = getTestCaseOutputsMap(testCase);
        String flowName = testCase.getTestFlowPath();

        Long executionId = slang.run(compilationArtifact, inputs, systemProperties);

        while (!globalListener.isFlowFinishedByExecutionId(executionId)) {
            poll();
        }

        String errorMessageFlowExecution = globalListener.getErrorMessageByExecutionId(executionId);

        String message;
        String testCaseReference = SlangTestCase.generateTestCaseReference(testCase);

        if (BooleanUtils.isTrue(testCase.getThrowsException())) {
            return handleExpectedExceptionCase(testCase, compilationArtifact, flowName, executionId,
                    errorMessageFlowExecution, testCaseReference);
        }

        if (StringUtils.isNotBlank(errorMessageFlowExecution)) {
            // unexpected exception occurred during flow execution
            message = "Error occurred while running test: " + testCaseReference + " - " +
                    testCase.getDescription() + "\n\t" + errorMessageFlowExecution;
            loggingService.logEvent(Level.INFO, message);
            throw new RuntimeException(message);
        }

        String executionResult = globalListener.getResultByExecutionId(executionId);
        if (result != null && !result.equals(executionResult)) {
            message = TEST_CASE_FAILED + testCaseReference + " - " + testCase.getDescription() +
                    "\n\tExpected result: " + result + "\n\tActual result: " + executionResult;
            loggingService.logEvent(Level.ERROR, message);
            throw new RuntimeException(message);
        }

        Map executionOutputs = globalListener.getOutputsByExecutionId(executionId);
        handleTestCaseFailuresFromOutputs(testCase, testCaseReference, outputs, executionOutputs);

        loggingService.logEvent(Level.INFO, TEST_CASE_PASSED + testCaseReference +
                ". Finished running: " + flowName + " with result: " + executionResult);
        return executionId;
    }

    private Long handleExpectedExceptionCase(SlangTestCase testCase, CompilationArtifact compilationArtifact,
                                             String flowName, Long executionId,
                                             String errorMessageFlowExecution, String testCaseReference) {
        String message;
        if (StringUtils.isBlank(errorMessageFlowExecution)) {

            message = TEST_CASE_FAILED + testCaseReference + " - " + testCase.getDescription() + "\n\tFlow " +
                    compilationArtifact.getExecutionPlan().getName() + " did not throw an exception as expected";
            loggingService.logEvent(Level.INFO, message);
            throw new RuntimeException(message);
        }
        loggingService.logEvent(Level.INFO, TEST_CASE_PASSED + testCaseReference +
                ". Finished running: " + flowName + " with exception as expected");
        return executionId;
    }

    private void poll() {
        try {
            Thread.sleep(200);
        } catch (InterruptedException ignore) {
        }
    }

    private void handleTestCaseFailuresFromOutputs(SlangTestCase testCase, String testCaseReference,
                                                   Map outputs,
                                                   Map executionOutputs) {
        String message;
        if (MapUtils.isNotEmpty(outputs)) {
            for (Map.Entry output : outputs.entrySet()) {
                String outputName = output.getKey();
                Serializable outputValue = output.getValue();
                Serializable executionOutputValue = executionOutputs.get(outputName);
                if (!executionOutputs.containsKey(outputName) ||
                        !outputsAreEqual(outputValue, executionOutputValue)) {
                    message = TEST_CASE_FAILED + testCaseReference + " - " + testCase.getDescription() +
                            "\n\tFor output: " + outputName + "\n\tExpected value: " + outputValue +
                            "\n\tActual value: " + executionOutputValue;

                    loggingService.logEvent(Level.ERROR, message);
                    throw new RuntimeException(message);
                }
            }
        }
    }

    private boolean outputsAreEqual(Serializable outputValue, Serializable executionOutputValue) {
        return executionOutputValue == outputValue ||
                StringUtils.equals(executionOutputValue.toString(), outputValue.toString());
    }

    private Set createListenerEventTypesSet() {
        Set handlerTypes = new HashSet<>();
        handlerTypes.add(ScoreLangConstants.EVENT_EXECUTION_FINISHED);
        handlerTypes.add(ScoreLangConstants.SLANG_EXECUTION_EXCEPTION);
        handlerTypes.add(ScoreLangConstants.EVENT_OUTPUT_END);
        handlerTypes.add(EventConstants.SCORE_ERROR_EVENT);
        handlerTypes.add(EventConstants.SCORE_FAILURE_EVENT);
        handlerTypes.add(EventConstants.SCORE_FINISHED_EVENT);
        return handlerTypes;
    }

    private String getResultFromFileName(String fileName) {

        int dashPosition = fileName.lastIndexOf('-');
        if (dashPosition > 0 && dashPosition < fileName.length()) {
            return fileName.substring(dashPosition + 1);
        } else {
            return null;
        }
    }
}