All Downloads are FREE. Search and download functionalities are using the official Maven repository.

csip.test.ServiceTest Maven / Gradle / Ivy

Go to download

The Cloud Services Integration Platform is a SoA implementation to offer a Model-as-a-Service framework, Application Programming Interface, deployment infrastructure, and service implementations for environmental modeling.

There is a newer version: 2.6.30
Show newest version
/*
 * $Id: ServiceTest.java 820b0b8fd71e 2020-07-14 od $
 *
 * This file is part of the Cloud Services Integration Platform (CSIP),
 * a Model-as-a-Service framework, API and application suite.
 *
 * 2012-2019, Olaf David and others, OMSLab, Colorado State University.
 *
 * OMSLab licenses this file to you under the MIT license.
 * See the LICENSE file in the project root for more information.
 */
package csip.test;

import csip.Client;
import csip.utils.JSONUtils;
import java.io.File;
import java.io.FileFilter;
import java.io.FileReader;
import java.io.Reader;
import java.net.URL;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.logging.SimpleFormatter;
import java.util.logging.StreamHandler;
import org.apache.commons.io.FileUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
import org.skyscreamer.jsonassert.JSONAssert;

/**
 * Service Testing.
 *
 * @author odavid
 */
public class ServiceTest {

    static final String KEYSONLY = "keysonly";
    static final String KEYVALUE = "keyvalue";
    static final String KEYVALUEORDER = "keyvalueorder";
    static final String NONE = "none";

    static final int NO_WARMUP = 0;
    static final int WARMUP_AND_RUN = 1;
    static final int WARMUP_ONLY = 2;

    static final List VALIDTESTING = Arrays.asList(KEYSONLY, KEYVALUE, KEYVALUEORDER, NONE);

    File folder;
    File[] files;
    String url;
    String testing;
    int concurrent;
    int workers;
    int timeout;
    int modelTimeout;
    int delay;
    int limit;
    int warmup;
    int warmupRuns;
    boolean downloadFiles;
    String logLevel = "WARNING";
    String config;
    boolean ignore = false;

    static final Logger log = Logger.getLogger("ServiceTest");


    static {
        log.setLevel(Level.SEVERE);
        SimpleFormatter fmt = new SimpleFormatter();
        StreamHandler sh = new StreamHandler(System.out, fmt);
        log.addHandler(sh);
    }


    private ServiceTest(File target) throws Exception {
        if (!target.exists()) {
            throw new IllegalArgumentException("Not a folder or file: " + target);
        }

        if (target.getName().endsWith("req.json") && target.isFile()) {
            folder = target.getParentFile();
            files = new File[]{target};
        } else if (target.isDirectory()) {
            folder = target;
            files = folder.listFiles(new FileFilter() {

                @Override
                public boolean accept(File pathname) {
                    return pathname.getName().endsWith("-req.json");
                }
            });
            // sort the file names.
            Arrays.sort(files, new Comparator() {
                @Override
                public int compare(File o1, File o2) {
                    int n1 = extractNumber(o1.getName());
                    int n2 = extractNumber(o2.getName());
                    if ((n1 == 0) || (n2 == 0)) {
                        return o1.compareTo(o2);
                    } else {
                        return n1 - n2;
                    }
                }

                // to do - could make char s and char e configurable
                // and this would enable custom ways to extract a number 
                // from the file name for sorting

                private int extractNumber(String name) {
                    int i = 0;
                    try {
                        int s = name.lastIndexOf('t') + 1;
                        int e = name.lastIndexOf('-');
                        if ((s < 0) || (e < 0)) {
                            s = name.lastIndexOf('_') + 1;
                            e = name.lastIndexOf('.');
                        }
                        String number = name.substring(s, e);
                        i = Integer.parseInt(number);
                    } catch (NumberFormatException nfe) {
                        i = 0;
                    }
                    return i;
                }
            });
        } else {
            throw new IllegalArgumentException("Invalid argument: " + target);
        }

        // load the service testing settings.
        Properties p = loadProperties(folder);
        testing = p.getProperty("verify", NONE);  // strict, keys
        if (!VALIDTESTING.contains(testing)) {
            throw new RuntimeException("invalid testing method: " + testing);
        }

        ignore = Boolean.parseBoolean(p.getProperty("ignore", "false"));

        url = p.getProperty("url");
        if (url == null) {
            String host = p.getProperty("host");
            String path = p.getProperty("path");
            if (host == null && path == null) {
                throw new RuntimeException("'url' (or host/path) not found in " + new File(folder, "service.properties"));
            }
            url = host + path;
        }

        concurrent = Integer.parseInt(p.getProperty("concurrency", "1"));
        if (concurrent < 1) {
            concurrent = 1;
        }
        timeout = Integer.parseInt(p.getProperty("timeout", "3600"));
        if (timeout < 1000) {
            timeout = 1000;
        }
        modelTimeout = Integer.parseInt(p.getProperty("modelTimeout", "600000"));  // default is 10 minutes
        if (modelTimeout <= 0) {
            modelTimeout = 600000;
        }
        delay = Integer.parseInt(p.getProperty("delay", "0"));
        if (delay < 0) {
            delay = 0;
        }
        workers = Integer.parseInt(p.getProperty("workers", "1"));
        if (workers < 0) {
            workers = 1;
        }
        // warmup phase: 0-off, 1-on
        warmup = Integer.parseInt(p.getProperty("warmup", "0"));
        if (warmup < 0) {
            warmup = NO_WARMUP;
        }
        warmupRuns = Integer.parseInt(p.getProperty("warmupRuns", "4"));
        if (warmupRuns < 0) {
            warmupRuns = 4;
        }
        downloadFiles = Boolean.parseBoolean(p.getProperty("download_files", "true"));

        logLevel = p.getProperty("logLevel", "WARNING");
        if ((logLevel != null) && (logLevel.length() > 0)) {
            log.setLevel(Level.parse(logLevel));
        }
        config = p.getProperty("config", "service-conf.json");
        limit = Integer.parseInt(p.getProperty("limit", "-1"));
        limit = (limit < 1) ? files.length : limit;
    }


    /**
     * Execute the test.
     *
     * @return
     */
    private Results test() {
        final Results results = new Results();

        if (ignore) {
            results.incIgnored();
            return results;
        }

        final Client client = new Client(timeout, log);

        // apply configuration
        if (config != null) {
            // changed with absolute name
            File conf = new File(config);
            if (!conf.exists()) {
                // relative name
                conf = new File(folder, config);
            }
            if (conf.exists()) {
                try {
                    JSONObject o = new JSONObject(FileUtils.readFileToString(conf, "UTF-8"));
                    client.doPUT(getBaseURLContext(url) + "/c/conf", o);
                } catch (Exception E) {
                    log.warning("Failed to load config: " + conf);
                }
            } else {
                // if there is no config... then assume service is pre-configured...
            }
        }

        int maxFiles = limit;

        final CountDownLatch latch = new CountDownLatch(maxFiles);
        final CountDownLatch warmupLatch = new CountDownLatch(workers * warmupRuns);
        final ConcurrentLinkedQueue fileQueue = new ConcurrentLinkedQueue<>();
        final ConcurrentLinkedQueue warmups = new ConcurrentLinkedQueue<>();
        for (int i = 0; i < maxFiles; i++) {
            if ((i < workers * warmupRuns) && (warmup != NO_WARMUP)) {
                warmups.add(files[i]);
            }
            fileQueue.add(files[i]);
        }

        ExecutorService warmupExecutor = null;
        final ExecutorService executor = Executors.newFixedThreadPool(concurrent, new ThreadFactory() {
            @Override
            public Thread newThread(Runnable r) {
                return new Thread(r, "exeuctor-thread");
            }
        });

//        final ExecutorService executor = new ThreadPoolExecutor(
//                concurrent, concurrent, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue()) {
//                    @Override
//                    protected void afterExecute(Runnable r, Throwable t) {
//                        super.afterExecute(r, t);
//                        if (t != null) {
//                            System.out.println("Perform exception handler logic");
//                        }
//                        final File file = fileQueue.poll();
//                        if (file != null) {
//                            if (delay > 0) {
//                                try {
//                                    System.out.println("THIS WORKER SLEEP FOR:" + delay);
//                                    Thread.sleep(delay);
//                                } catch (InterruptedException ex) {
//                                    System.out.println("Interruption during sleep - non-init!!!");
//                                }
//                            }
//                            // keep going
//                            submit(new Callable() {
//                                @Override
//                                public Void call() {
//                                    return t(file, results, client, latch);
//                                }
//                            });
//                        }
//                        else
//                        {
//                            System.out.println("THE FILE IS NULL - THIS SHOULD NOT HAPPEN UNTIL END, filequeue-size=" + fileQueue.size());
//                        }
//                    }
//                };
        // submit the first run twice as a "warmup" across the workers
        if (warmup != NO_WARMUP) {
            final Results warmupResults = new Results();
            warmupExecutor = Executors.newFixedThreadPool(workers * warmupRuns, new ThreadFactory() {
                @Override
                public Thread newThread(Runnable r) {
                    return new Thread(r, "warmup-thread");
                }
            });

            for (int i = 0; i < (workers * warmupRuns); i++) {
                final File file = warmups.poll();
                log.info("SUBMITTING WARM UP RUN=" + file);
                warmupExecutor.submit(new Callable() {
                    @Override
                    public Void call() {
                        return t(file, warmupResults, client, warmupLatch);
                    }
                });
            }
        }

        // submit the initial runs.
        if (warmup != WARMUP_ONLY) {
            //        for (int i = 0; i < Math.min(concurrent, maxFiles); i++) {
            for (int i = 0; i < maxFiles; i++) {
                if (delay > 0) {
                    try {
                        Thread.sleep(delay);
                    } catch (InterruptedException ex) {
                        log.warning("Interrupted during sleep- init runs!");
                    }
                }

                final File file = fileQueue.poll();
                log.info("SUBMITTING TO PROCESS FILE=" + file);
                executor.submit(new Callable() {
                    @Override
                    public Void call() {
                        return t(file, results, client, latch);
                    }
                });
            }
        }

        try {
            if (warmup != WARMUP_ONLY) {
                latch.await();
            }
            if (warmup != NO_WARMUP) {
                warmupLatch.await();
            }
        } catch (InterruptedException ex) {
            // whatever
        }
        executor.shutdown();
        if (warmup != NO_WARMUP) {
            warmupExecutor.shutdown();
        }
        return results;
    }


    /**
     *
     * @param reqFile
     * @param results
     * @param client
     * @param latch
     * @return
     */
    private Void t(File reqFile, Results results, Client client, CountDownLatch latch) {
        try {
            if (delay > 0) {
                try {
                    Thread.sleep(delay);
                } catch (InterruptedException ex) {
                    log.warning("Interrupted during sleep: pre-run-sleep");
                }
            }

            File res = getResFile(reqFile);
            // check if there is a async result with the status "Running"
            if (res.exists()) {
                res.delete();

//                // ASYNC PART
//                // check if the prev run was running.
//                JSONObject prev_json = new JSONObject(FileUtils.readFileToString(res));
//                if (JSONUtils.getStatus(prev_json).equals("Finished")) {
//                    log.warning("Skipped (Finished)" + " " + reqFile + "... ");
//                    results.incSkipped();
//                    log.info("EXITING test run,,, latch count-down?");
//                    return null;
//                }
//
//                if (JSONUtils.getStatus(prev_json).equals("Running") || JSONUtils.getStatus(prev_json).equals("Submitted")) {
//                    // it is running, need to check again.
//                    String q_url = getBaseURLContext(url) + "/q/" + JSONUtils.getSID(prev_json);
//
//                    // get the new
//                    client.doGET(q_url, res);
//
//                    // check the status again
//                    JSONObject this_json = new JSONObject(FileUtils.readFileToString(res));
//                    String status = JSONUtils.getStatus(this_json);
//                    log.info(status + " " + reqFile + "... ");
//                    if (status.equals("Finished")) {
//
//                        // dowload
//                        download(this_json, getResFolder(reqFile), client);
//
//                        if (testing.equals(NONE)) {
//                            results.incSuccess();
//                            log.info("ASYNC EXIT NONE");
//                            return null;
//                        }
//                        // test
//                        runtest(reqFile, this_json);
//                    }
//                    results.incSuccess();
//                    log.info("ASYNC EXIT END");
//                    return null;
//                }
            }

            JSONObject req_json = new JSONObject(FileUtils.readFileToString(reqFile, "UTF-8"));
            boolean isAsync = JSONUtils.isAsync(req_json);

            // input
            File inp = getInpFolder(reqFile);
            File[] attachments = (inp != null) ? inp.listFiles() : new File[0];

            // delete output if exist.
            FileUtils.deleteDirectory(getResFolderName(reqFile));

            // run the service
            JSONObject response = client.doPOST(url, req_json, attachments);
            FileUtils.writeStringToFile(getResFile(reqFile), response.toString(2).replace("\\/", "/"), "UTF-8");

            // print the error if failed.
            if (JSONUtils.getStatus(response).equals("Failed")) {
                JSONObject respose_metainfo = response.getJSONObject("metainfo");
                throw new Exception(respose_metainfo.getString("error"));
            }

            if (isAsync) {
                results.incSuccess();
                log.info("THREAD EXECUTOR TEST SUCCESS:" + reqFile);
                return null;
            }

            if (!JSONUtils.hasResult(response)) {
                throw new Exception("No 'result' in response");
            }

            if (downloadFiles) {
                download(response, getResFolder(reqFile), client);
            }

            if (testing.equals(NONE)) {
                results.incSuccess();
                log.info("EXITING test run,,, latch count-down?");
                return null;
            }

            runtest(reqFile, response);
            results.incSuccess();
        } catch (Exception | Error E) {
            System.out.println("  Failed: " + reqFile);
            log.warning("ERROR-" + E.toString());
            for (StackTraceElement ste : E.getStackTrace()) {
                log.warning(ste.getClassName() + ":" + ste.getMethodName() + ":" + ste.getLineNumber() + "---" + ste.toString());
            }
            System.out.println(E.getMessage());
            results.incFailed();
        } finally {
            log.info("THREAD EXECUTOR COUNT DOWN AFTER:" + reqFile);
            latch.countDown();
        }
        return null;
    }


    private void runtest(File reqFile, JSONObject response) throws Exception {
        JSONArray response_result = response.getJSONArray("result");
        // all real checking starts here.
        // golden response
        JSONObject golJson = new JSONObject(FileUtils.readFileToString(getResponse(reqFile), "UTF-8"));

        // check output files
        JSONArray response_golden = golJson.getJSONArray("result");
        Map rmap = JSONUtils.preprocess(response_result);

        Map golmap = JSONUtils.preprocess(response_golden);
        for (String key : rmap.keySet()) {
            JSONObject val = rmap.get(key);
            String url = val.getString("value");
            if (url.startsWith("http") && url.endsWith(key)) {
                response_result.remove(val);
                golmap.remove(golmap.get(key));
            }
        }

        File out = getRefFolder(reqFile);
        if (out != null) {
            File[] outFiles = out.listFiles();
            for (File file : outFiles) {
                String f = file.getName();
                if (!rmap.containsKey(f)) {
                    throw new Exception("Expecting file in result: " + f);
                }
                // TODO compare the output files.
            }
        }

        switch (testing) {
            case KEYVALUE:
                compareStrict(response_result, response_golden);
                break;
            case KEYSONLY:
                compareKeys(response_result, response_golden);
                break;
            default:
                throw new IllegalArgumentException(testing);
        }
    }

    /**
     *
     */
    public static class Results {

        int succeeded = 0;
        int failed = 0;
        int skipped = 0;
        int ignored = 0;


        void add(Results other) {
            succeeded += other.getSucceeded();
            failed += other.getFailed();
            skipped += other.getSkipped();
            ignored += other.getIgnored();
        }


        public int getTotal() {
            return succeeded + failed + skipped + ignored;
        }


        public int getFailed() {
            return failed;
        }


        public int getSkipped() {
            return skipped;
        }


        public int getSucceeded() {
            return succeeded;
        }


        public int getIgnored() {
            return ignored;
        }


        synchronized void incSuccess() {
            succeeded++;
        }


        synchronized void incFailed() {
            failed++;
        }


        synchronized void incSkipped() {
            skipped++;
        }


        synchronized void incIgnored() {
            ignored++;
        }


        @Override
        public String toString() {
            return "=================" + "\n"
                    + " Total:     " + getTotal() + "\n"
                    + " Failed:    " + getFailed() + "\n"
                    + " Succeeded: " + getSucceeded() + "\n"
                    + " Skipped:   " + getSkipped() + "\n"
                    + " Ignored:   " + getIgnored() + "\n";
        }

    }


    static void download(JSONObject resp, File resultFolder, Client client) throws Exception {
        JSONArray arr = resp.getJSONArray("result");
        // download files using the 'q' service.
        Map rmap = JSONUtils.preprocess(arr);
        for (String key : rmap.keySet()) {
            JSONObject val = rmap.get(key);
            String url = val.getString("value");
            if (url == null) {
                throw new Exception("Expecting file in result: " + key);
            }
            if (url.startsWith("http") && url.contains("/q/") && url.endsWith(key)) {
                if (!resultFolder.exists()) {
                    resultFolder.mkdirs();
                }
                File outFile = new File(resultFolder, key);
                client.doGET(url, outFile);
                if (!outFile.exists()) {
                    throw new Exception("Missing output file: " + key);
                }
            }
        }
    }


    static File getInpFolder(File request) {
        return getFolder(request, "-req");
    }


    static File getRefFolder(File request) {
        return getFolder(request, "-ref");
    }


    static File getResFolderName(File request) {
        String name = request.getPath();
        return new File(name.replace("-req.json", "-res"));
    }


    static File getResponse(File request) {
        String name = request.getPath();
        if (!name.endsWith("-req.json")) {
            return null;
        }
        File response = new File(name.replace("-req.json", "-res.json"));
        if (!response.exists() || !request.canRead() || !response.canRead()) {
            return null;
        }
        return response;
    }


    static File getResFile(File request) {
        String name = request.getPath();
        File res = new File(name.replace("-req.json", "-res.json"));
        return res;
    }


    static File getResFolder(File request) {
        String name = request.getPath();
        File res = new File(name.replace("-req.json", "-res"));
        return res;
    }


    static File getFolder(File request, String postfix) {
        String name = request.getPath();
        if (!name.endsWith("-req.json")) {
            return null;
        }
        File inp = new File(name.replace("-req.json", postfix));
        if (!inp.exists() || !inp.canRead() || !inp.isDirectory()) {
            return null;
        }
        return inp;
    }


    static void compareKeys(JSONArray res, JSONArray gol) throws Exception {
        Map preq = JSONUtils.preprocess(res);
        Map pgol = JSONUtils.preprocess(gol);
        for (String key : pgol.keySet()) {
            if (!preq.containsKey(key)) {
                throw new Exception("Missing key in result:" + key);
            }
            preq.remove(key);
        }
        if (!preq.keySet().isEmpty()) {
            throw new Exception("Extra key(s) in result:" + preq.keySet().toString());
        }
    }


    static void compareStrict(JSONArray res, JSONArray gol) throws Exception {
        JSONAssert.assertEquals(gol.toString(), res.toString(), false);
    }


    static Properties loadProperties(File folder) throws Exception {
        Properties p = new Properties();
        File props = new File(folder.getParentFile(), "service.properties");
        if (props.exists()) {
            Reader is = new FileReader(props);
            p.load(is);
            is.close();
        }
        props = new File(folder, "service.properties");
        if (!props.exists()) {
            throw new RuntimeException("Not found: " + props);
        }
        Reader is = new FileReader(props);
        p.load(is);
        is.close();
        return p;
    }


    static String getBaseURLContext(String url) {
        // this is the first part (host + port)
        String host = url.substring(0, url.indexOf("/", url.indexOf("://") + 3));
        // this is the second part (context)
        String context = url.substring(host.length() + 1, url.indexOf("/", host.length() + 1));

        return host + "/" + context;
    }


    public static Results run(String target) throws Exception {
        return new ServiceTest(new File(target)).test();
    }


    public static Results run(String[] targets) throws Exception {
        Results summary = new Results();
        for (String target : targets) {
            Results result = new ServiceTest(new File(target)).test();
            summary.add(result);
        }
        return summary;
    }


    public static void main(String[] args) throws Exception {
        System.out.println(run(args));
    }
}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy