be.bagofwords.jobs.JobRunner Maven / Gradle / Ivy

Go to download
package be.bagofwords.jobs;


import be.bagofwords.application.CloseableComponent;
import be.bagofwords.application.annotations.BowComponent;
import be.bagofwords.counts.WindowOfCounts;
import be.bagofwords.iterator.CloseableIterator;
import be.bagofwords.iterator.DataIterable;
import be.bagofwords.ui.UI;
import be.bagofwords.util.OccasionalAction;
import be.bagofwords.util.SafeThread;
import be.bagofwords.util.Utils;

import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;

@BowComponent
public class JobRunner implements CloseableComponent {

    private final List runningJobs;
    private boolean terminateRequested = false;

    public JobRunner() {
        runningJobs = new ArrayList<>();
    }

    public  void runPartitionedJobs(final int numOfPartitions, int numOfThreads, final String name, DataIterable iterable, final PartitionableJob job) {
        runPartitionedJobs(true, numOfPartitions, numOfThreads, name, iterable, job);
    }

    public  void runPartitionedJobs(final boolean printProgress, int numOfPartitions, int numOfThreads, final String name, DataIterable iterable, final PartitionableJob job) {
        final JobStatus jobStatus = new JobStatus(name, iterable.apprSize(), numOfPartitions);
        synchronized (runningJobs) {
            runningJobs.add(jobStatus);
        }
        OccasionalAction action = new OccasionalAction(10000) {
            @Override
            protected void doAction(T curr) {
                if (printProgress) {
                    String outS = createOutputString(jobStatus);
                    UI.write(outS);
                }
            }
        };
        for (; jobStatus.getCurrentPartition() < numOfPartitions; jobStatus.setCurrentPartition(jobStatus.getCurrentPartition() + 1)) {
            CloseableIterator iterator = iterable.iterator();
            ExecuteActionRunnable[] threads = new ExecuteActionRunnable[numOfThreads];
            for (int i = 0; i < threads.length; i++) {
                threads[i] = new ExecuteActionRunnable<>(jobStatus, name + "_" + i, job, iterator, action);
                threads[i].start();
            }
            boolean allFinished = false;
            Throwable receivedException = null;
            while (!allFinished) {
                Utils.threadSleep(200);
                allFinished = true;
                if (terminateRequested) {
                    for (ExecuteActionRunnable thread : threads) {
                        thread.terminateAndWaitForFinish();
                        receivedException = new RuntimeException("Termination of application was requested.");
                    }
                } else {
                    for (ExecuteActionRunnable thread : threads) {
                        if (thread.getExp() != null) {
                            //Terminate all other threads
                            for (ExecuteActionRunnable runner : threads) {
                                runner.terminateAndWaitForFinish();
                            }
                            receivedException = thread.getExp();
                        }
                        allFinished &= thread.isFinished();
                    }
                }
            }
            iterator.close();
            if (jobStatus.getCurrentPartition() == 0) {
                jobStatus.setNumberOfObjects(jobStatus.getWindowedCounts().getTotalCounts()); //Set estimated size of iterator to actual size
            }
            if (receivedException != null) {
                throw new RuntimeException(receivedException);
            }
        }
        synchronized (runningJobs) {
            runningJobs.remove(jobStatus);
        }
    }

    private  String createOutputString(JobStatus jobStatus) {
        long did = jobStatus.getWindowedCounts().getTotalCounts();
        long todo = jobStatus.getNumberOfObjects() * jobStatus.getNumberOfPartitions() - did;
        long didInIteration = did - jobStatus.getNumberOfObjects() * jobStatus.getCurrentPartition();
        String outS = "[Progress " + jobStatus.getName() + "]";
        if (jobStatus.getNumberOfPartitions() > 1) {
            outS += " " + jobStatus.getCurrentPartition() + "/" + jobStatus.getNumberOfPartitions();
        }
        outS += " did " + didInIteration + " of " + jobStatus.getNumberOfObjects();
        String endString;
        if (todo > 0) {
            endString = " end is " + new Date(System.currentTimeMillis() + jobStatus.getWindowedCounts().getNeededTime(todo));
        } else {
            endString = " should finish any second now";
        }
        return outS + endString;
    }

    public  void runJob(int numOfThreads, final String name, DataIterable iterable, final Job job) {
        runJob(true, numOfThreads, name, iterable, job);
    }

    public  void runJob(final boolean printProgress, int numOfThreads, final String name, DataIterable iterable, final Job job) {
        runPartitionedJobs(printProgress, 1, numOfThreads, name, iterable, (partition, target) -> job.doAction(target));
    }

    public  void runJob(final String name, DataIterable iterable, final Job job) {
        runJob(true, 1, name, iterable, job);
    }

    public String createHtmlStatus() {
        StringBuilder sb = new StringBuilder();
        sb.append("Job Runner");
        synchronized (runningJobs) {
            if (runningJobs.isEmpty()) {
                sb.append("No jobs currently running.");
            } else {
                sb.append("Currently running " + runningJobs.size() + " jobs
");
                for (JobStatus runningJob : runningJobs) {
                    sb.append("" + createOutputString(runningJob) + "");
                }
            }
        }
        return sb.toString();
    }

    @Override
    public void terminate() {
        terminateRequested = true;
    }

    private static class ExecuteActionRunnable extends SafeThread {
        private final PartitionableJob job;
        private final Iterator iterator;
        private Throwable exp;
        private final OccasionalAction action;
        private final JobStatus jobStatus;

        public ExecuteActionRunnable(JobStatus jobStatus, String nameOfThread, PartitionableJob job, Iterator iterator, OccasionalAction action) {
            super(nameOfThread, false);
            this.job = job;
            this.iterator = iterator;
            this.jobStatus = jobStatus;
            this.action = action;
        }

        public Throwable getExp() {
            return exp;
        }

        @Override
        public void runInt() {
            try {
                boolean finished = false;
                while (!finished && !isTerminateRequested()) {
                    T next = null;
                    synchronized (iterator) {
                        if (iterator.hasNext()) {
                            next = iterator.next();
                        } else {
                            finished = true;
                        }
                    }
                    if (next != null) {
                        job.doAction(jobStatus.getCurrentPartition(), next);
                        jobStatus.getWindowedCounts().addCount();
                        action.doOccasionalAction(next);
                    }

                }
            } catch (Throwable exp) {
                this.exp = exp;
            }
        }
    }

    private class JobStatus {
        private final String name;
        private final WindowOfCounts windowedCounts;
        private final int numberOfPartitions;
        private long numberOfObjects;
        private int currentPartition;

        private JobStatus(String name, long numberOfObjects, int numberOfPartitions) {
            this.name = name;
            this.numberOfObjects = numberOfObjects;
            this.numberOfPartitions = numberOfPartitions;
            this.windowedCounts = new WindowOfCounts(60000);
            this.currentPartition = 0;
        }

        public long getNumberOfObjects() {
            return numberOfObjects;
        }

        public void setNumberOfObjects(long numberOfObjects) {
            this.numberOfObjects = numberOfObjects;
        }

        public WindowOfCounts getWindowedCounts() {
            return windowedCounts;
        }

        public int getCurrentPartition() {
            return currentPartition;
        }

        public void setCurrentPartition(int currentPartition) {
            this.currentPartition = currentPartition;
        }

        public String getName() {
            return name;
        }

        public int getNumberOfPartitions() {
            return numberOfPartitions;
        }
    }
}