hudson.model.queue.BackFiller Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of hudson-core Show documentation
Contains the core Hudson code and view files to render HTML.
The newest version!
/*******************************************************************************
 *
 * Copyright (c) 2004-2010 Oracle Corporation.
 *
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *
 *
 *******************************************************************************/ 

package hudson.model.queue;

import com.google.common.collect.Iterables;
import hudson.Extension;
import hudson.model.Computer;
import hudson.model.Executor;
import hudson.model.Hudson;
import hudson.model.InvisibleAction;
import hudson.model.Queue.BuildableItem;
import hudson.model.queue.MappingWorksheet.ExecutorChunk;
import hudson.model.queue.MappingWorksheet.ExecutorSlot;
import hudson.model.queue.MappingWorksheet.Mapping;
import hudson.model.queue.MappingWorksheet.WorkChunk;
import hudson.util.TimeUnit2;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

/**
 * Experimental.
 *
 * @author Kohsuke Kawaguchi
 */
public class BackFiller extends LoadPredictor {

    private boolean recursion = false;

    @Override
    public Iterable predict(MappingWorksheet plan, Computer computer, long start, long end) {
        TimeRange timeRange = new TimeRange(start, end - start);
        List loads = new ArrayList();

        for (BuildableItem bi : Hudson.getInstance().getQueue().getBuildableItems()) {
            TentativePlan tp = bi.getAction(TentativePlan.class);
            if (tp == null) {// do this even for bi==plan.item ensures that we have FIFO semantics in tentative plans.
                tp = makeTentativePlan(bi);
                if (tp == null) {
                    continue;   // no viable plan.
                }
            }

            if (tp.isStale()) {
                // if the tentative plan is stale, just keep on pushing it to the current time
                // (if we recreate the plan, it'll be put at the end of the queue, whereas this job
                // should actually get priority over others)
                tp.range.shiftTo(System.currentTimeMillis());
            }

            // don't let its own tentative plan count when considering a scheduling for a job
            if (plan.item == bi) {
                continue;
            }


            // no overlap in the time span, meaning this plan is for a distant future
            if (!timeRange.overlapsWith(tp.range)) {
                continue;
            }

            // if this tentative plan has no baring on this computer, that's ignorable
            Integer i = tp.footprint.get(computer);
            if (i == null) {
                continue;
            }

            return Collections.singleton(tp.range.toFutureLoad(i));
        }

        return loads;
    }

    private static final class PseudoExecutorSlot extends ExecutorSlot {

        private Executor executor;

        private PseudoExecutorSlot(Executor executor) {
            this.executor = executor;
        }

        @Override
        public Executor getExecutor() {
            return executor;
        }

        @Override
        public boolean isAvailable() {
            return true;
        }

        // this slot isn't executable
        @Override
        protected void set(WorkUnit p) {
            throw new UnsupportedOperationException();
        }
    }

    private TentativePlan makeTentativePlan(BuildableItem bi) {
        if (recursion) {
            return null;
        }
        recursion = true;
        try {
            // pretend for now that all executors are available and decide some assignment that's executable.
            List slots = new ArrayList();
            for (Computer c : Hudson.getInstance().getComputers()) {
                if (c.isOffline()) {
                    continue;
                }
                for (Executor e : c.getExecutors()) {
                    slots.add(new PseudoExecutorSlot(e));
                }
            }

            // also ignore all load predictions as we just want to figure out some executable assignment
            // and we are not trying to figure out if this task is executable right now.
            MappingWorksheet worksheet = new MappingWorksheet(bi, slots, Collections.emptyList());
            Mapping m = Hudson.getInstance().getQueue().getLoadBalancer().map(bi.task, worksheet);
            if (m == null) {
                return null;
            }

            // figure out how many executors we need on each computer?
            Map footprint = new HashMap();
            for (Entry e : m.toMap().entrySet()) {
                Computer c = e.getValue().computer;
                Integer v = footprint.get(c);
                if (v == null) {
                    v = 0;
                }
                v += e.getKey().size();
                footprint.put(c, v);
            }

            // the point of a tentative plan is to displace other jobs to create a point in time
            // where this task can start executing. An incorrectly estimated duration is not
            // a problem in this regard, as we just need enough idle executors in the right moment.
            // The downside of guessing the duration wrong is that we can end up creating tentative plans
            // afterward that may be incorrect, but those plans will be rebuilt.
            long d = bi.task.getEstimatedDuration();
            if (d <= 0) {
                d = TimeUnit2.MINUTES.toMillis(5);
            }

            TimeRange slot = new TimeRange(System.currentTimeMillis(), d);

            // now, based on the real predicted loads, figure out the approximation of when we can
            // start executing this guy.
            for (Entry e : footprint.entrySet()) {
                Computer computer = e.getKey();
                Timeline timeline = new Timeline();
                for (LoadPredictor lp : LoadPredictor.all()) {
                    for (FutureLoad fl : Iterables.limit(lp.predict(worksheet, computer, slot.start, slot.end), 100)) {
                        timeline.insert(fl.startTime, fl.startTime + fl.duration, fl.numExecutors);
                    }
                }

                Long x = timeline.fit(slot.start, slot.duration, computer.countExecutors() - e.getValue());
                // if no suitable range was found in [slot.start,slot.end), slot.end would be a good approximation
                if (x == null) {
                    x = slot.end;
                }
                slot = slot.shiftTo(x);
            }

            TentativePlan tp = new TentativePlan(footprint, slot);
            bi.addAction(tp);
            return tp;
        } finally {
            recursion = false;
        }
    }

    /**
     * Represents a duration in time.
     */
    private static final class TimeRange {

        public final long start;
        public final long duration;
        public final long end;

        private TimeRange(long start, long duration) {
            this.start = start;
            this.duration = duration;
            this.end = start + duration;
        }

        public boolean overlapsWith(TimeRange that) {
            return (this.start <= that.start && that.start <= this.end)
                    || (that.start <= this.start && this.start <= that.end);
        }

        public FutureLoad toFutureLoad(int size) {
            return new FutureLoad(start, duration, size);
        }

        public TimeRange shiftTo(long newStart) {
            if (newStart == start) {
                return this;
            }
            return new TimeRange(newStart, duration);
        }
    }

    public static final class TentativePlan extends InvisibleAction {

        private final Map footprint;
        public final TimeRange range;

        public TentativePlan(Map footprint, TimeRange range) {
            this.footprint = footprint;
            this.range = range;
        }

        public Object writeReplace() {// don't persist
            return null;
        }

        public boolean isStale() {
            return range.end < System.currentTimeMillis();
        }
    }

    /**
     * Once this feature stabilizes, move it to the heavyjob plugin
     */
    @Extension
    public static BackFiller newInstance() {
        if (Boolean.getBoolean(BackFiller.class.getName())) {
            return new BackFiller();
        }
        return null;
    }
}