hudson.matrix.MatrixProject Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of hudson-core Show documentation
Contains the core Hudson code and view files to render HTML.
The newest version!
/*******************************************************************************
 *
 * Copyright (c) 2004-2011 Oracle Corporation.
 *
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 * 
 *    Kohsuke Kawaguchi, Jorg Heymans, Red Hat, Inc., id:cactusman, Anton Kozak, Nikita Levyankov
 *
 *
 *******************************************************************************/ 

package hudson.matrix;

import hudson.CopyOnWrite;
import hudson.Extension;
import hudson.Util;
import hudson.XmlFile;
import hudson.model.BaseBuildableProject;
import hudson.model.BuildableItemWithBuildWrappers;
import hudson.model.Descriptor;
import hudson.model.Descriptor.FormException;
import hudson.model.Hudson;
import hudson.model.Item;
import hudson.model.ItemGroup;
import hudson.model.Items;
import hudson.model.JDK;
import hudson.model.Job;
import hudson.model.Label;
import hudson.model.Queue.FlyweightTask;
import hudson.model.Result;
import hudson.model.SCMedItem;
import hudson.model.Saveable;
import hudson.model.TopLevelItem;
import hudson.tasks.Publisher;
import hudson.util.CascadingUtil;
import hudson.util.CopyOnWriteMap;
import hudson.util.DescribableList;
import hudson.util.DescribableListUtil;
import hudson.util.FormValidation;
import hudson.util.FormValidation.Kind;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.servlet.ServletException;
import net.sf.json.JSONObject;
import org.apache.commons.collections.CollectionUtils;
import org.eclipse.hudson.api.matrix.IMatrixProject;
import org.kohsuke.stapler.HttpResponse;
import org.kohsuke.stapler.StaplerRequest;
import org.kohsuke.stapler.StaplerResponse;
import org.kohsuke.stapler.TokenList;

/**
 * {@link Job} that allows you to run multiple different configurations from a
 * single setting.
 *
 * @author Kohsuke Kawaguchi
 */
public class MatrixProject extends BaseBuildableProject implements IMatrixProject, TopLevelItem,
        SCMedItem, ItemGroup, Saveable, FlyweightTask, BuildableItemWithBuildWrappers {

    public static final String HAS_COMBINATION_FILTER_PARAM = "hasCombinationFilter";
    public static final String HAS_TOUCH_STONE_COMBINATION_FILTER_PARAM = "hasTouchStoneCombinationFilter";
    public static final String TOUCH_STONE_COMBINATION_FILTER_PARAM = "touchStoneCombinationFilter";
    public static final String TOUCH_STONE_RESULT_CONDITION_PARAM = "touchStoneResultCondition";
    public static final String CUSTOM_WORKSPACE_PARAM = "customWorkspace";
    public static final String CUSTOM_WORKSPACE_DIRECTORY_PARAM = "customWorkspace.directory";
    public static final String RUN_SEQUENTIALLY_PROPERTY_NAME = "runSequentially";
    public static final String COMBINATION_FILTER_PROPERTY_NAME = "combinationFilter";
    public static final String TOUCH_STONE_COMBINATION_FILTER_PROPERTY_NAME = "touchStoneCombinationFilter";
    public static final String TOUCH_STONE_RESULT_CONDITION_PROPERTY_NAME = "touchStoneResultCondition";
    public static final String AXES_PROPERTY_NAME = "axes";
    protected static final String AXIS_CONFIGURATIONS_DIR = "configurations";
    /**
     * Configuration axes.
     *
     * @deprecated as of 2.2.0, use #getAxes() and #setAxes() instead
     */
    @Deprecated
    private volatile AxisList axes = new AxisList();
    /**
     * The filter that is applied to combinations. It is a Dynamic Language
     * Script if condition. This can be null, which means "true". Package
     * visible for the tests only.
     *
     * @deprecated as of 2.2.0, use #getCombinationFilter() and
     * #setCombinationFilter() instead
     */
    @Deprecated
    private volatile String combinationFilter;
    /**
     * All {@link MatrixConfiguration}s, keyed by their
     * {@link MatrixConfiguration#getName() names}.
     */
    private transient /*final*/ Map configurations = new CopyOnWriteMap.Tree();
    /**
     * @see #getActiveConfigurations()
     */
    @CopyOnWrite
    private transient /*final*/ Set activeConfigurations = new LinkedHashSet();
    /**
     * @deprecated as of 2.2.0, use #isRunSequentially() and
     * #setRunSequentially() instead
     */
    @Deprecated
    private boolean runSequentially;
    /**
     * Filter to select a number of combinations to build first
     *
     * @deprecated as of 2.2.0, use #getTouchStoneCombinationFilter() and
     * #setTouchStoneCombinationFilter() instead
     */
    @Deprecated
    private String touchStoneCombinationFilter;
    /**
     * Required result on the touchstone combinations, in order to continue with
     * the rest
     *
     * @deprecated as of 2.2.0, use #getTouchStoneResultCondition() and
     * #setTouchStoneResultCondition() instead
     */
    @Deprecated
    private Result touchStoneResultCondition;
    /**
     * @deprecated as of 2.2.0, use #getCustomWorkspace() and
     * #setCustomWorkspace() instead
     */
    @Deprecated
    private String customWorkspace;

    public MatrixProject(String name) {
        this(Hudson.getInstance(), name);
    }

    public MatrixProject(ItemGroup parent, String name) {
        super(parent, name);
    }

    /**
     * @inheritDoc
     */
    public AxisList getAxes() {
        return CascadingUtil.getAxesListProjectProperty(this, AXES_PROPERTY_NAME).getValue();
    }

    /**
     * @inheritDoc
     */
    public void setAxes(AxisList axes) throws IOException {
        CascadingUtil.getAxesListProjectProperty(this, AXES_PROPERTY_NAME).setValue(axes);
        rebuildConfigurations();
        save();
    }

    /**
     * @inheritDoc
     */
    public boolean isRunSequentially() {
        return CascadingUtil.getBooleanProjectProperty(this, RUN_SEQUENTIALLY_PROPERTY_NAME).getValue();
    }

    /**
     * @inheritDoc
     */
    public void setRunSequentially(boolean runSequentially) throws IOException {
        CascadingUtil.getBooleanProjectProperty(this, RUN_SEQUENTIALLY_PROPERTY_NAME).setValue(runSequentially);
        save();
    }

    /**
     * @inheritDoc
     */
    public String getCombinationFilter() {
        return CascadingUtil.getStringProjectProperty(this, COMBINATION_FILTER_PROPERTY_NAME).getValue();
    }

    /**
     * @inheritDoc
     */
    public void setCombinationFilter(String combinationFilter) throws IOException {
        CascadingUtil.getStringProjectProperty(this, COMBINATION_FILTER_PROPERTY_NAME).setValue(combinationFilter);
        rebuildConfigurations();
        save();
    }

    /**
     * @inheritDoc
     */
    public String getTouchStoneCombinationFilter() {
        return CascadingUtil.getStringProjectProperty(this, TOUCH_STONE_COMBINATION_FILTER_PROPERTY_NAME).getValue();
    }

    /**
     * @inheritDoc
     */
    public void setTouchStoneCombinationFilter(String touchStoneCombinationFilter) {
        CascadingUtil.getStringProjectProperty(this, TOUCH_STONE_COMBINATION_FILTER_PROPERTY_NAME)
                .setValue(touchStoneCombinationFilter);
    }

    /**
     * @inheritDoc
     */
    public Result getTouchStoneResultCondition() {
        return CascadingUtil.getResultProjectProperty(this, TOUCH_STONE_RESULT_CONDITION_PROPERTY_NAME).getValue();
    }

    /**
     * @inheritDoc
     */
    public void setTouchStoneResultCondition(Result touchStoneResultCondition) {
        CascadingUtil.getResultProjectProperty(this,
                TOUCH_STONE_RESULT_CONDITION_PROPERTY_NAME).setValue(touchStoneResultCondition);
    }

    /**
     * @inheritDoc
     */
    public String getCustomWorkspace() {
        return CascadingUtil.getStringProjectProperty(this, CUSTOM_WORKSPACE_PROPERTY_NAME).getValue();
    }

    /**
     * @inheritDoc
     */
    public void setCustomWorkspace(String customWorkspace) throws IOException {
        CascadingUtil.getStringProjectProperty(this, CUSTOM_WORKSPACE_PROPERTY_NAME).setValue(customWorkspace);
    }

    @Override
    protected void buildProjectProperties() throws IOException {
        super.buildProjectProperties();
        //Convert legacy properties to IProjectProperty logic
        if (null != axes && null == getProperty(AXES_PROPERTY_NAME)) {
            setAxes(axes);
            axes = null; //Reset to null. No longer needed.
        }
        if (null != combinationFilter && null == getProperty(COMBINATION_FILTER_PROPERTY_NAME)) {
            setCombinationFilter(combinationFilter);
            combinationFilter = null; //Reset to null. No longer needed.
        }
        if (null == getProperty(RUN_SEQUENTIALLY_PROPERTY_NAME)) {
            setRunSequentially(runSequentially);
            runSequentially = false;
        }
        if (null != touchStoneCombinationFilter && null == getProperty(TOUCH_STONE_COMBINATION_FILTER_PROPERTY_NAME)) {
            setTouchStoneCombinationFilter(touchStoneCombinationFilter);
            touchStoneCombinationFilter = null; //Reset to null. No longer needed.
        }
        if (null != touchStoneResultCondition && null == getProperty(TOUCH_STONE_RESULT_CONDITION_PROPERTY_NAME)) {
            setTouchStoneResultCondition(touchStoneResultCondition);
            touchStoneResultCondition = null; //Reset to null. No longer needed.
        }
        if (null != customWorkspace && null == getProperty(CUSTOM_WORKSPACE_PROPERTY_NAME)) {
            setCustomWorkspace(customWorkspace);
            customWorkspace = null; //Reset to null. No longer needed.
        }
        save();
        rebuildConfigurations();
    }

    /**
     * Gets the subset of {@link AxisList} that are not system axes.
     *
     * @deprecated as of 1.373 System vs user difference are generalized into
     * extension point.
     */
    public List getUserAxes() {
        List r = new ArrayList();
        for (Axis a : getAxes()) {
            if (!a.isSystem()) {
                r.add(a);
            }
        }
        return r;
    }

    public Layouter getLayouter() {
        return new Layouter(getAxes()) {
            protected MatrixConfiguration getT(Combination c) {
                return getItem(c);
            }
        };
    }

    @Override
    public void onLoad(ItemGroup parent, String name) throws IOException {
        super.onLoad(parent, name);
        AxisList axes = getAxes();
        if (!CollectionUtils.isEmpty(axes)) {
            // perhaps the file was edited on disk and the sort order might have been broken
            Collections.sort(getAxes());
        }
        rebuildConfigurations();
    }

    @Override
    public void logRotate() throws IOException, InterruptedException {
        super.logRotate();
        // perform the log rotation of inactive configurations to make sure
        // their logs get eventually discarded
        for (MatrixConfiguration config : configurations.values()) {
            if (!config.isActiveConfiguration()) {
                config.logRotate();
            }
        }
    }

    /**
     * Recursively search for configuration and put them to the map
     *
     *  The directory structure would be axis-a/b/axis-c/d/axis-e/f
     * for combination [a=b,c=d,e=f]. Note that two combinations [a=b,c=d] and
     * [a=b,c=d,e=f] can both co-exist (where one is an archived record and the
     * other is live, for example) so search needs to be thorough.
     *
     * @param dir Directory to be searched.
     * @param result Receives the loaded {@link MatrixConfiguration}s.
     * @param combination Combination of key/values discovered so far while
     * traversing the directories. Read-only.
     */
    private void loadConfigurations(File dir, CopyOnWriteMap.Tree result, Map combination) {
        File[] axisDirs = dir.listFiles(new FileFilter() {
            public boolean accept(File child) {
                return child.isDirectory() && child.getName().startsWith("axis-");
            }
        });
        if (axisDirs == null) {
            return;
        }

        for (File subdir : axisDirs) {
            String axis = subdir.getName().substring(5);    // axis name

            File[] valuesDir = subdir.listFiles(new FileFilter() {
                public boolean accept(File child) {
                    return child.isDirectory();
                }
            });
            if (valuesDir == null) {
                continue;   // no values here
            }
            for (File v : valuesDir) {
                Map c = new HashMap(combination);
                c.put(axis, TokenList.decode(v.getName()));

                try {
                    XmlFile config = Items.getConfigFile(v);
                    if (config.exists()) {
                        Combination comb = new Combination(c);
                        // if we already have this in memory, just use it.
                        // otherwise load it
                        MatrixConfiguration item = null;
                        if (this.configurations != null) {
                            item = this.configurations.get(comb);
                        }
                        if (item == null) {
                            item = (MatrixConfiguration) config.read();
                            item.setCombination(comb);
                            item.onLoad(this, v.getName());
                        }
                        result.put(item.getCombination(), item);
                    }
                } catch (IOException e) {
                    LOGGER.log(Level.WARNING, "Failed to load matrix configuration " + v, e);
                }
                loadConfigurations(v, result, c);
            }
        }
    }

    /**
     * Rebuilds the {@link #configurations} list and
     * {@link #activeConfigurations}.
     */
    void rebuildConfigurations() throws IOException {
        // backward compatibility check to see if there's any data in the old structure
        // if so, bring them to the newer structure.
        File[] oldDirs = getConfigurationsDir().listFiles(new FileFilter() {
            public boolean accept(File child) {
                return child.isDirectory() && !child.getName().startsWith("axis-");
            }
        });
        //TODO seems oldDir is always null and old matrix configuration is not cleared.
        if (oldDirs != null) {
            // rename the old directory to the new one
            for (File dir : oldDirs) {
                try {
                    Combination c = Combination.fromString(dir.getName());
                    dir.renameTo(getRootDirFor(c));
                } catch (IllegalArgumentException e) {
                    // it's not a configuration dir. Just ignore.
                }
            }
        }

        CopyOnWriteMap.Tree configurations =
                new CopyOnWriteMap.Tree();
        loadConfigurations(getConfigurationsDir(), configurations, Collections.emptyMap());
        this.configurations = configurations;

        // find all active configurations
        Set active = new LinkedHashSet();
        AxisList axes = getAxes();
        if (!CollectionUtils.isEmpty(axes)) {
            for (Combination c : axes.list()) {
                String combinationFilter = getCombinationFilter();
                if (c.evalScriptExpression(axes, combinationFilter)) {
                    LOGGER.fine("Adding configuration: " + c);
                    MatrixConfiguration config = configurations.get(c);
                    if (config == null) {
                        config = new MatrixConfiguration(this, c);
                        config.save();
                        configurations.put(config.getCombination(), config);
                    }
                    active.add(config);
                }
            }
        }
        this.activeConfigurations = active;
    }

    private File getConfigurationsDir() {
        return new File(getRootDir(), AXIS_CONFIGURATIONS_DIR);
    }

    /**
     * Gets all active configurations.  In contract, inactive configurations
     * are those that are left for archival purpose and no longer built when a
     * new {@link MatrixBuild} is executed.
     */
    public Collection getActiveConfigurations() {
        return activeConfigurations;
    }

    public Collection getItems() {
        return configurations.values();
    }

    @Override
    public Collection getAllJobs() {
        Set jobs = new HashSet(getItems());
        jobs.add(this);
        return jobs;
    }

    public String getUrlChildPrefix() {
        return ".";
    }

    public MatrixConfiguration getItem(String name) {
        return getItem(Combination.fromString(name));
    }

    public MatrixConfiguration getItem(Combination c) {
        return configurations.get(c);
    }

    public File getRootDirFor(MatrixConfiguration child) {
        return getRootDirFor(child.getCombination());
    }

    public void onRenamed(MatrixConfiguration item, String oldName, String newName) throws IOException {
        throw new UnsupportedOperationException();
    }

    public void onDeleted(MatrixConfiguration item) throws IOException {
        // noop
    }

    public File getRootDirFor(Combination combination) {
        File f = getConfigurationsDir();
        for (Entry e : combination.entrySet()) {
            f = new File(f, "axis-" + e.getKey() + '/' + Util.rawEncode(e.getValue()));
        }
        f.getParentFile().mkdirs();
        return f;
    }

    /**
     * @see #getJDKs()
     */
    @Override
    @Deprecated
    public JDK getJDK() {
        return super.getJDK();
    }

    /**
     * Gets the {@link JDK}s where the builds will be run.
     *
     * @return never null but can be empty
     */
    public Set getJDKs() {
        Axis a = getAxes().find("jdk");
        if (a == null) {
            return Collections.emptySet();
        }
        Set r = new HashSet();
        for (String j : a) {
            JDK jdk = Hudson.getInstance().getJDK(j);
            if (jdk != null) {
                r.add(jdk);
            }
        }
        return r;
    }

    /**
     * Gets the {@link Label}s where the builds will be run.
     *
     * @return never null
     */
    public Set getLabels() {
        Set r = new HashSet();
        for (Combination c : getAxes().subList(LabelAxis.class).list()) {
            r.add(Hudson.getInstance().getLabel(Util.join(c.values(), "&&")));
        }
        return r;
    }

    public Publisher getPublisher(Descriptor descriptor) {
        for (Publisher p : getPublishersList()) {
            if (p.getDescriptor() == descriptor) {
                return p;
            }
        }
        return null;
    }

    protected Class getBuildClass() {
        return MatrixBuild.class;
    }

    public boolean isFingerprintConfigured() {
        return false;
    }

    public MatrixProject asProject() {
        return this;
    }

    @Override
    public Object getDynamic(String token, StaplerRequest req, StaplerResponse rsp) {
        try {
            MatrixConfiguration item = getItem(token);
            if (item != null) {
                return item;
            }
        } catch (IllegalArgumentException _) {
            // failed to parse the token as Combination. Must be something else
        }
        return super.getDynamic(token, req, rsp);
    }
    
    private boolean configSubmit = false;

    @Override
    protected void submit(StaplerRequest req, StaplerResponse rsp) throws IOException, ServletException, FormException {
        configSubmit = true;
        super.submit(req, rsp);

        JSONObject json = req.getSubmittedForm();

        setCombinationFilter(
                req.getParameter(HAS_COMBINATION_FILTER_PARAM) != null ? Util.nullify(req.getParameter(
                COMBINATION_FILTER_PROPERTY_NAME)) : null);

        if (req.getParameter(HAS_TOUCH_STONE_COMBINATION_FILTER_PARAM) != null) {
            setTouchStoneCombinationFilter(Util.nullify(req.getParameter(TOUCH_STONE_COMBINATION_FILTER_PARAM)));
            setTouchStoneResultCondition(Result.fromString(req.getParameter(TOUCH_STONE_RESULT_CONDITION_PARAM)));
        } else {
            setTouchStoneCombinationFilter(null);
        }

        setCustomWorkspace(
                req.hasParameter(CUSTOM_WORKSPACE_PARAM) ? req.getParameter(CUSTOM_WORKSPACE_DIRECTORY_PARAM) : null);

        // parse system axes
        DescribableList newAxes = DescribableListUtil.buildFromHetero(this, req, json, "axis",
                Axis.all());
        checkAxisNames(newAxes);
        setAxes(new AxisList(newAxes.toList()));

        setRunSequentially(json.has(RUN_SEQUENTIALLY_PROPERTY_NAME));

        rebuildConfigurations();
        configSubmit = false;
    }

    /**
     * Verifies that Axis names are valid and unique.
     */
    private void checkAxisNames(Iterable newAxes) throws FormException {
        HashSet axisNames = new HashSet();
        for (Axis a : newAxes) {
            FormValidation fv = a.getDescriptor().doCheckName(a.getName());
            if (fv.kind != Kind.OK) {
                throw new FormException(Messages.MatrixProject_DuplicateAxisName(), fv, "axis.name");
            }

            if (axisNames.contains(a.getName())) {
                throw new FormException(Messages.MatrixProject_DuplicateAxisName(), "axis.name");
            }
            axisNames.add(a.getName());
        }
    }

    /**
     * Also delete all the workspaces of the configuration, too.
     */
    @Override
    public HttpResponse doDoWipeOutWorkspace() throws IOException, ServletException, InterruptedException {
        HttpResponse rsp = super.doDoWipeOutWorkspace();
        for (MatrixConfiguration c : configurations.values()) {
            c.doDoWipeOutWorkspace();
        }
        return rsp;
    }

    public DescriptorImpl getDescriptor() {
        return DESCRIPTOR;
    }
    @Extension
    public static final DescriptorImpl DESCRIPTOR = new DescriptorImpl();

    public static final class DescriptorImpl extends AbstractProjectDescriptor {

        public String getDisplayName() {
            return Messages.MatrixProject_DisplayName();
        }

        public MatrixProject newInstance(ItemGroup parent, String name) {
            return new MatrixProject(parent, name);
        }

        /**
         * All {@link AxisDescriptor}s that contribute to the UI.
         */
        public List getAxisDescriptors() {
            List r = new ArrayList();
            for (AxisDescriptor d : Axis.all()) {
                if (d.isInstantiable()) {
                    r.add(d);
                }
            }
            return r;
        }
    }

    /**
     * For the unit tests only. Sets cascadingProject for the job.
     *
     * @param cascadingProject parent job
     */
    void setCascadingProject(MatrixProject cascadingProject) {
        this.cascadingProject = cascadingProject;
    }

    @Override
    public void setCascadingProjectName(String cascadingProjectName) throws IOException {
        super.setCascadingProjectName(cascadingProjectName);
        try {
            // Config submit will rebuild configuration, so do not do it here
            if (!configSubmit) {
                rebuildConfigurations();
            }
        } catch (IOException e) {
            LOGGER.log(Level.WARNING, "Failed to rebuild matrix configuration", e);
        }
    }
    private static final Logger LOGGER = Logger.getLogger(MatrixProject.class.getName());
}