• Home
• net.sf.filepiper
• file-piper
• 1.2
• source code
• Pipeline.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

net.sf.filePiper.model.Pipeline Maven / Gradle / Ivy

package net.sf.filePiper.model;


import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import net.sf.filePiper.processors.HeadProcessor;
import net.sf.sfac.file.FilePathUtils;
import net.sf.sfac.file.InvalidPathException;
import net.sf.sfac.setting.Settings;
import net.sf.sfac.setting.SubSettingsList;
import net.sf.sfac.setting.SubSettingsProxy;

import org.apache.log4j.Logger;


/**
 * The pipeline is the object controlling the overall processing. 

 * The pipeline is composed of:
 * 

 * 
A pipeline start: The object selecting the input file(s) and creating the initial input stream(s).
 * 
A list of ProcessorThread: each ProcessorThread is controlling the job of one FileProcessor object. If there are several
 * processorThreads, they will create Piped streams to communicate.
 * 
A pipeline end: The component of the pipeline writing the resulting output stream(s) to file(s).
 * 
 * 
 * @author berol
 */
public class Pipeline {


    private static final String SETTING_PROCESSOR_CLASS = "processor.class";

    private static final String SETTING_SOURCE_FILE = "pipeline.source.file";
    private static final String SETTING_SOURCE_MULTI_FILE = "pipeline.source.multi.file";
    private static final String SETTING_SOURCE_INCLUDES = "pipeline.source.includes";
    private static final String SETTING_SOURCE_EXCLUDES = "pipeline.source.excludes";

    private static final String SETTING_OUTPUT_NAME_CHOICE = "pipeline.output.type";
    private static final String SETTING_OUTPUT_FILE = "pipeline.output.file";
    private static final String SETTING_OUTPUT_DESTINATION = "pipeline.output.destination";

    /** Possible options for ouput name choice */
    public static final int OUTPUT_NAME_CURRENT = 0;
    public static final int OUTPUT_NAME_PROPOSED = 1;
    public static final int OUTPUT_NAME_NEW = 2;

    /** Possible option for ouput destination */
    public static final int OUTPUT_TO_CONSOLE = 10;
    public static final int OUTPUT_TO_FILE = 11;

    static Logger log = Logger.getLogger(Pipeline.class);

    private List processors;
    private Settings sett;
    private SubSettingsList subSettings;

    private boolean sourceMultiFile;
    /** Source file if single input and source base directory if multiple input */
    private File sourceFileOrDirectory;
    private String includesPattern;
    private String excludesPattern;

    private int outputDestination;
    private int outputNameChoice;
    /** Output file if single output and output base directory if multiple output */
    private File outputFileOrDirectory;


    public Pipeline(Settings setts) {
        sett = setts;
        subSettings = (sett == null) ? null : new SubSettingsList(sett, "processor");
        processors = new ArrayList();
        init();
    }


    public void reset() {
        subSettings.synchronizeWithSettings();
        processors.clear();
        init();
    }


    /**
     * Init from Settings.
     */
    private void init() {
        // source
        sourceMultiFile = sett.getBooleanProperty(SETTING_SOURCE_MULTI_FILE, false);
        String sourceFileStr = sett.getFileProperty(SETTING_SOURCE_FILE, null);
        sourceFileOrDirectory = (sourceFileStr == null) ? null : new File(sourceFileStr);
        includesPattern = sett.getStringProperty(SETTING_SOURCE_INCLUDES, null);
        excludesPattern = sett.getStringProperty(SETTING_SOURCE_EXCLUDES, null);
        // output
        outputNameChoice = sett.getIntProperty(SETTING_OUTPUT_NAME_CHOICE, OUTPUT_NAME_PROPOSED);
        outputDestination = sett.getIntProperty(SETTING_OUTPUT_DESTINATION, OUTPUT_TO_FILE);
        String outputFileStr = sett.getFileProperty(SETTING_OUTPUT_FILE, null);
        outputFileOrDirectory = (outputFileStr == null) ? null : new File(outputFileStr);
        // processors
        int nbrProcessor = (subSettings == null) ? 0 : subSettings.getSize();
        if (nbrProcessor > 0) {
            log.info("Loading pipeline from settings: " + nbrProcessor + " processors");
            for (int i = 0; i < nbrProcessor; i++) {
                Settings subSett = subSettings.getSubSettingAt(i);
                String className = subSett.getStringProperty(SETTING_PROCESSOR_CLASS,
                        "net.sf.filePiper.processors.HeadProcessor");
                try {
                    if (log.isDebugEnabled()) log.debug("  * add " + className);
                    FileProcessor fp = (FileProcessor) Class.forName(className).newInstance();
                    fp.init(subSett);
                    processors.add(fp);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        } else {
            log.info("Add default processor for pipeline");
            addProcessor(new HeadProcessor());
        }
    }


    public boolean isSourceMultiFile() {
        return sourceMultiFile;
    }


    public void setSourceMultiFile(boolean multi) {
        sourceMultiFile = multi;
        sett.setBooleanProperty(SETTING_SOURCE_MULTI_FILE, sourceMultiFile);
    }


    public int getOutputDestination() {
        return outputDestination;
    }


    public void setOutputDestination(int newOutputDestination) {
        outputDestination = newOutputDestination;
        sett.setIntProperty(SETTING_OUTPUT_DESTINATION, outputDestination);
    }


    public int getOutputNameChoice() {
        return outputNameChoice;
    }


    public void setOutputNameChoice(int newOutputNameChoice) {
        outputNameChoice = newOutputNameChoice;
        sett.setIntProperty(SETTING_OUTPUT_NAME_CHOICE, outputNameChoice);
    }


    public File getSource() {
        return sourceFileOrDirectory;
    }


    public void setSource(File newSource) {
        try {
            sett.setFileProperty(SETTING_SOURCE_FILE, (newSource == null) ? null : newSource.getAbsolutePath());
            sourceFileOrDirectory = newSource;
        } catch (InvalidPathException ipe) {
            log.warn("Invalid source path: " + newSource, ipe);
        }
    }


    public String getIncludesPattern() {
        return includesPattern;
    }


    public void setIncludesPattern(String newIncludesPattern) {
        includesPattern = newIncludesPattern;
        sett.setStringProperty(SETTING_SOURCE_INCLUDES, includesPattern);
    }


    public String getExcludesPattern() {
        return excludesPattern;
    }


    public void setExcludesPattern(String newExcludesPattern) {
        excludesPattern = newExcludesPattern;
        sett.setStringProperty(SETTING_SOURCE_EXCLUDES, excludesPattern);
    }


    public File getOutputFile() {
        return outputFileOrDirectory;
    }


    public void setOutputFile(File newOutputFile) {
        if (log.isDebugEnabled()) log.debug("Set destination to: " + newOutputFile);
        try {
            sett.setFileProperty(SETTING_OUTPUT_FILE, (newOutputFile == null) ? null : newOutputFile.getAbsolutePath());
            outputFileOrDirectory = newOutputFile;
        } catch (InvalidPathException ipe) {
            log.warn("Invalid destination path: " + newOutputFile, ipe);
        }
    }


    public Settings getSettings() {
        return sett;
    }


    public void changeProcessorAt(int processorIndex, FileProcessor newProto) {
        FileProcessor odlFp = processors.get(processorIndex);
        SubSettingsProxy sub = (SubSettingsProxy) subSettings.getSubSettingAt(processorIndex);
        try {
            FileProcessor newFp = newProto.getClass().newInstance();
            sub.clear();
            sub.setStringProperty(SETTING_PROCESSOR_CLASS, newFp.getClass().getName());
            newFp.init(sub);
            log.info("Edit pipeline at " + processorIndex + ":  replace " + odlFp + " by " + newFp);
            processors.set(processorIndex, newFp);
        } catch (Exception e) {
            log.error("Cannot instantiate new FileProcessor of " + newProto.getClass(), e);
        }

    }


    public void removeProcessor(int index) {
        FileProcessor odlFp = processors.remove(index);
        subSettings.removeSubSettingAt(index);
        log.info("Remove " + odlFp + " from pipeline at " + index);
    }


    public void addProcessor(FileProcessor fp) {
        addProcessor(processors.size(), fp);
    }


    public void addProcessor(int index, FileProcessor fp) {
        processors.add(index, fp);
        if (subSettings != null) {
            Settings sub = subSettings.addSubSetting(index);
            sub.setStringProperty(SETTING_PROCESSOR_CLASS, fp.getClass().getName());
            fp.init(sub);
        }
    }


    public void duplicateProcessor(int index) {
        FileProcessor currentProcessor = processors.get(index);
        try {
            FileProcessor newProcessor = currentProcessor.getClass().newInstance();
            processors.add(index, newProcessor);
            if (subSettings != null) {
                SubSettingsProxy sourceSub = (SubSettingsProxy) subSettings.getSubSettingAt(index);
                SubSettingsProxy newSub = (SubSettingsProxy) subSettings.addSubSetting(index);
                newSub.copyValues(sourceSub);
                newProcessor.init(newSub);
            }
        } catch (Exception e) {
            log.error("Cannot create duplicate of " + currentProcessor, e);
            throw new IllegalStateException("Unable to create new processor", e);
        }
    }


    public List getProcessors() {
        return processors;
    }


    public void process(final PipelineEnvironment reporter) throws IOException {
        Thread processThead = new Thread(new Runnable() {


            public void run() {
                try {
                    reporter.startProcessing();
                    processAsynch(reporter);
                    reporter.finished(null);
                } catch (Exception e) {
                    log.error("Asynchrone processing stopped by exception", e);
                    reporter.finished(e);
                }
            }

        }, "MainProcessing");
        processThead.start();
    }


    void processAsynch(PipelineEnvironment reporter) throws IOException {
        log.info("Start file processing batch");

        // create the processing threads
        PipelineStart pipeStart = new PipelineStart(this);
        List threads = startThreads(reporter);
        if (log.isDebugEnabled()) log.debug("Threads created");

        // search for files and process them
        ProcessorThread firstProcessorThread = threads.get(0);
        pipeStart.processInputFiles(firstProcessorThread, reporter);
        firstProcessorThread.finished();

        // wait until all threads are finished
        if (log.isDebugEnabled()) log.debug("Waiting for all threads");
        for (ProcessorThread pt : threads) {
            try {
                if (log.isDebugEnabled()) log.debug("  * Waiting for thread of " + pt);
                pt.join();
            } catch (Exception e) {
                log.warn("Exception during join", e);
            }
        }
        log.info("File processing batch done");
        reporter.finished(null);
    }


    private List startThreads(PipelineEnvironment reporter) {
        List threads = new ArrayList();
        // create end component
        FilePathUtils toBaseDir = null;
        if ((getOutputDestination() == OUTPUT_TO_FILE) && (getOutputCardinality() == FileProcessor.MANY)) {
            if (outputFileOrDirectory == null) throw new IllegalArgumentException("You must define a destination directory");
            toBaseDir = new FilePathUtils(outputFileOrDirectory);
        }
        PipeComponent next = new PipelineEnd(this, toBaseDir, reporter);
        // create one ProcessorThread per processor, attach it to the next pipeline component and start its Thread.
        int len = processors.size();
        if (len == 0) throw new IllegalStateException("You must define at least one file processor");
        log.info("Create " + len + " processing threads");
        for (int i = len - 1; i >= 0; i--) {
            if (log.isDebugEnabled()) log.debug("Create thread for " + processors.get(i));
            ProcessorThread pt = new ProcessorThread(processors.get(i), this, next, reporter);
            threads.add(0, pt);
            pt.start();
            next = pt;
        }
        return threads;
    }


    public int getOutputCardinality() {
        int cardinality = sourceMultiFile ? FileProcessor.MANY : FileProcessor.ONE;
        for (FileProcessor fp : processors) {
            cardinality = fp.getOutputCardinality(cardinality);
        }
        return cardinality;
    }


}