org.dspace.scripts.ProcessServiceImpl Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of dspace-api Show documentation
DSpace core data model and service APIs.
The newest version!
/**
 * The contents of this file are subject to the license and copyright
 * detailed in the LICENSE and NOTICE files at the root of the source
 * tree and available online at
 *
 * http://www.dspace.org/license/
 */
package org.dspace.scripts;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException;
import java.time.Instant;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.regex.Pattern;

import org.apache.commons.collections4.ListUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.Logger;
import org.dspace.authorize.AuthorizeException;
import org.dspace.authorize.service.AuthorizeService;
import org.dspace.content.Bitstream;
import org.dspace.content.Item;
import org.dspace.content.MetadataField;
import org.dspace.content.MetadataValue;
import org.dspace.content.ProcessStatus;
import org.dspace.content.dao.ProcessDAO;
import org.dspace.content.service.BitstreamFormatService;
import org.dspace.content.service.BitstreamService;
import org.dspace.content.service.MetadataFieldService;
import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.core.LogHelper;
import org.dspace.eperson.EPerson;
import org.dspace.eperson.Group;
import org.dspace.scripts.service.ProcessService;
import org.dspace.services.ConfigurationService;
import org.springframework.beans.factory.annotation.Autowired;

/**
 * The implementation for the {@link ProcessService} class
 */
public class ProcessServiceImpl implements ProcessService {

    private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(ProcessService.class);

    @Autowired
    private ProcessDAO processDAO;

    @Autowired
    private BitstreamService bitstreamService;

    @Autowired
    private BitstreamFormatService bitstreamFormatService;

    @Autowired
    private AuthorizeService authorizeService;

    @Autowired
    private MetadataFieldService metadataFieldService;

    @Autowired
    private ConfigurationService configurationService;

    @Override
    public Process create(Context context, EPerson ePerson, String scriptName,
                          List parameters,
                          final Set specialGroups) throws SQLException {

        Process process = new Process();
        process.setEPerson(ePerson);
        process.setName(scriptName);
        process.setParameters(DSpaceCommandLineParameter.concatenate(parameters));
        process.setCreationTime(Instant.now());
        Optional.ofNullable(specialGroups)
            .ifPresent(sg -> {
                // we use a set to be sure no duplicated special groups are stored with process
                Set specialGroupsSet = new HashSet<>(sg);
                process.setGroups(new ArrayList<>(specialGroupsSet));
            });

        Process createdProcess = processDAO.create(context, process);

        if (ePerson != null) {
            log.info(LogHelper.getHeader(context, "process_create",
                "Process has been created for eperson with email " + ePerson.getEmail()
                    + " with ID " + createdProcess.getID() + " and scriptName " +
                    scriptName + " and parameters " + parameters));
        } else {
            log.info(LogHelper.getHeader(context, "process_create",
                "Process has been created for command-line user with ID " + createdProcess.getID()
                    + " and scriptName " + scriptName + " and parameters " + parameters));
        }
        return createdProcess;
    }

    @Override
    public Process find(Context context, int processId) throws SQLException {
        return processDAO.findByID(context, Process.class, processId);
    }

    @Override
    public List findAll(Context context) throws SQLException {
        return processDAO.findAll(context, Process.class);
    }

    @Override
    public List findAll(Context context, int limit, int offset) throws SQLException {
        return processDAO.findAll(context, limit, offset);
    }

    @Override
    public List findAllSortByScript(Context context) throws SQLException {
        return processDAO.findAllSortByScript(context);
    }

    @Override
    public List findAllSortByStartTime(Context context) throws SQLException {
        List processes = findAll(context);
        Comparator comparing = Comparator
            .comparing(Process::getStartTime, Comparator.nullsLast(Comparator.naturalOrder()));
        comparing = comparing.thenComparing(Process::getID);
        processes.sort(comparing);
        return processes;
    }

    @Override
    public List findByUser(Context context, EPerson eperson, int limit, int offset) throws SQLException {
        return processDAO.findByUser(context, eperson, limit, offset);
    }

    @Override
    public void start(Context context, Process process) throws SQLException {
        process.setProcessStatus(ProcessStatus.RUNNING);
        process.setStartTime(Instant.now());
        update(context, process);
        log.info(LogHelper.getHeader(context, "process_start", "Process with ID " + process.getID()
            + " and name " + process.getName() + " has started"));

    }

    @Override
    public void fail(Context context, Process process) throws SQLException {
        process.setProcessStatus(ProcessStatus.FAILED);
        process.setFinishedTime(Instant.now());
        update(context, process);
        log.info(LogHelper.getHeader(context, "process_fail", "Process with ID " + process.getID()
            + " and name " + process.getName() + " has failed"));

    }

    @Override
    public void complete(Context context, Process process) throws SQLException {
        process.setProcessStatus(ProcessStatus.COMPLETED);
        process.setFinishedTime(Instant.now());
        update(context, process);
        log.info(LogHelper.getHeader(context, "process_complete", "Process with ID " + process.getID()
            + " and name " + process.getName() + " has been completed"));

    }

    @Override
    public void appendFile(Context context, Process process, InputStream is, String type, String fileName)
        throws IOException, SQLException, AuthorizeException {
        Bitstream bitstream = bitstreamService.create(context, is);
        if (getBitstream(context, process, type) != null) {
            throw new IllegalArgumentException("Cannot create another file of type: " + type + " for this process" +
                                                   " with id: " + process.getID());
        }
        bitstream.setName(context, fileName);
        bitstreamService.setFormat(context, bitstream, bitstreamFormatService.guessFormat(context, bitstream));
        MetadataField dspaceProcessFileTypeField = metadataFieldService
            .findByString(context, Process.BITSTREAM_TYPE_METADATAFIELD, '.');
        bitstreamService.addMetadata(context, bitstream, dspaceProcessFileTypeField, null, type);
        authorizeService.addPolicy(context, bitstream, Constants.READ, context.getCurrentUser());
        authorizeService.addPolicy(context, bitstream, Constants.WRITE, context.getCurrentUser());
        authorizeService.addPolicy(context, bitstream, Constants.DELETE, context.getCurrentUser());
        bitstreamService.update(context, bitstream);
        process.addBitstream(bitstream);
        update(context, process);
    }

    @Override
    public void delete(Context context, Process process) throws SQLException, IOException, AuthorizeException {

        for (Bitstream bitstream : ListUtils.emptyIfNull(process.getBitstreams())) {
            bitstreamService.delete(context, bitstream);
        }
        processDAO.delete(context, process);
        log.info(LogHelper.getHeader(context, "process_delete", "Process with ID " + process.getID()
            + " and name " + process.getName() + " has been deleted"));
    }

    @Override
    public void update(Context context, Process process) throws SQLException {
        processDAO.save(context, process);
    }

    @Override
    public List getParameters(Process process) {
        if (StringUtils.isBlank(process.getParameters())) {
            return Collections.emptyList();
        }

        String[] parameterArray = process.getParameters().split(Pattern.quote(DSpaceCommandLineParameter.SEPARATOR));
        List parameterList = new ArrayList<>();

        for (String parameter : parameterArray) {
            parameterList.add(new DSpaceCommandLineParameter(parameter));
        }

        return parameterList;
    }

    @Override
    public Bitstream getBitstreamByName(Context context, Process process, String bitstreamName) {
        for (Bitstream bitstream : getBitstreams(context, process)) {
            if (StringUtils.equals(bitstream.getName(), bitstreamName)) {
                return bitstream;
            }
        }

        return null;
    }

    @Override
    public Bitstream getBitstream(Context context, Process process, String type) {
        List allBitstreams = process.getBitstreams();

        if (type == null) {
            return null;
        } else {
            if (allBitstreams != null) {
                for (Bitstream bitstream : allBitstreams) {
                    if (StringUtils.equals(bitstreamService.getMetadata(bitstream,
                                                                        Process.BITSTREAM_TYPE_METADATAFIELD), type)) {
                        return bitstream;
                    }
                }
            }
        }
        return null;
    }

    @Override
    public List getBitstreams(Context context, Process process) {
        return process.getBitstreams();
    }

    public int countTotal(Context context) throws SQLException {
        return processDAO.countRows(context);
    }

    @Override
    public List getFileTypesForProcessBitstreams(Context context, Process process) {
        List list = getBitstreams(context, process);
        Set fileTypesSet = new HashSet<>();
        for (Bitstream bitstream : list) {
            List metadata = bitstreamService.getMetadata(bitstream,
                                                                        Process.BITSTREAM_TYPE_METADATAFIELD, Item.ANY);
            if (metadata != null && !metadata.isEmpty()) {
                fileTypesSet.add(metadata.get(0).getValue());
            }
        }
        return new ArrayList<>(fileTypesSet);
    }

    @Override
    public List search(Context context, ProcessQueryParameterContainer processQueryParameterContainer,
                                int limit, int offset) throws SQLException {
        return processDAO.search(context, processQueryParameterContainer, limit, offset);
    }

    @Override
    public int countSearch(Context context, ProcessQueryParameterContainer processQueryParameterContainer)
        throws SQLException {
        return processDAO.countTotalWithParameters(context, processQueryParameterContainer);
    }


    @Override
    public void appendLog(int processId, String scriptName, String output, ProcessLogLevel processLogLevel)
            throws IOException {
        File logsDir = getLogsDirectory();
        File tempFile = new File(logsDir, processId + "-" + scriptName + ".log");
        FileWriter out = new FileWriter(tempFile, true);
        try {
            try (BufferedWriter writer = new BufferedWriter(out)) {
                writer.append(formatLogLine(processId, scriptName, output, processLogLevel));
                writer.newLine();
            }
        } finally {
            out.close();
        }
    }

    @Override
    public void createLogBitstream(Context context, Process process)
            throws IOException, SQLException, AuthorizeException {
        File logsDir = getLogsDirectory();
        File tempFile = new File(logsDir, process.getID() + "-" + process.getName() + ".log");
        if (tempFile.exists()) {
            FileInputStream inputStream = FileUtils.openInputStream(tempFile);
            appendFile(context, process, inputStream, Process.OUTPUT_TYPE,
                       process.getID() + "-" + process.getName() + ".log");
            inputStream.close();
            tempFile.delete();
        }
    }

    @Override
    public List findByStatusAndCreationTimeOlderThan(Context context, List statuses,
        Instant date) throws SQLException {
        return this.processDAO.findByStatusAndCreationTimeOlderThan(context, statuses, date);
    }

    @Override
    public int countByUser(Context context, EPerson user) throws SQLException {
        return processDAO.countByUser(context, user);
    }

    @Override
    public void failRunningProcesses(Context context) throws SQLException, IOException, AuthorizeException {
        List processesToBeFailed = findByStatusAndCreationTimeOlderThan(
                context, List.of(ProcessStatus.RUNNING, ProcessStatus.SCHEDULED), Instant.now());
        for (Process process : processesToBeFailed) {
            context.setCurrentUser(process.getEPerson());
            // Fail the process.
            log.info("Process with ID {} did not complete before tomcat shutdown, failing it now.", process.getID());
            fail(context, process);
            // But still attach its log to the process.
            appendLog(process.getID(), process.getName(),
                      "Process did not complete before tomcat shutdown.",
                      ProcessLogLevel.ERROR);
            createLogBitstream(context, process);
        }
    }

    private String formatLogLine(int processId, String scriptName, String output, ProcessLogLevel processLogLevel) {
        StringBuilder sb = new StringBuilder();
        sb.append(DateTimeFormatter.ISO_INSTANT.format(Instant.now()));
        sb.append(" ");
        sb.append(processLogLevel);
        sb.append(" ");
        sb.append(scriptName);
        sb.append(" - ");
        sb.append(processId);
        sb.append(" @ ");
        sb.append(output);
        return  sb.toString();
    }

    private File getLogsDirectory() {
        String pathStr = configurationService.getProperty("dspace.dir")
            + File.separator + "log" + File.separator + "processes";
        File logsDir = new File(pathStr);
        if (!logsDir.exists()) {
            if (!logsDir.mkdirs()) {
                throw new RuntimeException("Couldn't create [dspace.dir]/log/processes/ directory.");
            }
        }
        return logsDir;
    }
}