• Home
• io.kestra.plugin
• plugin-git
• 0.18.0
• source code
• AbstractPushTask.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

io.kestra.plugin.git.AbstractPushTask Maven / Gradle / Ivy

package io.kestra.plugin.git;

import io.kestra.core.exceptions.IllegalVariableEvaluationException;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.tasks.RunnableTask;
import io.kestra.core.runners.RunContext;
import io.kestra.core.serializers.JacksonMapper;
import io.kestra.plugin.git.services.GitService;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.annotation.Nullable;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
import org.eclipse.jgit.api.AddCommand;
import org.eclipse.jgit.api.DiffCommand;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.RmCommand;
import org.eclipse.jgit.api.errors.EmptyCommitException;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.diff.DiffEntry;
import org.eclipse.jgit.diff.DiffFormatter;
import org.eclipse.jgit.diff.Edit;
import org.eclipse.jgit.diff.EditList;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.treewalk.CanonicalTreeParser;
import org.slf4j.Logger;

import java.io.*;
import java.net.URI;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.util.*;
import java.util.function.Supplier;
import java.util.stream.Stream;

import static io.kestra.core.utils.Rethrow.*;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;

@SuperBuilder(toBuilder = true)
@NoArgsConstructor
@Getter
public abstract class AbstractPushTask extends AbstractCloningTask implements RunnableTask {
    @PluginProperty(dynamic = true)
    protected String commitMessage;

    @Schema(
        title = "If `true`, the task will only output modifications without pushing any file to Git yet. If `false` (default), all listed files will be pushed to Git immediately."
    )
    @PluginProperty
    @Builder.Default
    private boolean dryRun = false;

    @Schema(
        title = "The commit author email.",
        description = "If null, no author will be set on this commit."
    )
    @PluginProperty(dynamic = true)
    private String authorEmail;

    @Schema(
        title = "The commit author name.",
        description = "If null, the username will be used instead.",
        defaultValue = "`username`"
    )
    @PluginProperty(dynamic = true)
    private String authorName;

    public abstract String getCommitMessage();

    public abstract String getGitDirectory();

    public abstract Object globs();

    public abstract String fetchedNamespace();

    private Path createGitDirectory(RunContext runContext) throws IllegalVariableEvaluationException {
        Path flowDirectory = runContext.workingDir().resolve(Path.of(runContext.render(this.getGitDirectory())));
        flowDirectory.toFile().mkdirs();
        return flowDirectory;
    }

    protected abstract Map> instanceResourcesContentByPath(RunContext runContext, Path baseDirectory, List globs) throws Exception;

    /**
     * Removes any file from the remote that is no longer present on the instance
     */
    private void deleteOutdatedResources(Git git, Path basePath, Map> contentByPath, List globs) throws IOException, GitAPIException {
        try (Stream paths = Files.walk(basePath)) {
            Stream filteredPathsStream = paths.filter(path ->
                !contentByPath.containsKey(path) &&
                    !path.getFileName().toString().equals(".git") &&
                    !path.equals(basePath)
            );

            if (globs != null) {
                List matchers = globs.stream().map(glob -> FileSystems.getDefault().getPathMatcher("glob:" + glob)).toList();
                filteredPathsStream = filteredPathsStream.filter(path -> matchers.stream().anyMatch(matcher ->
                    matcher.matches(path) ||
                        matcher.matches(path.getFileName())
                ));
            }

            List filteredPaths = filteredPathsStream
                .map(path -> git.getRepository().getWorkTree().toPath().relativize(path).toString())
                .toList();
            if (filteredPaths.isEmpty()) {
                return;
            }

            RmCommand rm = git.rm();
            filteredPaths.forEach(rm::addFilepattern);
            rm.call();
        }
    }

    private void writeResourceFiles(Map> contentByPath) throws Exception {
        contentByPath.forEach(throwBiConsumer((path, content) -> this.writeResourceFile(path, content.get())));
    }

    protected void writeResourceFile(Path path, InputStream inputStream) throws IOException {
        if (!path.getParent().toFile().exists()) {
            path.getParent().toFile().mkdirs();
        }
        Files.copy(inputStream, path, REPLACE_EXISTING);
    }

    private URI createDiffFile(RunContext runContext, Git git) throws IOException, GitAPIException {
        File diffFile = runContext.workingDir().createTempFile(".ion").toFile();
        try (DiffFormatter diffFormatter = new DiffFormatter(null);
             BufferedWriter diffWriter = new BufferedWriter(new FileWriter(diffFile))) {
            diffFormatter.setRepository(git.getRepository());

            DiffCommand diff = git.diff();
            if (this.dryRun) {
                diff = diff.setCached(true);
            } else {
                diff = diff.setOldTree(treeIterator(git, "HEAD~1"))
                    .setNewTree(treeIterator(git, "HEAD"));
            }

            diff.call()
                .stream().sorted(Comparator.comparing(AbstractPushTask::getPath))
                .map(throwFunction(diffEntry -> {
                    EditList editList = diffFormatter.toFileHeader(diffEntry).toEditList();
                    int additions = 0;
                    int deletions = 0;
                    int changes = 0;
                    for (Edit edit : editList) {
                        int modifications = edit.getLengthB() - edit.getLengthA();
                        if (modifications > 0) {
                            additions += modifications;
                        } else if (modifications < 0) {
                            deletions += -modifications;
                        } else {
                            changes += edit.getLengthB();
                        }
                    }


                    return Map.of(
                        "file", AbstractPushTask.getPath(diffEntry),
                        "additions", "+" + additions,
                        "deletions", "-" + deletions,
                        "changes", Integer.toString(changes)
                    );
                }))
                .map(throwFunction(JacksonMapper.ofIon()::writeValueAsString))
                .forEach(throwConsumer(ionDiff -> {
                    diffWriter.write(ionDiff);
                    diffWriter.write("\n");
                    runContext.logger().debug(ionDiff);
                }));

            diffWriter.flush();
        }

        return runContext.storage().putFile(diffFile);
    }

    private static CanonicalTreeParser treeIterator(Git git, String ref) throws IOException {
        ObjectReader reader = git.getRepository().newObjectReader();
        CanonicalTreeParser treeIter = new CanonicalTreeParser();
        ObjectId oldTree = git.getRepository().resolve(ref + "^{tree}");
        if (oldTree != null) {
            treeIter.reset(reader, oldTree);
        }
        return treeIter;
    }

    private static String getPath(DiffEntry diffEntry) {
        return diffEntry.getChangeType() == DiffEntry.ChangeType.DELETE ? diffEntry.getOldPath() : diffEntry.getNewPath();
    }

    private Output push(Git git, RunContext runContext, GitService gitService) throws Exception {
        Logger logger = runContext.logger();

        String commitURL = null;
        String commitId = null;
        ObjectId commit;
        try {
            String httpUrl = gitService.getHttpUrl(runContext.render(this.url));
            if (this.isDryRun()) {
                logger.info(
                    "Dry run — no changes will be pushed to {} for now until you set the `dryRun` parameter to false",
                    httpUrl
                );
            } else {
                String renderedBranch = runContext.render(this.getBranch());
                logger.info(
                    "Pushing to {} on branch {}",
                    httpUrl,
                    renderedBranch
                );

                String message = runContext.render(this.getCommitMessage());
                ObjectId head = git.getRepository().resolve(Constants.HEAD);
                commit = git.commit()
                    .setAllowEmpty(false)
                    .setMessage(message)
                    .setAuthor(author(runContext))
                    .call()
                    .getId();
                if (head == null) {
                    git.branchRename().setNewName(renderedBranch).call();
                }
                authentified(git.push(), runContext).call();

                commitId = commit.getName();
                commitURL = buildCommitUrl(httpUrl, renderedBranch, commitId);

                logger.info("Pushed to " + commitURL);
            }
        } catch (EmptyCommitException e) {
            logger.info("No changes to commit. Skipping push.");
        }

        return Output.builder()
            .commitURL(commitURL)
            .commitId(commitId)
            .build();
    }

    private PersonIdent author(RunContext runContext) throws IllegalVariableEvaluationException {
        String name = Optional.ofNullable(this.authorName).orElse(runContext.render(this.username));
        String authorEmail = this.authorEmail;
        if (authorEmail == null || name == null) {
            return null;
        }

        return new PersonIdent(runContext.render(name), runContext.render(authorEmail));
    }

    private String buildCommitUrl(String httpUrl, String branch, String commitId) {
        if (commitId == null) {
            return null;
        }

        String commitSubroute = httpUrl.contains("bitbucket.org") ? "commits" : "commit";
        String commitUrl = httpUrl + "/" + commitSubroute + "/" + commitId;
        if (commitUrl.contains("azure.com")) {
            commitUrl = commitUrl + "?refName=refs%2Fheads%2F" + branch;
        }

        return commitUrl;
    }

    public O run(RunContext runContext) throws Exception {
        GitService gitService = new GitService(this);

        gitService.namespaceAccessGuard(runContext, this.fetchedNamespace());
        this.detectPasswordLeaks();

        Git git = gitService.cloneBranch(runContext, runContext.render(this.getBranch()), this.cloneSubmodules);

        Path localGitDirectory = this.createGitDirectory(runContext);

        List globs = Optional.ofNullable(this.globs())
            .map(globObject -> globObject instanceof List ? (List) globObject : Collections.singletonList((String) globObject))
            .map(throwFunction(runContext::render))
            .orElse(null);
        Map> contentByPath = this.instanceResourcesContentByPath(runContext, localGitDirectory, globs);

        this.deleteOutdatedResources(git, localGitDirectory, contentByPath, globs);

        this.writeResourceFiles(contentByPath);

        AddCommand add = git.add();
        add.addFilepattern(runContext.render(this.getGitDirectory()));
        add.call();

        Output pushOutput = this.push(git, runContext, gitService);

        URI diffFileStorageUri = this.createDiffFile(runContext, git);

        git.close();

        return output(pushOutput, diffFileStorageUri);
    }

    protected abstract O output(Output pushOutput, URI diffFileStorageUri);

    @SuperBuilder
    @Getter
    public static class Output implements io.kestra.core.models.tasks.Output {
        @Schema(
            title = "ID of the commit pushed."
        )
        @Nullable
        private String commitId;

        @Schema(
            title = "URL to see what’s included in the commit.",
            description = "Example format for GitHub: https://github.com/username/your_repo/commit/{commitId}."
        )
        @Nullable
        private String commitURL;

        public URI diffFileUri() {
            return null;
        }
    }
}