io.kestra.plugin.gcp.cli.GCloudCLI Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of plugin-gcp Show documentation
Integrate Google Cloud Platform services with Kestra data workflows.
There is a newer version: 0.21.2
package io.kestra.plugin.gcp.cli;

import io.kestra.core.exceptions.IllegalVariableEvaluationException;
import io.kestra.core.models.annotations.Example;
import io.kestra.core.models.annotations.Plugin;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.tasks.runners.ScriptService;
import io.kestra.core.models.tasks.*;
import io.kestra.core.models.tasks.runners.TaskRunner;
import io.kestra.core.runners.RunContext;
import io.kestra.plugin.scripts.exec.scripts.models.DockerOptions;
import io.kestra.plugin.scripts.exec.scripts.models.RunnerType;
import io.kestra.plugin.scripts.exec.scripts.models.ScriptOutput;
import io.kestra.plugin.scripts.exec.scripts.runners.CommandsWrapper;
import io.kestra.plugin.scripts.runner.docker.Docker;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import lombok.*;
import lombok.experimental.SuperBuilder;

import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import java.io.IOException;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

@SuperBuilder
@ToString
@EqualsAndHashCode
@Getter
@NoArgsConstructor
@Schema(
        title = "Execute gcloud commands."
)
@Plugin(
        examples = {
                @Example(
                        title = "Create a cluster then list them using a service account.",
                        full = true,
                        code = """
                            id: gcp_g_cloud_cli
                            namespace: company.team
            
                            tasks:
                              - id: g_cloud_cli
                                type: io.kestra.plugin.gcp.cli.GCloudCLI
                                projectId: my-gcp-project
                                serviceAccount: "{{ secret('gcp-sa') }}"
                                commands:
                                  - gcloud container clusters create simple-cluster --region=europe-west3
                                  - gcloud container clusters list
                            """
                ),
                @Example(
                        title = "Create a GCS bucket.",
                        full = true,
                        code = """
                            id: gcp_g_cloud_cli
                            namespace: company.team
            
                            tasks:
                              - id: g_cloud_cli
                                type: io.kestra.plugin.gcp.cli.GCloudCLI
                                projectId: my-gcp-project
                                serviceAccount: "{{ secret('gcp-sa') }}"
                                commands:
                                  - gcloud storage buckets create gs://my-bucket
                            """
                ),
                @Example(
                        title = "Output the result of a command.",
                        full = true,
                        code = """
                            id: gcp_g_cloud_cli
                            namespace: company.team
            
                            tasks:
                              - id: g_cloud_cli
                                type: io.kestra.plugin.gcp.cli.GCloudCLI
                                projectId: my-gcp-project
                                serviceAccount: "{{ secret('gcp-sa') }}"
                                commands:
                                  # Outputs as a flow output for UI display
                                  - gcloud pubsub topics list --format=json | tr -d '\n ' | xargs -0 -I {} echo '::{"outputs":{"gcloud":{}}}::'
                                
                                  # Outputs as a file, preferred way for large payloads
                                  - gcloud storage ls --json > storage.json
                            """
                )
        }
)
public class GCloudCLI extends Task implements RunnableTask, NamespaceFilesInterface, InputFilesInterface, OutputFilesInterface {
    private static final String DEFAULT_IMAGE = "google/cloud-sdk";

    @Schema(
        title = "The full service account JSON key to use to authenticate to gcloud."
    )
    @PluginProperty(dynamic = true)
    protected String serviceAccount;

    @Schema(
        title = "The GCP project ID to scope the commands to."
    )
    @PluginProperty(dynamic = true)
    protected String projectId;

    @Schema(
        title = "The commands to run."
    )
    @PluginProperty(dynamic = true)
    @NotNull
    @NotEmpty
    protected List commands;

    @Schema(
        title = "Additional environment variables for the current process."
    )
    @PluginProperty(
            additionalProperties = String.class,
            dynamic = true
    )
    protected Map env;

    @Schema(
        title = "Deprecated, use 'taskRunner' instead"
    )
    @PluginProperty
    @Deprecated
    private DockerOptions docker;

    @Schema(
        title = "The task runner to use.",
        description = "Task runners are provided by plugins, each have their own properties."
    )
    @PluginProperty
    @Builder.Default
    @Valid
    private TaskRunner taskRunner = Docker.instance();

    @Schema(title = "The task runner container image, only used if the task runner is container-based.")
    @PluginProperty(dynamic = true)
    @Builder.Default
    private String containerImage = DEFAULT_IMAGE;

    private NamespaceFiles namespaceFiles;

    private Object inputFiles;

    private List outputFiles;

    @Override
    public ScriptOutput run(RunContext runContext) throws Exception {

        CommandsWrapper commands = new CommandsWrapper(runContext)
            .withWarningOnStdErr(true)
            .withDockerOptions(injectDefaults(getDocker()))
            .withTaskRunner(this.taskRunner)
            .withContainerImage(this.containerImage)
            .withCommands(
                    ScriptService.scriptCommands(
                            List.of("/bin/sh", "-c"),
                            null,
                            this.commands)
            )
            .withEnv(this.getEnv(runContext))
            .withNamespaceFiles(namespaceFiles)
            .withInputFiles(inputFiles)
            .withOutputFiles(outputFiles);

        return commands.run();
    }

    private DockerOptions injectDefaults(DockerOptions original) {
        if (original == null) {
            return null;
        }

        var builder = original.toBuilder();
        if (original.getImage() == null) {
            builder.image(DEFAULT_IMAGE);
        }

        return builder.build();
    }

    private Map getEnv(RunContext runContext) throws IOException, IllegalVariableEvaluationException {
        Map envs = new HashMap<>();

        if (serviceAccount != null) {
            Path serviceAccountPath = runContext.workingDir().createTempFile(runContext.render(this.serviceAccount).getBytes());
            envs.putAll(Map.of(
                    "GOOGLE_APPLICATION_CREDENTIALS", serviceAccountPath.toString(),
                    "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE", serviceAccountPath.toString()
            ));
        }

        if (projectId != null) {
            envs.put("CLOUDSDK_CORE_PROJECT", runContext.render(this.projectId));
        }

        if (this.env != null) {
            envs.putAll(this.env);
        }

        return envs;
    }
}