io.kestra.plugin.gcp.gcs.List Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of plugin-gcp Show documentation

Integrate Google Cloud Platform services with Kestra data workflows.

There is a newer version: 0.21.2

package io.kestra.plugin.gcp.gcs;

import com.google.cloud.storage.Storage;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.*;
import lombok.experimental.SuperBuilder;
import io.kestra.core.models.annotations.Example;
import io.kestra.core.models.annotations.Plugin;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.executions.metrics.Counter;
import io.kestra.core.models.tasks.RunnableTask;
import io.kestra.core.runners.RunContext;
import io.kestra.plugin.gcp.gcs.models.Blob;
import org.slf4j.Logger;

import java.net.URI;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;

@SuperBuilder
@ToString
@EqualsAndHashCode
@Getter
@NoArgsConstructor
@Plugin(
    examples = {
        @Example(
            title = "List files in a bucket",
            full = true,
            code = """
                id: gcp_gcs_list
                namespace: company.team

                tasks:
                  - id: list
                    type: io.kestra.plugin.gcp.gcs.List
                    from: "gs://my_bucket/dir/"
                """
        )
    }
)
@Schema(
    title = "List file on a GCS bucket."
)
public class List extends AbstractList implements RunnableTask, ListInterface {
    @Schema(
        title = "The filter files or directory"
    )
    @Builder.Default
    @PluginProperty
    protected final Filter filter = Filter.BOTH;

    @Override
    public Output run(RunContext runContext) throws Exception {
        Storage connection = this.connection(runContext);
        Logger logger = runContext.logger();

        URI from = encode(runContext, this.from);
        String regExp = runContext.render(this.regExp);

        java.util.List blobs = StreamSupport
            .stream(this.iterator(connection, from), false)
            .filter(blob -> this.filter(blob, regExp))
            .map(Blob::of)
            .collect(Collectors.toList());

        runContext.metric(Counter.of("size", blobs.size()));

        logger.debug("Found '{}' blobs from '{}'", blobs.size(), from);

        return Output
            .builder()
            .blobs(blobs)
            .build();
    }

    @Override
    protected boolean filter(com.google.cloud.storage.Blob blob, String regExp) {
        boolean b = filter == Filter.DIRECTORY ? blob.isDirectory() :
            (filter != Filter.FILES || !blob.isDirectory());

        if (!b) {
            return false;
        }

        return super.filter(blob, regExp);
    }

    @Builder
    @Getter
    public static class Output implements io.kestra.core.models.tasks.Output {
        @Schema(
            title = "The list of blobs"
        )
        private final java.util.List blobs;
    }
}