All Downloads are FREE. Search and download functionalities are using the official Maven repository.

io.kestra.plugin.compress.ArchiveCompress Maven / Gradle / Ivy

The newest version!
package io.kestra.plugin.compress;

import com.fasterxml.jackson.core.type.TypeReference;
import io.kestra.core.exceptions.IllegalVariableEvaluationException;
import io.kestra.core.models.annotations.Example;
import io.kestra.core.models.annotations.Plugin;
import io.kestra.core.models.annotations.PluginProperty;
import io.kestra.core.models.tasks.RunnableTask;
import io.kestra.core.runners.RunContext;
import io.kestra.core.serializers.JacksonMapper;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotNull;
import lombok.*;
import lombok.experimental.SuperBuilder;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.ArchiveOutputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.compressors.CompressorOutputStream;
import org.apache.commons.io.IOUtils;

import java.io.*;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;

@SuperBuilder
@ToString
@EqualsAndHashCode
@Getter
@NoArgsConstructor
@Schema(
    title = "Compress an archive file."
)
@Plugin(
    examples = {
        @Example(
            full = true,
            code = """
                id: archive_compress
                namespace: company.team

                inputs:
                  - id: file
                    type: FILE

                tasks:
                  - id: "archive_compress"
                    type: "io.kestra.plugin.compress.ArchiveCompress"
                    from:
                      myfile.txt: "{{ inputs.file }}"
                    algorithm: ZIP
                """
        ),
        @Example(
            full = true,
            code = """
                id: archive_compress
                namespace: company.team

                tasks:
                  - id: products_download
                    type: io.kestra.plugin.core.http.Download
                    uri: "https://raw.githubusercontent.com/kestra-io/datasets/main/csv/products.csv"

                  - id: orders_download
                    type: io.kestra.plugin.core.http.Download
                    uri: "https://raw.githubusercontent.com/kestra-io/datasets/main/csv/orders.csv"

                  - id: archive_compress
                    type: "io.kestra.plugin.compress.ArchiveCompress"
                    from:
                      products.csv: "{{ outputs.products_download.uri }}"
                      orders.csv: "{{ outputs.orders_download.uri }}"
                    algorithm: TAR
                    compression: GZIP
                """
        )
    }
)
public class ArchiveCompress extends AbstractArchive implements RunnableTask {
    @Schema(
        title = "The files to compress.",
        description = "The key must be a valid path in the archive and can contain `/` to represent the directory, " +
            "the value must be a Kestra internal storage URI.\n"+
            "The value can also be a JSON containing multiple keys/values."
    )
    @PluginProperty(dynamic = true, additionalProperties = String.class)
    @NotNull
    private Object from;

    public Output run(RunContext runContext) throws Exception {
        File tempFile = runContext.workingDir().createTempFile().toFile();

        try (BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(tempFile))) {
            if (this.compression != null) {
                try (
                    CompressorOutputStream compressorOutputStream = this.compressorOutputStream(
                        runContext.render(this.compression).as(CompressionAlgorithm.class).orElseThrow(),
                        outputStream
                    );
                    ArchiveOutputStream archiveInputStream = this.archiveOutputStream(compressorOutputStream, runContext)
                ) {
                    this.writeArchive(runContext, archiveInputStream);
                }
            } else {
                try (ArchiveOutputStream archiveOutputStream = this.archiveOutputStream(outputStream, runContext)) {
                    this.writeArchive(runContext, archiveOutputStream);
                }
            }
        }

        return Output.builder()
            .uri(runContext.storage().putFile(tempFile))
            .build();
    }

    @SuppressWarnings("unchecked")
    private void writeArchive(RunContext runContext, ArchiveOutputStream archiveInputStream) throws IOException, IllegalVariableEvaluationException {
        Map from = this.from instanceof String ?
            JacksonMapper.ofJson().readValue(
                runContext.render((String) this.from),
                new TypeReference>() {}
            ) :
            (Map) this.from;

        for (Map.Entry current : from.entrySet()) {
            // temp file and path
            String finalPath = runContext.render(current.getKey());
            File tempFile = runContext.workingDir().resolve(Path.of(finalPath)).toFile();
            new File(tempFile.getParent()).mkdirs();

            // write to temp file
            String render = runContext.render(current.getValue());
            OutputStream fileOutputStream = new BufferedOutputStream(new FileOutputStream(tempFile));
            InputStream inputStream = runContext.storage().getFile(URI.create(render));

            IOUtils.copy(inputStream, fileOutputStream);
            fileOutputStream.flush();

            // create archive entry
            ArchiveEntry entry = archiveInputStream.createArchiveEntry(tempFile, finalPath);
            archiveInputStream.putArchiveEntry(entry);

            // write archive entry
            try (InputStream i = Files.newInputStream(tempFile.toPath())) {
                IOUtils.copy(i, archiveInputStream);
            }
            archiveInputStream.closeArchiveEntry();
        }

        archiveInputStream.finish();
    }

    @Builder
    @Getter
    public static class Output implements io.kestra.core.models.tasks.Output {
        @Schema(
            title = "URI of the compressed archive file on Kestra's internal storage."
        )
        private final URI uri;
    }
}




© 2015 - 2025 Weber Informatics LLC | Privacy Policy