com.marklogic.flux.impl.importdata.ImportRdfFilesCommand Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of flux-api Show documentation
Flux API for data movement with MarkLogic
There is a newer version: 1.0.0.ea1
/*
 * Copyright © 2024 MarkLogic Corporation. All Rights Reserved.
 */
package com.marklogic.flux.impl.importdata;

import com.marklogic.flux.api.CompressionType;
import com.marklogic.flux.api.RdfFilesImporter;
import com.marklogic.flux.impl.AbstractCommand;
import com.marklogic.flux.impl.OptionsUtil;
import com.marklogic.spark.Options;
import picocli.CommandLine;

import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Supplier;

@CommandLine.Command(
    name = "import-rdf-files",
    description = "Read RDF data from local, HDFS, and S3 files and write the data as managed triples documents in MarkLogic."
)
public class ImportRdfFilesCommand extends AbstractImportFilesCommand implements RdfFilesImporter {

    @CommandLine.Mixin
    private ReadRdfFilesParams readParams = new ReadRdfFilesParams();

    @CommandLine.Mixin
    private WriteTriplesDocumentsParams writeParams = new WriteTriplesDocumentsParams();

    @Override
    protected String getReadFormat() {
        return AbstractCommand.MARKLOGIC_CONNECTOR;
    }

    @Override
    protected ReadFilesParams getReadParams() {
        return readParams;
    }

    @Override
    protected Supplier> getWriteParams() {
        return writeParams;
    }

    public static class ReadRdfFilesParams extends ReadFilesParams implements ReadRdfFilesOptions {

        @CommandLine.Option(names = "--compression", description = "When importing compressed files, specify the type of compression used. "
            + OptionsUtil.VALID_VALUES_DESCRIPTION)
        private CompressionType compressionType;

        @CommandLine.Option(names = "--partitions", description = "Specifies the number of partitions used for reading files.")
        private int partitions;

        @Override
        public Map makeOptions() {
            return OptionsUtil.addOptions(super.makeOptions(),
                Options.READ_FILES_TYPE, "rdf",
                Options.READ_FILES_COMPRESSION, compressionType != null ? compressionType.name() : null,
                Options.READ_NUM_PARTITIONS, OptionsUtil.intOption(partitions)
            );
        }

        @Override
        public ReadRdfFilesOptions compressionType(CompressionType compressionType) {
            this.compressionType = compressionType;
            return this;
        }

        @Override
        public ReadRdfFilesOptions partitions(int partitions) {
            this.partitions = partitions;
            return this;
        }
    }

    public static class WriteTriplesDocumentsParams extends WriteDocumentParams implements WriteTriplesDocumentsOptions {

        @CommandLine.Option(names = "--graph", description = "Specify the graph URI for each triple not already associated with a graph. If not set, " +
            "triples will be added to the default MarkLogic graph - http://marklogic.com/semantics#default-graph . ")
        private String graph;

        @CommandLine.Option(names = "--graph-override", description = "Specify the graph URI for each triple to be included in, " +
            "even if is already associated with a graph.")
        private String graphOverride;

        @Override
        public Map makeOptions() {
            return OptionsUtil.addOptions(super.makeOptions(),
                Options.WRITE_GRAPH, graph,
                Options.WRITE_GRAPH_OVERRIDE, graphOverride
            );
        }

        @Override
        public WriteTriplesDocumentsOptions graph(String graph) {
            this.graph = graph;
            return this;
        }

        @Override
        public WriteTriplesDocumentsOptions graphOverride(String graphOverride) {
            this.graphOverride = graphOverride;
            return this;
        }
    }

    @Override
    public RdfFilesImporter from(Consumer consumer) {
        consumer.accept(readParams);
        return this;
    }

    @Override
    public RdfFilesImporter from(String... paths) {
        readParams.paths(paths);
        return this;
    }

    @Override
    public RdfFilesImporter to(Consumer consumer) {
        consumer.accept(writeParams);
        return this;
    }
}