com.marklogic.contentpump.test.SimpleSequenceFileCompressCreator Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of mlcp Show documentation

MarkLogic Content Pump

There is a newer version: 11.3.1

package com.marklogic.contentpump.test;

import java.io.BufferedReader;
import java.io.FileReader;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.GzipCodec;

import com.marklogic.contentpump.examples.SimpleSequenceFileKey;
import com.marklogic.contentpump.examples.SimpleSequenceFileValue;
import com.marklogic.mapreduce.DocumentURI;

public class SimpleSequenceFileCompressCreator {
    public static void main(String args[]) throws Exception {
        System.out.println("Sequence File Creator");
        String uri = args[0]; // output sequence file name
        String filePath = args[1]; // text file to read from; Odd line is key,
                                   // even line is value

        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(URI.create(uri), conf);
        Path path = new Path(uri);
        SequenceFile.Writer writer = null;
        SimpleSequenceFileKey key = new SimpleSequenceFileKey();

        String line = null;

        try (BufferedReader buffer = 
                 new BufferedReader(new FileReader(filePath))) {
            SimpleSequenceFileValue value = 
                 new SimpleSequenceFileValue();
            writer = SequenceFile.createWriter(fs, conf, path, key.getClass(),
                value.getClass(), CompressionType.RECORD, new GzipCodec());
            while ((line = buffer.readLine()) != null) {
                key.setDocumentURI(new DocumentURI(line));
                if ((line = buffer.readLine()) == null) {
                    break;
                }
                value.setValue(new Text(line));
                writer.append(key, value);
            }
        } finally {
            IOUtils.closeStream(writer);
        }
    }
}