com.datastax.data.common.hadoop.HDFSUtil Maven / Gradle / Ivy

Go to download
package com.datastax.data.common.hadoop;
import java.io.File;
import java.io.FileInputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class HDFSUtil {
    public HDFSUtil() {
    }

    public static void upload(Configuration conf, String localFilePath, String hdfsFilePath) throws Exception {
        FileSystem fs = FileSystem.get(conf);
        Path localPath = new Path(localFilePath);
        Path dstPath = new Path(hdfsFilePath);
        fs.copyFromLocalFile(localPath, dstPath);
    }

    public static void delete(Configuration conf, String hdfsFilePath) throws Exception {
        FileSystem fs = FileSystem.get(conf);
        Path dstPath = new Path(hdfsFilePath);
        fs.delete(dstPath, true);
    }

    public static void download(Configuration conf, String hdfsFilePath, String localFilePath) throws Exception {
        FileSystem fs = FileSystem.get(conf);
        Path localPath = new Path(localFilePath);
        Path hdfsPath = new Path(hdfsFilePath);
        fs.copyToLocalFile(hdfsPath, localPath);
    }

    public static void append(Configuration conf, String hdfsFilePath, String localFilePath, boolean fromNewLine) throws Exception {
        FileSystem fs = FileSystem.get(conf);
        File localFile = new File(localFilePath);
        if (localFile.isDirectory()) {
            throw new Exception("不支持文件夹的append操作");
        } else {
            Path hdfsPath = new Path(hdfsFilePath);
            FSDataOutputStream outputStream = fs.append(hdfsPath);
            if (fromNewLine) {
                String newLine = "\n";
                outputStream.write(newLine.getBytes());
            }

            int bufferSize = 4096;
            byte[] buffer = new byte[bufferSize];
            FileInputStream inputStream = new FileInputStream(new File(localFilePath));
            boolean var11 = true;

            int size;
            while((size = inputStream.read(buffer)) > 0) {
                outputStream.write(buffer, 0, size);
            }

            inputStream.close();
            outputStream.close();
        }
    }

    public static boolean isFileExist(Configuration conf, String hdfsFolder, String filePreffix) throws Exception {
        boolean exist = false;
        Path parentPath = new Path(hdfsFolder);
        FileSystem fileSystem = FileSystem.get(conf);
        FileStatus[] fileStatus = fileSystem.listStatus(parentPath);

        for(int i = 0; i < fileStatus.length; ++i) {
            String name = fileStatus[i].getPath().getName();
            if (name.startsWith(filePreffix)) {
                exist = true;
                break;
            }
        }

        return exist;
    }

    public static void createEmptyFile(Configuration conf, String hdfsFileName) throws Exception {
        Path filePath = new Path(hdfsFileName);
        FileSystem fileSystem = FileSystem.get(conf);
        FSDataOutputStream stream = fileSystem.create(filePath);
        stream.write("".getBytes());
        stream.close();
    }

}