• Home
• com.clickhouse
• clickhouse-data
• 0.6.3
• source code
• ClickHouseUtils.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

com.clickhouse.data.ClickHouseUtils Maven / Gradle / Ivy

package com.clickhouse.data;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.lang.reflect.InvocationTargetException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Deque;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.function.Consumer;
import java.util.function.Supplier;
import java.util.function.UnaryOperator;

public final class ClickHouseUtils {
    private static final boolean IS_UNIX;
    private static final boolean IS_WINDOWS;

    private static final String HOME_DIR;

    static {
        final String osName = System.getProperty("os.name", "");

        // https://github.com/apache/commons-lang/blob/5a3904c8678574a4ddb8502ebbc606be1091fb3f/src/main/java/org/apache/commons/lang3/SystemUtils.java#L1370
        IS_UNIX = osName.startsWith("AIX") || osName.startsWith("HP-UX") || osName.startsWith("OS/400")
                || osName.startsWith("Irix") || osName.startsWith("Linux") || osName.startsWith("LINUX")
                || osName.startsWith("Mac OS X") || osName.startsWith("Solaris") || osName.startsWith("SunOS")
                || osName.startsWith("FreeBSD") || osName.startsWith("OpenBSD") || osName.startsWith("NetBSD");
        IS_WINDOWS = osName.toLowerCase(Locale.ROOT).contains("windows");

        HOME_DIR = IS_WINDOWS
                ? Paths.get(System.getenv("APPDATA"), "clickhouse").toFile().getAbsolutePath()
                : Paths.get(System.getProperty("user.home"), ".clickhouse").toFile().getAbsolutePath();
    }

    /**
     * Default charset.
     */
    public static final String DEFAULT_CHARSET = StandardCharsets.UTF_8.name();

    public static final int MIN_CORE_THREADS = 4;

    public static final CompletableFuture NULL_FUTURE = CompletableFuture.completedFuture(null);
    public static final Supplier NULL_SUPPLIER = () -> null;

    public static final String VARIABLE_PREFIX = "{{";
    public static final String VARIABLE_SUFFIX = "}}";

    private static  T findFirstService(Class serviceInterface) {
        ClickHouseChecker.nonNull(serviceInterface, "serviceInterface");

        T service = null;

        for (T s : ServiceLoader.load(serviceInterface, ClickHouseUtils.class.getClassLoader())) {
            if (s != null) {
                service = s;
                break;
            }
        }

        return service;
    }

    public static String applyVariables(String template, UnaryOperator applyFunc) {
        if (template == null) {
            template = "";
        }

        if (applyFunc == null) {
            return template;
        }

        StringBuilder sb = new StringBuilder();

        for (int i = 0, len = template.length(); i < len; i++) {
            int index = template.indexOf(VARIABLE_PREFIX, i);
            if (index != -1) {
                sb.append(template.substring(i, index));

                i = index;
                index = template.indexOf(VARIABLE_SUFFIX, i);

                if (index != -1) {
                    String variable = template.substring(i + VARIABLE_PREFIX.length(), index).trim();
                    String value = applyFunc.apply(variable);
                    if (value == null) {
                        i += VARIABLE_PREFIX.length() - 1;
                        sb.append(VARIABLE_PREFIX);
                    } else {
                        i = index + VARIABLE_SUFFIX.length() - 1;
                        sb.append(value);
                    }
                } else {
                    sb.append(template.substring(i));
                    break;
                }
            } else {
                sb.append(template.substring(i));
                break;
            }
        }

        return sb.toString();
    }

    public static String applyVariables(String template, Map variables) {
        return applyVariables(template, variables == null || variables.isEmpty() ? null : variables::get);
    }

    /**
     * Creates a temporary file. Same as {@code createTempFile(null, null, true)}.
     *
     * @return non-null temporary file
     * @throws IOException when failed to create the temporary file
     */
    public static File createTempFile() throws IOException {
        return createTempFile(null, null, true);
    }

    /**
     * Creates a temporary file with given prefix and suffix. Same as
     * {@code createTempFile(prefix, suffix, true)}.
     *
     * @param prefix prefix, could be null
     * @param suffix suffix, could be null
     * @return non-null temporary file
     * @throws IOException when failed to create the temporary file
     */
    public static File createTempFile(String prefix, String suffix) throws IOException {
        return createTempFile(prefix, suffix, true);
    }

    /**
     * Creates a temporary file with the given prefix and suffix. The file has only
     * read and write access granted to the owner.
     *
     * @param prefix       prefix, null or empty string is taken as {@code "ch"}
     * @param suffix       suffix, null or empty string is taken as {@code ".data"}
     * @param deleteOnExit whether the file be deleted on exit
     * @return non-null temporary file
     * @throws IOException when failed to create the temporary file
     */
    public static File createTempFile(String prefix, String suffix, boolean deleteOnExit) throws IOException {
        if (prefix == null || prefix.isEmpty()) {
            prefix = "ch";
        }
        if (suffix == null || suffix.isEmpty()) {
            suffix = ".data";
        }

        final File f;
        if (IS_UNIX) {
            FileAttribute> attr = PosixFilePermissions
                    .asFileAttribute(PosixFilePermissions.fromString("rw-------"));
            f = Files.createTempFile(prefix, suffix, attr).toFile();
        } else {
            f = Files.createTempFile(prefix, suffix).toFile(); // NOSONAR
            f.setReadable(true, true); // NOSONAR
            f.setWritable(true, true); // NOSONAR
            f.setExecutable(false, false); // NOSONAR
        }

        if (deleteOnExit) {
            f.deleteOnExit();
        }
        return f;
    }

    /**
     * Decode given string using {@link URLDecoder} and
     * {@link StandardCharsets#UTF_8}.
     *
     * @param encodedString encoded string
     * @return non-null decoded string
     */
    public static String decode(String encodedString) {
        if (ClickHouseChecker.isNullOrEmpty(encodedString)) {
            return "";
        }

        try {
            return URLDecoder.decode(encodedString, DEFAULT_CHARSET);
        } catch (UnsupportedEncodingException e) {
            return encodedString;
        }
    }

    /**
     * Encode given string using {@link URLEncoder} and
     * {@link StandardCharsets#UTF_8}.
     *
     * @param str string to encode
     * @return non-null encoded string
     */
    public static String encode(String str) {
        if (ClickHouseChecker.isNullOrEmpty(str)) {
            return "";
        }

        try {
            return URLEncoder.encode(str, DEFAULT_CHARSET);
        } catch (UnsupportedEncodingException e) {
            return str;
        }
    }

    /**
     * Extracts parameters, usually key-value pairs, from the given query string.
     *
     * @param query  non-empty query string
     * @param params mutable map for extracted parameters, a new {@link HashMap}
     *               will be created when it's null
     * @return map with extracted parameters, usually same as {@code params}
     */
    public static Map extractParameters(String query, Map params) {
        if (params == null) {
            params = new HashMap<>();
        }
        if (ClickHouseChecker.isNullOrEmpty(query)) {
            return params;
        }
        int len = query.length();
        for (int i = 0; i < len; i++) {
            int index = query.indexOf('&', i);
            if (index == i) {
                continue;
            }

            String param;
            if (index < 0) {
                param = query.substring(i);
                i = len;
            } else {
                param = query.substring(i, index);
                i = index;
            }
            index = param.indexOf('=');
            String key;
            String value;
            if (index < 0) {
                key = decode(param);
                if (key.charAt(0) == '!') {
                    key = key.substring(1);
                    value = Boolean.FALSE.toString();
                } else {
                    value = Boolean.TRUE.toString();
                }
            } else {
                key = decode(param.substring(0, index));
                value = decode(param.substring(index + 1));
            }

            // any multi-value option? cluster?
            if (!ClickHouseChecker.isNullOrEmpty(value)) {
                params.put(key, value);
            }
        }
        return params;
    }

    public static  T newInstance(String className, Class returnType, Class callerClass) {
        if (className == null || className.isEmpty() || returnType == null) {
            throw new IllegalArgumentException("Non-empty class name and return type are required");
        } else if (callerClass == null) {
            callerClass = returnType;
        }

        try {
            Class clazz = Class.forName(className, false, callerClass.getClassLoader());
            if (!returnType.isAssignableFrom(clazz)) {
                throw new IllegalArgumentException(
                        format("Invalid %s class type. Input class should be a superclass of %s.", className,
                                returnType));
            }

            return returnType.cast(clazz.getConstructor().newInstance());
        } catch (ClassNotFoundException e) {
            throw new IllegalArgumentException(format("Class %s is not found in the classpath.", className));
        } catch (IllegalAccessException | NoSuchMethodException e) {
            throw new IllegalArgumentException(
                    format("Class %s does not have a public constructor without any argument.", className));
        } catch (InstantiationException | InvocationTargetException e) {
            throw new IllegalArgumentException(format("Error while creating an %s class instance.", className), e);
        }
    }

    /**
     * Gets absolute and normalized path to the given file.
     *
     * @param file non-empty file
     * @return non-null absolute and normalized path to the file
     */
    public static Path getFile(String file) {
        if (file == null || file.isEmpty()) {
            throw new IllegalArgumentException("Non-empty file is required");
        } else if (file.startsWith("~/")) {
            return Paths.get(System.getProperty("user.home"), file.substring(2)).normalize();
        }
        return Paths.get(file).toAbsolutePath().normalize();
    }

    /**
     * Finds files according to the given pattern and path.
     *
     * @param pattern non-empty pattern may or may have syntax prefix as in
     *                {@link java.nio.file.FileSystem#getPathMatcher(String)},
     *                defaults to {@code glob} syntax
     * @param paths   path to search, defaults to current work directory
     * @return non-null list of normalized absolute paths matching the pattern
     * @throws IOException when failed to find files
     */
    public static List findFiles(String pattern, String... paths) throws IOException {
        if (pattern == null || pattern.isEmpty()) {
            throw new IllegalArgumentException("Non-empty pattern is required");
        } else if (pattern.startsWith("~/")) {
            return Collections
                    .singletonList(Paths.get(System.getProperty("user.home"), pattern.substring(2)).normalize());
        }

        if (!pattern.startsWith("glob:") && !pattern.startsWith("regex:")) {
            Path path = Paths.get(pattern);
            if (path.isAbsolute()) {
                return Collections.singletonList(path);
            } else {
                pattern = "glob:" + pattern;
            }
        }

        final Path searchPath;
        if (paths == null || paths.length == 0) {
            searchPath = Paths.get("");
        } else {
            String root = paths[0];
            Path rootPath = root.startsWith("~/")
                    ? Paths.get(System.getProperty("user.home"), root.substring(2)).normalize()
                    : Paths.get(root);
            searchPath = paths.length < 2 ? rootPath
                    : Paths.get(rootPath.toFile().getAbsolutePath(), Arrays.copyOfRange(paths, 1, paths.length))
                            .normalize();
        }

        final List files = new ArrayList<>();
        final PathMatcher matcher = FileSystems.getDefault().getPathMatcher(pattern);
        Files.walkFileTree(searchPath, new SimpleFileVisitor() {
            @Override
            public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException {
                if (matcher.matches(path)) {
                    files.add(path.normalize());
                }
                return FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult visitFileFailed(Path file, IOException exc)
                    throws IOException {
                return FileVisitResult.CONTINUE;
            }
        });
        return files;
    }

    public static String toJavaByteArrayExpression(byte[] bytes) {
        if (bytes == null) {
            return "null";
        }
        int len = bytes.length;
        if (len == 0) {
            return "{}";
        }

        String prefix = "(byte)0x";
        StringBuilder builder = new StringBuilder(10 * len).append('{');
        for (int i = 0; i < len; i++) {
            builder.append(prefix).append(String.format("%02X", 0xFF & bytes[i])).append(',');
        }
        builder.setCharAt(builder.length() - 1, '}');
        return builder.toString();
    }

    public static ExecutorService newThreadPool(Object owner, int maxThreads, int maxRequests) {
        return newThreadPool(owner, maxThreads, 0, maxRequests, 0L, true);
    }

    public static ExecutorService newThreadPool(Object owner, int coreThreads, int maxThreads, int maxRequests,
            long keepAliveTimeoutMs, boolean allowCoreThreadTimeout) {
        final BlockingQueue queue;
        if (coreThreads < MIN_CORE_THREADS) {
            coreThreads = MIN_CORE_THREADS;
        }
        if (maxRequests > 0) {
            queue = new ArrayBlockingQueue<>(maxRequests);
            if (maxThreads <= coreThreads) {
                maxThreads = coreThreads * 2;
            }
        } else {
            queue = new LinkedBlockingQueue<>();
            if (maxThreads != coreThreads) {
                maxThreads = coreThreads;
            }
        }
        if (keepAliveTimeoutMs <= 0L) {
            keepAliveTimeoutMs = allowCoreThreadTimeout ? 1000L : 0L;
        }

        ThreadPoolExecutor pool = new ThreadPoolExecutor(coreThreads, maxThreads, keepAliveTimeoutMs,
                TimeUnit.MILLISECONDS, queue, new ClickHouseThreadFactory(owner), new ThreadPoolExecutor.AbortPolicy());
        if (allowCoreThreadTimeout) {
            pool.allowCoreThreadTimeOut(true);
        }
        return pool;
    }

    public static boolean isCloseBracket(char ch) {
        return ch == ')' || ch == ']' || ch == '}';
    }

    public static boolean isOpenBracket(char ch) {
        return ch == '(' || ch == '[' || ch == '{';
    }

    public static boolean isQuote(char ch) {
        return ch == '\'' || ch == '`' || ch == '"';
    }

    public static boolean isSeparator(char ch) {
        return ch == ',' || ch == ';';
    }

    /**
     * Escape quotes in given string.
     *
     * @param str   string
     * @param quote quote to escape
     * @return escaped string
     */
    public static String escape(String str, char quote) {
        if (str == null) {
            return str;
        }

        int len = str.length();
        StringBuilder sb = new StringBuilder(len + 10);

        for (int i = 0; i < len; i++) {
            char ch = str.charAt(i);
            if (ch == quote || ch == '\\') {
                sb.append('\\');
            }
            sb.append(ch);
        }

        return sb.toString();
    }

    /**
     * Unescape quoted string.
     *
     * @param str quoted string
     * @return unescaped string
     */
    public static String unescape(String str) {
        if (ClickHouseChecker.isNullOrEmpty(str)) {
            return str;
        }

        int len = str.length();
        char quote = str.charAt(0);
        if (!isQuote(quote) || quote != str.charAt(len - 1)) { // not a quoted string
            return str;
        }

        StringBuilder sb = new StringBuilder(len = len - 1);
        for (int i = 1; i < len; i++) {
            char ch = str.charAt(i);

            if (++i >= len) {
                sb.append(ch);
            } else {
                char nextChar = str.charAt(i);
                if (ch == '\\' || (ch == quote && nextChar == quote)) {
                    sb.append(nextChar);
                } else {
                    sb.append(ch);
                    i--;
                }
            }
        }

        return sb.toString();
    }

    /**
     * Wrapper of {@code String.format(Locale.ROOT, ...)}.
     *
     * @param template string to format
     * @param args     arguments used in substitution
     * @return formatted string
     */
    public static String format(String template, Object... args) {
        return String.format(Locale.ROOT, template, args);
    }

    /**
     * Normalizes given directory by appending back slash if it does exist.
     *
     * @param dir original directory
     * @return normalized directory
     */
    public static String normalizeDirectory(String dir) {
        if (dir == null || dir.isEmpty()) {
            return "./";
        }

        return dir.charAt(dir.length() - 1) == '/' ? dir : dir.concat("/");
    }

    private static int readJsonArray(String json, List