io.questdb.cutlass.text.TextMetadataParser Maven / Gradle / Ivy

Go to download
/*******************************************************************************
 *     ___                  _   ____  ____
 *    / _ \ _   _  ___  ___| |_|  _ \| __ )
 *   | | | | | | |/ _ \/ __| __| | | |  _ \
 *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
 *    \__\_\\__,_|\___||___/\__|____/|____/
 *
 *  Copyright (c) 2014-2019 Appsicle
 *  Copyright (c) 2019-2020 QuestDB
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 ******************************************************************************/

package io.questdb.cutlass.text;

import io.questdb.cairo.ColumnType;
import io.questdb.cutlass.json.JsonException;
import io.questdb.cutlass.json.JsonLexer;
import io.questdb.cutlass.json.JsonParser;
import io.questdb.cutlass.text.types.TypeAdapter;
import io.questdb.cutlass.text.types.TypeManager;
import io.questdb.griffin.SqlKeywords;
import io.questdb.log.Log;
import io.questdb.log.LogFactory;
import io.questdb.std.*;
import io.questdb.std.datetime.DateLocale;
import io.questdb.std.datetime.DateLocaleFactory;
import io.questdb.std.datetime.microtime.TimestampFormatFactory;
import io.questdb.std.datetime.millitime.DateFormatFactory;
import io.questdb.std.str.AbstractCharSequence;

import java.io.Closeable;

public class TextMetadataParser implements JsonParser, Mutable, Closeable {
    private static final Log LOG = LogFactory.getLog(TextMetadataParser.class);

    private static final int S_NEED_ARRAY = 1;
    private static final int S_NEED_OBJECT = 2;
    private static final int S_NEED_PROPERTY = 3;
    private static final int P_NAME = 1;
    private static final int P_TYPE = 2;
    private static final int P_PATTERN = 3;
    private static final int P_LOCALE = 4;
    private static final int P_UTF8 = 5;
    private static final CharSequenceIntHashMap propertyNameMap = new CharSequenceIntHashMap();
    private final DateLocaleFactory dateLocaleFactory;
    private final ObjectPool csPool;
    private final DateFormatFactory dateFormatFactory;
    private final TimestampFormatFactory timestampFormatFactory;
    private final ObjList columnNames;
    private final ObjList columnTypes;
    private final TypeManager typeManager;
    private final DateLocale dateLocale;
    private int state = S_NEED_ARRAY;
    private CharSequence name;
    private int type = -1;
    private CharSequence pattern;
    private CharSequence locale;
    private int propertyIndex;
    private long buf;
    private long bufCapacity = 0;
    private int bufSize = 0;
    private CharSequence tableName;
    private int localePosition;
    private boolean utf8 = false;

    public TextMetadataParser(TextConfiguration textConfiguration, TypeManager typeManager) {
        this.columnNames = new ObjList<>();
        this.columnTypes = new ObjList<>();
        this.csPool = new ObjectPool<>(FloatingCharSequence::new, textConfiguration.getMetadataStringPoolCapacity());
        this.dateLocaleFactory = typeManager.getInputFormatConfiguration().getDateLocaleFactory();
        this.dateFormatFactory = typeManager.getInputFormatConfiguration().getDateFormatFactory();
        this.timestampFormatFactory = typeManager.getInputFormatConfiguration().getTimestampFormatFactory();
        this.typeManager = typeManager;
        this.dateLocale = textConfiguration.getDefaultDateLocale();
    }

    @Override
    public void clear() {
        bufSize = 0;
        state = S_NEED_ARRAY;
        columnNames.clear();
        columnTypes.clear();
        csPool.clear();
        clearStage();
    }

    @Override
    public void close() {
        clear();
        if (bufCapacity > 0) {
            Unsafe.free(buf, bufCapacity);
            bufCapacity = 0;
        }
    }

    public ObjList getColumnNames() {
        return columnNames;
    }

    public ObjList getColumnTypes() {
        return columnTypes;
    }

    @Override
    public void onEvent(int code, CharSequence tag, int position) throws JsonException {
        switch (code) {
            case JsonLexer.EVT_ARRAY_START:
                if (state != S_NEED_ARRAY) {
                    throw JsonException.$(position, "Unexpected array");
                }
                state = S_NEED_OBJECT;
                break;
            case JsonLexer.EVT_OBJ_START:
                if (state != S_NEED_OBJECT) {
                    throw JsonException.$(position, "Unexpected object");
                }
                state = S_NEED_PROPERTY;
                break;
            case JsonLexer.EVT_NAME:
                this.propertyIndex = propertyNameMap.get(tag);
                if (this.propertyIndex == -1) {
                    LOG.info().$("unknown [table=").$(tableName).$(", tag=").$(tag).$(']').$();
                }
                break;
            case JsonLexer.EVT_VALUE:
                switch (propertyIndex) {
                    case P_NAME:
                        name = copy(tag);
                        break;
                    case P_TYPE:
                        type = ColumnType.columnTypeOf(tag);
                        if (type == -1) {
                            throw JsonException.$(position, "Invalid type");
                        }
                        break;
                    case P_PATTERN:
                        pattern = copy(tag);
                        break;
                    case P_LOCALE:
                        locale = copy(tag);
                        localePosition = position;
                        break;
                    case P_UTF8:
                        utf8 = SqlKeywords.isTrueKeyword(tag);
                        break;
                    default:
                        LOG.info().$("ignoring [table=").$(tableName).$(", value=").$(tag).$(']').$();
                        break;
                }
                break;
            case JsonLexer.EVT_OBJ_END:
                state = S_NEED_OBJECT;
                createImportedType(position);
                break;
            case JsonLexer.EVT_ARRAY_VALUE:
                throw JsonException.$(position, "Must be an object");
            default:
                break;
        }
    }

    private static void strcpyw(final CharSequence value, final int len, final long address) {
        for (int i = 0; i < len; i++) {
            Unsafe.getUnsafe().putChar(address + ((long) i << 1), value.charAt(i));
        }
    }

    private static void checkInputs(int position, CharSequence name, int type) throws JsonException {
        if (name == null) {
            throw JsonException.$(position, "Missing 'name' property");
        }

        if (type == -1) {
            throw JsonException.$(position, "Missing 'type' property");
        }
    }

    private void clearStage() {
        name = null;
        type = -1;
        pattern = null;
        locale = null;
        localePosition = 0;
        utf8 = false;
    }

    private CharSequence copy(CharSequence tag) {
        final int l = tag.length() * 2;
        final long n = bufSize + l;
        if (n > bufCapacity) {
            long ptr = Unsafe.malloc(n * 2);
            Vect.memcpy(buf, ptr, bufSize);
            if (bufCapacity > 0) {
                Unsafe.free(buf, bufCapacity);
            }
            buf = ptr;
            bufCapacity = n * 2;
        }

        strcpyw(tag, l / 2, buf + bufSize);
        CharSequence cs = csPool.next().of(bufSize, l / 2);
        bufSize += l;
        return cs;
    }

    private void createImportedType(int position) throws JsonException {
        checkInputs(position, name, type);

        columnNames.add(name);

        switch (type) {
            case ColumnType.DATE:
                DateLocale dateLocale = locale == null ? this.dateLocale : dateLocaleFactory.getLocale(locale);

                if (dateLocale == null) {
                    throw JsonException.$(localePosition, "Invalid date locale");
                }

                // date pattern is required
                if (pattern == null) {
                    throw JsonException.$(0, "DATE format pattern is required");
                }
                columnTypes.add(typeManager.nextDateAdapter().of(dateFormatFactory.get(pattern), dateLocale));
                break;
            case ColumnType.TIMESTAMP:
                DateLocale timestampLocale =
                        locale == null ?
                                this.dateLocale
                                : dateLocaleFactory.getLocale(locale);
                if (timestampLocale == null) {
                    throw JsonException.$(localePosition, "Invalid timestamp locale");
                }

                // timestamp pattern is required
                if (pattern == null) {
                    throw JsonException.$(0, "TIMESTAMP format pattern is required");
                }
                columnTypes.add(typeManager.nextTimestampAdapter(utf8, timestampFormatFactory.get(pattern), timestampLocale));
                break;
            default:
                columnTypes.add(typeManager.getTypeAdapter(type));
                break;
        }
        // prepare for next iteration
        clearStage();
    }

    void setTableName(CharSequence tableName) {
        this.tableName = tableName;
    }

    private class FloatingCharSequence extends AbstractCharSequence implements Mutable {

        private int offset;
        private int len;

        @Override
        public void clear() {
        }

        @Override
        public int length() {
            return len;
        }

        @Override
        public char charAt(int index) {
            return Unsafe.getUnsafe().getChar(buf + offset + index * 2L);
        }

        CharSequence of(int lo, int len) {
            this.offset = lo;
            this.len = len;
            return this;
        }
    }

    static {
        propertyNameMap.put("name", P_NAME);
        propertyNameMap.put("type", P_TYPE);
        propertyNameMap.put("pattern", P_PATTERN);
        propertyNameMap.put("locale", P_LOCALE);
        propertyNameMap.put("utf8", P_UTF8);
    }
}