• Home
• com.hazelcast
• hazelcast
• 5.1-BETA-1
• source code
• ScannerImpl.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

com.hazelcast.org.snakeyaml.engine.v2.scanner.ScannerImpl Maven / Gradle / Ivy

/*
 * Copyright (c) 2018, http://www.snakeyaml.org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.hazelcast.org.snakeyaml.engine.v2.scanner;

import com.hazelcast.org.snakeyaml.engine.v2.api.LoadSettings;
import com.hazelcast.org.snakeyaml.engine.v2.comments.CommentType;
import com.hazelcast.org.snakeyaml.engine.v2.common.Anchor;
import com.hazelcast.org.snakeyaml.engine.v2.common.ArrayStack;
import com.hazelcast.org.snakeyaml.engine.v2.common.CharConstants;
import com.hazelcast.org.snakeyaml.engine.v2.common.ScalarStyle;
import com.hazelcast.org.snakeyaml.engine.v2.common.UriEncoder;
import com.hazelcast.org.snakeyaml.engine.v2.exceptions.Mark;
import com.hazelcast.org.snakeyaml.engine.v2.exceptions.ScannerException;
import com.hazelcast.org.snakeyaml.engine.v2.exceptions.YamlEngineException;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.AliasToken;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.AnchorToken;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.BlockEndToken;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.BlockEntryToken;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.BlockMappingStartToken;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.BlockSequenceStartToken;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.CommentToken;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.DirectiveToken;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.DocumentEndToken;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.DocumentStartToken;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.FlowEntryToken;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.FlowMappingEndToken;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.FlowMappingStartToken;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.FlowSequenceEndToken;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.FlowSequenceStartToken;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.KeyToken;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.ScalarToken;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.StreamEndToken;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.StreamStartToken;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.TagToken;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.TagTuple;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.Token;
import com.hazelcast.org.snakeyaml.engine.v2.tokens.ValueToken;

import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Optional;
import java.util.regex.Pattern;

import static com.hazelcast.org.snakeyaml.engine.v2.common.CharConstants.ESCAPES;
import static com.hazelcast.org.snakeyaml.engine.v2.common.CharConstants.ESCAPE_CODES;
import static com.hazelcast.org.snakeyaml.engine.v2.common.CharConstants.ESCAPE_REPLACEMENTS;

/**
 * 
 * Scanner produces tokens of the following types:
 * STREAM-START
 * STREAM-END
 * COMMENT
 * DIRECTIVE(name, value)
 * DOCUMENT-START
 * DOCUMENT-END
 * BLOCK-SEQUENCE-START
 * BLOCK-MAPPING-START
 * BLOCK-END
 * FLOW-SEQUENCE-START
 * FLOW-MAPPING-START
 * FLOW-SEQUENCE-END
 * FLOW-MAPPING-END
 * BLOCK-ENTRY
 * FLOW-ENTRY
 * KEY
 * VALUE
 * ALIAS(value)
 * ANCHOR(value)
 * TAG(value)
 * SCALAR(value, plain, style)
 * Read comments in the Scanner code for more details.
 * 
 */
public final class ScannerImpl implements Scanner {

    private static final String DIRECTIVE_PREFIX = "while scanning a directive";
    private static final String EXPECTED_ALPHA_ERROR_PREFIX = "expected alphabetic or numeric character, but found ";
    private static final String SCANNING_SCALAR = "while scanning a block scalar";
    private static final String SCANNING_PREFIX = "while scanning a ";
    /**
     * A regular expression matching characters which are not in the hexadecimal
     * set (0-9, A-F, a-f).
     */
    private static final Pattern NOT_HEXA = Pattern.compile("[^0-9A-Fa-f]");

    private final StreamReader reader;
    // Had we reached the end of the stream?
    private boolean done = false;

    // The number of unclosed '{' and '['. `flow_level == 0` means block
    // context.
    private int flowLevel = 0;

    // List of processed tokens that are not yet emitted.
    private final List tokens;

    // Number of tokens that were emitted through the `get_token` method.
    private int tokensTaken = 0;

    // The current indentation level.
    private int indent = -1;

    // Past indentation levels.
    private final ArrayStack indents;

    // Variables related to simple keys treatment.

    /**
     * 
     * A simple key is a key that is not denoted by the '?' indicator.
     * Example of simple keys:
     *   ---
     *   block simple key: value
     *   ? not a simple key:
     *   : { flow simple key: value }
     * We emit the KEY token before all keys, so when we find a potential
     * simple key, we try to locate the corresponding ':' indicator.
     * Simple keys should be limited to a single line and 1024 characters.
     *
     * Can a simple key start at the current position? A simple key may
     * start:
     * - at the beginning of the line, not counting indentation spaces
     *       (in block context),
     * - after '{', '[', ',' (in the flow context),
     * - after '?', ':', '-' (in the block context).
     * In the block context, this flag also signifies if a block collection
     * may start at the current position.
     * 
     */
    private boolean allowSimpleKey = true;

    /*
     * Keep track of possible simple keys. This is a dictionary. The key is
     * `flow_level`; there can be no more that one possible simple key for each
     * level. The value is a SimpleKey record: (token_number, required, index,
     * line, column, mark) A simple key may start with ALIAS, ANCHOR, TAG,
     * SCALAR(flow), '[', or '{' tokens.
     */
    private final Map possibleSimpleKeys;

    private final LoadSettings settings;

    /**
     * @deprecated use the other constructor with LoadSettings first
     */
    public ScannerImpl(StreamReader reader, LoadSettings settings) {
        this(settings, reader);
    }

    public ScannerImpl(LoadSettings settings, StreamReader reader) {
        this.reader = reader;
        this.settings = settings;
        this.tokens = new ArrayList<>(100);
        this.indents = new ArrayStack<>(10);
        // The order in possibleSimpleKeys is kept for nextPossibleSimpleKey()
        this.possibleSimpleKeys = new LinkedHashMap<>();
        fetchStreamStart();// Add the STREAM-START token.
    }

    /**
     * @deprecated it should be used with LoadSettings
     */
    public ScannerImpl(StreamReader reader) {
        this(LoadSettings.builder().build(), reader);
    }

    /**
     * Check whether the next token is one of the given types.
     */
    public boolean checkToken(Token.ID... choices) {
        while (needMoreTokens()) {
            fetchMoreTokens();
        }
        if (!this.tokens.isEmpty()) {
            if (choices.length == 0) {
                return true;
            }
            // since profiler puts this method on top (it is used a lot), we
            // should not use 'foreach' here because of the performance reasons
            Token firstToken = this.tokens.get(0);
            Token.ID first = firstToken.getTokenId();
            for (int i = 0; i < choices.length; i++) {
                if (first == choices[i]) {
                    return true;
                }
            }
        }
        return false;
    }

    /**
     * Return the next token, but do not delete it from the queue.
     */
    public Token peekToken() {
        while (needMoreTokens()) {
            fetchMoreTokens();
        }
        return this.tokens.get(0);
    }

    @Override
    public boolean hasNext() {
        return checkToken();
    }

    /**
     * Return the next token, removing it from the queue.
     */
    public Token next() {
        this.tokensTaken++;
        if (this.tokens.isEmpty()) {
            throw new NoSuchElementException("No more Tokens found.");
        } else {
            return this.tokens.remove(0);
        }
    }

    // Private methods.

    /**
     * Returns true if more tokens should be scanned.
     */
    private boolean needMoreTokens() {
        // If we are done, we do not require more tokens.
        if (this.done) {
            return false;
        }
        // If we aren't done, but we have no tokens, we need to scan more.
        if (this.tokens.isEmpty()) {
            return true;
        }
        // The current token may be a potential simple key, so we
        // need to look further.
        stalePossibleSimpleKeys();
        return nextPossibleSimpleKey() == this.tokensTaken;
    }

    /**
     * Fetch one or more tokens from the StreamReader.
     */
    private void fetchMoreTokens() {
        // Eat whitespaces and process comments until we reach the next token.
        scanToNextToken();
        // Remove obsolete possible simple keys.
        stalePossibleSimpleKeys();
        // Compare the current indentation and column. It may add some tokens
        // and decrease the current indentation level.
        unwindIndent(reader.getColumn());
        // Peek the next code point, to decide what the next group of tokens
        // will look like.
        int c = reader.peek();
        switch (c) {
            case '\0':
                // Is it the end of stream?
                fetchStreamEnd();
                return;
            case '%':
                // Is it a directive?
                if (checkDirective()) {
                    fetchDirective();
                    return;
                }
                break;
            case '-':
                // Is it the document start?
                if (checkDocumentStart()) {
                    fetchDocumentStart();
                    return;
                    // Is it the block entry indicator?
                } else if (checkBlockEntry()) {
                    fetchBlockEntry();
                    return;
                }
                break;
            case '.':
                // Is it the document end?
                if (checkDocumentEnd()) {
                    fetchDocumentEnd();
                    return;
                }
                break;
            case '[':
                // Is it the flow sequence start indicator?
                fetchFlowSequenceStart();
                return;
            case '{':
                // Is it the flow mapping start indicator?
                fetchFlowMappingStart();
                return;
            case ']':
                // Is it the flow sequence end indicator?
                fetchFlowSequenceEnd();
                return;
            case '}':
                // Is it the flow mapping end indicator?
                fetchFlowMappingEnd();
                return;
            case ',':
                // Is it the flow entry indicator?
                fetchFlowEntry();
                return;
            // see block entry indicator above
            case '?':
                // Is it the key indicator?
                if (checkKey()) {
                    fetchKey();
                    return;
                }
                break;
            case ':':
                // Is it the value indicator?
                if (checkValue()) {
                    fetchValue();
                    return;
                }
                break;
            case '*':
                // Is it an alias?
                fetchAlias();
                return;
            case '&':
                // Is it an anchor?
                fetchAnchor();
                return;
            case '!':
                // Is it a tag?
                fetchTag();
                return;
            case '|':
                // Is it a literal scalar?
                if (this.flowLevel == 0) {
                    fetchLiteral();
                    return;
                }
                break;
            case '>':
                // Is it a folded scalar?
                if (this.flowLevel == 0) {
                    fetchFolded();
                    return;
                }
                break;
            case '\'':
                // Is it a single quoted scalar?
                fetchSingle();
                return;
            case '"':
                // Is it a double quoted scalar?
                fetchDouble();
                return;
            default:
                // It must be a plain scalar then.
        }
        if (checkPlain()) {
            fetchPlain();
            return;
        }
        // No? It's an error. Let's produce a nice error message. We do this by
        // converting escaped characters into their escape sequences. This is a
        // backwards use of the ESCAPE_REPLACEMENTS map.
        String chRepresentation = String.valueOf(Character.toChars(c));
        if (ESCAPES.containsKey(Character.valueOf((char) c))) {
            chRepresentation = "\\" + ESCAPES.get(Character.valueOf((char) c));
        }
        if (c == '\t')
            chRepresentation += "(TAB)";
        String text = String
                .format("found character '%s' that cannot start any token. (Do not use %s for indentation)",
                        chRepresentation, chRepresentation);
        throw new ScannerException("while scanning for the next token", Optional.empty(), text, reader.getMark());
    }

    // Simple keys treatment.

    /**
     * Return the number of the nearest possible simple key. Actually we don't
     * need to loop through the whole dictionary.
     */
    private int nextPossibleSimpleKey() {
        /*
         * Because this.possibleSimpleKeys is ordered we can simply take the first key
         */
        if (!this.possibleSimpleKeys.isEmpty()) {
            return this.possibleSimpleKeys.values().iterator().next().getTokenNumber();
        }
        return -1;
    }

    /**
     * 
     * Remove entries that are no longer possible simple keys. According to
     * the YAML specification, simple keys
     * - should be limited to a single line,
     * - should be no longer than 1024 characters.
     * Disabling this procedure will allow simple keys of any length and
     * height (may cause problems if indentation is broken though).
     * 
     */
    private void stalePossibleSimpleKeys() {
        if (!this.possibleSimpleKeys.isEmpty()) {
            for (Iterator iterator = this.possibleSimpleKeys.values().iterator(); iterator
                    .hasNext(); ) {
                SimpleKey key = iterator.next();
                if ((key.getLine() != reader.getLine())
                        || (reader.getIndex() - key.getIndex() > 1024)) {
                    // If the key is not on the same line as the current
                    // position OR the difference in column between the token
                    // start and the current position is more than the maximum
                    // simple key length, then this cannot be a simple key.
                    if (key.isRequired()) {
                        // If the key was required, this implies an error
                        // condition.
                        throw new ScannerException("while scanning a simple key", key.getMark(),
                                "could not find expected ':'", reader.getMark());
                    }
                    iterator.remove();
                }
            }
        }
    }

    /**
     * The next token may start a simple key. We check if it's possible and save
     * its position. This function is called for ALIAS, ANCHOR, TAG,
     * SCALAR(flow), '[', and '{'.
     */
    private void savePossibleSimpleKey() {
        // The next token may start a simple key. We check if it's possible
        // and save its position. This function is called for
        // ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.

        // Check if a simple key is required at the current position.
        // A simple key is required if this position is the root flowLevel, AND
        // the current indentation level is the same as the last indent-level.
        boolean required = (this.flowLevel == 0) && (this.indent == this.reader.getColumn());

        if (allowSimpleKey || !required) {
            // A simple key is required only if it is the first token in the
            // current line. Therefore it is always allowed.
        } else {
            throw new YamlEngineException(
                    "A simple key is required only if it is the first token in the current line");
        }

        // The next token might be a simple key. Let's save it's number and
        // position.
        if (this.allowSimpleKey) {
            removePossibleSimpleKey();
            int tokenNumber = this.tokensTaken + this.tokens.size();
            SimpleKey key = new SimpleKey(tokenNumber, required, reader.getIndex(),
                    reader.getLine(), this.reader.getColumn(), this.reader.getMark());
            this.possibleSimpleKeys.put(this.flowLevel, key);
        }
    }

    /**
     * Remove the saved possible key position at the current flow level.
     */
    private void removePossibleSimpleKey() {
        SimpleKey key = possibleSimpleKeys.remove(flowLevel);
        if (key != null && key.isRequired()) {
            throw new ScannerException("while scanning a simple key", key.getMark(),
                    "could not find expected ':'", reader.getMark());
        }
    }

    // Indentation functions.

    /**
     * * Handle implicitly ending multiple levels of block nodes by decreased
     * indentation. This function becomes important on lines 4 and 7 of this
     * example:
     *
     * 
     * 1) book one:
     * 2)   part one:
     * 3)     chapter one
     * 4)   part two:
     * 5)     chapter one
     * 6)     chapter two
     * 7) book two:
     * 
     * 

     * In flow context, tokens should respect indentation. Actually the
     * condition should be `self.indent >= column` according to the spec. But
     * this condition will prohibit intuitively correct constructions such as
     * key : { } 

     * *(anchor name)
     * 

     * &(anchor name)
     * 

         * A plain scalar may start with any non-space character except:
         *   '-', '?', ':', ',', '[', ']', '{', '}',
         *   '#', '&', '*', '!', '|', '>', '\'', '\"',
         *   '%', '@', '`'.
         *
         * It may also start with
         *   '-', '?', ':'
         * if it is followed by a non-space character.
         *
         * Note that we limit the last rule to the block context (except the
         * '-' character) because we want the flow context to be space
         * independent.
         * 

     * We ignore spaces, line breaks and comments.
     * If we find a line break in the block context, we set the flag
     * `allow_simple_key` on.
     * The byte order mark is stripped if it's the first character in the
     * stream. We do not yet support BOM inside the stream as the
     * specification requires. Any such mark will be considered as a part
     * of the document.
     * TODO: We need to make tab handling rules more sane. A good rule is
     *   Tabs cannot precede tokens
     *   BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END,
     *   KEY(block), VALUE(block), BLOCK-ENTRY
     * So the checking code is
     *   if <TAB>:
     *       self.allow_simple_keys = False
     * We also need to add the check for `allow_simple_keys == True` to
     * `unwind_indent` before issuing BLOCK-END.
     * Scanners for block, flow, and plain scalars need to be modified.
     * 

     * s-ignored-space+ c-tag-handle s-ignored-space+ ns-tag-prefix s-l-comments
     * 

     * The YAML 1.2 specification does not restrict characters for anchors and
     * aliases. This may lead to problems.
     * see https://bitbucket.org/asomov/snakeyaml/issues/485/alias-names-are-too-permissive-compared-to
     * This implementation tries to follow https://github.com/yaml/yaml-spec/blob/master/rfc/RFC-0003.md
     * 

     * See the specification for details.
     * Note that we loose indentation rules for quoted scalars. Quoted
     * scalars don't need to adhere indentation because " and ' clearly
     * mark the beginning and the end of them. Therefore we are less
     * restrictive then the specification requires. We only need to check
     * that document separators are not included in scalars.
     * 

     * See the specification for details.
     * We add an additional restriction for the flow context:
     *   plain scalars in the flow context cannot contain ',', ':' and '?'.
     * We also keep track of the `allow_simple_key` flag here.
     * Indentation rules are loosed for the flow context.
     * 

     * "!" (c-primary-tag-handle)
     * "!!" (ns-secondary-tag-handle)
     * "!(name)!" (c-named-tag-handle)
     * 

     * See the specification for details.
     * For some strange reasons, the specification does not allow '_' in
     * tag handles. I have allowed it anyway.
     * 

     * '\r\n' : '\n'
     * '\r' : '\n'
     * '\n' : '\n'
     * '\x85' : '\n'
     * default : ''
     *