org.fryske_akademy.exist.lucene.NoSemiColonTokenizer Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of exist-db-addons Show documentation

Functions for exist-db: module to load properties from filesystem, cronjobs to import data from filesystem to collection, some analyzers for lucene.

The newest version!

package org.fryske_akademy.exist.lucene;

/*-
 * #%L
 * exist-db-addons
 * %%
 * Copyright (C) 2020 - 2021 Fryske Akademy
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

import org.apache.lucene.analysis.util.CharTokenizer;
import org.apache.lucene.util.AttributeFactory;

import java.io.Reader;

/**
 * For this tokenizer "; " or ";" are token separators, and therefore no token characters.
 */
public class NoSemiColonTokenizer extends CharTokenizer {

    public NoSemiColonTokenizer(Reader input) {
        super(input);
    }

    public NoSemiColonTokenizer(AttributeFactory factory, Reader input) {
        super(factory, input);
    }

    private boolean prevsemicolon = false;

    @Override
    protected boolean isTokenChar(int c) {
        boolean semicolon = c == ';';
        boolean rv = c == ' ' ? !prevsemicolon : !semicolon;
        prevsemicolon=semicolon;
        return rv;
    }
}