org.languagetool.tokenizers.SrxTools Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of languagetool-core Show documentation

LanguageTool is an Open Source proofreading software for English, French, German, Polish, Romanian, and more than 20 other languages. It finds many errors that a simple spell checker cannot detect like mixing up there/their and it detects some grammar problems.

There is a newer version: 6.5

Show newest version

/* LanguageTool, a natural language style checker 
 * Copyright (C) 2014 Daniel Naber (http://www.danielnaber.de)
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.tokenizers;

import net.loomchild.segment.TextIterator;
import net.loomchild.segment.srx.SrxDocument;
import net.loomchild.segment.srx.SrxParser;
import net.loomchild.segment.srx.SrxTextIterator;
import net.loomchild.segment.srx.io.Srx2SaxParser;
import org.languagetool.JLanguageTool;

import java.io.*;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * Tools for loading an SRX tokenizer file.
 * @since 2.6
 */
final class SrxTools {

  private SrxTools() {
  }

  static SrxDocument createSrxDocument(String path) {
    try {
      try (
        InputStream inputStream = JLanguageTool.getDataBroker().getFromResourceDirAsStream(path);
        BufferedReader srxReader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))
      ) {
        Map parserParameters = new HashMap<>();
        parserParameters.put(Srx2SaxParser.VALIDATE_PARAMETER, true);
        SrxParser srxParser = new Srx2SaxParser(parserParameters);
        return srxParser.parse(srxReader);
      }
    } catch (IOException e) {
      throw new RuntimeException("Could not load SRX rules", e);
    }
  }

  static List tokenize(String text, SrxDocument srxDocument, String code) {
    List segments = new ArrayList<>();
    TextIterator textIterator = new SrxTextIterator(srxDocument, code, text);
    while (textIterator.hasNext()) {
      segments.add(textIterator.next());
    }
    return segments;
  }

}