org.apache.lucene.analysis.cn.smart.Utility Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.analysis.cn.smart;
import org.apache.lucene.analysis.cn.smart.hhmm.SegTokenFilter; // for javadoc
import static java.lang.Character.isSurrogate;
/**
* SmartChineseAnalyzer utility constants and methods
* @lucene.experimental
*/
public class Utility {
public static final char[] STRING_CHAR_ARRAY = "未##串"
.toCharArray();
public static final char[] NUMBER_CHAR_ARRAY = "未##数"
.toCharArray();
public static final char[] START_CHAR_ARRAY = "始##始"
.toCharArray();
public static final char[] END_CHAR_ARRAY = "末##末".toCharArray();
/**
* Delimiters will be filtered to this character by {@link SegTokenFilter}
*/
public static final char[] COMMON_DELIMITER = new char[] { ',' };
/**
* Space-like characters that need to be skipped: such as space, tab, newline, carriage return.
*/
public static final String SPACES = " \t\r\n";
/**
* Maximum bigram frequency (used in the smoothing function).
*/
public static final int MAX_FREQUENCE = 2079997 + 80000;
/**
* compare two arrays starting at the specified offsets.
*
* @param larray left array
* @param lstartIndex start offset into larray
* @param rarray right array
* @param rstartIndex start offset into rarray
* @return 0 if the arrays are equal，1 if larray > rarray, -1 if larray < rarray
*/
public static int compareArray(char[] larray, int lstartIndex, char[] rarray,
int rstartIndex) {
if (larray == null) {
if (rarray == null || rstartIndex >= rarray.length)
return 0;
else
return -1;
} else {
// larray != null
if (rarray == null) {
if (lstartIndex >= larray.length)
return 0;
else
return 1;
}
}
int li = lstartIndex, ri = rstartIndex;
while (li < larray.length && ri < rarray.length && larray[li] == rarray[ri]) {
li++;
ri++;
}
if (li == larray.length) {
if (ri == rarray.length) {
// Both arrays are equivalent, return 0.
return 0;
} else {
// larray < rarray because larray has ended first.
return -1;
}
} else {
// differing lengths
if (ri == rarray.length) {
// larray > rarray because rarray has ended first.
return 1;
} else {
// determine by comparison
if (larray[li] > rarray[ri])
return 1;
else
return -1;
}
}
}
/**
* Compare two arrays, starting at the specified offsets, but treating shortArray as a prefix to longArray.
* As long as shortArray is a prefix of longArray, return 0.
* Otherwise, behave as {@link Utility#compareArray(char[], int, char[], int)}
*
* @param shortArray prefix array
* @param shortIndex offset into shortArray
* @param longArray long array (word)
* @param longIndex offset into longArray
* @return 0 if shortArray is a prefix of longArray, otherwise act as {@link Utility#compareArray(char[], int, char[], int)}
*/
public static int compareArrayByPrefix(char[] shortArray, int shortIndex,
char[] longArray, int longIndex) {
// a null prefix is a prefix of longArray
if (shortArray == null)
return 0;
else if (longArray == null)
return (shortIndex < shortArray.length) ? 1 : 0;
int si = shortIndex, li = longIndex;
while (si < shortArray.length && li < longArray.length
&& shortArray[si] == longArray[li]) {
si++;
li++;
}
if (si == shortArray.length) {
// shortArray is a prefix of longArray
return 0;
} else {
// shortArray > longArray because longArray ended first.
if (li == longArray.length)
return 1;
else
// determine by comparison
return (shortArray[si] > longArray[li]) ? 1 : -1;
}
}
/**
* Return the internal {@link CharType} constant of a given character.
* @param ch input character
* @return constant from {@link CharType} describing the character type.
*
* @see CharType
*/
public static int getCharType(char ch) {
if (isSurrogate(ch))
return CharType.SURROGATE;
// Most (but not all!) of these are Han Ideographic Characters
if (ch >= 0x4E00 && ch <= 0x9FA5)
return CharType.HANZI;
if ((ch >= 0x0041 && ch <= 0x005A) || (ch >= 0x0061 && ch <= 0x007A))
return CharType.LETTER;
if (ch >= 0x0030 && ch <= 0x0039)
return CharType.DIGIT;
if (ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' || ch == ' ')
return CharType.SPACE_LIKE;
// Punctuation Marks
if ((ch >= 0x0021 && ch <= 0x00BB) || (ch >= 0x2010 && ch <= 0x2642)
|| (ch >= 0x3001 && ch <= 0x301E))
return CharType.DELIMITER;
// Full-Width range
if ((ch >= 0xFF21 && ch <= 0xFF3A) || (ch >= 0xFF41 && ch <= 0xFF5A))
return CharType.FULLWIDTH_LETTER;
if (ch >= 0xFF10 && ch <= 0xFF19)
return CharType.FULLWIDTH_DIGIT;
if (ch >= 0xFE30 && ch <= 0xFF63)
return CharType.DELIMITER;
return CharType.OTHER;
}
}