com.ibm.icu.text.RuleBasedTransliterator Maven / Gradle / Ivy
Show all versions of virtdata-lib-realer Show documentation
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html#License
/*
*******************************************************************************
* Copyright (C) 1996-2016, International Business Machines Corporation and *
* others. All Rights Reserved. *
*******************************************************************************
*/
package com.ibm.icu.text;
import java.util.HashMap;
import java.util.Map;
/**
* RuleBasedTransliterator is a transliterator
* that reads a set of rules in order to determine how to perform
* translations. Rule sets are stored in resource bundles indexed by
* name. Rules within a rule set are separated by semicolons (';').
* To include a literal semicolon, prefix it with a backslash ('\').
* Unicode Pattern_White_Space is ignored.
* If the first non-blank character on a line is '#',
* the entire line is ignored as a comment.
*
* Each set of rules consists of two groups, one forward, and one
* reverse. This is a convention that is not enforced; rules for one
* direction may be omitted, with the result that translations in
* that direction will not modify the source text. In addition,
* bidirectional forward-reverse rules may be specified for
* symmetrical transformations.
*
*
Rule syntax
*
*
Rule statements take one of the following forms:
*
*
* $alefmadda=\u0622;- Variable definition. The name on the
* left is assigned the text on the right. In this example,
* after this statement, instances of the left hand name,
* "
$alefmadda", will be replaced by
* the Unicode character U+0622. Variable names must begin
* with a letter and consist only of letters, digits, and
* underscores. Case is significant. Duplicate names cause
* an exception to be thrown, that is, variables cannot be
* redefined. The right hand side may contain well-formed
* text of any length, including no text at all ("$empty=;").
* The right hand side may contain embedded UnicodeSet
* patterns, for example, "$softvowel=[eiyEIY]".
* -
* ai>$alefmadda;- Forward translation rule. This rule
* states that the string on the left will be changed to the
* string on the right when performing forward
* transliteration.
* -
* ai<$alefmadda;- Reverse translation rule. This rule
* states that the string on the right will be changed to
* the string on the left when performing reverse
* transliteration.
*
*
*
* ai<>$alefmadda;- Bidirectional translation rule. This
* rule states that the string on the right will be changed
* to the string on the left when performing forward
* transliteration, and vice versa when performing reverse
* transliteration.
*
*
* Translation rules consist of a match pattern and an output
* string. The match pattern consists of literal characters,
* optionally preceded by context, and optionally followed by
* context. Context characters, like literal pattern characters,
* must be matched in the text being transliterated. However, unlike
* literal pattern characters, they are not replaced by the output
* text. For example, the pattern "abc{def}"
* indicates the characters "def" must be
* preceded by "abc" for a successful match.
* If there is a successful match, "def" will
* be replaced, but not "abc". The final '}'
* is optional, so "abc{def" is equivalent to
* "abc{def}". Another example is "{123}456"
* (or "123}456") in which the literal
* pattern "123" must be followed by "456".
*
*
The output string of a forward or reverse rule consists of
* characters to replace the literal pattern characters. If the
* output string contains the character '|', this is
* taken to indicate the location of the cursor after
* replacement. The cursor is the point in the text at which the
* next replacement, if any, will be applied. The cursor is usually
* placed within the replacement text; however, it can actually be
* placed into the precending or following context by using the
* special character '@'. Examples:
*
*
* a {foo} z > | @ bar; # foo -> bar, move cursor
* before a
* {foo} xyz > bar @@|; # foo -> bar, cursor between
* y and z
*
*
* UnicodeSet
*
*
UnicodeSet patterns may appear anywhere that
* makes sense. They may appear in variable definitions.
* Contrariwise, UnicodeSet patterns may themselves
* contain variable references, such as "$a=[a-z];$not_a=[^$a]",
* or "$range=a-z;$ll=[$range]".
*
*
UnicodeSet patterns may also be embedded directly
* into rule strings. Thus, the following two rules are equivalent:
*
*
* $vowel=[aeiou]; $vowel>'*'; # One way to do this
* [aeiou]>'*';
* #
* Another way
*
*
* See {@link UnicodeSet} for more documentation and examples.
*
*
Segments
*
*
Segments of the input string can be matched and copied to the
* output string. This makes certain sets of rules simpler and more
* general, and makes reordering possible. For example:
*
*
* ([a-z]) > $1 $1;
* #
* double lowercase letters
* ([:Lu:]) ([:Ll:]) > $2 $1; # reverse order of Lu-Ll pairs
*
*
* The segment of the input string to be copied is delimited by
* "(" and ")". Up to
* nine segments may be defined. Segments may not overlap. In the
* output string, "$1" through "$9"
* represent the input string segments, in left-to-right order of
* definition.
*
*
Anchors
*
*
Patterns can be anchored to the beginning or the end of the text. This is done with the
* special characters '^' and '$'. For example:
*
*
* ^ a > 'BEG_A'; # match 'a' at start of text
* a > 'A'; # match other instances
* of 'a'
* z $ > 'END_Z'; # match 'z' at end of text
* z > 'Z'; # match other instances
* of 'z'
*
*
* It is also possible to match the beginning or the end of the text using a UnicodeSet.
* This is done by including a virtual anchor character '$' at the end of the
* set pattern. Although this is usually the match chafacter for the end anchor, the set will
* match either the beginning or the end of the text, depending on its placement. For
* example:
*
*
* $x = [a-z$]; # match 'a' through 'z' OR anchor
* $x 1 > 2; # match '1' after a-z or at the start
* 3 $x > 4; # match '3' before a-z or at the end
*
*
* Example
*
*
The following example rules illustrate many of the features of
* the rule language.
*
*
*
* Rule 1.
* abc{def}>x|y
*
* Rule 2.
* xyz>r
*
* Rule 3.
* yz>q
*
*
* Applying these rules to the string "adefabcdefz"
* yields the following results:
*
*
*
* |adefabcdefzInitial state, no rules match. Advance
* cursor.
*
*
* a|defabcdefzStill no match. Rule 1 does not match
* because the preceding context is not present.
*
*
* ad|efabcdefzStill no match. Keep advancing until
* there is a match...
*
*
* ade|fabcdefz...
*
*
* adef|abcdefz...
*
*
* adefa|bcdefz...
*
*
* adefab|cdefz...
*
*
* adefabc|defzRule 1 matches; replace "def"
* with "xy" and back up the cursor
* to before the 'y'.
*
*
* adefabcx|yzAlthough "xyz" is
* present, rule 2 does not match because the cursor is
* before the 'y', not before the 'x'.
* Rule 3 does match. Replace "yz"
* with "q".
*
*
* adefabcxq|The cursor is at the end;
* transliteration is complete.
*
*
*
* The order of rules is significant. If multiple rules may match
* at some point, the first matching rule is applied.
*
*
Forward and reverse rules may have an empty output string.
* Otherwise, an empty left or right hand side of any statement is a
* syntax error.
*
*
Single quotes are used to quote any character other than a
* digit or letter. To specify a single quote itself, inside or
* outside of quotes, use two single quotes in a row. For example,
* the rule "'>'>o''clock" changes the
* string ">" to the string "o'clock".
*
*
Notes
*
*
While a RuleBasedTransliterator is being built, it checks that
* the rules are added in proper order. For example, if the rule
* "a>x" is followed by the rule "ab>y",
* then the second rule will throw an exception. The reason is that
* the second rule can never be triggered, since the first rule
* always matches anything it matches. In other words, the first
* rule masks the second rule.
*
* @author Alan Liu
* @internal
* @deprecated This API is ICU internal only.
*/
@Deprecated
public class RuleBasedTransliterator extends Transliterator {
private final Data data;
// /**
// * Constructs a new transliterator from the given rules.
// * @param rules rules, separated by ';'
// * @param direction either FORWARD or REVERSE.
// * @exception IllegalArgumentException if rules are malformed
// * or direction is invalid.
// */
// public RuleBasedTransliterator(String ID, String rules, int direction,
// UnicodeFilter filter) {
// super(ID, filter);
// if (direction != FORWARD && direction != REVERSE) {
// throw new IllegalArgumentException("Invalid direction");
// }
//
// TransliteratorParser parser = new TransliteratorParser();
// parser.parse(rules, direction);
// if (parser.idBlockVector.size() != 0 ||
// parser.compoundFilter != null) {
// throw new IllegalArgumentException("::ID blocks illegal in RuleBasedTransliterator constructor");
// }
//
// data = (Data)parser.dataVector.get(0);
// setMaximumContextLength(data.ruleSet.getMaximumContextLength());
// }
// /**
// * Constructs a new transliterator from the given rules in the
// * FORWARD direction.
// * @param rules rules, separated by ';'
// * @exception IllegalArgumentException if rules are malformed
// * or direction is invalid.
// */
// public RuleBasedTransliterator(String ID, String rules) {
// this(ID, rules, FORWARD, null);
// }
RuleBasedTransliterator(String ID, Data data, UnicodeFilter filter) {
super(ID, filter);
this.data = data;
setMaximumContextLength(data.ruleSet.getMaximumContextLength());
}
/**
* Implements {@link Transliterator#handleTransliterate}.
* @internal
* @deprecated This API is ICU internal only.
*/
@Override
@Deprecated
protected void handleTransliterate(Replaceable text,
Position index, boolean incremental) {
/* We keep start and limit fixed the entire time,
* relative to the text -- limit may move numerically if text is
* inserted or removed. The cursor moves from start to limit, with
* replacements happening under it.
*
* Example: rules 1. ab>x|y
* 2. yc>z
*
* |eabcd start - no match, advance cursor
* e|abcd match rule 1 - change text & adjust cursor
* ex|ycd match rule 2 - change text & adjust cursor
* exz|d no match, advance cursor
* exzd| done
*/
/* A rule like
* a>b|a
* creates an infinite loop. To prevent that, we put an arbitrary
* limit on the number of iterations that we take, one that is
* high enough that any reasonable rules are ok, but low enough to
* prevent a server from hanging. The limit is 16 times the
* number of characters n, unless n is so large that 16n exceeds a
* uint32_t.
*/
synchronized(data) {
int loopCount = 0;
int loopLimit = (index.limit - index.start) << 4;
if (loopLimit < 0) {
loopLimit = 0x7FFFFFFF;
}
while (index.start < index.limit &&
loopCount <= loopLimit &&
data.ruleSet.transliterate(text, index, incremental)) {
++loopCount;
}
}
}
static class Data {
public Data() {
variableNames = new HashMap();
ruleSet = new TransliterationRuleSet();
}
/**
* Rule table. May be empty.
*/
public TransliterationRuleSet ruleSet;
/**
* Map variable name (String) to variable (char[]). A variable name
* corresponds to zero or more characters, stored in a char[] array in
* this hash. One or more of these chars may also correspond to a
* UnicodeSet, in which case the character in the char[] in this hash is
* a stand-in: it is an index for a secondary lookup in
* data.variables. The stand-in also represents the UnicodeSet in
* the stored rules.
*/
Map variableNames;
/**
* Map category variable (Character) to UnicodeMatcher or UnicodeReplacer.
* Variables that correspond to a set of characters are mapped
* from variable name to a stand-in character in data.variableNames.
* The stand-in then serves as a key in this hash to lookup the
* actual UnicodeSet object. In addition, the stand-in is
* stored in the rule text to represent the set of characters.
* variables[i] represents character (variablesBase + i).
*/
Object[] variables;
/**
* The character that represents variables[0]. Characters
* variablesBase through variablesBase +
* variables.length - 1 represent UnicodeSet objects.
*/
char variablesBase;
/**
* Return the UnicodeMatcher represented by the given character, or
* null if none.
*/
public UnicodeMatcher lookupMatcher(int standIn) {
int i = standIn - variablesBase;
return (i >= 0 && i < variables.length)
? (UnicodeMatcher) variables[i] : null;
}
/**
* Return the UnicodeReplacer represented by the given character, or
* null if none.
*/
public UnicodeReplacer lookupReplacer(int standIn) {
int i = standIn - variablesBase;
return (i >= 0 && i < variables.length)
? (UnicodeReplacer) variables[i] : null;
}
}
/**
* Return a representation of this transliterator as source rules.
* These rules will produce an equivalent transliterator if used
* to construct a new transliterator.
* @param escapeUnprintable if TRUE then convert unprintable
* character to their hex escape representations, \\uxxxx or
* \\Uxxxxxxxx. Unprintable characters are those other than
* U+000A, U+0020..U+007E.
* @return rules string
* @internal
* @deprecated This API is ICU internal only.
*/
@Override
@Deprecated
public String toRules(boolean escapeUnprintable) {
return data.ruleSet.toRules(escapeUnprintable);
}
// /**
// * Return the set of all characters that may be modified by this
// * Transliterator, ignoring the effect of our filter.
// */
// protected UnicodeSet handleGetSourceSet() {
// return data.ruleSet.getSourceTargetSet(false, unicodeFilter);
// }
//
// /**
// * Returns the set of all characters that may be generated as
// * replacement text by this transliterator.
// */
// public UnicodeSet getTargetSet() {
// return data.ruleSet.getSourceTargetSet(true, unicodeFilter);
// }
/**
* @internal
* @deprecated This API is ICU internal only.
*/
@Deprecated
@Override
public void addSourceTargetSet(UnicodeSet filter, UnicodeSet sourceSet, UnicodeSet targetSet) {
data.ruleSet.addSourceTargetSet(filter, sourceSet, targetSet);
}
/**
* Temporary hack for registry problem. Needs to be replaced by better architecture.
* @internal
* @deprecated This API is ICU internal only.
*/
@Deprecated
public Transliterator safeClone() {
UnicodeFilter filter = getFilter();
if (filter != null && filter instanceof UnicodeSet) {
filter = new UnicodeSet((UnicodeSet)filter);
}
return new RuleBasedTransliterator(getID(), data, filter);
}
}