com.groupbyinc.flux.common.apache.lucene.analysis.wikipedia.WikipediaTokenizer.class Maven / Gradle / Ivy
???? 4u Ncom/groupbyinc/flux/common/apache/lucene/analysis/wikipedia/WikipediaTokenizer ;com/groupbyinc/flux/common/apache/lucene/analysis/Tokenizer WikipediaTokenizer.java Ccom/groupbyinc/flux/common/apache/lucene/util/AttributeSource$State =com/groupbyinc/flux/common/apache/lucene/util/AttributeSource State
INTERNAL_LINK Ljava/lang/String; il
EXTERNAL_LINK el EXTERNAL_LINK_URL elu CITATION ci CATEGORY c BOLD b ITALICS i BOLD_ITALICS bi " HEADING h % SUB_HEADING sh ( ALPHANUM_ID I
APOSTROPHE_ID
ACRONYM_ID
COMPANY_ID EMAIL_ID HOST_ID NUM_ID CJ_ID INTERNAL_LINK_ID EXTERNAL_LINK_ID CITATION_ID
CATEGORY_ID BOLD_ID
ITALICS_ID
BOLD_ITALICS_ID
HEADING_ID SUB_HEADING_ID EXTERNAL_LINK_URL_ID TOKEN_TYPES [Ljava/lang/String; TOKENS_ONLY UNTOKENIZED_ONLY BOTH UNTOKENIZED_TOKEN_FLAG scanner TLcom/groupbyinc/flux/common/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl; tokenOutput untokenizedTypes Ljava/util/Set; #Ljava/util/Set; tokens Ljava/util/Iterator; [Ljava/util/Iterator; offsetAtt SLcom/groupbyinc/flux/common/apache/lucene/analysis/tokenattributes/OffsetAttribute; typeAtt QLcom/groupbyinc/flux/common/apache/lucene/analysis/tokenattributes/TypeAttribute;
posIncrAtt ^Lcom/groupbyinc/flux/common/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute; termAtt ULcom/groupbyinc/flux/common/apache/lucene/analysis/tokenattributes/CharTermAttribute; flagsAtt RLcom/groupbyinc/flux/common/apache/lucene/analysis/tokenattributes/FlagsAttribute; first Z ()V java/util/Collections l emptySet ()Ljava/util/Set; n o
m p (ILjava/util/Set;)V j r
s this PLcom/groupbyinc/flux/common/apache/lucene/analysis/wikipedia/WikipediaTokenizer; j k
w W + y X Y { [ \ } Qcom/groupbyinc/flux/common/apache/lucene/analysis/tokenattributes/OffsetAttribute addAttribute L(Ljava/lang/Class;)Lcom/groupbyinc/flux/common/apache/lucene/util/Attribute; ? ?
? ^ _ ? Ocom/groupbyinc/flux/common/apache/lucene/analysis/tokenattributes/TypeAttribute ? ` a ? \com/groupbyinc/flux/common/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute ? b c ? Scom/groupbyinc/flux/common/apache/lucene/analysis/tokenattributes/CharTermAttribute ? d e ? Pcom/groupbyinc/flux/common/apache/lucene/analysis/tokenattributes/FlagsAttribute ? f g ? Rcom/groupbyinc/flux/common/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl ? input Ljava/io/Reader; ? ? ? (Ljava/io/Reader;)V j ?
? ? U V ? init ? r
? S(Lcom/groupbyinc/flux/common/apache/lucene/util/AttributeFactory;ILjava/util/Set;)V C(Lcom/groupbyinc/flux/common/apache/lucene/util/AttributeFactory;)V j ?
? factory @Lcom/groupbyinc/flux/common/apache/lucene/util/AttributeFactory; "java/lang/IllegalArgumentException ? 9tokenOutput must be TOKENS_ONLY, UNTOKENIZED_ONLY or BOTH ? (Ljava/lang/String;)V j ?
? ? incrementToken ()Z java/io/IOException ? java/util/Iterator ? hasNext ? ? ? ? next ()Ljava/lang/Object; ? ? ? ? restoreState H(Lcom/groupbyinc/flux/common/apache/lucene/util/AttributeSource$State;)V ? ?
? clearAttributes ? k
? getNextToken ()I ? ?
? ? O P ? ?
java/util/Set ? contains (Ljava/lang/Object;)Z ? ? ? ? java/lang/String ?
setupToken ? k
? collapseTokens (I)V ? ?
? collapseAndSaveTokens (ILjava/lang/String;)V ? ?
? getPositionIncrement ? ?
? ? h i ? setPositionIncrement ? ? ? ? setType ? ? ? ? state ELcom/groupbyinc/flux/common/apache/lucene/util/AttributeSource$State; tokenType type posinc java/lang/StringBuilder ? j ?
? ? setText (Ljava/lang/StringBuilder;)I ? ?
? ? yychar ? ?
? ? java/util/ArrayList ?
? w setupSavedToken ? ?
? captureState G()Lcom/groupbyinc/flux/common/apache/lucene/util/AttributeSource$State;
java/util/List add ? getNumWikiTokensSeen ?
?
append (C)Ljava/lang/StringBuilder;
? toString ()Ljava/lang/String;
? trim
? setEmpty W()Lcom/groupbyinc/flux/common/apache/lucene/analysis/tokenattributes/CharTermAttribute; ? i(Ljava/lang/String;)Lcom/groupbyinc/flux/common/apache/lucene/analysis/tokenattributes/CharTermAttribute; ?
correctOffset (I)I
length " ?
?# setOffset (II)V %& ?' setFlags ) ? ?* yylength , ?
?-
yypushback / ?
?0 iterator ()Ljava/util/Iterator; 23 4 currPos buffer Ljava/lang/StringBuilder; numAdded theStart lastPos
tmpTokType numSeen tmp WLjava/util/List; Ljava/util/List; s positionInc getText X(Lcom/groupbyinc/flux/common/apache/lucene/analysis/tokenattributes/CharTermAttribute;)V CD
?E ?# start close I k
J yyreset L ?
?M reset O k
P
?P end S k
T finalOffset
X Z \ ^ ` b d f ?
ConstantValue Signature Code LocalVariableTable LineNumberTable LocalVariableTypeTable '(ILjava/util/Set;)V g(Lcom/groupbyinc/flux/common/apache/lucene/util/AttributeFactory;ILjava/util/Set;)V
StackMapTable
Exceptions
SourceFile InnerClasses 1 + i i i i i i i ! i # $ i & ' i ) * + i , - + i . / + i 0 1 + i 2 3 + i 4 5 + i 6 7 + i 8 9 + i : ; + i < = + i > ? + i @ A + i B C + i D E + i F G + i H I + i J K + i L M + i N O P Q + i , R + i . S + i 0 T + i . U V W + X Y j Z [ \ j ] ^ _ ` a b c d e f g h i
j k k 7 *? q? t? l u v m
? ? j r k ? l*? x*? z*? q? |*? ~**?? ?? ?? ?**?? ?? ?? ?**?? ?? ?? ?**?? ?? ?? ?**?? ?? ?? ?*? ?Y*? ?? ?? ?*,? ?? l l u v l W + l X Y n l X Z m 2 ? p q r t " u / v < w I x V ? e ? k ?j o j ? k ? m*+? ?*? z*? q? |*? ~**?? ?? ?? ?**?? ?? ?? ?**?? ?? ?? ?**?? ?? ?? ?**?? ?? ?? ?*? ?Y*? ?? ?? ?* -? ?? l * m u v m ? ? m W + m X Y n m X Z m 2 ? p
q r t # u 0 v = w J x W ? f ? l ?j p ? r k ? #? ? ?
? ?Y?? ??*? z*,? |? l # u v # W + # X Y n # X Z m ? ? ? ? " ?q j o ? ? k w ?*? ~? #*? ~? ? ? *? ~? ? ? L*+? ??*? ?*? ?? ?<? ?? ?2M*? z? *? |,? ? ?
*? ֧ /*? z? *? |,? ? ? *? ڧ *? z? *,? ?*? ?? ?>*? ? ? >*? ? ? ? *? ?,? ? *? ?? l 4 ? ? ? u v 3 ? ? + @ w ? ? ( ? + m Z ? ? ? % ? ' ? + ? 3 ? 8 ? : ? @ ? T ? [ ? q ? y ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?q '? ? ?
? r ? ? ? k a ? ?Y ? ?N*? ?-? ?6*? ?? ?6`66? ?Y? ?:*,? ?*?? W*? ?? ?Y6 ? g ? a*? ?? ? U*? ?? ?6
6
d? - ?W? ???*? ?-? ?6**? ?? ?,? ?*?? W?
`6???-??:
*? ??
? W*? ?*?!*
?$`?!?( *? ??+ ? *? ?*? ??.?1*?5 ? ~? l ?
n + k F6 +
u v ? + ?
78 ?9 + ?: + $ ?; + L ?< + ' ?= + 0 ?>@ ? VA
n 0 ?>? m n ?
? ? ? $ ? ' ? 0 ? 6 ? B ? b ? k ? x ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?q , ? B ? ? ? +? ? .? R ?r ? ? ? k c *? ?*? ?? ? *? ?,? ? ? l u v B + ? m ? ? ? ? ? ? k ?
ٻ ?Y ? ?M*? ?,? ?>*? ?? ?6 `66*? ?? ?Y6? M? G*? ?? ? ;*? ?? ?66 d? , ?W? ???*? ?,? ?>? `6???,??:*? ?? ? W*? ?*?!*?$`?!?( *? ??+ ? *? ?*? ??.?1? *? ~? l p Q + N ,6 + ? u v ? ? +
?78 ?9 + ?: + " ?; + / ?< + % ?= + ? SA m Z ?
? ? " % E N [ b h
q t z
} ? ? ? ? ? ? ?q ' ? % ? ? +? ? ? U ?r ? ? k k q 1*? ?*? ??F*? ?? ?<*? ?*?!**? ??G `?!?( ? l 1 u v H + m 0 I k k B *?K*? ?*? ??N? l u v m $ % &r ? O k k _ !*?Q*? ?*? ??N*? ~*? ??R*? ? l ! u v m / 0 1 2 3 4r ? S k k d $*?U**? ?? ?*? ??.`?!<*? ??( ? l $ u v V + m 8 : ; #<r ? W k k ? w? ?YYSY[SY]SY_SYaSYcSYeSYgSYSY SY
SY SY SY
SY#SY&SY)SYS?h? m F s t
© 2015 - 2025 Weber Informatics LLC | Privacy Policy