• Home
• org.bigml
• mimir
• 0.8.3
• source code
• TokenStreamFactory.class

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

org.bigml.mimir.nlp.tokenization.TokenStreamFactory.class Maven / Gradle / Ivy

????7
-
^?	,?	,?	,?	,?	,?	,??
??
????
?????
???
???
???
H???
??
?
??
????
,??
 ??????
,?
,???
*????
.???
1??
3??
5??
7???
:??
<??
>??
@?
??
C???????
F??
J?
,?????
O???????
O?
M?
,?
???
X?
H?
H?????
CombinationStreamFactory
InnerClasses
NullStreamFactory
DelimitedStreamFactory
RegexStreamFactory
DictionaryStreamFactory
CharNGramStreamFactory
NGramStreamFactory
BigramStreamFactory
ChineseNGramStreamFactory
ChineseStreamFactory
JapaneseStreamFactory
LetterNumberStreamFactory
ALL_MODE
Ljava/lang/String;

ConstantValue
FULL_TERMS_MODE
CHARACTER_MODE
_gram

I
_case

Z
	_fullTerm

_stopWords
Ljava/util/HashSet;
	Signature
'Ljava/util/HashSet;
	_patterns
[Ljava/util/regex/Pattern;

_delimiter
_MAX_FULL_TERM_LENGTH


()V
Code
LineNumberTable
	getStream
B(Ljava/lang/String;)Lorg/bigml/mimir/nlp/tokenization/TokenStream;
getTextBuilder
`(Lcom/fasterxml/jackson/databind/JsonNode;)Lorg/bigml/mimir/nlp/tokenization/TokenStreamFactory;

StackMapTable??
getItemsBuilder

getBuilder
Y(Ljava/lang/String;ZLjava/util/Set;)Lorg/bigml/mimir/nlp/tokenization/TokenStreamFactory;
m(Ljava/lang/String;ZLjava/util/Set;)Lorg/bigml/mimir/nlp/tokenization/TokenStreamFactory;
|(Ljava/lang/String;Ljava/lang/String;IZLjava/util/Set;Ljava/util/List;)Lorg/bigml/mimir/nlp/tokenization/TokenStreamFactory;
?(Ljava/lang/String;Ljava/lang/String;IZLjava/util/Set;Ljava/util/List;)Lorg/bigml/mimir/nlp/tokenization/TokenStreamFactory;
toDoubleArrayTrie
<(Ljava/util/Collection;)Lorg/trie4j/doublearray/DoubleArray;?
P(Ljava/util/Collection;)Lorg/trie4j/doublearray/DoubleArray;
getTokenList
$(Ljava/lang/String;)Ljava/util/List;??
8(Ljava/lang/String;)Ljava/util/List;
setTokenMode
(Ljava/lang/String;)V

SourceFile
TokenStreamFactory.java
NestMembers?qrstutvwz{|m

term_analysis????
language
none???

token_mode
tokens_only
case_sensitive?

ngrams



bigrams


zh


	tag_cloud
java/util/HashSet


?
	




??
Jorg/bigml/mimir/nlp/tokenization/TokenStreamFactory$DelimitedStreamFactory

item_analysis
	separator
text
optype
items????
BootstrapMethods





"java/lang/IllegalArgumentException?
3org/bigml/mimir/nlp/tokenization/TokenStreamFactory

characters
Jorg/bigml/mimir/nlp/tokenization/TokenStreamFactory$CharNGramStreamFactory
full_terms_only
Eorg/bigml/mimir/nlp/tokenization/TokenStreamFactory$NullStreamFactory
Korg/bigml/mimir/nlp/tokenization/TokenStreamFactory$DictionaryStreamFactory

Morg/bigml/mimir/nlp/tokenization/TokenStreamFactory$ChineseNGramStreamFactory
Horg/bigml/mimir/nlp/tokenization/TokenStreamFactory$ChineseStreamFactory
ja
Iorg/bigml/mimir/nlp/tokenization/TokenStreamFactory$JapaneseStreamFactory
Gorg/bigml/mimir/nlp/tokenization/TokenStreamFactory$BigramStreamFactory
Forg/bigml/mimir/nlp/tokenization/TokenStreamFactory$NGramStreamFactory
Morg/bigml/mimir/nlp/tokenization/TokenStreamFactory$LetterNumberStreamFactory

Forg/bigml/mimir/nlp/tokenization/TokenStreamFactory$RegexStreamFactory
java/util/regex/Pattern?

java/lang/String


Lorg/bigml/mimir/nlp/tokenization/TokenStreamFactory$CombinationStreamFactory
??
"org/trie4j/doublearray/DoubleArray



 org/trie4j/patricia/PatriciaTrie






?
??
 
!
java/util/ArrayList
"

#

all
java/lang/Object
'com/fasterxml/jackson/databind/JsonNode

java/util/Set
java/util/Iterator
java/util/List
,org/bigml/mimir/nlp/tokenization/TokenStream
has
(Ljava/lang/String;)Z
get
=(Ljava/lang/String;)Lcom/fasterxml/jackson/databind/JsonNode;
org/bigml/mimir/utils/Json
	getString
a(Lcom/fasterxml/jackson/databind/JsonNode;Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;

getBoolean
?(Lcom/fasterxml/jackson/databind/JsonNode;Ljava/lang/String;Z)Z
asInt
()I
	asBoolean
()Z
equals
(Ljava/lang/Object;)Z
size
(I)V
,(I)Lcom/fasterxml/jackson/databind/JsonNode;
asText
()Ljava/lang/String;
add

$
%
Cannot tokenize optype '
'!
makeConcatWithConstants
&(Ljava/lang/String;)Ljava/lang/String;
(Ljava/lang/Iterable;)V
(Ljava/util/Collection;)V
(I)Ljava/lang/Object;
compile
-(Ljava/lang/String;)Ljava/util/regex/Pattern;
m(Lorg/bigml/mimir/nlp/tokenization/TokenStreamFactory;Lorg/bigml/mimir/nlp/tokenization/TokenStreamFactory;)V
java/util/Collection
isEmpty
iterator
()Ljava/util/Iterator;
hasNext
next
()Ljava/lang/Object;
insert
(Lorg/trie4j/Trie;)V
toList
()Ljava/util/List;
length
toLowerCase
&

)
$java/lang/invoke/StringConcatFactory
+
Lookup
?(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;Ljava/lang/invoke/MethodType;Ljava/lang/String;[Ljava/lang/Object;)Ljava/lang/invoke/CallSite;
,
%java/lang/invoke/MethodHandles$Lookup
java/lang/invoke/MethodHandles!,^
lm
n]om
n0pm
n-qrstutvw
xyz{|m}r
n~

?
?S
#*?
*?*?*?*
?*
?*
??
?+	,-./0
??	??
?
L	?*L*?	?
*?
L+?
M+?
N+?66+?	?+?
?6?+?	?+?
??6
:,?K,??B*?
:?Y??:6??????W?
???,-
???N35689$:-<0>9?G@\A_CbEoFwG?I?J?I?M?+???4??HH

?(??
?%	??
?K#? Y?!L*"?
#?
?M+,?+?+?
?QRTU!W	??
??B$L*%?	?
*%?
?L+&??*?'?+$??*?(?+?)M?*Y,?+??&	[]^`a$b-c2e9f?
?H

	???~:
N*-???.Y?/N? *0???1Y?2N??3Y,?4N-?-??"prstu*w3z8{?
?,x?	???
?

<
:
:+-???.Y?/:??+0???1Y?2:?q*?)*?? ??5Y?6:?S?7Y?8:?G*?*9???:Y?;:?.??Y??:??@Y?A:???*?*???Y?B??]?CY?D:??E
?F?6		?E
? ?		?G?H?IS?	
??ڻJY?K:?
?+?L:???!?????$?0?=?N?Z?g?s?x???????????????????????????
?
?
&?
)?
/?
5?
9??"?,,
?%,
?(?x?	????N*?M?*?M?*??*?N
?6?OY?PL*?Q
M,?R
?,?S
?HN+-?T???MY+?U?
?????!?C?L???O???x?
????b
M*+?VN-?-?WM??XY?YM*??@+?Z
?6+:*??
?[:,?E
?,?G??,?\W,??*
?????-?0?>?W?`??????!H?x???
?O +]??+0??*??*???????????J1 C3.><57:@`j
J,_
1,a
 ,b
C,c
3,d
.,e
>,f
<,g
5,h
7,i
:,j
@,k

'
*
(?
?
?