Misc.make_ucnhashdat.py Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of jython Show documentation
Show all versions of jython Show documentation
Jython is an implementation of the high-level, dynamic, object-oriented
language Python written in 100% Pure Java, and seamlessly integrated with
the Java platform. It thus allows you to run Python on any Java platform.
# Copyright Finn Bock
# Updated 2017 by Stefan Richthofer to support Unicode 9.0
#
# Generate a ucnhash.dat file with mapping from unicode
# names to codepoints.
#
# python make_ucnhashdat.py UnicodeData.txt mph.exe
#
# The "mph" program must be available on the path.
# This program is used to create the minimum perfect
# hash used by the wordhash table.
#
# I've used 1.2 from:
#
# http://www.ibiblio.org/pub/Linux/devel/lang/c/!INDEX.short.html
#
import fileinput, re, os, sys, struct, cStringIO
mph_exec = 'mph' # Formerly: 'mph.exe'
def debug(str):
print >>debugFile, str
def splitIntoWords(name):
wordlist = []
wordstart = 0
l = len(name)
for i in range(l):
c = name[i]
n = None
if c == ' ' or c == '-':
n = name[wordstart:i]
elif i == l-1:
n = name[wordstart:i+1]
if n:
wordstart = i
if c == '-' and n != '':
n += '-'
if c == ' ' or c == '-':
wordstart = i+1
wordlist.append(n)
return wordlist
def readUnicodeDict(file):
d = {}
for l in fileinput.input(file):
l = l.strip().split(";");
v,name = l[0:2]
if name == "":
name = l[10]
if name == '':
continue
if name[0] == "<":
continue
#handled by code in ucnhash
if name.startswith("CJK COMPATIBILITY IDEOGRAPH-"):
continue
wordlist = splitIntoWords(name)
d[name] = (int(v, 16), wordlist, [])
return d
def count(dict, index):
c = dict.get(index)
if c is None: c = 0
c += 1
dict[index] = c
def dumpUnicodeDict(title, dict):
lst = []
i = 0
for k,(v,wordlist, rawlist) in dict.items():
p = wordlist[:]
lst.append((v, k, p))
lst.sort()
print "=======", title
for v,k,p in lst:
print "%.4X %s %s" % (v, k, p)
class MphEmitter:
def readint(self):
return int(self.inf.readline().strip())
def readfloat(self):
return float(self.inf.readline().strip())
def readconst(self):
global d, n, m, c, maxlen, minklen, maxklen, minchar, maxchar, alphasz
self.inf.readline()
self.d = self.readint()
self.n = self.readint()
self.m = self.readint()
self.c = self.readfloat()
self.maxlen = self.readint()
self.minklen = self.readint()
self.maxklen = self.readint()
self.minchar = self.readint()
self.maxchar = self.readint()
self.loop = self.readint()
self.numiter= self.readint()
self.readint()
self.readint()
debug(" * d=%d" % self.d)
debug(" * n=%d" % self.n)
debug(" * m=%d" % self.m)
debug(" * c=%g" % self.c)
debug(" * maxlen=%d" % self.maxlen)
debug(" * minklen=%d" % self.minklen)
debug(" * maxklen=%d" % self.maxklen)
debug(" * minchar=%d" % self.minchar)
debug(" * maxchar=%d" % self.maxchar)
self.alphasz = self.maxchar - self.minchar+1;
def readg(self):
data = Table()
for i in range(self.n):
v = self.readint()
data.write_Short(v)
return data
def readT(self, t):
data = Table()
for i in range(self.maxlen):
for j in range(256):
v = self.readint()
if j < self.minchar or j > self.maxchar:
continue
data.write_Short(v)
return data
def writeFile(self, inf, outf):
self.inf = inf
self.readconst();
outf.write(struct.pack("!hhhhh", self.n,
self.m,
self.minchar,
self.alphasz,
self.maxlen))
G = self.readg()
debug("G len: %d" % (G.size()/2))
G.writeto(outf)
outf.write(struct.pack("!h", self.d))
for t in range(self.d):
T = self.readT(t)
debug("T%d len: %d" % (t, T.size()/2))
T.writeto(outf)
class Table:
def __init__(self):
self.buf = cStringIO.StringIO()
def write_Str(self, str):
self.buf.write(str)
def write_Short(self, v):
self.buf.write(struct.pack("!h", v))
def write_UShort(self, v):
self.buf.write(struct.pack("!H", v))
def write_Int32(self, v):
self.buf.write(struct.pack("!i", v))
def write_UInt32(self, v):
self.buf.write(struct.pack("!I", v))
def writeto(self, file):
file.write('t')
file.write(struct.pack("!I", self.size()))
file.write(self.buf.getvalue())
def size(self):
return self.buf.tell()
def calculateSize(dict):
cnt = 0
for name in dict.keys():
cnt += len(name)
return cnt
def calculateWords(unicodeDict):
words = {}
for key, (value, wordlist, rawlist) in unicodeDict.items():
for name in wordlist:
wordlist = words.setdefault(name, [])
wordlist.append(key)
return words
def replaceWord(word, index, charlist):
replaced = 0
for char in charlist:
(v, wordlist, rawlist) = unicodeDict[char]
try:
i = wordlist.index(word)
except ValueError:
continue
wordlist[i] = index
replaced = 1
return replaced
def compress():
#dumpUnicodeDict("UnicodeDict before", unicodeDict)
words = calculateWords(unicodeDict)
lenp = [(len(v), k, v) for k, v in words.items()]
lenp.sort()
lenp.reverse()
wordidx = len(chars)
for (length, word, value) in lenp:
# Do not lookup single char words or words only used once
if len(word) == 1 or len(value) == 1:
continue
# Do not lookup two char words of the replacement would
# be just as big.
if len(word) == 2 and wordidx >= 238:
continue
replaceWord(word, wordidx, value)
wordmap[wordidx] = word
wordidx += 1
#dumpUnicodeDict("UnicodeDict after", unicodeDict)
def writeUcnhashDat():
cutoff = 255 - ((len(chars) + len(wordmap)) >> 8)
debug("wordmap entries: %d" % len(wordmap))
debug("wordmap cutoffs: %d" % cutoff)
worddata = Table()
wordoffs = Table()
wordfile = open("words.in", "wt");
size = 0
l = [(k,v) for k,v in wordmap.items()]
l.sort()
for k,v in l:
print >>wordfile, v
wordoffs.write_UShort(worddata.size())
mapv = ''.join(map(lambda x: chr(chardict.get(x)), v))
worddata.write_Str(mapv)
wordfile.close()
os.system(mph_exec+" -d3 -S1 -m4 -a < words.in > words.hash")
outf = open("ucnhash.dat", "wb+")
m = MphEmitter()
m.writeFile(open("words.hash"), outf)
debug("wordhash size %d" % outf.tell())
debug("wordoffs size %d" % (wordoffs.size()/2))
debug("worddata size %d" % (worddata.size()))
wordoffs.writeto(outf)
worddata.writeto(outf)
maxklen = 0
lst = []
for key, (value, wordlist, rawlist) in unicodeDict.items():
savewordlist = wordlist[:]
# Map remaining strings to a list of bytes in chardict
# range: range(0, 40)
l = len(wordlist)
for i in range(l-1, -1, -1):
part = wordlist[i]
if type(part) == type(""):
ipart = map(chardict.get, part)
if i > 0 and type(wordlist[i-1]) == type(""):
ipart[0:0] = [0] # index of space
wordlist[i:i+1] = ipart
# Encode high values as two bytes
for v in wordlist:
if v <= cutoff:
rawlist.append(v)
else:
rawlist.append((v>>8) + cutoff)
rawlist.append(v & 0xFF)
if value in debugChars:
print key, savewordlist, rawlist
lst.append((rawlist, wordlist, key, value))
maxklen = max(maxklen, len(key))
lst.sort()
outf.write(struct.pack("!hhh", len(chars), cutoff, maxklen));
raw = Table()
datasize = []
i = 0
for (rawlist, wordlist, key, value) in lst:
for r in rawlist:
raw.write_Str(chr(r))
datasize.append((len(rawlist), value))
#debug("%d %s %r" % (i, key, rawlist))
i += 1
debug("Raw size = %d" % raw.size())
raw.writeto(outf)
rawindex = Table()
codepoint = Table()
offset = 0
maxlen = 0
maxvl = 0 # for debugging
# Formerly it was sufficient for rawindex and codepoint
# to be 16 bit.
# We leave the old 16 bit write instructions here as
# comments in case future debugging is necessary.
# Note that rawindex blocksize therefore shrunk from
# 5 to 3, which was adjusted in ucnhash.java accordingly.
# In line 'v = v | (long(size) << (j*5))' the '5' seemingly
# refers to blocksize, but for some reason it must not be
# adjusted to 3.
# (3 would break things, while it works well with 5)
for i in range(0, len(datasize), 12):
saveoffset = offset
#rawindex.write_UShort(offset)
rawindex.write_UInt32(offset)
v = 0L
j = 0
for (size, value) in datasize[i:i+12]:
# we keep track of max value to confirm
# that 32 bit codepoint table is needed
if value > maxvl:
maxvl = value
offset += size
v = v | (long(size) << (j*5))
maxlen = max(maxlen, size)
#codepoint.write_UShort(value)
codepoint.write_UInt32(value)
j += 1
#debug("%d %d %x" % (i/ 12, saveoffset, v))
#rawindex.write_UShort((v >> 48) & 0xFFFF)
#rawindex.write_UShort((v >> 32) & 0xFFFF)
#rawindex.write_UShort((v >> 16) & 0xFFFF)
#rawindex.write_UShort(v & 0xFFFF)
rawindex.write_UInt32((v >> 32) & 0xFFFFFFFF)
rawindex.write_UInt32(v & 0xFFFFFFFF)
debug("maxval % d" % maxvl)
debug("rawindex size % d" % (rawindex.size()/4))
debug("codepoint size % d" % (codepoint.size()/4))
rawindex.writeto(outf)
codepoint.writeto(outf)
debug("raw entries %d" % len(datasize))
outf.close();
if __name__ == "__main__":
chars = " ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-()"
chardict = {}
for c in chars:
chardict[c] = chars.index(c)
debugChars = [] # [0x41, 0x20AC]
debugFile = open("ucnhash.lst", "wt")
#debugFile = sys.stdout
wordmap = {}
# Called 2017 with UnicodeData.txt for Unicode 9.0
# Called 2024 with https://www.unicode.org/Public/15.1.0/ucd/UnicodeData.txt
unicodeDataFile = "UnicodeData.txt"
if len(sys.argv) > 1:
unicodeDataFile = sys.argv[1]
if len(sys.argv) > 2:
mph_exec = sys.argv[2]
unicodeDict = readUnicodeDict(unicodeDataFile)
print "Size:", calculateSize(unicodeDict)
compress()
print "compressed"
writeUcnhashDat()
print "done"
sys.exit(0)
# Debugging-hints:
#-----------------
# (with debugFile = sys.stdout, omitting mph-output)
# Output for "UnicodeData-3.0.0.txt":
# Size: 259126
# compressed
# wordmap entries: 1384
# wordmap cutoffs: 250
# * d=3
# * n=1703
# * m=1384
# * c=1.23
# * maxlen=4
# * minklen=2
# * maxklen=18
# * minchar=45
# * maxchar=90
# G len: 1703
# T0 len: 184
# T1 len: 184
# T2 len: 184
# wordhash size 4542
# wordoffs size 1384
# worddata size 7375
# Raw size = 58531
# maxval 65533
# rawindex size 2577
# codepoint size 10298
# raw entries 10298
# done
# Output for "UnicodeData.txt", Unicode 9.0:
# Size: 755323
# compressed
# wordmap entries: 3708
# wordmap cutoffs: 241
# * d=3
# * n=4561
# * m=3708
# * c=1.23
# * maxlen=4
# * minklen=2
# * maxklen=18
# * minchar=32
# * maxchar=90
# G len: 4561
# T0 len: 236
# T1 len: 236
# T2 len: 236
# wordhash size 10570
# wordoffs size 3708
# worddata size 19399
# Raw size = 184818
# maxval 917999
# rawindex size 7389
# codepoint size 29545
# raw entries 29545
# done