Misc.make_ucnhashdat.py Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of jython-standalone Show documentation
Show all versions of jython-standalone Show documentation
Jython is an implementation of the high-level, dynamic, object-oriented
language Python written in 100% Pure Java, and seamlessly integrated with
the Java platform. It thus allows you to run Python on any Java platform.
# Copyright Finn Bock
#
# Generate a ucnhash.dat file with mapping from unicode
# names to codepoints.
#
# python mkucnhashdat.py UnicodeData-3.0.0.txt
#
# The "mph" program must be available on the path.
# This program is used to create the minimum perfect
# hash used by the wordhash table.
#
# I've used 1.2 from:
#
# http://www.ibiblio.org/pub/Linux/devel/lang/c/!INDEX.short.html
#
import fileinput, re, os, sys, struct, cStringIO
def debug(str):
print >>debugFile, str
def splitIntoWords(name):
wordlist = []
wordstart = 0
l = len(name)
for i in range(l):
c = name[i]
n = None
if c == ' ' or c == '-':
n = name[wordstart:i]
elif i == l-1:
n = name[wordstart:i+1]
if n:
#print " ", i, c, n
wordstart = i
if c == '-' and n != '':
n += '-'
if c == ' ' or c == '-':
wordstart = i+1
#print " ", n
wordlist.append(n)
return wordlist
def readUnicodeDict(file):
d = {}
for l in fileinput.input(file):
l = l.strip().split(";");
v,name = l[0:2]
if name == "":
name = l[10]
if name == '':
continue
if name[0] == "<":
continue
#handled by code in ucnhash
if name.startswith("CJK COMPATIBILITY IDEOGRAPH-"):
continue
wordlist = splitIntoWords(name)
#print name, wordlist
d[name] = (int(v, 16), wordlist, [])
return d
#readUnicodeDict("nametest.txt")
#sys.exit()
def count(dict, index):
c = dict.get(index)
if c is None: c = 0
c += 1
dict[index] = c
def dumpUnicodeDict(title, dict):
lst = []
i = 0
for k,(v,wordlist, rawlist) in dict.items():
p = wordlist[:]
lst.append((v, k, p))
lst.sort()
print "=======", title
for v,k,p in lst:
print "%.4X %s %s" % (v, k, p)
class MphEmitter:
def readint(self):
return int(self.inf.readline().strip())
def readfloat(self):
return float(self.inf.readline().strip())
def readconst(self):
global d, n, m, c, maxlen, minklen, maxklen, minchar, maxchar, alphasz
self.inf.readline()
self.d = self.readint()
self.n = self.readint()
self.m = self.readint()
self.c = self.readfloat()
self.maxlen = self.readint()
self.minklen = self.readint()
self.maxklen = self.readint()
self.minchar = self.readint()
self.maxchar = self.readint()
self.loop = self.readint()
self.numiter= self.readint()
self.readint()
self.readint()
debug(" * d=%d" % self.d)
debug(" * n=%d" % self.n)
debug(" * m=%d" % self.m)
debug(" * c=%g" % self.c)
debug(" * maxlen=%d" % self.maxlen)
debug(" * minklen=%d" % self.minklen)
debug(" * maxklen=%d" % self.maxklen)
debug(" * minchar=%d" % self.minchar)
debug(" * maxchar=%d" % self.maxchar)
self.alphasz = self.maxchar - self.minchar+1;
def readg(self):
data = Table()
for i in range(self.n):
v = self.readint()
data.write_Short(v)
return data
def readT(self, t):
data = Table()
for i in range(self.maxlen):
for j in range(256):
v = self.readint()
if j < self.minchar or j > self.maxchar:
continue
data.write_Short(v)
return data
def writeFile(self, inf, outf):
self.inf = inf
self.readconst();
outf.write(struct.pack("!hhhhhh", self.n,
self.m,
self.minchar,
self.maxchar,
self.alphasz,
self.maxlen))
self.readg().writeto(outf)
outf.write(struct.pack("!h", self.d))
for t in range(self.d):
self.readT(t).writeto(outf)
class Table:
def __init__(self):
self.buf = cStringIO.StringIO()
def write_Str(self, str):
self.buf.write(str)
def write_Short(self, v):
self.buf.write(struct.pack("!h", v))
def write_UShort(self, v):
self.buf.write(struct.pack("!H", v))
def writeto(self, file):
file.write('t')
file.write(struct.pack("!H", self.size()))
file.write(self.buf.getvalue())
def size(self):
return self.buf.tell()
def calculateSize(dict):
cnt = 0
for name in dict.keys():
cnt += len(name)
return cnt
def calculateWords(unicodeDict):
words = {}
for key, (value, wordlist, rawlist) in unicodeDict.items():
for name in wordlist:
wordlist = words.setdefault(name, [])
wordlist.append(key)
return words
def replaceWord(word, index, charlist):
replaced = 0
for char in charlist:
(v, wordlist, rawlist) = unicodeDict[char]
try:
i = wordlist.index(word)
except ValueError:
continue
wordlist[i] = index
replaced = 1
return replaced
def compress():
#dumpUnicodeDict("UnicodeDict before", unicodeDict)
words = calculateWords(unicodeDict)
lenp = [(len(v), k, v) for k, v in words.items()]
lenp.sort()
lenp.reverse()
wordidx = len(chars)
for (length, word, value) in lenp:
# Do not lookup single char words or words only used once
if len(word) == 1 or len(value) == 1:
continue
# Do not lookup two char words of the replacement would
# be just as big.
if len(word) == 2 and wordidx >= 238:
continue
#print length, word, len(value)
replaceWord(word, wordidx, value)
wordmap[wordidx] = word
wordidx += 1
#dumpUnicodeDict("UnicodeDict after", unicodeDict)
def writeUcnhashDat():
cutoff = 255 - ((len(chars) + len(wordmap)) >> 8)
debug("wordmap entries: %d" % len(wordmap))
debug("wordmap cutoffs: %d" % cutoff)
worddata = Table()
wordoffs = Table()
wordfile = open("words.in", "wt");
size = 0
l = [(k,v) for k,v in wordmap.items()]
l.sort()
for k,v in l:
print >>wordfile, v
wordoffs.write_UShort(worddata.size())
mapv = ''.join(map(lambda x: chr(chardict.get(x)), v))
worddata.write_Str(mapv)
wordfile.close()
os.system("mph.exe -d3 -S1 -m4 -a < words.in > words.hash")
outf = open("ucnhash.dat", "wb+")
m = MphEmitter()
m.writeFile(open("words.hash"), outf)
debug("wordhash size %d" % outf.tell())
debug("wordoffs size %d" % wordoffs.size())
debug("worddata size %d" % worddata.size())
wordoffs.writeto(outf)
worddata.writeto(outf)
maxklen = 0
lst = []
for key, (value, wordlist, rawlist) in unicodeDict.items():
savewordlist = wordlist[:]
# Map remaining strings to a list of bytes in chardict
# range: range(0,37)
l = len(wordlist)
for i in range(l-1, -1, -1):
part = wordlist[i]
if type(part) == type(""):
ipart = map(chardict.get, part)
if i > 0 and type(wordlist[i-1]) == type(""):
ipart[0:0] = [0] # index of space
wordlist[i:i+1] = ipart
# Encode high values as two bytes
for v in wordlist:
if v <= cutoff:
rawlist.append(v)
else:
rawlist.append((v>>8) + cutoff)
rawlist.append(v & 0xFF)
if value in debugChars:
print key, savewordlist, rawlist
lst.append((rawlist, wordlist, key, value))
maxklen = max(maxklen, len(key))
lst.sort()
outf.write(struct.pack("!hhh", len(chars), cutoff, maxklen));
raw = Table()
datasize = []
i = 0
for (rawlist, wordlist, key, value) in lst:
for r in rawlist:
raw.write_Str(chr(r))
datasize.append((len(rawlist), value))
debug("%d %s %r" % (i, key, rawlist))
i += 1
debug("Raw size = %d" % raw.size())
raw.writeto(outf)
rawindex = Table()
codepoint = Table()
offset = 0
maxlen = 0
for i in range(0, len(datasize), 12):
saveoffset = offset
rawindex.write_UShort(offset)
v = 0L
j = 0
for (size, value) in datasize[i:i+12]:
offset += size
v = v | (long(size) << (j*5))
maxlen = max(maxlen, size)
codepoint.write_UShort(value)
j += 1
debug("%d %d %x" % (i/ 12, saveoffset, v))
rawindex.write_UShort((v >> 48) & 0xFFFF)
rawindex.write_UShort((v >> 32) & 0xFFFF)
rawindex.write_UShort((v >> 16) & 0xFFFF)
rawindex.write_UShort(v & 0xFFFF)
debug("rawindex size % d" % rawindex.size())
rawindex.writeto(outf)
codepoint.writeto(outf)
debug("raw entries %d" % len(datasize))
outf.close();
if __name__ == "__main__":
chars = " ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-"
chardict = {}
for c in chars:
chardict[c] = chars.index(c)
debugChars = [] # [0x41, 0x20AC]
debugFile = open("ucnhash.lst", "wt")
wordmap = {}
unicodeDataFile = "UnicodeData-3.0.0.txt"
if len(sys.argv) > 1:
unicodeDataFile = sys.argv[1]
unicodeDict = readUnicodeDict(unicodeDataFile)
print "Size:", calculateSize(unicodeDict)
compress()
print "compressed"
writeUcnhashDat()
print "done"
sys.exit(0)