#! /usr/bin/python
#
#  A script to parse the CEDICT file into a sqlite
#  database.
#
#	Author:	Benny Wong <bwong.net>
#	Date:	2008.04.27
 
import re 
from pysqlite2 import dbapi2 as sqlite
 
columns = ['Traditional', 'Simplified', 'Pinyin', 'Definition']
 
f = open('cedict_ts.u8', 'r')
p = re.compile('(.*) (.*) \[(.*)\] /(.*)/')
 
conn = sqlite.connect('Cedict.sqlite')
cursor = conn.cursor()
 
cursor.execute("DROP TABLE IF EXISTS Cedict")
cursor.execute("CREATE TABLE Cedict (" + ", ".join([i + " TEXT" for i in columns]) + ")")
 
for line in f:
	if not line.startswith('#'):
		tokens = p.match(line)
		traditional = tokens.group(1)
		simplified = tokens.group(2)
		pinyin = tokens.group(3)
		definition = tokens.group(4).replace('/', '|')
 
		cursor.execute("INSERT INTO Cedict (" + ', '.join(columns) + \
			") VALUES (?, ?, ?, ?)", \
			[traditional, simplified, pinyin, definition]);
 
f.close()
 
conn.commit()

#! /usr/bin/python
#
#  A script to convert/pivot the Unihan.txt file into a sqlite
#  database.
#
#	Author:	Benny Wong <bwong.net>
#	Date:	2008.04.27
 
from pysqlite2 import dbapi2 as sqlite
 
charmap = {}
keys = set()
keys.add('Character')
 
f = open('Unihan.txt', 'r')
 
for line in f:
	if not line.startswith('#'):
		tokens = line.split()
		key = tokens[0].replace('U+', '')
		if not charmap.has_key(tokens[0]):
			charmap[tokens[0]] = {}
 
			unichar = tokens[0].replace('U+', '0x')
			unichar = unichr(long(unichar, 16))
 
			charmap[tokens[0]]['Character'] = unichar.encode('utf8')
 
		charmap[tokens[0]][tokens[1]] = " ".join(tokens[2:])
		keys.add(tokens[1])
 
f.close()
 
keystring = ", ".join(key + " TEXT" for key in keys)
 
conn = sqlite.connect('Unihan.sqlite')
cursor = conn.cursor()
 
cursor.execute("DROP TABLE IF EXISTS Unihan")
cursor.execute("CREATE TABLE Unihan (key TEXT, " + keystring + ")")
 
while len(charmap) > 0:
	key, values = charmap.popitem()
 
	columns = ",".join(values.keys())
	cells = '","'.join(values.values())
 
	sql = 'INSERT INTO Unihan (key, ' + columns + ') VALUES ("' + key + '", "' + cells + '")'.encode('utf8')
	cursor.execute(sql)
 
cursor.execute("CREATE INDEX key ON Unihan(key)")
for key in keys:
	cursor.execute("CREATE INDEX " + key + " ON Unihan(" + key + ")")
 
conn.commit()