oxjs/tools/unicode/unicode.py
2012-09-07 03:55:22 +02:00

170 lines
4.8 KiB
Python

import json
import ox
import re
remove = [
'ABOVE', 'BAR', 'BELOW', 'CEDILLA', 'DIGRAPH', 'LONGA', 'ROTUNDA'
]
special = {
'ACCOUNT OF': 'a/o',
'ADDRESSED TO THE SUBJECT': 'a/s',
'AM': 'a.m.',
'ANGSTROM SIGN': 'A',
'C OVER KG': 'c/kg',
'CADA UNA': 'c/u',
'CARE OF': 'c/o',
'CM CUBED': 'cm3',
'CM SQUARED': 'cm2',
'CO': 'Co.',
'DM CUBED': 'dm3',
'DM SQUARED': 'dm2',
'EIGHT': '8',
'ELEVEN': '11',
'EULER CONSTANT': 'E',
'FACSIMILE SIGN': 'FAX',
'FEMININE ORDINAL INDICATOR': 'a',
'FIFTY': '50',
'FIVE': '5',
'FIVE HUNDRED': '500',
'FOUR': '4',
'INFORMATION SOURCE': 'I',
'KCAL': 'kcal',
'KELVIN SIGN': 'K',
'KK': 'K.K.',
'KM CAPITAL': 'KM',
'KM CUBED': 'km3',
'KM SQUARED': 'km2',
'LATIN SMALL LETTER N PRECEDED BY APOSTROPHE': '\'n',
'LIMITED LIABILITY SIGN': 'LTD',
'M CUBED': 'm3',
'M OVER S SQUARED': 'm/s2',
'M SQUARED': 'm2',
'MASCULINE ORDINAL INDICATOR': 'o',
'MB SMALL': 'mb',
'MM CUBED': 'mm3',
'MM SQUARED': 'mm2',
'MV MEGA': 'MV',
'MW MEGA': 'MW',
'NINE': '9',
'NUMERO SIGN': 'No',
'ONE': '1',
'ONE HUNDRED': '100',
'ONE THOUSAND': '1000',
'PA AMPS': 'pA',
'PARTNERSHIP SIGN': 'PTE',
'PLANCK CONSTANT': 'h',
'PLANCK CONSTANT OVER PI': 'h',
'PLANCK CONSTANT OVER TWO PI': 'h',
'PM': 'p.m.',
'RAD OVER S SQUARED': 'rad/s2',
'RUPEE SIGN': 'Rs',
'S T': 'st',
'SERVICE MARK': 'SM',
'SEVEN': '7',
'SIX': '6',
'TELEPHONE SIGN': 'TEL',
'TEN': '10',
'THREE': '3',
'TRADE MARK SIGN': 'TM',
'TWELVE': '12',
'TWO': '2'
}
special_keys = sorted(special.keys(), key=lambda x: -len(x))
units = [
'bar', 'Bq',
'cal', 'cd', 'cm',
'da', 'dB', 'dl',
'ffi', 'ffl', 'fm',
'GHz', 'GPa', 'Gy',
'ha', 'hPa', 'Hz',
'in',
'kA', 'kcal', 'kg', 'KHz', 'kl', 'km', 'KPa', 'kt', 'kV', 'kW',
'log', 'lm', 'ln', 'lx',
'mA', 'mg', 'MHz', 'mil', 'ml', 'mm', 'mol', 'MPa', 'ms', 'mV', 'mW',
'nA', 'nF', 'nm', 'ns', 'nV', 'nW',
'oV',
'Pa', 'pc', 'pH', 'PPM', 'ps', 'pV', 'pW',
'rad',
'sr', 'Sv',
'THz',
'wb'
]
txt = ox.cache.read_url('http://unicode.org/Public/UNIDATA/NamesList.txt', unicode=True)
lines = txt.split('\n')
length = len(lines)
chars = {}
sections = []
types = []
for i, line in enumerate(lines):
results = re.compile('^@@\t[0-9A-Z]{4}\t(.+)\t[0-9A-Z]{4}').findall(line)
if results:
# section
section = results[0].upper()
sections.append(section)
else:
results = re.compile('^@\t\t(.+)').findall(line)
if results:
# type
type = results[0].upper()
types.append(type)
else:
results = re.compile('^([0-9A-Z]{4})\t(.+)').findall(line)
if results:
# char + name
char = unichr(int(results[0][0], 16))
name = results[0][1]
chars[char] = {
'names': [] if name[0] == '<' else [name],
'section': section,
'type': type
}
if char == '\uFFFF':
break
else:
results = re.compile('^\t= (.+)').findall(line)
if results:
# name
for name in results[0].upper().split(', '):
chars[char]['names'].append(name)
html = ox.cache.readUrlUnicode('http://unicode.org/charts/uca/chart_Latin.html')
results = re.compile("title='(.+):.+<tt>([0-9A-Z]{4})</tt>").findall(html)
no_ascii = []
for result in results:
code = result[1]
if int(code, 16) > 127:
char = unichr(int(code, 16))
name = result[0]
words = name.split(' ')
ascii = ''
for key in special_keys:
if name == key or name.endswith(' ' + key):
ascii = special[key]
break
if not ascii:
for unit in units:
if words[-1] == unit.upper():
ascii = unit
break;
if not ascii:
name = re.sub(' WITH .+', '', name)
for word in remove:
name = re.sub(' ' + word, '', name)
words = name.split(' ')
if len(words[-1]) <= 2:
ascii = words[-1]
else:
no_ascii.append(name)
if ascii:
if 'SMALL' in words and not 'CAPITAL' in words:
ascii = ascii.lower()
chars[char]['ascii'] = ascii
f = open('../../source/Ox.Unicode/json/Ox.Unicode.json', 'w')
f.write(json.dumps(chars, indent=4, sort_keys=True))
f.close()
f = open('json/no_ascii.json', 'w')
f.write(json.dumps(sorted(no_ascii), indent=4))
f.close()