update geo tools

tools/geo/html/geo.html

@@ -64,7 +64,7 @@
         <script>
             /*
                 this adds the properties
-                    area, east, imageURL, lat, lng, north, south, west
+                    area, east, lat, lng, north, south, west
             */
             Ox.load('UI', {debug: true}, function() {
                 Ox.getJSON('../json/geo.json', function(geo) {
@@ -222,8 +222,11 @@
                         var addresses = geo.google_query[country.name] || [country.name],
                             length = addresses.length,
                             union;
+                        // fixme: this might be too much data
+                        // check if map really needs this preloaded
+                        // (and even if, country markers may not be necessary)
                         getImageURLs(country, function(imageURLs) {
-                            $.extend(country, {
+                            Ox.extend(country, {
                                 imageURLs: imageURLs
                             });
                             callGeocode();
@@ -273,14 +276,14 @@
                         }
                     }
                     function getImageURLs(country, callback) {
-                        Ox.print(country, '../png/marker/32/' + country.code + '.png')
+                        Ox.print(country, '../png/icon/16/' + country.code + '.png')
                         var image = new Image();
                         image.onload = function() {
                             callback({
-                                marker: Ox.canvas(image).canvas.toDataURL()
+                                icon16: Ox.canvas(image).canvas.toDataURL()
                             });
                         };
-                        image.src = '../png/markers/32/' + country.code + '.png';
+                        image.src = '../png/icons/16/' + country.code + '.png';
                     }
                 });
             });

tools/geo/py/geo.py

@@ -222,7 +222,7 @@ geo = {
         'Diego Garcia': 'Flag_of_the_British_Indian_Ocean_Territory.svg',
         'French Guiana': 'Flag_of_French_Guiana.svg',
         'Korea': 'Flag_of_Korea_1882.svg',
-        'Libya': 'Flag_of_the_Libyan_Jamahiriya_1977.svg',
+        # 'Libya': 'Flag_of_the_Libyan_Jamahiriya_1977.svg',
         'Metropolitan France': 'Flag_of_France.svg',
         'Neutral Zone': 'Flag_of_the_United_Nations.svg',
         'New Hebrides': 'Flag_of_Anglo-French_Joint_Naval_Commission.svg',
@@ -260,7 +260,7 @@ geo = {
         'Curaçao': ['Banda Abou, Curaçao'],
         'Czechoslovakia': ['Czech Republic', 'Slovakia'],
         'East Germany': [
-            'Brandenburg', 'Mecklenburg Vorpommern', 'Saxony', 'Saxony Anhalt', 'Thuringia'
+            'Mecklenburg Vorpommern', 'Saxony', 'Thuringia'
         ],
         'Dahomey': ['Benin'],
         'European Union': ['Europe'],
@@ -303,14 +303,15 @@ geo = {
             'Latvia', 'Lithuania', 'Kazakhstan', 'Kyrgyzstan', 'Moldova',
             'Russia', 'Tajikistan', 'Turkmenistan', 'Ukraine', 'Uzbekistan'
         ],
-        'United Kingdom': ['England', 'Northern Ireland', 'Scotland', 'Wales'],
+        'United Kingdom': ['England', 'Northern Ireland', 'Scotland', 'Wales, United Kingdom'],
         'United States Minor Outlying Islands': ['Midway Islands'],
         # Baker Island, Howland Island, Jarvis Island, Kingman Reef, Palmyra Atoll --
         # but the first three each return the full United States Minor Outlying Islands
         'United States Miscellaneous Pacific Islands': ['Kingman Reef', 'Palmyra Atoll'],
-        'UK': ['England', 'Northern Ireland', 'Scotland', 'Wales'],
+        'UK': ['England', 'Northern Ireland', 'Scotland', 'Wales, United Kingdom'],
         'Upper Volta': ['Burkina Faso'],
         'Wake Island': ['Wake Atoll'],
+        'Wales': ['Wales, United Kingdom'],
         'West Germany': ['Schleswig Holstein', 'Northrhine Westphalia', 'Bavaria, Germany'],
         'Yugoslavia': [
             'Bosnia and Herzegovina', 'Croatia', 'Serbia', 'Slovenia', 'Macedonia',
@@ -634,6 +635,324 @@ geo = {
         'ZRCD': 'crop()',
         'ZW': 'crop(left)'
     },
+
+    # import json
+    # import re    
+    # from ox.net import readUrl
+    # html = readUrl('http://www.imdb.com/language/')
+    # results = re.compile(re.compile('<a href="/language/.*?">(.*?)</a>').findall(html))
+    # languages = {}
+    # for result in results:
+    #     languages[result] = ''
+    # print json.dumps(languages, sort_keys=True)
+
+    # http://www.imdb.com/language/
+    'languages': {
+        'Abkhazian': 'Abkhazia',
+        'Aboriginal': 'Australia',
+        'Aché': '',
+        'Acholi': '',
+        'Afrikaans': 'South Africa',
+        'Aidoukrou': '',
+        'Akan': '',
+        'Albanian': 'Albania',
+        'Algonquin': '',
+        'American': 'United States',
+        'Amharic': '',
+        'Apache': '',
+        'Arabic': 'Saudi Arabia',
+        'Aragonese': '',
+        'Aramaic': 'Syria',
+        'Arapaho': '',
+        'Armenian': 'Armenia',
+        'Assamese': 'India',
+        'Assyrian Neo-Aramaic': '',
+        'Athapascan': '',
+        'Australian': 'Australia',
+        'Awadhi': '',
+        'Aymara': 'Bolivia',
+        'Azerbaijani': 'Azerbaijan',
+        'Bable': '',
+        'Baka': '',
+        'Balinese': 'Indonesia',
+        'Bambara': 'Mali',
+        'Basque': '',
+        'Bassari': '',
+        'Belarusian': 'Belarus',
+        'Bemba': '',
+        'Bengali': 'Bangladesh',
+        'Berber': '',
+        'Bhojpuri': '',
+        'Bicolano': 'Philippines',
+        'Bodo': '',
+        'Bosnian': 'Bosnia and Herzegovina',
+        'Brazilian': 'Brazil',
+        'Breton': 'France',
+        'British': 'United Kingdom',
+        'Bulgarian': 'Bulgaria',
+        'Burmese': 'Burma',
+        'Cantonese': 'China',
+        'Catalan': 'Spain',
+        'Central Khmer': 'Cambodia',
+        'Chaozhou': '',
+        'Chechen': 'Russia',
+        'Cherokee': 'United Stated',
+        'Cheyenne': '',
+        'Chhattisgarhi': '',
+        'Chinese': 'China',
+        'Cornish': '',
+        'Corsican': 'France',
+        'Cree': 'Canada',
+        'Creek': '',
+        'Creole': '',
+        'Creoles and pidgins': '',
+        'Croatian': 'Croatia',
+        'Crow': '',
+        'Czech': 'Czech Republic',
+        'Danish': 'Denmark',
+        'Dari': 'Afghanistan',
+        'Desiya': '',
+        'Dinka': '',
+        'Djerma': '',
+        'Dogri': '',
+        'Dutch': 'Netherlands',
+        'Dyula': '',
+        'Dzongkha': '',
+        'East-Greenlandic': 'Greenland',
+        'Eastern Frisian': 'Germany',
+        'Egyptian (Ancient)': 'Egypt',
+        'English': 'United Kingdom',
+        'Esperanto': '',
+        'Estonian': 'Estonia',
+        'Ewe': '',
+        'Faliasch': '',
+        'Faroese': 'Faroe Islands',
+        'Filipino': 'Philippines',
+        'Finnish': 'Finland',
+        'Flemish': 'Belgium',
+        'Fon': '',
+        'French': 'France',
+        'Fulah': '',
+        'Fur': '',
+        'Gaelic': 'Ireland',
+        'Galician': '',
+        'Georgian': 'Georgia',
+        'German': 'Germany',
+        'Grebo': '',
+        'Greek': 'Greece',
+        'Greek, Ancient (to 1453)': 'Greece',
+        'Greenlandic': 'Greenland',
+        'Guarani': 'Paraguay',
+        'Gujarati': 'India',
+        'Gumatj': '',
+        'Gunwinggu': '',
+        'Haitian': 'Haiti',
+        'Hakka': 'China',
+        'Haryanvi': '',
+        'Hassanya': '',
+        'Hausa': '',
+        'Hawaiian': 'United States',
+        'Hebrew': 'Israel',
+        'Hindi': 'India',
+        'Hmong': '',
+        'Hokkien': '',
+        'Hopi': 'United States',
+        'Hungarian': 'Hungary',
+        'Iban': '',
+        'Ibo': 'Nigeria',
+        'Icelandic': 'Iceland',
+        'Indian': 'India',
+        'Indonesian': 'Indonesia',
+        'Inuktitut': '',
+        'Inupiaq': '',
+        'Irish Gaelic': 'Ireland',
+        'Italian': 'Italy',
+        'Japanese': 'Japan',
+        'Jola-Fonyi': '',
+        'Ju\'hoan': '',
+        'Kaado': '',
+        'Kabuverdianu': 'Cape Verde',
+        'Kabyle': '',
+        'Kalmyk-Oirat': '',
+        'Kannada': 'India',
+        'Karajá': '',
+        'Karbi': '',
+        'Karen': '',
+        'Kazakh': 'Kazakhstan',
+        'Khanty': 'Russia',
+        'Khasi': '',
+        'Kikuyu': '',
+        'Kinyarwanda': '',
+        'Kirundi': '',
+        'Klingon': '',
+        'Kodava': '',
+        'Konkani': 'India',
+        'Korean': 'South Korea',
+        'Korowai': 'Papua New Guinea',
+        'Kriolu': '',
+        'Kru': '',
+        'Kudmali': '',
+        'Kuna': '',
+        'Kurdish': 'Turkey', # ? one of multiple
+        'Kwakiutl': '',
+        'Kyrgyz': 'Kyrgyzstan',
+        'Ladakhi': '',
+        'Ladino': '',
+        'Lao': 'Laos',
+        'Latin': 'Italy',
+        'Latvian': 'Latvia',
+        'Limbu': '',
+        'Lingala': '',
+        'Lithuanian': 'Lithuania',
+        'Low German': 'Germany',
+        'Luxembourgish': 'Luxemburg',
+        'Macedonian': 'Macedonia',
+        'Macro-Jê': '',
+        'Magahi': '',
+        'Maithili': '',
+        'Malagasy': 'Madagascar',
+        'Malay': 'Malaysia',
+        'Malayalam': '',
+        'Malecite-Passamaquoddy': '',
+        'Malinka': '',
+        'Maltese': 'Malta',
+        'Manchu': '',
+        'Mandarin': 'China',
+        'Mandingo': '',
+        'Manipuri': '',
+        'Maori': 'New Zealand',
+        'Mapudungun': 'Chile',
+        'Marathi': 'India',
+        'Marshallese': 'Marshall Islands',
+        'Masai': '',
+        'Masalit': '',
+        'Maya': 'Mexico',
+        'Mende': 'Sierra Leone',
+        'Micmac': '',
+        'Middle English': 'England',
+        'Min Nan': '',
+        'Minangkabau': '',
+        'Mirandese': '',
+        'Mizo': '',
+        'Mohawk': '',
+        'Mongolian': 'Mongolia',
+        'Montagnais': '',
+        'More': 'Burkina Faso',
+        'Morisyen': '',
+        'Nagpuri': '',
+        'Nahuatl': '',
+        'Nama': '',
+        'Navajo': '',
+        'Naxi': 'China',
+        'Ndebele': '',
+        'Neapolitan': 'Italy',
+        'Nenets': '',
+        'Nepali': 'Nepal',
+        'Nisga\'a': '',
+        'None': '',
+        'Norse, Old': '',
+        'North American Indian': '',
+        'Norwegian': 'Norway',
+        'Nushi': '',
+        'Nyaneka': '',
+        'Nyanja': 'Malawi',
+        'Occitan': '',
+        'Ojibwa': '',
+        'Ojihimba': '',
+        'Old English': 'England',
+        'Oriya': '',
+        'Papiamento': '',
+        'Parsee': 'Iran',
+        'Pashtu': 'Afghanistan',
+        'Pawnee': '',
+        'Persian': 'Iran',
+        'Peul': '',
+        'Polish': 'Poland',
+        'Polynesian': '',
+        'Portuguese': 'Portugal',
+        'Pular': '',
+        'Punjabi': 'India',
+        'Purepecha': '',
+        'Quechua': 'Peru',
+        'Quenya': '',
+        'Rajasthani': 'India',
+        'Rawan': '',
+        'Romanian': 'Romania',
+        'Romansh': 'Switzerland',
+        'Romany': 'Romania',
+        'Rotuman': '',
+        'Russian': 'Russia',
+        'Ryukyuan': 'Japan',
+        'Saami': 'Finland',
+        'Samoan': 'Samoa',
+        'Sanskrit': '',
+        'Sardinian': 'Italy',
+        'Scanian': '',
+        'Serbian': 'Serbia',
+        'Serbo-Croatian': 'Yugoslavia',
+        'Serer': '',
+        'Shanghainese': 'China',
+        'Shanxi': '',
+        'Shona': 'Zimbabwe',
+        'Shoshoni': '',
+        'Sicilian': 'Italy',
+        'Sindarin': '',
+        'Sindhi': '',
+        'Sinhala': 'Sri Lanka',
+        'Sioux': '',
+        'Slovak': 'Slovakia',
+        'Slovenian': 'Slovenia',
+        'Somali': 'Somalia',
+        'Songhay': '',
+        'Soninke': '',
+        'Sorbian': 'Germany',
+        'Sotho': '',
+        'Sousson': '',
+        'Spanish': 'Spain',
+        'Sranan': '',
+        'Swahili': '',
+        'Swedish': 'Sweden',
+        'Swiss German': 'Switzerland',
+        'Sylheti': '',
+        'Tagalog': 'Philippines',
+        'Tajik': 'Tajikistan',
+        'Tamashek': '',
+        'Tamil': 'Sri Lanka',
+        'Tarahumara': '',
+        'Tatar': 'Russia',
+        'Telugu': 'India',
+        'Teochew': '',
+        'Thai': 'Thailand',
+        'Tibetan': 'China',
+        'Tigrigna': '',
+        'Tlingit': '',
+        'Tok Pisin': '',
+        'Tonga (Tonga Islands)': 'Tonga',
+        'Tsonga': '',
+        'Tswa': '',
+        'Tswana': '',
+        'Tulu': '',
+        'Tupi': '',
+        'Turkish': 'Turkey',
+        'Turkmen': 'Turkmenistan',
+        'Tuvinian': '',
+        'Tzotzil': '',
+        'Ukrainian': 'Ukraine',
+        'Ungwatsi': '',
+        'Urdu': 'Pakistan',
+        'Uzbek': 'Uzbekistan',
+        'Vietnamese': 'Vietnam',
+        'Visayan': '',
+        'Washoe': '',
+        'Welsh': 'Wales',
+        'Wolof': 'Senegal', # ? one of multiple
+        'Xhosa': 'South Africa',
+        'Yakut': '',
+        'Yapese': '',
+        'Yiddish': 'Israel',
+        'Yoruba': '',
+        'Zulu': 'South Africa'
+    },
     'other': ['European Union', 'Metropolitan France', 'UK'],
     'wikipedia_name': {
         # ambiguous on wikipedia
@@ -751,6 +1070,11 @@ def get_countries():
                 break
         get_country_flag(country['code'], country['flagURL'])
         get_country_icon(country['code'])
+        # languages
+        country['languages'] = []
+        for language, language_country in geo['languages'].iteritems():
+            if language_country == country['name']:
+                country['languages'].append(language)
     write_json('../json/countries.json', countries)
     return countries
 
@@ -774,7 +1098,7 @@ def get_country_flag(code, url):
         write_image(file, png)
     for width in [256, 32]:
         file = '../png/flags/' + str(width) + '/' + code + '.png'
-        if not os.path.exists(file):
+        if not os.path.exists(file) or True:
             png_ = png.resize((width, int(round(width / png.size[0] * png.size[1]))), Image.ANTIALIAS)
             write_image(file, png_)
     
@@ -911,7 +1235,7 @@ def get_country_icon(code):
         icon.paste(crop_right, (width, 0))
         return icon
     file = '../png/icons/1024/' + code + '.png'
-    if not os.path.exists(file):
+    if not os.path.exists(file) or True:
         flag = Image.open('../png/flags/2048/' + code + '.png')
         icon = None
         functions = geo['icon'][code].split(' # ')[0].split('; ')
@@ -950,7 +1274,7 @@ def get_country_icon(code):
         icon = Image.open(file)
     for width in [256, 16]:
         file = '../png/icons/' + str(width) + '/' + code + '.png'
-        if not os.path.exists(file):
+        if not os.path.exists(file) or True:
             write_image(file, icon.resize((width, width), Image.ANTIALIAS))
     size = 256
     mask_ = Image.open('../png/circle.png')
@@ -961,7 +1285,9 @@ def get_country_icon(code):
         icon_ = icon.resize((size, size), Image.ANTIALIAS)
         mask_ = mask.resize((size, size), Image.ANTIALIAS)
         marker.paste(icon_, (0, 0), mask_)
-        write_image('../png/markers/' + str(size) + '/' + code + '.png', marker)
+        file = '../png/markers/' + str(size) + '/' + code + '.png'
+        if not os.path.exists(file) or True:
+            write_image(file, marker)
 
 def get_oxjs_languages():
     # geonames_countries = get_geonames_countries()
@@ -1127,6 +1453,10 @@ import wikipedia
 
 if __name__ == '__main__':
 
+    write_json('../json/geo.json', geo)
+    countries = get_countries()
+    sys.exit()
+
     file = '../txt/geonames.org/cities1000.txt'
     geo['keys'] = read_json('../json/oxjs.org/geonames.keys.json')
     keys = geo['keys']['place']['geonames.org']
@@ -1139,10 +1469,6 @@ if __name__ == '__main__':
     print len(data)
     sys.exit()
 
-    write_json('../json/geo.json', geo)
-
-    countries = get_countries()
-    sys.exit()
 
     cities = get_cities()
     print cities, len(cities)