normalize data to NFC for siteposter, fixes #2428

pandora/item/models.py

@@ -97,6 +97,7 @@ def get_item(info, user=None):
         if key in info and info[key]:
             item_data[key] = info[key]
 
+    item_data = utils.normalize_dict('NFC', item_data)
     if settings.USE_IMDB:
         if 'imdbId' in info and info['imdbId']:
             try:
@@ -1473,6 +1474,7 @@ class Item(models.Model):
         if os.path.exists(timeline):
             data['timeline'] = timeline
         data['oxdbId'] = self.oxdbId or self.oxdb_id() or self.public_id
+        data = utils.normalize_dict('NFC', data)
         ox.makedirs(os.path.join(settings.MEDIA_ROOT, self.path()))
         p = subprocess.Popen(cmd, stdin=subprocess.PIPE, close_fds=True)
         p.communicate(json.dumps(data, default=fields.to_json).encode('utf-8'))

pandora/item/utils.py

@@ -3,8 +3,11 @@
 #
 from decimal import Decimal
 import re
+import unicodedata
+
 import ox
 from ox import sort_string
+from six import PY2
 
 def safe_filename(filename):
     filename = filename.replace(': ', '_ ')
@@ -86,3 +89,17 @@ def get_by_key(objects, key, value):
 
 def get_by_id(objects, id):
     return get_by_key(objects, 'id', id)
+
+def normalize_dict(encoding, data):
+    if PY2:
+        string_type = unicode
+    else:
+        string_type = str
+    if isinstance(data, string_type):
+        data = unicodedata.normalize(encoding, data)
+    elif isinstance(data, dict):
+        for key in data:
+            data[key] = normalize_dict(encoding, data[key])
+    elif isinstance(data, list):
+        return [normalize_dict(encoding, value) for value in data]
+    return data