0x2620/pandora
2
1
Fork 1
Code Issues 1.3k Pull requests Releases Wiki Activity

normalize data to NFC for siteposter, fixes #2428

Browse source
This commit is contained in:
j 2017-07-30 10:56:57 +02:00
parent 5de40bdfdd
commit b82f222c73
2 changed files with 20 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
pandora/item/models.py
View file

@ -97,6 +97,7 @@ def get_item(info, user=None):
if key in info and info[key]: if key in info and info[key]:
item_data[key] = info[key] item_data[key] = info[key]
item_data = utils.normalize_dict('NFC', item_data)
if settings.USE_IMDB: if settings.USE_IMDB:
if 'imdbId' in info and info['imdbId']: if 'imdbId' in info and info['imdbId']:
try: try:
@ -1473,6 +1474,7 @@ class Item(models.Model):
if os.path.exists(timeline): if os.path.exists(timeline):
data['timeline'] = timeline data['timeline'] = timeline
data['oxdbId'] = self.oxdbId or self.oxdb_id() or self.public_id data['oxdbId'] = self.oxdbId or self.oxdb_id() or self.public_id
data = utils.normalize_dict('NFC', data)
ox.makedirs(os.path.join(settings.MEDIA_ROOT, self.path())) ox.makedirs(os.path.join(settings.MEDIA_ROOT, self.path()))
p = subprocess.Popen(cmd, stdin=subprocess.PIPE, close_fds=True) p = subprocess.Popen(cmd, stdin=subprocess.PIPE, close_fds=True)
p.communicate(json.dumps(data, default=fields.to_json).encode('utf-8')) p.communicate(json.dumps(data, default=fields.to_json).encode('utf-8'))

17
pandora/item/utils.py
View file

@ -3,8 +3,11 @@
# #
from decimal import Decimal from decimal import Decimal
import re import re
import unicodedata
import ox import ox
from ox import sort_string from ox import sort_string
from six import PY2
def safe_filename(filename): def safe_filename(filename):
filename = filename.replace(': ', '_ ') filename = filename.replace(': ', '_ ')
@ -86,3 +89,17 @@ def get_by_key(objects, key, value):
def get_by_id(objects, id): def get_by_id(objects, id):
return get_by_key(objects, 'id', id) return get_by_key(objects, 'id', id)
def normalize_dict(encoding, data):
if PY2:
string_type = unicode
else:
string_type = str
if isinstance(data, string_type):
data = unicodedata.normalize(encoding, data)
elif isinstance(data, dict):
for key in data:
data[key] = normalize_dict(encoding, data[key])
elif isinstance(data, list):
return [normalize_dict(encoding, value) for value in data]
return data