normalize find values and make lower case, also normalize and lowercase quieries

This commit is contained in:
j 2012-11-18 20:26:13 +01:00
parent 1bc21588a6
commit ec5158c03a
7 changed files with 32 additions and 10 deletions

View file

@ -1,5 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4 # vi:si:et:sw=4:sts=4:ts=4
import unicodedata
from django.db.models import Q, Manager from django.db.models import Q, Manager
from ox.django.query import QuerySet from ox.django.query import QuerySet
@ -71,6 +73,8 @@ def parseCondition(condition, user):
}.get(op, '__icontains')) }.get(op, '__icontains'))
key = str(key) key = str(key)
if isinstance(v, unicode):
v = unicodedata.normalize('NFKD', v).lower()
if exclude: if exclude:
q = ~Q(**{key: v}) q = ~Q(**{key: v})
else: else:

View file

@ -2,6 +2,7 @@
# vi:si:et:sw=4:sts=4:ts=4 # vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, with_statement from __future__ import division, with_statement
import re import re
import unicodedata
from django.db import models from django.db import models
from django.db.models import Q from django.db.models import Q
@ -136,6 +137,7 @@ class Annotation(models.Model):
if self.value: if self.value:
self.value = utils.cleanup_value(self.value, layer['type']) self.value = utils.cleanup_value(self.value, layer['type'])
self.findvalue = ox.decode_html(ox.strip_tags(re.sub('<br */?>\n?', ' ', self.value))).replace('\n', ' ') self.findvalue = ox.decode_html(ox.strip_tags(re.sub('<br */?>\n?', ' ', self.value))).replace('\n', ' ')
self.findvalue = unicodedata.normalize('NFKD', self.findvalue).lower()
sortvalue = sort_string(self.findvalue) sortvalue = sort_string(self.findvalue)
if sortvalue: if sortvalue:
self.sortvalue = sortvalue[:900] self.sortvalue = sortvalue[:900]

View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4 # vi:si:et:sw=4:sts=4:ts=4
import unicodedata
from django.db.models import Q, Manager from django.db.models import Q, Manager
from ox.django.query import QuerySet from ox.django.query import QuerySet
@ -31,6 +32,8 @@ def parseCondition(condition, user):
}.get(op,'__icontains')) }.get(op,'__icontains'))
key = str(key) key = str(key)
if isinstance(v, unicode):
v = unicodedata.normalize('NFKD', v).lower()
if exclude: if exclude:
q = ~Q(**{k: v}) q = ~Q(**{k: v})
else: else:

View file

@ -1,6 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4 # vi:si:et:sw=4:sts=4:ts=4
from datetime import datetime from datetime import datetime
import unicodedata
from django.db.models import Q, Manager from django.db.models import Q, Manager
from django.conf import settings from django.conf import settings
@ -130,6 +131,8 @@ def parseCondition(condition, user):
k = str(k) k = str(k)
value_key = str(value_key) value_key = str(value_key)
if isinstance(v, unicode):
v = unicodedata.normalize('NFKD', v).lower()
if k == '*': if k == '*':
q = Q(**{value_key: v}) q = Q(**{value_key: v})
elif in_find: elif in_find:

View file

@ -239,11 +239,11 @@ class Item(models.Model):
else: else:
k = filter(lambda i: i['id'] == key, settings.CONFIG['itemKeys']) k = filter(lambda i: i['id'] == key, settings.CONFIG['itemKeys'])
ktype = k and k[0].get('type') or '' ktype = k and k[0].get('type') or ''
if ktype == 'text': if ktype == 'text':
self.data[key] = ox.sanitize_html(data[key]) self.data[key] = ox.sanitize_html(data[key])
elif ktype == '[text]': elif ktype == '[text]':
self.data[key] = [ox.sanitize_html(t) for t in data[key]] self.data[key] = [ox.sanitize_html(t) for t in data[key]]
elif ktype == '[string]': elif ktype == '[string]':
self.data[key] = [ox.escape_html(t) for t in data[key]] self.data[key] = [ox.escape_html(t) for t in data[key]]
elif key in ('episodeTitle', 'seriesTitle', 'episodeDirector', 'seriesYear'): elif key in ('episodeTitle', 'seriesTitle', 'episodeDirector', 'seriesYear'):
self.data[key] = ox.escape_html(data[key]) self.data[key] = ox.escape_html(data[key])
@ -344,7 +344,7 @@ class Item(models.Model):
update_poster = True update_poster = True
if len(self.itemId) != 7: if len(self.itemId) != 7:
update_ids = True update_ids = True
#id changed, what about existing item with new id? #id changed, what about existing item with new id?
if settings.USE_IMDB and len(self.itemId) != 7 and self.oxdbId != self.itemId: if settings.USE_IMDB and len(self.itemId) != 7 and self.oxdbId != self.itemId:
self.itemId = self.oxdbId self.itemId = self.oxdbId
@ -471,7 +471,7 @@ class Item(models.Model):
'selected': p == pos, 'selected': p == pos,
'url': '/%s/posterframe%d.jpg' %(self.itemId, p), 'url': '/%s/posterframe%d.jpg' %(self.itemId, p),
'height': f['height'], 'height': f['height'],
'width': f['width'] 'width': f['width']
}) })
p += 1 p += 1
return frames return frames
@ -600,6 +600,7 @@ class Item(models.Model):
value = value and 'true' or 'false' value = value and 'true' or 'false'
if isinstance(value, basestring): if isinstance(value, basestring):
value = ox.decode_html(ox.strip_tags(value.strip())) value = ox.decode_html(ox.strip_tags(value.strip()))
value = unicodedata.normalize('NFKD', value).lower()
f.value = value f.value = value
f.save() f.save()
else: else:
@ -719,7 +720,7 @@ class Item(models.Model):
sort_type = key.get('sortType', key['type']) sort_type = key.get('sortType', key['type'])
if 'value' in key: if 'value' in key:
if 'layer' in key['value']: if 'layer' in key['value']:
continue continue
source = key['value']['key'] source = key['value']['key']
sort_type = key['value'].get('type', sort_type) sort_type = key['value'].get('type', sort_type)
if isinstance(sort_type, list): if isinstance(sort_type, list):
@ -818,7 +819,7 @@ class Item(models.Model):
s.cutsperminute = s.numberofcuts / (s.duration/60) s.cutsperminute = s.numberofcuts / (s.duration/60)
s.wordsperminute = s.words / (s.duration / 60) s.wordsperminute = s.words / (s.duration / 60)
else: else:
s.cutsperminute = None s.cutsperminute = None
s.wordsperminute = None s.wordsperminute = None
s.timesaccessed = self.accessed.aggregate(Sum('accessed'))['accessed__sum'] s.timesaccessed = self.accessed.aggregate(Sum('accessed'))['accessed__sum']
if not s.timesaccessed: if not s.timesaccessed:
@ -1177,7 +1178,7 @@ class Item(models.Model):
def select_frame(self): def select_frame(self):
frames = self.poster_frames() frames = self.poster_frames()
if frames: if frames:
heat = [ox.image.getImageHeat(f['path']) for f in frames] heat = [ox.image.getImageHeat(f['path']) for f in frames]
self.poster_frame = heat.index(max(heat)) self.poster_frame = heat.index(max(heat))
def get_poster_frame_path(self): def get_poster_frame_path(self):
@ -1232,7 +1233,7 @@ class Item(models.Model):
elif '' in languages: elif '' in languages:
language = '' language = ''
else: else:
language = languages[0] language = languages[0]
#loop over all videos #loop over all videos
for f in self.files.filter(Q(is_audio=True)|Q(is_video=True)) \ for f in self.files.filter(Q(is_audio=True)|Q(is_video=True)) \
@ -1393,7 +1394,7 @@ class Facet(models.Model):
does not perform to well if total number of items goes above 10k does not perform to well if total number of items goes above 10k
this happens for keywords in 0xdb right now this happens for keywords in 0xdb right now
''' '''
class Meta: class Meta:
unique_together = ("item", "key", "value") unique_together = ("item", "key", "value")

View file

@ -1,5 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4 # vi:si:et:sw=4:sts=4:ts=4
import unicodedata
from django.db.models import Q, Manager from django.db.models import Q, Manager
from ox.django.query import QuerySet from ox.django.query import QuerySet
@ -60,6 +62,8 @@ def parseCondition(condition, user):
}.get(op,'__icontains')) }.get(op,'__icontains'))
key = str(key) key = str(key)
if isinstance(v, unicode):
v = unicodedata.normalize('NFKD', v).lower()
if exclude: if exclude:
q = ~Q(**{key: v}) q = ~Q(**{key: v})
else: else:

View file

@ -1,5 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4 # vi:si:et:sw=4:sts=4:ts=4
import unicodedata
from django.db.models import Q, Manager from django.db.models import Q, Manager
from ox.django.query import QuerySet from ox.django.query import QuerySet
@ -59,6 +61,9 @@ def parseCondition(condition, user):
}.get(op,'__icontains')) }.get(op,'__icontains'))
key = str(key) key = str(key)
if isinstance(v, unicode):
v = unicodedata.normalize('NFKD', v).lower()
if exclude: if exclude:
q = ~Q(**{key: v}) q = ~Q(**{key: v})
else: else: