From 508e84b73cc23da6eeb5035cfee170e90b295d86 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Mon, 3 Jan 2011 19:44:54 +0530 Subject: [PATCH] dynamic sort table, not creating so far, needs sqldiff to update --- pandora/app/models.py | 12 ++- pandora/item/managers.py | 45 ++++---- pandora/item/models.py | 227 +++++++++++++++++++++++++++++---------- pandora/item/utils.py | 1 + pandora/item/views.py | 76 ++++++++++--- 5 files changed, 267 insertions(+), 94 deletions(-) diff --git a/pandora/app/models.py b/pandora/app/models.py index 16bc42893..dd0198668 100644 --- a/pandora/app/models.py +++ b/pandora/app/models.py @@ -1,7 +1,10 @@ # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 -from django.db import models +import os +from django.db import models +from django.conf import settings +from ox.utils import json class Page(models.Model): created = models.DateTimeField(auto_now_add=True) @@ -19,3 +22,10 @@ class SiteSettings(models.Model): def __unicode__(self): return self.key + +with open(os.path.join(settings.PROJECT_ROOT, 'templates', 'site.json')) as f: + site_config = json.load(f) + site_config['keys'] = {} + for key in site_config['sortKeys']: + site_config['keys'][key['id']] = key + diff --git a/pandora/item/managers.py b/pandora/item/managers.py index 06f0d3ba2..1b1e20351 100644 --- a/pandora/item/managers.py +++ b/pandora/item/managers.py @@ -7,16 +7,6 @@ from django.db.models import Q, Manager import models -def keyType(key): - if key in ('released', ): - return "date" - if key in ('year', 'cast.length'): - return "int" - if key in ('rating', 'votes'): - return "float" - return "string" - - def parseCondition(condition): ''' condition: { @@ -37,29 +27,40 @@ def parseCondition(condition): v = condition['value'] op = condition.get('operator', None) if not op: - op = '~' + op = '' if op.startswith('!'): op = op[1:] exclude = True else: exclude = False - if keyType(k) == "string": + + key_type = models.site_config['keys'].get(k, 'string') + return { + 'title': 'string', + 'person': 'string' + }.get(key_type, key_type) + + if key_type == "string": in_find=True value_key = 'find__value' - if op == '=': - if k in models.Item.facet_keys: - in_find=False + if k in models.Item.facet_keys: + in_find = False + if op == '=': v = models.Item.objects.filter(facets__key=k, facets__value=v) - k = 'id__in' - else: - value_key = 'find__value__iexact' + elif op == '^': + v = models.Item.objects.filter(facets__key=k, facets__value__istartswith=v) + elif op == '$': + v = models.Item.objects.filter(facets__key=k, facets__value__iendswith=v) + k = 'id__in' + elif op == '=': + value_key = 'find__value__iexact' elif op == '^': v = v[1:] value_key = 'find__value__istartswith' elif op == '$': v = v[:-1] value_key = 'find__value__iendswith' - else: # elif op == '~': + else: # default value_key = 'find__value__icontains' k = str(k) if exclude: @@ -82,7 +83,7 @@ def parseCondition(condition): if op == '-': v1 = v[1] v2 = v[2] - if keyType(k) == "date": + if key_type == "date": v1 = parseDate(v1.split('.')) v2 = parseDate(v2.split('.')) @@ -95,7 +96,7 @@ def parseCondition(condition): k2 = 'value__lt' return Q(**{'find__key': k, k1: v1})&Q(**{'find__key': k, k2: v2}) else: - if keyType(k) == "date": + if key_type == "date": v = parseDate(v.split('.')) if op == '=': vk = 'value__exact' @@ -107,6 +108,8 @@ def parseCondition(condition): vk = 'value__lt' elif op == '<=': vk = 'value__lte' + elif op == '': + vk = 'value__exact' vk = 'find__%s' % vk vk = str(vk) diff --git a/pandora/item/models.py b/pandora/item/models.py index 6dda19510..444a3eacc 100644 --- a/pandora/item/models.py +++ b/pandora/item/models.py @@ -6,6 +6,7 @@ from datetime import datetime import os.path import subprocess from glob import glob +import unicodedata from django.db import models from django.core.files.base import ContentFile @@ -24,7 +25,7 @@ from archive import extract from annotaion.models import Annotation, Layer from person.models import get_name_sort - +from app.models import site_config def siteJson(): r = {} @@ -218,7 +219,6 @@ class Property(models.Model): class Item(models.Model): person_keys = ('director', 'writer', 'producer', 'editor', 'cinematographer', 'actor', 'character') - facet_keys = person_keys + ('country', 'language', 'genre', 'keyword') created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) published = models.DateTimeField(default=datetime.now, editable=False) @@ -271,7 +271,19 @@ class Item(models.Model): def update_imdb(self): if len(self.itemId) == 7: - self.external_data = ox.web.imdb.Imdb(self.itemId) + data = ox.web.imdb.Imdb(self.itemId) + #FIXME: all this should be in ox.web.imdb.Imdb + for key in ('directors', 'writers', 'editors', 'producers', 'cinematographers', 'languages', 'genres', 'keywords'): + if key in data: + data[key[:-1]] = data.pop(key) + if 'countries' in data: + data['country'] = data.pop('countries') + if 'release date' in data: + data['releasedate'] = min(data.pop('release date')) + if 'plot' in data: + data['summary'] = data.pop('plot') + data['actor'] = [c[0] for c in data['cast']] + self.external_data = data self.save() def __unicode__(self): @@ -517,36 +529,113 @@ class Item(models.Model): sort_value = '' return sort_value - #title - title = canonicalTitle(self.get('title')) - s.title = utils.sort_title(title) + base_keys = ( + 'id', + 'aspectratio', + 'duration', + 'color', + 'saturation', + 'brightness', + 'volume', + 'clips', + 'cuts', + 'cutsperminute', + 'words', + 'wordsperminute', + 'resolution', + 'pixels', + 'size', + 'bitrate', + 'files', + 'filename', + 'published', + 'modified', + 'popularity' + ) + for key in site_config['sortKeys']: + name = key['id'] + field_type = key['type'] + if name not in base_keys: + if field_type == 'title': + value = utils.sort_title(canonicalTitle(self.get(name))) + value = unicodedata.normalize('NFKD', value) + setattr(s, name, value) + if not value: + value = 'zzzzzzzzzzzzzzzzzzzzzzzzz' + setattr(s, '%s_desc'%name, value) + elif field_type == 'person': + value = sortNames(self.get(name, [])) + value = unicodedata.normalize('NFKD', value) + setattr(s, name, value) + if not value: + value = 'zzzzzzzzzzzzzzzzzzzzzzzzz' + setattr(s, '%s_desc'%name, value) + elif field_type == 'text': + #FIXME: what use pural_key? + value = self.get(name, '') + if isinstance(value, list): + value = ','.join(value) + value = unicodedata.normalize('NFKD', value) + setattr(s, name, value) + if not value: + value = 'zzzzzzzzzzzzzzzzzzzzzzzzz' + setattr(s, '%s_desc'%name, value) + elif field_type == 'length': + setattr(s, name, len(self.get(name, ''))) + elif field_type == 'integer': + max_int = 9223372036854775807L + value = self.get(name, -max_int) + if isinstance(value, list): + value = len(value) + setattr(s, name, value) + if value == -max_int: + value = max_int + setattr(s, '%s_desc'%name, value) + elif field_type == 'float': + max_float = 9223372036854775807L + value = self.get(name, -max_float) + if isinstance(value, list): + value = sum(value) + setattr(s, name, value) + if value == -max_float: + value = max_float + setattr(s, '%s_desc'%name, value) + elif field_type == 'words': + value = self.get(name, '') + if isinstance(value, list): + value = '\n'.join(value) + if value: + value = len(value.split(' ')) + else: + value = 0 + setattr(s, name, value) + elif field_type == 'year': + value = self.get(name, '') + setattr(s, name, value) + if not value: + value = '9999' + setattr(s, '%s_desc'%name, value) + elif field_type == 'date': + value = self.get(name, None) + if isinstance(value, basestring): + value = datetime.strptime(value, '%Y-%m-%d') + setattr(s, name, value) + if not value: + value = datetime.strptime('9999-12-12', '%Y-%m-%d') + setattr(s, '%s_desc'%name, value) - s.country = ','.join(self.get('countries', [])) - s.year = self.get('year', '') - s.year_desc = s.year + #sort keys based on database, these will always be available + s.id = self.itemId.replace('0x', 'xx') + s.modified = self.modified + s.modified_desc = self.modified + s.published = self.published + s.published_desc = self.published - for key in self.person_keys: - setattr(s, key, sortNames(self.get(utils.plural_key(key), []))) - - for key in ('language', 'country'): - setattr(s, key, ','.join(self.get(utils.plural_key(key), []))) - - s.runtime = self.get('runtime', 0) - - for key in ('keywords', 'genres', 'cast', 'summary', 'trivia', 'connections'): - setattr(s, key, len(self.get(key, ''))) - - s.itemId = self.itemId.replace('0x', 'xx') - s.rating = self.get('rating', -1) - s.votes = self.get('votes', -1) - - # data from related subtitles - s.scenes = 0 #FIXME - s.dialog = 0 #FIXME - s.words = 0 #FIXME - s.wpm = 0 #FIXME - s.risk = 0 #FIXME - # data from related files + # sort values based on data from videos + s.words = 0 #FIXME: get words from all layers or something + s.wordsperminute = 0 + s.clips = 0 #FIXME: get clips from all layers or something + s.popularity = 0 #FIXME: get popularity from somewhere videos = self.main_videos() if len(videos) > 0: s.duration = sum([v.duration for v in videos]) @@ -557,8 +646,10 @@ class Item(models.Model): s.bitrate = videos[0].info['bitrate'] s.pixels = sum([v.pixels for v in videos]) s.filename = ' '.join([v.name for v in videos]) + s.filename_desc = ' '.join([v.name for v in videos]) s.files = self.files.all().count() s.size = sum([v.size for v in videos]) #FIXME: only size of movies? + s.volume = 0 else: s.duration = 0 s.resolution = 0 @@ -568,6 +659,7 @@ class Item(models.Model): s.filename = 0 s.files = 0 s.size = 0 + s.volume = 0 s.color = int(sum(self.data.get('color', []))) s.saturation = 0 #FIXME @@ -578,14 +670,6 @@ class Item(models.Model): s.cutsperminute = s.cuts / (s.duration/60) else: s.cutsperminute = 0 - for key in ('title', 'language', 'country') + self.person_keys: - setattr(s, '%s_desc'%key, getattr(s, key)) - if not getattr(s, key): - setattr(s, key, u'zzzzzzzzzzzzzzzzzzzzzzzzz') - if not s.year: - s.year_desc = '' - s.year = '9999' - #FIXME: also deal with number based rows like genre, keywords etc s.save() def update_facets(self): @@ -781,6 +865,14 @@ class Item(models.Model): p.wait() return posters.keys() +Item.facet_keys = [] +Item.person_keys = [] +for key in site_config['findKeys']: + name = key['id'] + if key.get('autocomplete', False) and not site_config['keys'].get(name, {'type': None})['type'] == 'title': + Item.facet_keys.append(name) + if name in site_config['keys'] and site_config['keys'][name]['type'] == 'person': + Item.person_keys.append(name) class ItemFind(models.Model): """ @@ -796,7 +888,40 @@ class ItemFind(models.Model): key = models.CharField(max_length=200, db_index=True) value = models.TextField(blank=True) +attrs = { + '__module__': 'item.models', + 'item': models.OneToOneField('Item', related_name='sort', primary_key=True), +} +for key in site_config['sortKeys']: + name = key['id'] + field_type = key['type'] + if field_type in ('string', 'title'): + attrs[name] = models.CharField(max_length=1000, db_index=True) + attrs['%s_desc'%name] = models.CharField(max_length=1000, db_index=True) + elif field_type == 'year': + attrs[name] = models.CharField(max_length=4, db_index=True) + attrs['%s_desc'%name] = models.CharField(max_length=4, db_index=True) + elif field_type in ('text', 'person'): + attrs[name] = models.TextField(blank=True, db_index=True) + attrs['%s_desc'%name] = models.TextField(blank=True, db_index=True) + elif field_type in ('integer', 'words', 'length'): + attrs[name] = models.BigIntegerField(blank=True, db_index=True) + attrs['%s_desc'%name] = models.BigIntegerField(blank=True, db_index=True) + elif field_type == 'float': + attrs[name] = models.FloatField(blank=True, db_index=True) + attrs['%s_desc'%name] = models.FloatField(blank=True, db_index=True) + elif field_type == 'date': + attrs[name] = models.DateTimeField(blank=True, db_index=True) + attrs['%s_desc'%name] = models.DateTimeField(blank=True, db_index=True) + else: + print field_type + print key +ItemSort = type('ItemSort', (models.Model,), attrs) +ItemSort.fields = filter(lambda x: not x.endswith('_desc'), [f.name for f in ItemSort._meta.fields]) +ItemSort.descending_fields = filter(lambda x: x.endswith('_desc'), [f.name for f in ItemSort._meta.fields]) + +''' class ItemSort(models.Model): #FIXME: make sort based on site.json """ @@ -848,7 +973,6 @@ class ItemSort(models.Model): cuts = models.IntegerField(blank=True, db_index=True) cutsperminute = models.FloatField(blank=True, db_index=True) - #required to move empty values to the bottom for both asc and desc sort title_desc = models.CharField(max_length=1000, db_index=True) director_desc = models.TextField(blank=True, db_index=True) @@ -862,25 +986,18 @@ class ItemSort(models.Model): language_desc = models.TextField(blank=True, db_index=True) - _private_fields = ('id', 'item') - #return available sort fields - #FIXME: should return mapping name -> verbose_name - def fields(self): - fields = [] - for f in self._meta.fields: - if f.name not in self._private_fields: - name = f.verbose_name - name = name[0].capitalize() + name[1:] - fields.append(name) - return tuple(fields) - fields = classmethod(fields) - +ItemSort.fields = filter(lambda x: not x.endswith('_desc'), [f.name for f in ItemSort._meta.fields]) +ItemSort.descending_fields = filter(lambda x: x.endswith('_desc'), [f.name for f in ItemSort._meta.fields]) +''' class Facet(models.Model): + class Meta: + unique_together = ("item", "key", "value") + item = models.ForeignKey('Item', related_name='facets') key = models.CharField(max_length=200, db_index=True) - value = models.CharField(max_length=200) - value_sort = models.CharField(max_length=200) + value = models.CharField(max_length=200, db_index=True) + value_sort = models.CharField(max_length=200, db_index=True) def save(self, *args, **kwargs): if not self.value_sort: diff --git a/pandora/item/utils.py b/pandora/item/utils.py index d6d130505..f1119cfd5 100644 --- a/pandora/item/utils.py +++ b/pandora/item/utils.py @@ -201,3 +201,4 @@ def sort_title(title): #pad numbered titles title = re.sub('(\d+)', lambda x: '%010d' % int(x.group(0)), title) return title.strip() + diff --git a/pandora/item/views.py b/pandora/item/views.py index b10514b8b..190b769a0 100644 --- a/pandora/item/views.py +++ b/pandora/item/views.py @@ -42,13 +42,14 @@ def _order_query(qs, sort, prefix='sort__'): sort.append({'operator': '+', 'key': 'director'}) sort.append({'operator': '-', 'key': 'year'}) sort.append({'operator': '+', 'key': 'title'}) + + for e in sort: operator = e['operator'] if operator != '-': operator = '' key = {'id': 'itemId'}.get(e['key'], e['key']) - #FIXME: this should be a property of models.ItemSort!!! - if operator=='-' and key in ('title', 'director', 'writer', 'producer', 'editor', 'cinematographer', 'language', 'country', 'year'): + if operator=='-' and '%s_desc'%key in models.ItemSort.descending_fields: key = '%s_desc' % key order = '%s%s%s' % (operator, prefix, key) order_by.append(order) @@ -150,35 +151,33 @@ Positions if 'sort' in query: if len(query['sort']) == 1 and query['sort'][0]['key'] == 'items': if query['group'] == "year": - query['sort'].append({'key': 'name', 'operator':'-'}) + order_by = query['sort'][0]['operator'] == '-' and 'items' or '-items' else: - query['sort'].append({'key': 'name', 'operator':'+'}) + order_by = query['sort'][0]['operator'] == '-' and '-items' or 'items' + if query['group'] != "keyword": + order_by = (order_by, 'value_sort') + else: + order_by = (order_by,) + else: + order_by = query['sort'][0]['operator'] == '-' and '-value_sort' or 'value_sort' + order_by = (order_by, 'items') else: - query['sort'] = [{'key': 'name', 'operator':'+'}] + order_by = ('-value_sort', 'items') response['data']['items'] = [] items = 'items' item_qs = query['qs'] qs = models.Facet.objects.filter(key=query['group']).filter(item__id__in=item_qs) - qs = qs.values('value').annotate(items=Count('id')).order_by() - name = 'value' - name_sort = 'value_sort' + qs = qs.values('value').annotate(items=Count('id')).order_by(**order_by) - #replace normalized items/name sort with actual db value - for i in range(0, len(query['sort'])): - if query['sort'][i]['key'] == 'name': - query['sort'][i]['key'] = name_sort - elif query['sort'][i]['key'] == 'items': - query['sort'][i]['key'] = items - qs = _order_query(qs, query['sort'], prefix='') if 'ids' in query: #FIXME: this does not scale for larger results response['data']['positions'] = {} - ids = [j[name] for j in qs] + ids = [j['value'] for j in qs] response['data']['positions'] = _get_positions(ids, query['ids']) elif 'range' in data: qs = qs[query['range'][0]:query['range'][1]] - response['data']['items'] = [{'name': i[name], 'items': i[items]} for i in qs] + response['data']['items'] = [{'name': i['value'], 'items': i[items]} for i in qs] else: response['data']['items'] = qs.count() elif 'ids' in query: @@ -222,7 +221,50 @@ Positions actions.register(find) +def autocomplete(request): + ''' + param data + key + value + operator '', '^', '$' + range + return + ''' + data = json.loads(request.POST['data']) + if not 'range' in data: + data['range'] = [0, 10] + op = data.get('operator', '') + if models.site_config['keys'][data['key']]['type'] == 'title': + qs = models.Item.objects.filter(available=True) #does this need more limiting? user etc + if data['value']: + if op == '': + qs = qs.filter(find__key=data['key'], find__value__icontains=data['value']) + elif op == '^': + qs = qs.filter(find__key=data['key'], find__value__istartswith=data['value']) + elif op == '$': + qs = qs.filter(find__key=data['key'], find__value__iendswith=data['value']) + qs = qs.order_by('-sort__%s'%models.site_config['keys'][data['key']]['autocompleteSortKey']) + qs = qs[data['range'][0]:data['range'][1]] + response = json_response({}) + response['data']['items'] = [i.get(data['key']) for i in qs] + else: + qs = models.Facet.objects.filter(key=data['key']) + if data['value']: + if op == '': + qs = qs.filter(value__icontains=data['value']) + elif op == '^': + qs = qs.filter(value__istartswith=data['value']) + elif op == '$': + qs = qs.filter(value__iendswith=data['value']) + qs = qs.values('value').annotate(items=Count('id')).order_by('-items') + qs = qs[data['range'][0]:data['range'][1]] + response = json_response({}) + response['data']['items'] = [i['value'] for i in qs] + return render_to_json_response(response) +actions.register(autocomplete) + + def getItem(request): ''' param data