dynamic sort table, not creating so far, needs sqldiff to update

This commit is contained in:
j 2011-01-03 19:44:54 +05:30
parent eacaa943d7
commit 508e84b73c
5 changed files with 267 additions and 94 deletions

View file

@ -1,7 +1,10 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4 # vi:si:et:sw=4:sts=4:ts=4
from django.db import models import os
from django.db import models
from django.conf import settings
from ox.utils import json
class Page(models.Model): class Page(models.Model):
created = models.DateTimeField(auto_now_add=True) created = models.DateTimeField(auto_now_add=True)
@ -19,3 +22,10 @@ class SiteSettings(models.Model):
def __unicode__(self): def __unicode__(self):
return self.key return self.key
with open(os.path.join(settings.PROJECT_ROOT, 'templates', 'site.json')) as f:
site_config = json.load(f)
site_config['keys'] = {}
for key in site_config['sortKeys']:
site_config['keys'][key['id']] = key

View file

@ -7,16 +7,6 @@ from django.db.models import Q, Manager
import models import models
def keyType(key):
if key in ('released', ):
return "date"
if key in ('year', 'cast.length'):
return "int"
if key in ('rating', 'votes'):
return "float"
return "string"
def parseCondition(condition): def parseCondition(condition):
''' '''
condition: { condition: {
@ -37,21 +27,32 @@ def parseCondition(condition):
v = condition['value'] v = condition['value']
op = condition.get('operator', None) op = condition.get('operator', None)
if not op: if not op:
op = '~' op = ''
if op.startswith('!'): if op.startswith('!'):
op = op[1:] op = op[1:]
exclude = True exclude = True
else: else:
exclude = False exclude = False
if keyType(k) == "string":
key_type = models.site_config['keys'].get(k, 'string')
return {
'title': 'string',
'person': 'string'
}.get(key_type, key_type)
if key_type == "string":
in_find=True in_find=True
value_key = 'find__value' value_key = 'find__value'
if op == '=':
if k in models.Item.facet_keys: if k in models.Item.facet_keys:
in_find = False in_find = False
if op == '=':
v = models.Item.objects.filter(facets__key=k, facets__value=v) v = models.Item.objects.filter(facets__key=k, facets__value=v)
elif op == '^':
v = models.Item.objects.filter(facets__key=k, facets__value__istartswith=v)
elif op == '$':
v = models.Item.objects.filter(facets__key=k, facets__value__iendswith=v)
k = 'id__in' k = 'id__in'
else: elif op == '=':
value_key = 'find__value__iexact' value_key = 'find__value__iexact'
elif op == '^': elif op == '^':
v = v[1:] v = v[1:]
@ -59,7 +60,7 @@ def parseCondition(condition):
elif op == '$': elif op == '$':
v = v[:-1] v = v[:-1]
value_key = 'find__value__iendswith' value_key = 'find__value__iendswith'
else: # elif op == '~': else: # default
value_key = 'find__value__icontains' value_key = 'find__value__icontains'
k = str(k) k = str(k)
if exclude: if exclude:
@ -82,7 +83,7 @@ def parseCondition(condition):
if op == '-': if op == '-':
v1 = v[1] v1 = v[1]
v2 = v[2] v2 = v[2]
if keyType(k) == "date": if key_type == "date":
v1 = parseDate(v1.split('.')) v1 = parseDate(v1.split('.'))
v2 = parseDate(v2.split('.')) v2 = parseDate(v2.split('.'))
@ -95,7 +96,7 @@ def parseCondition(condition):
k2 = 'value__lt' k2 = 'value__lt'
return Q(**{'find__key': k, k1: v1})&Q(**{'find__key': k, k2: v2}) return Q(**{'find__key': k, k1: v1})&Q(**{'find__key': k, k2: v2})
else: else:
if keyType(k) == "date": if key_type == "date":
v = parseDate(v.split('.')) v = parseDate(v.split('.'))
if op == '=': if op == '=':
vk = 'value__exact' vk = 'value__exact'
@ -107,6 +108,8 @@ def parseCondition(condition):
vk = 'value__lt' vk = 'value__lt'
elif op == '<=': elif op == '<=':
vk = 'value__lte' vk = 'value__lte'
elif op == '':
vk = 'value__exact'
vk = 'find__%s' % vk vk = 'find__%s' % vk
vk = str(vk) vk = str(vk)

View file

@ -6,6 +6,7 @@ from datetime import datetime
import os.path import os.path
import subprocess import subprocess
from glob import glob from glob import glob
import unicodedata
from django.db import models from django.db import models
from django.core.files.base import ContentFile from django.core.files.base import ContentFile
@ -24,7 +25,7 @@ from archive import extract
from annotaion.models import Annotation, Layer from annotaion.models import Annotation, Layer
from person.models import get_name_sort from person.models import get_name_sort
from app.models import site_config
def siteJson(): def siteJson():
r = {} r = {}
@ -218,7 +219,6 @@ class Property(models.Model):
class Item(models.Model): class Item(models.Model):
person_keys = ('director', 'writer', 'producer', 'editor', 'cinematographer', 'actor', 'character') person_keys = ('director', 'writer', 'producer', 'editor', 'cinematographer', 'actor', 'character')
facet_keys = person_keys + ('country', 'language', 'genre', 'keyword')
created = models.DateTimeField(auto_now_add=True) created = models.DateTimeField(auto_now_add=True)
modified = models.DateTimeField(auto_now=True) modified = models.DateTimeField(auto_now=True)
published = models.DateTimeField(default=datetime.now, editable=False) published = models.DateTimeField(default=datetime.now, editable=False)
@ -271,7 +271,19 @@ class Item(models.Model):
def update_imdb(self): def update_imdb(self):
if len(self.itemId) == 7: if len(self.itemId) == 7:
self.external_data = ox.web.imdb.Imdb(self.itemId) data = ox.web.imdb.Imdb(self.itemId)
#FIXME: all this should be in ox.web.imdb.Imdb
for key in ('directors', 'writers', 'editors', 'producers', 'cinematographers', 'languages', 'genres', 'keywords'):
if key in data:
data[key[:-1]] = data.pop(key)
if 'countries' in data:
data['country'] = data.pop('countries')
if 'release date' in data:
data['releasedate'] = min(data.pop('release date'))
if 'plot' in data:
data['summary'] = data.pop('plot')
data['actor'] = [c[0] for c in data['cast']]
self.external_data = data
self.save() self.save()
def __unicode__(self): def __unicode__(self):
@ -517,36 +529,113 @@ class Item(models.Model):
sort_value = '' sort_value = ''
return sort_value return sort_value
#title base_keys = (
title = canonicalTitle(self.get('title')) 'id',
s.title = utils.sort_title(title) 'aspectratio',
'duration',
'color',
'saturation',
'brightness',
'volume',
'clips',
'cuts',
'cutsperminute',
'words',
'wordsperminute',
'resolution',
'pixels',
'size',
'bitrate',
'files',
'filename',
'published',
'modified',
'popularity'
)
for key in site_config['sortKeys']:
name = key['id']
field_type = key['type']
if name not in base_keys:
if field_type == 'title':
value = utils.sort_title(canonicalTitle(self.get(name)))
value = unicodedata.normalize('NFKD', value)
setattr(s, name, value)
if not value:
value = 'zzzzzzzzzzzzzzzzzzzzzzzzz'
setattr(s, '%s_desc'%name, value)
elif field_type == 'person':
value = sortNames(self.get(name, []))
value = unicodedata.normalize('NFKD', value)
setattr(s, name, value)
if not value:
value = 'zzzzzzzzzzzzzzzzzzzzzzzzz'
setattr(s, '%s_desc'%name, value)
elif field_type == 'text':
#FIXME: what use pural_key?
value = self.get(name, '')
if isinstance(value, list):
value = ','.join(value)
value = unicodedata.normalize('NFKD', value)
setattr(s, name, value)
if not value:
value = 'zzzzzzzzzzzzzzzzzzzzzzzzz'
setattr(s, '%s_desc'%name, value)
elif field_type == 'length':
setattr(s, name, len(self.get(name, '')))
elif field_type == 'integer':
max_int = 9223372036854775807L
value = self.get(name, -max_int)
if isinstance(value, list):
value = len(value)
setattr(s, name, value)
if value == -max_int:
value = max_int
setattr(s, '%s_desc'%name, value)
elif field_type == 'float':
max_float = 9223372036854775807L
value = self.get(name, -max_float)
if isinstance(value, list):
value = sum(value)
setattr(s, name, value)
if value == -max_float:
value = max_float
setattr(s, '%s_desc'%name, value)
elif field_type == 'words':
value = self.get(name, '')
if isinstance(value, list):
value = '\n'.join(value)
if value:
value = len(value.split(' '))
else:
value = 0
setattr(s, name, value)
elif field_type == 'year':
value = self.get(name, '')
setattr(s, name, value)
if not value:
value = '9999'
setattr(s, '%s_desc'%name, value)
elif field_type == 'date':
value = self.get(name, None)
if isinstance(value, basestring):
value = datetime.strptime(value, '%Y-%m-%d')
setattr(s, name, value)
if not value:
value = datetime.strptime('9999-12-12', '%Y-%m-%d')
setattr(s, '%s_desc'%name, value)
s.country = ','.join(self.get('countries', [])) #sort keys based on database, these will always be available
s.year = self.get('year', '') s.id = self.itemId.replace('0x', 'xx')
s.year_desc = s.year s.modified = self.modified
s.modified_desc = self.modified
s.published = self.published
s.published_desc = self.published
for key in self.person_keys: # sort values based on data from videos
setattr(s, key, sortNames(self.get(utils.plural_key(key), []))) s.words = 0 #FIXME: get words from all layers or something
s.wordsperminute = 0
for key in ('language', 'country'): s.clips = 0 #FIXME: get clips from all layers or something
setattr(s, key, ','.join(self.get(utils.plural_key(key), []))) s.popularity = 0 #FIXME: get popularity from somewhere
s.runtime = self.get('runtime', 0)
for key in ('keywords', 'genres', 'cast', 'summary', 'trivia', 'connections'):
setattr(s, key, len(self.get(key, '')))
s.itemId = self.itemId.replace('0x', 'xx')
s.rating = self.get('rating', -1)
s.votes = self.get('votes', -1)
# data from related subtitles
s.scenes = 0 #FIXME
s.dialog = 0 #FIXME
s.words = 0 #FIXME
s.wpm = 0 #FIXME
s.risk = 0 #FIXME
# data from related files
videos = self.main_videos() videos = self.main_videos()
if len(videos) > 0: if len(videos) > 0:
s.duration = sum([v.duration for v in videos]) s.duration = sum([v.duration for v in videos])
@ -557,8 +646,10 @@ class Item(models.Model):
s.bitrate = videos[0].info['bitrate'] s.bitrate = videos[0].info['bitrate']
s.pixels = sum([v.pixels for v in videos]) s.pixels = sum([v.pixels for v in videos])
s.filename = ' '.join([v.name for v in videos]) s.filename = ' '.join([v.name for v in videos])
s.filename_desc = ' '.join([v.name for v in videos])
s.files = self.files.all().count() s.files = self.files.all().count()
s.size = sum([v.size for v in videos]) #FIXME: only size of movies? s.size = sum([v.size for v in videos]) #FIXME: only size of movies?
s.volume = 0
else: else:
s.duration = 0 s.duration = 0
s.resolution = 0 s.resolution = 0
@ -568,6 +659,7 @@ class Item(models.Model):
s.filename = 0 s.filename = 0
s.files = 0 s.files = 0
s.size = 0 s.size = 0
s.volume = 0
s.color = int(sum(self.data.get('color', []))) s.color = int(sum(self.data.get('color', [])))
s.saturation = 0 #FIXME s.saturation = 0 #FIXME
@ -578,14 +670,6 @@ class Item(models.Model):
s.cutsperminute = s.cuts / (s.duration/60) s.cutsperminute = s.cuts / (s.duration/60)
else: else:
s.cutsperminute = 0 s.cutsperminute = 0
for key in ('title', 'language', 'country') + self.person_keys:
setattr(s, '%s_desc'%key, getattr(s, key))
if not getattr(s, key):
setattr(s, key, u'zzzzzzzzzzzzzzzzzzzzzzzzz')
if not s.year:
s.year_desc = ''
s.year = '9999'
#FIXME: also deal with number based rows like genre, keywords etc
s.save() s.save()
def update_facets(self): def update_facets(self):
@ -781,6 +865,14 @@ class Item(models.Model):
p.wait() p.wait()
return posters.keys() return posters.keys()
Item.facet_keys = []
Item.person_keys = []
for key in site_config['findKeys']:
name = key['id']
if key.get('autocomplete', False) and not site_config['keys'].get(name, {'type': None})['type'] == 'title':
Item.facet_keys.append(name)
if name in site_config['keys'] and site_config['keys'][name]['type'] == 'person':
Item.person_keys.append(name)
class ItemFind(models.Model): class ItemFind(models.Model):
""" """
@ -796,7 +888,40 @@ class ItemFind(models.Model):
key = models.CharField(max_length=200, db_index=True) key = models.CharField(max_length=200, db_index=True)
value = models.TextField(blank=True) value = models.TextField(blank=True)
attrs = {
'__module__': 'item.models',
'item': models.OneToOneField('Item', related_name='sort', primary_key=True),
}
for key in site_config['sortKeys']:
name = key['id']
field_type = key['type']
if field_type in ('string', 'title'):
attrs[name] = models.CharField(max_length=1000, db_index=True)
attrs['%s_desc'%name] = models.CharField(max_length=1000, db_index=True)
elif field_type == 'year':
attrs[name] = models.CharField(max_length=4, db_index=True)
attrs['%s_desc'%name] = models.CharField(max_length=4, db_index=True)
elif field_type in ('text', 'person'):
attrs[name] = models.TextField(blank=True, db_index=True)
attrs['%s_desc'%name] = models.TextField(blank=True, db_index=True)
elif field_type in ('integer', 'words', 'length'):
attrs[name] = models.BigIntegerField(blank=True, db_index=True)
attrs['%s_desc'%name] = models.BigIntegerField(blank=True, db_index=True)
elif field_type == 'float':
attrs[name] = models.FloatField(blank=True, db_index=True)
attrs['%s_desc'%name] = models.FloatField(blank=True, db_index=True)
elif field_type == 'date':
attrs[name] = models.DateTimeField(blank=True, db_index=True)
attrs['%s_desc'%name] = models.DateTimeField(blank=True, db_index=True)
else:
print field_type
print key
ItemSort = type('ItemSort', (models.Model,), attrs)
ItemSort.fields = filter(lambda x: not x.endswith('_desc'), [f.name for f in ItemSort._meta.fields])
ItemSort.descending_fields = filter(lambda x: x.endswith('_desc'), [f.name for f in ItemSort._meta.fields])
'''
class ItemSort(models.Model): class ItemSort(models.Model):
#FIXME: make sort based on site.json #FIXME: make sort based on site.json
""" """
@ -848,7 +973,6 @@ class ItemSort(models.Model):
cuts = models.IntegerField(blank=True, db_index=True) cuts = models.IntegerField(blank=True, db_index=True)
cutsperminute = models.FloatField(blank=True, db_index=True) cutsperminute = models.FloatField(blank=True, db_index=True)
#required to move empty values to the bottom for both asc and desc sort #required to move empty values to the bottom for both asc and desc sort
title_desc = models.CharField(max_length=1000, db_index=True) title_desc = models.CharField(max_length=1000, db_index=True)
director_desc = models.TextField(blank=True, db_index=True) director_desc = models.TextField(blank=True, db_index=True)
@ -862,25 +986,18 @@ class ItemSort(models.Model):
language_desc = models.TextField(blank=True, db_index=True) language_desc = models.TextField(blank=True, db_index=True)
_private_fields = ('id', 'item') ItemSort.fields = filter(lambda x: not x.endswith('_desc'), [f.name for f in ItemSort._meta.fields])
#return available sort fields ItemSort.descending_fields = filter(lambda x: x.endswith('_desc'), [f.name for f in ItemSort._meta.fields])
#FIXME: should return mapping name -> verbose_name '''
def fields(self):
fields = []
for f in self._meta.fields:
if f.name not in self._private_fields:
name = f.verbose_name
name = name[0].capitalize() + name[1:]
fields.append(name)
return tuple(fields)
fields = classmethod(fields)
class Facet(models.Model): class Facet(models.Model):
class Meta:
unique_together = ("item", "key", "value")
item = models.ForeignKey('Item', related_name='facets') item = models.ForeignKey('Item', related_name='facets')
key = models.CharField(max_length=200, db_index=True) key = models.CharField(max_length=200, db_index=True)
value = models.CharField(max_length=200) value = models.CharField(max_length=200, db_index=True)
value_sort = models.CharField(max_length=200) value_sort = models.CharField(max_length=200, db_index=True)
def save(self, *args, **kwargs): def save(self, *args, **kwargs):
if not self.value_sort: if not self.value_sort:

View file

@ -201,3 +201,4 @@ def sort_title(title):
#pad numbered titles #pad numbered titles
title = re.sub('(\d+)', lambda x: '%010d' % int(x.group(0)), title) title = re.sub('(\d+)', lambda x: '%010d' % int(x.group(0)), title)
return title.strip() return title.strip()

View file

@ -42,13 +42,14 @@ def _order_query(qs, sort, prefix='sort__'):
sort.append({'operator': '+', 'key': 'director'}) sort.append({'operator': '+', 'key': 'director'})
sort.append({'operator': '-', 'key': 'year'}) sort.append({'operator': '-', 'key': 'year'})
sort.append({'operator': '+', 'key': 'title'}) sort.append({'operator': '+', 'key': 'title'})
for e in sort: for e in sort:
operator = e['operator'] operator = e['operator']
if operator != '-': if operator != '-':
operator = '' operator = ''
key = {'id': 'itemId'}.get(e['key'], e['key']) key = {'id': 'itemId'}.get(e['key'], e['key'])
#FIXME: this should be a property of models.ItemSort!!! if operator=='-' and '%s_desc'%key in models.ItemSort.descending_fields:
if operator=='-' and key in ('title', 'director', 'writer', 'producer', 'editor', 'cinematographer', 'language', 'country', 'year'):
key = '%s_desc' % key key = '%s_desc' % key
order = '%s%s%s' % (operator, prefix, key) order = '%s%s%s' % (operator, prefix, key)
order_by.append(order) order_by.append(order)
@ -150,35 +151,33 @@ Positions
if 'sort' in query: if 'sort' in query:
if len(query['sort']) == 1 and query['sort'][0]['key'] == 'items': if len(query['sort']) == 1 and query['sort'][0]['key'] == 'items':
if query['group'] == "year": if query['group'] == "year":
query['sort'].append({'key': 'name', 'operator':'-'}) order_by = query['sort'][0]['operator'] == '-' and 'items' or '-items'
else: else:
query['sort'].append({'key': 'name', 'operator':'+'}) order_by = query['sort'][0]['operator'] == '-' and '-items' or 'items'
if query['group'] != "keyword":
order_by = (order_by, 'value_sort')
else: else:
query['sort'] = [{'key': 'name', 'operator':'+'}] order_by = (order_by,)
else:
order_by = query['sort'][0]['operator'] == '-' and '-value_sort' or 'value_sort'
order_by = (order_by, 'items')
else:
order_by = ('-value_sort', 'items')
response['data']['items'] = [] response['data']['items'] = []
items = 'items' items = 'items'
item_qs = query['qs'] item_qs = query['qs']
qs = models.Facet.objects.filter(key=query['group']).filter(item__id__in=item_qs) qs = models.Facet.objects.filter(key=query['group']).filter(item__id__in=item_qs)
qs = qs.values('value').annotate(items=Count('id')).order_by() qs = qs.values('value').annotate(items=Count('id')).order_by(**order_by)
name = 'value'
name_sort = 'value_sort'
#replace normalized items/name sort with actual db value
for i in range(0, len(query['sort'])):
if query['sort'][i]['key'] == 'name':
query['sort'][i]['key'] = name_sort
elif query['sort'][i]['key'] == 'items':
query['sort'][i]['key'] = items
qs = _order_query(qs, query['sort'], prefix='')
if 'ids' in query: if 'ids' in query:
#FIXME: this does not scale for larger results #FIXME: this does not scale for larger results
response['data']['positions'] = {} response['data']['positions'] = {}
ids = [j[name] for j in qs] ids = [j['value'] for j in qs]
response['data']['positions'] = _get_positions(ids, query['ids']) response['data']['positions'] = _get_positions(ids, query['ids'])
elif 'range' in data: elif 'range' in data:
qs = qs[query['range'][0]:query['range'][1]] qs = qs[query['range'][0]:query['range'][1]]
response['data']['items'] = [{'name': i[name], 'items': i[items]} for i in qs] response['data']['items'] = [{'name': i['value'], 'items': i[items]} for i in qs]
else: else:
response['data']['items'] = qs.count() response['data']['items'] = qs.count()
elif 'ids' in query: elif 'ids' in query:
@ -222,6 +221,49 @@ Positions
actions.register(find) actions.register(find)
def autocomplete(request):
'''
param data
key
value
operator '', '^', '$'
range
return
'''
data = json.loads(request.POST['data'])
if not 'range' in data:
data['range'] = [0, 10]
op = data.get('operator', '')
if models.site_config['keys'][data['key']]['type'] == 'title':
qs = models.Item.objects.filter(available=True) #does this need more limiting? user etc
if data['value']:
if op == '':
qs = qs.filter(find__key=data['key'], find__value__icontains=data['value'])
elif op == '^':
qs = qs.filter(find__key=data['key'], find__value__istartswith=data['value'])
elif op == '$':
qs = qs.filter(find__key=data['key'], find__value__iendswith=data['value'])
qs = qs.order_by('-sort__%s'%models.site_config['keys'][data['key']]['autocompleteSortKey'])
qs = qs[data['range'][0]:data['range'][1]]
response = json_response({})
response['data']['items'] = [i.get(data['key']) for i in qs]
else:
qs = models.Facet.objects.filter(key=data['key'])
if data['value']:
if op == '':
qs = qs.filter(value__icontains=data['value'])
elif op == '^':
qs = qs.filter(value__istartswith=data['value'])
elif op == '$':
qs = qs.filter(value__iendswith=data['value'])
qs = qs.values('value').annotate(items=Count('id')).order_by('-items')
qs = qs[data['range'][0]:data['range'][1]]
response = json_response({})
response['data']['items'] = [i['value'] for i in qs]
return render_to_json_response(response)
actions.register(autocomplete)
def getItem(request): def getItem(request):
''' '''