dynamic sort table, not creating so far, needs sqldiff to update

This commit is contained in:
j 2011-01-03 19:44:54 +05:30
parent eacaa943d7
commit 508e84b73c
5 changed files with 267 additions and 94 deletions

View file

@ -1,7 +1,10 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from django.db import models
import os
from django.db import models
from django.conf import settings
from ox.utils import json
class Page(models.Model):
created = models.DateTimeField(auto_now_add=True)
@ -19,3 +22,10 @@ class SiteSettings(models.Model):
def __unicode__(self):
return self.key
with open(os.path.join(settings.PROJECT_ROOT, 'templates', 'site.json')) as f:
site_config = json.load(f)
site_config['keys'] = {}
for key in site_config['sortKeys']:
site_config['keys'][key['id']] = key

View file

@ -7,16 +7,6 @@ from django.db.models import Q, Manager
import models
def keyType(key):
if key in ('released', ):
return "date"
if key in ('year', 'cast.length'):
return "int"
if key in ('rating', 'votes'):
return "float"
return "string"
def parseCondition(condition):
'''
condition: {
@ -37,29 +27,40 @@ def parseCondition(condition):
v = condition['value']
op = condition.get('operator', None)
if not op:
op = '~'
op = ''
if op.startswith('!'):
op = op[1:]
exclude = True
else:
exclude = False
if keyType(k) == "string":
key_type = models.site_config['keys'].get(k, 'string')
return {
'title': 'string',
'person': 'string'
}.get(key_type, key_type)
if key_type == "string":
in_find=True
value_key = 'find__value'
if op == '=':
if k in models.Item.facet_keys:
in_find=False
if k in models.Item.facet_keys:
in_find = False
if op == '=':
v = models.Item.objects.filter(facets__key=k, facets__value=v)
k = 'id__in'
else:
value_key = 'find__value__iexact'
elif op == '^':
v = models.Item.objects.filter(facets__key=k, facets__value__istartswith=v)
elif op == '$':
v = models.Item.objects.filter(facets__key=k, facets__value__iendswith=v)
k = 'id__in'
elif op == '=':
value_key = 'find__value__iexact'
elif op == '^':
v = v[1:]
value_key = 'find__value__istartswith'
elif op == '$':
v = v[:-1]
value_key = 'find__value__iendswith'
else: # elif op == '~':
else: # default
value_key = 'find__value__icontains'
k = str(k)
if exclude:
@ -82,7 +83,7 @@ def parseCondition(condition):
if op == '-':
v1 = v[1]
v2 = v[2]
if keyType(k) == "date":
if key_type == "date":
v1 = parseDate(v1.split('.'))
v2 = parseDate(v2.split('.'))
@ -95,7 +96,7 @@ def parseCondition(condition):
k2 = 'value__lt'
return Q(**{'find__key': k, k1: v1})&Q(**{'find__key': k, k2: v2})
else:
if keyType(k) == "date":
if key_type == "date":
v = parseDate(v.split('.'))
if op == '=':
vk = 'value__exact'
@ -107,6 +108,8 @@ def parseCondition(condition):
vk = 'value__lt'
elif op == '<=':
vk = 'value__lte'
elif op == '':
vk = 'value__exact'
vk = 'find__%s' % vk
vk = str(vk)

View file

@ -6,6 +6,7 @@ from datetime import datetime
import os.path
import subprocess
from glob import glob
import unicodedata
from django.db import models
from django.core.files.base import ContentFile
@ -24,7 +25,7 @@ from archive import extract
from annotaion.models import Annotation, Layer
from person.models import get_name_sort
from app.models import site_config
def siteJson():
r = {}
@ -218,7 +219,6 @@ class Property(models.Model):
class Item(models.Model):
person_keys = ('director', 'writer', 'producer', 'editor', 'cinematographer', 'actor', 'character')
facet_keys = person_keys + ('country', 'language', 'genre', 'keyword')
created = models.DateTimeField(auto_now_add=True)
modified = models.DateTimeField(auto_now=True)
published = models.DateTimeField(default=datetime.now, editable=False)
@ -271,7 +271,19 @@ class Item(models.Model):
def update_imdb(self):
if len(self.itemId) == 7:
self.external_data = ox.web.imdb.Imdb(self.itemId)
data = ox.web.imdb.Imdb(self.itemId)
#FIXME: all this should be in ox.web.imdb.Imdb
for key in ('directors', 'writers', 'editors', 'producers', 'cinematographers', 'languages', 'genres', 'keywords'):
if key in data:
data[key[:-1]] = data.pop(key)
if 'countries' in data:
data['country'] = data.pop('countries')
if 'release date' in data:
data['releasedate'] = min(data.pop('release date'))
if 'plot' in data:
data['summary'] = data.pop('plot')
data['actor'] = [c[0] for c in data['cast']]
self.external_data = data
self.save()
def __unicode__(self):
@ -517,36 +529,113 @@ class Item(models.Model):
sort_value = ''
return sort_value
#title
title = canonicalTitle(self.get('title'))
s.title = utils.sort_title(title)
base_keys = (
'id',
'aspectratio',
'duration',
'color',
'saturation',
'brightness',
'volume',
'clips',
'cuts',
'cutsperminute',
'words',
'wordsperminute',
'resolution',
'pixels',
'size',
'bitrate',
'files',
'filename',
'published',
'modified',
'popularity'
)
for key in site_config['sortKeys']:
name = key['id']
field_type = key['type']
if name not in base_keys:
if field_type == 'title':
value = utils.sort_title(canonicalTitle(self.get(name)))
value = unicodedata.normalize('NFKD', value)
setattr(s, name, value)
if not value:
value = 'zzzzzzzzzzzzzzzzzzzzzzzzz'
setattr(s, '%s_desc'%name, value)
elif field_type == 'person':
value = sortNames(self.get(name, []))
value = unicodedata.normalize('NFKD', value)
setattr(s, name, value)
if not value:
value = 'zzzzzzzzzzzzzzzzzzzzzzzzz'
setattr(s, '%s_desc'%name, value)
elif field_type == 'text':
#FIXME: what use pural_key?
value = self.get(name, '')
if isinstance(value, list):
value = ','.join(value)
value = unicodedata.normalize('NFKD', value)
setattr(s, name, value)
if not value:
value = 'zzzzzzzzzzzzzzzzzzzzzzzzz'
setattr(s, '%s_desc'%name, value)
elif field_type == 'length':
setattr(s, name, len(self.get(name, '')))
elif field_type == 'integer':
max_int = 9223372036854775807L
value = self.get(name, -max_int)
if isinstance(value, list):
value = len(value)
setattr(s, name, value)
if value == -max_int:
value = max_int
setattr(s, '%s_desc'%name, value)
elif field_type == 'float':
max_float = 9223372036854775807L
value = self.get(name, -max_float)
if isinstance(value, list):
value = sum(value)
setattr(s, name, value)
if value == -max_float:
value = max_float
setattr(s, '%s_desc'%name, value)
elif field_type == 'words':
value = self.get(name, '')
if isinstance(value, list):
value = '\n'.join(value)
if value:
value = len(value.split(' '))
else:
value = 0
setattr(s, name, value)
elif field_type == 'year':
value = self.get(name, '')
setattr(s, name, value)
if not value:
value = '9999'
setattr(s, '%s_desc'%name, value)
elif field_type == 'date':
value = self.get(name, None)
if isinstance(value, basestring):
value = datetime.strptime(value, '%Y-%m-%d')
setattr(s, name, value)
if not value:
value = datetime.strptime('9999-12-12', '%Y-%m-%d')
setattr(s, '%s_desc'%name, value)
s.country = ','.join(self.get('countries', []))
s.year = self.get('year', '')
s.year_desc = s.year
#sort keys based on database, these will always be available
s.id = self.itemId.replace('0x', 'xx')
s.modified = self.modified
s.modified_desc = self.modified
s.published = self.published
s.published_desc = self.published
for key in self.person_keys:
setattr(s, key, sortNames(self.get(utils.plural_key(key), [])))
for key in ('language', 'country'):
setattr(s, key, ','.join(self.get(utils.plural_key(key), [])))
s.runtime = self.get('runtime', 0)
for key in ('keywords', 'genres', 'cast', 'summary', 'trivia', 'connections'):
setattr(s, key, len(self.get(key, '')))
s.itemId = self.itemId.replace('0x', 'xx')
s.rating = self.get('rating', -1)
s.votes = self.get('votes', -1)
# data from related subtitles
s.scenes = 0 #FIXME
s.dialog = 0 #FIXME
s.words = 0 #FIXME
s.wpm = 0 #FIXME
s.risk = 0 #FIXME
# data from related files
# sort values based on data from videos
s.words = 0 #FIXME: get words from all layers or something
s.wordsperminute = 0
s.clips = 0 #FIXME: get clips from all layers or something
s.popularity = 0 #FIXME: get popularity from somewhere
videos = self.main_videos()
if len(videos) > 0:
s.duration = sum([v.duration for v in videos])
@ -557,8 +646,10 @@ class Item(models.Model):
s.bitrate = videos[0].info['bitrate']
s.pixels = sum([v.pixels for v in videos])
s.filename = ' '.join([v.name for v in videos])
s.filename_desc = ' '.join([v.name for v in videos])
s.files = self.files.all().count()
s.size = sum([v.size for v in videos]) #FIXME: only size of movies?
s.volume = 0
else:
s.duration = 0
s.resolution = 0
@ -568,6 +659,7 @@ class Item(models.Model):
s.filename = 0
s.files = 0
s.size = 0
s.volume = 0
s.color = int(sum(self.data.get('color', [])))
s.saturation = 0 #FIXME
@ -578,14 +670,6 @@ class Item(models.Model):
s.cutsperminute = s.cuts / (s.duration/60)
else:
s.cutsperminute = 0
for key in ('title', 'language', 'country') + self.person_keys:
setattr(s, '%s_desc'%key, getattr(s, key))
if not getattr(s, key):
setattr(s, key, u'zzzzzzzzzzzzzzzzzzzzzzzzz')
if not s.year:
s.year_desc = ''
s.year = '9999'
#FIXME: also deal with number based rows like genre, keywords etc
s.save()
def update_facets(self):
@ -781,6 +865,14 @@ class Item(models.Model):
p.wait()
return posters.keys()
Item.facet_keys = []
Item.person_keys = []
for key in site_config['findKeys']:
name = key['id']
if key.get('autocomplete', False) and not site_config['keys'].get(name, {'type': None})['type'] == 'title':
Item.facet_keys.append(name)
if name in site_config['keys'] and site_config['keys'][name]['type'] == 'person':
Item.person_keys.append(name)
class ItemFind(models.Model):
"""
@ -796,7 +888,40 @@ class ItemFind(models.Model):
key = models.CharField(max_length=200, db_index=True)
value = models.TextField(blank=True)
attrs = {
'__module__': 'item.models',
'item': models.OneToOneField('Item', related_name='sort', primary_key=True),
}
for key in site_config['sortKeys']:
name = key['id']
field_type = key['type']
if field_type in ('string', 'title'):
attrs[name] = models.CharField(max_length=1000, db_index=True)
attrs['%s_desc'%name] = models.CharField(max_length=1000, db_index=True)
elif field_type == 'year':
attrs[name] = models.CharField(max_length=4, db_index=True)
attrs['%s_desc'%name] = models.CharField(max_length=4, db_index=True)
elif field_type in ('text', 'person'):
attrs[name] = models.TextField(blank=True, db_index=True)
attrs['%s_desc'%name] = models.TextField(blank=True, db_index=True)
elif field_type in ('integer', 'words', 'length'):
attrs[name] = models.BigIntegerField(blank=True, db_index=True)
attrs['%s_desc'%name] = models.BigIntegerField(blank=True, db_index=True)
elif field_type == 'float':
attrs[name] = models.FloatField(blank=True, db_index=True)
attrs['%s_desc'%name] = models.FloatField(blank=True, db_index=True)
elif field_type == 'date':
attrs[name] = models.DateTimeField(blank=True, db_index=True)
attrs['%s_desc'%name] = models.DateTimeField(blank=True, db_index=True)
else:
print field_type
print key
ItemSort = type('ItemSort', (models.Model,), attrs)
ItemSort.fields = filter(lambda x: not x.endswith('_desc'), [f.name for f in ItemSort._meta.fields])
ItemSort.descending_fields = filter(lambda x: x.endswith('_desc'), [f.name for f in ItemSort._meta.fields])
'''
class ItemSort(models.Model):
#FIXME: make sort based on site.json
"""
@ -848,7 +973,6 @@ class ItemSort(models.Model):
cuts = models.IntegerField(blank=True, db_index=True)
cutsperminute = models.FloatField(blank=True, db_index=True)
#required to move empty values to the bottom for both asc and desc sort
title_desc = models.CharField(max_length=1000, db_index=True)
director_desc = models.TextField(blank=True, db_index=True)
@ -862,25 +986,18 @@ class ItemSort(models.Model):
language_desc = models.TextField(blank=True, db_index=True)
_private_fields = ('id', 'item')
#return available sort fields
#FIXME: should return mapping name -> verbose_name
def fields(self):
fields = []
for f in self._meta.fields:
if f.name not in self._private_fields:
name = f.verbose_name
name = name[0].capitalize() + name[1:]
fields.append(name)
return tuple(fields)
fields = classmethod(fields)
ItemSort.fields = filter(lambda x: not x.endswith('_desc'), [f.name for f in ItemSort._meta.fields])
ItemSort.descending_fields = filter(lambda x: x.endswith('_desc'), [f.name for f in ItemSort._meta.fields])
'''
class Facet(models.Model):
class Meta:
unique_together = ("item", "key", "value")
item = models.ForeignKey('Item', related_name='facets')
key = models.CharField(max_length=200, db_index=True)
value = models.CharField(max_length=200)
value_sort = models.CharField(max_length=200)
value = models.CharField(max_length=200, db_index=True)
value_sort = models.CharField(max_length=200, db_index=True)
def save(self, *args, **kwargs):
if not self.value_sort:

View file

@ -201,3 +201,4 @@ def sort_title(title):
#pad numbered titles
title = re.sub('(\d+)', lambda x: '%010d' % int(x.group(0)), title)
return title.strip()

View file

@ -42,13 +42,14 @@ def _order_query(qs, sort, prefix='sort__'):
sort.append({'operator': '+', 'key': 'director'})
sort.append({'operator': '-', 'key': 'year'})
sort.append({'operator': '+', 'key': 'title'})
for e in sort:
operator = e['operator']
if operator != '-':
operator = ''
key = {'id': 'itemId'}.get(e['key'], e['key'])
#FIXME: this should be a property of models.ItemSort!!!
if operator=='-' and key in ('title', 'director', 'writer', 'producer', 'editor', 'cinematographer', 'language', 'country', 'year'):
if operator=='-' and '%s_desc'%key in models.ItemSort.descending_fields:
key = '%s_desc' % key
order = '%s%s%s' % (operator, prefix, key)
order_by.append(order)
@ -150,35 +151,33 @@ Positions
if 'sort' in query:
if len(query['sort']) == 1 and query['sort'][0]['key'] == 'items':
if query['group'] == "year":
query['sort'].append({'key': 'name', 'operator':'-'})
order_by = query['sort'][0]['operator'] == '-' and 'items' or '-items'
else:
query['sort'].append({'key': 'name', 'operator':'+'})
order_by = query['sort'][0]['operator'] == '-' and '-items' or 'items'
if query['group'] != "keyword":
order_by = (order_by, 'value_sort')
else:
order_by = (order_by,)
else:
order_by = query['sort'][0]['operator'] == '-' and '-value_sort' or 'value_sort'
order_by = (order_by, 'items')
else:
query['sort'] = [{'key': 'name', 'operator':'+'}]
order_by = ('-value_sort', 'items')
response['data']['items'] = []
items = 'items'
item_qs = query['qs']
qs = models.Facet.objects.filter(key=query['group']).filter(item__id__in=item_qs)
qs = qs.values('value').annotate(items=Count('id')).order_by()
name = 'value'
name_sort = 'value_sort'
qs = qs.values('value').annotate(items=Count('id')).order_by(**order_by)
#replace normalized items/name sort with actual db value
for i in range(0, len(query['sort'])):
if query['sort'][i]['key'] == 'name':
query['sort'][i]['key'] = name_sort
elif query['sort'][i]['key'] == 'items':
query['sort'][i]['key'] = items
qs = _order_query(qs, query['sort'], prefix='')
if 'ids' in query:
#FIXME: this does not scale for larger results
response['data']['positions'] = {}
ids = [j[name] for j in qs]
ids = [j['value'] for j in qs]
response['data']['positions'] = _get_positions(ids, query['ids'])
elif 'range' in data:
qs = qs[query['range'][0]:query['range'][1]]
response['data']['items'] = [{'name': i[name], 'items': i[items]} for i in qs]
response['data']['items'] = [{'name': i['value'], 'items': i[items]} for i in qs]
else:
response['data']['items'] = qs.count()
elif 'ids' in query:
@ -222,7 +221,50 @@ Positions
actions.register(find)
def autocomplete(request):
'''
param data
key
value
operator '', '^', '$'
range
return
'''
data = json.loads(request.POST['data'])
if not 'range' in data:
data['range'] = [0, 10]
op = data.get('operator', '')
if models.site_config['keys'][data['key']]['type'] == 'title':
qs = models.Item.objects.filter(available=True) #does this need more limiting? user etc
if data['value']:
if op == '':
qs = qs.filter(find__key=data['key'], find__value__icontains=data['value'])
elif op == '^':
qs = qs.filter(find__key=data['key'], find__value__istartswith=data['value'])
elif op == '$':
qs = qs.filter(find__key=data['key'], find__value__iendswith=data['value'])
qs = qs.order_by('-sort__%s'%models.site_config['keys'][data['key']]['autocompleteSortKey'])
qs = qs[data['range'][0]:data['range'][1]]
response = json_response({})
response['data']['items'] = [i.get(data['key']) for i in qs]
else:
qs = models.Facet.objects.filter(key=data['key'])
if data['value']:
if op == '':
qs = qs.filter(value__icontains=data['value'])
elif op == '^':
qs = qs.filter(value__istartswith=data['value'])
elif op == '$':
qs = qs.filter(value__iendswith=data['value'])
qs = qs.values('value').annotate(items=Count('id')).order_by('-items')
qs = qs[data['range'][0]:data['range'][1]]
response = json_response({})
response['data']['items'] = [i['value'] for i in qs]
return render_to_json_response(response)
actions.register(autocomplete)
def getItem(request):
'''
param data