From ffa955eef06d9bd5794a14f47a23ce2eb2543125 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Mon, 22 Feb 2010 14:55:29 +0530 Subject: [PATCH] make find api more jsonic --- apache/vhost.conf.in | 1 + pandora/backend/extract.py | 7 +- pandora/backend/managers.py | 145 +++++++++++++++++++++++++++++++----- pandora/backend/models.py | 4 +- pandora/backend/views.py | 102 +++++++++++++++++-------- 5 files changed, 206 insertions(+), 53 deletions(-) diff --git a/apache/vhost.conf.in b/apache/vhost.conf.in index 8811d4b5..b484bab2 100644 --- a/apache/vhost.conf.in +++ b/apache/vhost.conf.in @@ -19,6 +19,7 @@ Alias /media __PREFIX__/pandora/media Alias /admin/media __PREFIX__/src/django/django/contrib/admin/media Alias /static/js/jquery.js __PREFIX__/src/django/django/contrib/admin/media/js/jquery.min.js + WSGIScriptAlias / __PREFIX__/wsgi/django.wsgi WSGIDaemonProcess pandora user=pandora group=pandora threads=25 python-path=__PREFIX__/lib/python2.6/site-packages/ diff --git a/pandora/backend/extract.py b/pandora/backend/extract.py index 3bfaf388..ebd14f20 100644 --- a/pandora/backend/extract.py +++ b/pandora/backend/extract.py @@ -38,9 +38,12 @@ def frame(videoFile, position, baseFolder, width=128, redo=False): width of frame redo boolean to extract file even if it exists ''' + def frame_path(size): + return os.path.join(baseFolder, "%s.%s.%s" % (oxlib.ms2time(position*1000), size, img_extension)) + #not using input file, to slow to extract frame right now base_size = 320 - frame = os.path.join(baseFolder, "%f.%s.%s" % (position, base_size, img_extension)) + frame = frame_path(base_size) if exists(videoFile): if redo or not exists(frame): @@ -50,7 +53,7 @@ def frame(videoFile, position, baseFolder, width=128, redo=False): run_command(cmd) if width != base_size: frame_base = frame - frame = os.path.join(baseFolder, "%f.%s.%s" % (position, width, img_extension)) + frame = frame_path(width) if not exists(frame): resize_image(frame_base, frame, width) return frame diff --git a/pandora/backend/managers.py b/pandora/backend/managers.py index 3227910a..eeaeff8f 100644 --- a/pandora/backend/managers.py +++ b/pandora/backend/managers.py @@ -25,7 +25,130 @@ class MovieManager(Manager): def get_query_set(self): return super(MovieManager, self).get_query_set() - def find(self, request): + def filter_list(self, qs, l, user): + if l != "all": + l = l.split(":") + only_public = True + if not user.is_anonymous(): + if len(l) == 1: l = [request.user.username] + l + if request.user.username == l[0]: + only_public = False + if len(l) == 2: + lqs = models.List.objects.filter(name=l[1], user__username=l[0]) + if only_public: + lqs = qls.filter(public=True) + if lqs.count() == 1: + qs = qs.filter(listitem__list__id=lqs[0].id) + return qs + + def find(self, data, user): + ''' + query: { + conditions: [ + { + value: "war"" + } + { + key: "year", + value: "1970-1980, + operator: "!=" + }, + { + key: "country", + value: "f", + operator: "^" + } + ], + operator: "&" + } + ''' + query_operator = data['query'].get('operator', '&') + conditions = [] + for condition in data['query']['conditions']: + k = condition.get('key', 'all') + v = condition['value'] + op = condition.get('operator', None) + if op.startswith('!'): + op = op[1:] + exclude = True + else: + exclude = False + if keyType(k) == "string": + startswith = op.startswith('^') + endswith = op.endswith('$') + if op == '=': + k = '%s__iexact' % k + elif op == '^': + v = v[1:] + k = '%s__istartswith' % k + elif op == '$': + v = v[:-1] + k = '%s__iendswith' % k + else: # elif op == '~': + k = '%s__icontains' % k + k = 'find__%s' % k + k = str(k) + if exclude: + conditions.append(~Q(**{k:v})) + else: + conditions.append(Q(**{k:v})) + else: #number or date + def parseDate(d): + while len(d) < 3: + d.append(1) + return datetime(*[int(i) for i in d]) + if op == '-': + v1 = v[1] + v2 = v[2] + if keyType(k) == "date": + v1 = parseDate(v1.split('.')) + v2 = parseDate(v2.split('.')) + if exclude: #!1960-1970 + k1 = str('%s__lt' % k) + k2 = str('%s__gte' % k) + conditions.append(Q(**{k1:v1})|Q(**{k2:v2})) + else: #1960-1970 + k1 = str('%s__gte' % k) + k2 = str('%s__lt' % k) + conditions.append(Q(**{k1:v1})&Q(**{k2:v2})) + else: + if keyType(k) == "date": + v = parseDate(v.split('.')) + if op == '>': + k = str('%s__gt' % k) + elif op == '>=': + k = str('%s__gte' % k) + elif op == '<': + k = str('%s__lt' % k) + elif op == '<=': + k = str('%s__lte' % k) + + k = str(k) + if exclude: #!1960 + conditions.append(~Q(**{k:v})) + else: #1960 + conditions.append(Q(**{k:v})) + + #join query with operator + qs = self.get_query_set() + #only include movies that have hard metadata + qs = qs.filter(available=True) + if conditions: + q = conditions[0] + for c in conditions[1:]: + if query_operator == '|': + q = q | c + else: + q = q & c + qs = qs.filter(q) + + # filter list, works for own or public lists + l = data.get('list', 'all') + qs = self.filter_list(qs, l, user) + return qs + + """ + def find(self, data, user): ''' construct query set from q value in request, also checks for lists. @@ -37,8 +160,7 @@ class MovieManager(Manager): if i.startswith('q='): q = i[2:] ''' - q = json.loads(request.POST['data'])['q'] - print q + q = data['q'] op = ',' if '|' in q: op = '|' @@ -115,21 +237,10 @@ class MovieManager(Manager): qs = qs.filter(q) # filter list, works for own or public lists - l = request.GET.get('l', 'all') - if l != "all": - l = l.split(":") - only_public = True - if not request.user.is_anonymous(): - if len(l) == 1: l = [request.user.username] + l - if request.user.username == l[0]: - only_public = False - if len(l) == 2: - lqs = models.List.objects.filter(name=l[1], user__username=l[0]) - if only_public: - lqs = qls.filter(public=True) - if lqs.count() == 1: - qs = qs.filter(listitem__list__id=lqs[0].id) + l = data.get('l', 'all') + qs = self.filter_list(qs, l, user) return qs + """ class FileManager(Manager): def get_query_set(self): diff --git a/pandora/backend/models.py b/pandora/backend/models.py index e7f610a0..7bf0fe16 100644 --- a/pandora/backend/models.py +++ b/pandora/backend/models.py @@ -442,7 +442,9 @@ class Movie(models.Model): except MovieFind.DoesNotExist: f = MovieFind(movie=self) - f.title = self.get('title') + ' '.join([t.title for t in self.alternative_titles()]) + f.title = self.get('title') + #FIXME: filter us/int title + #f.title += ' '.join([t.title for t in self.alternative_titles()]) f.director = ' '.join([i.name for i in self.directors()]) f.country = ' '.join([i.name for i in self.countries()]) f.year = self.get('year', '') diff --git a/pandora/backend/views.py b/pandora/backend/views.py index f23dd1a5..db553521 100644 --- a/pandora/backend/views.py +++ b/pandora/backend/views.py @@ -77,9 +77,26 @@ def api_error(request): success = error_is_success return render_to_json_response({}) -def _order_query(qs, s, prefix='sort__'): +def _order_query(qs, sort, prefix='sort__'): order_by = [] - for e in s.split(','): + if isinstance(sort, basestring): + sort = [sort, ] + for e in sort: + desc = '' + if e.startswith('-'): + e = e[1:] + desc = '-' + order = {'id': 'movieId'}.get(e, e) + order = '%s%s%s' % (desc, prefix, order) + order_by.append(order) + if order_by: + qs = qs.order_by(*order_by) + return qs + +''' +def _order_query(qs, sort, prefix='sort__'): + order_by = [] + for e in sort.split(','): o = e.split(':') if len(o) == 1: o.append('asc') order = {'id': 'movieId'}.get(o[0], o[0]) @@ -90,9 +107,22 @@ def _order_query(qs, s, prefix='sort__'): if order_by: qs = qs.order_by(*order_by) return qs - -def _parse_query(request): - get = json.loads(request.POST['data']) +''' +def _parse_query(data, user): + query = {} + query['range'] = [0, 100] + query['sort'] = (['title', 'asc']) + _dicts = ['p', ] + _ints = ['n', ] + for key in ('sort', 'keys', 'group', 'list', 'range', 'n'): + if key in data: + query[key] = data[key] + print query + query['qs'] = models.Movie.objects.find(data, user) + #group by only allows sorting by name or number of itmes + return query +''' +def _parse_query(data, user): query = {} query['i'] = 0 query['o'] = 100 @@ -103,16 +133,18 @@ def _parse_query(request): _dicts = ['p', ] _ints = ['n', ] for key in ('s', 'p', 'g', 'l', 'n'): - if key in get: + if key in data: if key in _ints: - query[key] = int(get[key]) + query[key] = int(data[key]) elif key in _dicts: - query[key] = parse_dict(get[key]) + query[key] = parse_dict(data[key]) else: - query[key] = get[key] - query['q'] = models.Movie.objects.find(request) - if 'r' in get: - r = get['r'].split(':') + query[key] = data[key] + + print query + query['q'] = models.Movie.objects.find(data, user) + if 'r' in data: + r = data['r'].split(':') if len(r) == 1: r.append(0) if r[0] == '': r[0] = 0 if r[1] == '': r[0] = -1 @@ -120,44 +152,48 @@ def _parse_query(request): query['o'] = int(r[1]) #group by only allows sorting by name or number of itmes return query +''' def api_find(request): ''' param data - {'q': query, 's': sort, 'r': range} + {'query': query, 'sort': string, 'range': array} - q: query string, can contain field:search more on query syntax at + query: query object, more on query syntax at http://wiki.0xdb.org/wiki/QuerySyntax - s: comma seperated list of field:order, default: director:asc,year:desc - r: result ragne, from:to or from - p: properties to return, if obmited stats are returned - g: group elements by, country, genre, director... + sort: string or arrays of keys; to sort key in descending order prefix with - + default: ['director', '-year'] + range: result range, array [from, to] + keys: array of keys to return + group: group elements by, country, genre, director... - with p, items is list of dicts with requested properties: + with keys, items is list of dicts with requested properties: return {'status': {'code': int, 'text': string}, 'data': {items: array}} - with g, items contains list of {'title': string, 'items': int}: + with group and keys(possible values: name, items) + items contains list of {'name': string, 'items': int}: return {'status': {'code': int, 'text': string}, 'data': {items: array}} - with g + n=1, return number of items in given query + with group without keys: return number of items in given query return {'status': {'code': int, 'text': string}, 'data': {items: int}} - without p or g, return stats about query: + without keys or group, return stats about query: return {'status': {'code': int, 'text': string}, 'data': {items=int, files=int, pixels=int, size=int, duration=int}} ''' - query = _parse_query(request) + data = json.loads(request.POST['data']) + query = _parse_query(data, request.user) response = json_response({}) - if 'p' in query: + if 'keys' in query: response['data']['items'] = [] - qs = _order_query(query['q'], query['s']) + qs = _order_query(query['qs'], query['sort']) if 'n' in query: response = {'items': qs.count()} else: - _p = query['p'] + _p = query['keys'] def only_p(m): r = {} if m: @@ -165,11 +201,11 @@ def api_find(request): for p in _p: r[p] = m[p] return r - qs = qs[query['i']:query['o']] + qs = qs[query['range'][0]:query['range'][1]] response['data']['items'] = [only_p(m['json']) for m in qs.values('json')] - elif 'g' in query: + elif 'group' in query: if query['s'].split(':')[0] not in ('name', 'items'): query['s'] = 'name' #FIXME: also filter lists here @@ -183,14 +219,12 @@ def api_find(request): 'language': models.Language.objects, 'director': models.Person.objects.filter(cast__role='directors'), } - if query['g'] in _objects: + if query['group'] in _objects: qs = _objects[query['g']].filter(movies__id__in=movie_qs).values('name').annotate(movies=Count('movies')) - elif query['g'] == "year": + elif query['group'] == "year": qs = movie_qs.values('imdb__year').annotate(movies=Count('id')) name='imdb__year' - if 'n' in query: - response['data']['items'] = qs.count() - else: + if 'keys' in query: #replace normalized items/name sort with actual db value order_by = query['s'].split(":") if len(order_by) == 1: @@ -203,6 +237,8 @@ def api_find(request): qs = qs[query['i']:query['o']] response['data']['items'] = [{'title': i[name], 'items': i[items]} for i in qs] + else: + response['data']['items'] = qs.count() else: #FIXME: also filter lists here