From cf3161d4bd5be24f48a0f7dff4aeacb83c4f156f Mon Sep 17 00:00:00 2001 From: j Date: Tue, 30 Aug 2016 18:55:30 +0200 Subject: [PATCH] better findMedia --- pandora/archive/managers.py | 120 ++++++++++++++++++++++++++++++++++++ pandora/archive/models.py | 3 + pandora/archive/views.py | 54 +++++----------- pandora/oxdjango/fields.py | 9 ++- 4 files changed, 145 insertions(+), 41 deletions(-) create mode 100644 pandora/archive/managers.py diff --git a/pandora/archive/managers.py b/pandora/archive/managers.py new file mode 100644 index 000000000..81815e45c --- /dev/null +++ b/pandora/archive/managers.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 + +from django.db.models import Q, Manager + +from oxdjango.managers import get_operator +from oxdjango.query import QuerySet +from oxdjango.fields import DictField + +keymap = { + 'id': 'item__public_id', +} +default_key = 'oshash' + +def parseCondition(condition, user): + ''' + ''' + k = condition.get('key', default_key) + k = keymap.get(k, k) + if not k: + k = default_key + v = condition.get('value', '') + op = condition.get('operator') + if not op: + op = '=' + if op.startswith('!'): + op = op[1:] + exclude = True + else: + exclude = False + if isinstance(v, bool): + key = k + elif k == 'url': + key = 'info' + get_operator('=', 'istr') + v = DictField.dumps({'url': v})[1:-1] + else: + key = k + get_operator(op, 'istr') + key = str(key) + if exclude: + q = ~Q(**{key: v}) + else: + q = Q(**{key: v}) + return q + +def parseConditions(conditions, operator, user): + ''' + conditions: [ + { + value: "war" + } + { + key: "year", + value: "1970-1980, + operator: "!=" + }, + { + key: "country", + value: "f", + operator: "^" + } + ], + operator: "&" + ''' + conn = [] + for condition in conditions: + if 'conditions' in condition: + q = parseConditions(condition['conditions'], + condition.get('operator', '&'), user) + if q: + conn.append(q) + pass + else: + conn.append(parseCondition(condition, user)) + if conn: + q = conn[0] + for c in conn[1:]: + if operator == '|': + q = q | c + else: + q = q & c + return q + return None + + +class FileManager(Manager): + + def get_query_set(self): + return QuerySet(self.model) + + def find(self, data, user): + ''' + query: { + conditions: [ + { + value: "war" + } + { + key: "year", + value: "1970-1980, + operator: "!=" + }, + { + key: "country", + value: "f", + operator: "^" + } + ], + operator: "&" + } + ''' + + # join query with operator + qs = self.get_query_set() + query = data.get('query', {}) + conditions = parseConditions(query.get('conditions', []), + query.get('operator', '&'), + user) + if conditions: + qs = qs.filter(conditions) + return qs diff --git a/pandora/archive/models.py b/pandora/archive/models.py index f45e5082d..ce561e1c1 100644 --- a/pandora/archive/models.py +++ b/pandora/archive/models.py @@ -25,6 +25,7 @@ from taskqueue.models import Task from .chunk import save_chunk from . import extract +from . import managers if not PY2: unicode = str @@ -100,6 +101,8 @@ class File(models.Model): data = models.FileField(null=True, blank=True, upload_to=data_path) + objects = managers.FileManager() + def __unicode__(self): return self.path diff --git a/pandora/archive/views.py b/pandora/archive/views.py index ea2aa0745..9e76b1891 100644 --- a/pandora/archive/views.py +++ b/pandora/archive/views.py @@ -605,6 +605,17 @@ def _order_query(qs, sort, prefix=''): qs = qs.order_by(*order_by) return qs +def parse_query(data, user): + query = {} + query['range'] = [0, 100] + query['sort'] = [{'key': 'path', 'operator': '+'}] + for key in ('sort', 'keys', 'group', 'range', 'position', 'positions'): + if key in data: + query[key] = data[key] + if [r for r in query['range'] if not isinstance(r, int)]: + query['range'] = [0, 0] + query['qs'] = models.File.objects.find(data, user) + return query def findMedia(request, data): ''' @@ -625,40 +636,9 @@ def findMedia(request, data): response = json_response({}) if 'group' in query: - if 'sort' in query: - if len(query['sort']) == 1 and query['sort'][0]['key'] == 'items': - if query['group'] == "year": - order_by = query['sort'][0]['operator'] == '-' and 'items' or '-items' - else: - order_by = query['sort'][0]['operator'] == '-' and '-items' or 'items' - if query['group'] != "keyword": - order_by = (order_by, 'sortvalue') - else: - order_by = (order_by,) - else: - order_by = query['sort'][0]['operator'] == '-' and '-sortvalue' or 'sortvalue' - order_by = (order_by, 'items') - else: - order_by = ('-sortvalue', 'items') - response['data']['items'] = [] - items = 'items' - item_qs = query['qs'] - qs = models.Facet.objects.filter(key=query['group']).filter(item__id__in=item_qs) - qs = qs.values('value').annotate(items=Count('id')).order_by(*order_by) - - if 'positions' in query: - response['data']['positions'] = {} - ids = [j['value'] for j in qs] - response['data']['positions'] = utils.get_positions(ids, query['positions']) - - elif 'range' in data: - qs = qs[query['range'][0]:query['range'][1]] - response['data']['items'] = [{'path': i['value'], 'items': i[items]} for i in qs] - else: - response['data']['items'] = qs.count() + print('findMedia does not support group query') elif 'positions' in query: - qs = models.File.objects.filter(item__in=query['qs']) - qs = _order_query(qs, query['sort']) + qs = _order_query(query['qs'], query['sort']) response['data']['positions'] = {} ids = list(qs.values_list('oshash', flat=True)) @@ -666,21 +646,19 @@ def findMedia(request, data): elif 'keys' in query: response['data']['items'] = [] - qs = models.File.objects.filter(item__in=query['qs']) - qs = _order_query(qs, query['sort']) + qs = _order_query(query['qs'], query['sort']) qs = qs.select_related() keys = query['keys'] qs = qs[query['range'][0]:query['range'][1]] response['data']['items'] = [f.json(keys) for f in qs] else: # otherwise stats - items = query['qs'] - files = models.File.objects.filter(item__in=query['qs']) + files = query['qs'] response['data']['items'] = files.count() return render_to_json_response(response) actions.register(findMedia) -def parsePath(request, data): # parse path and return info +def parsePath(request, data): # parse path and return info ''' Parses a path takes { diff --git a/pandora/oxdjango/fields.py b/pandora/oxdjango/fields.py index 31f957e14..fdb5722da 100644 --- a/pandora/oxdjango/fields.py +++ b/pandora/oxdjango/fields.py @@ -48,10 +48,12 @@ def from_json(json_object): class DictField(models.TextField): _type = dict - def loads(self, value): + @classmethod + def loads(cls, value): return json.loads(value, object_hook=from_json) - def dumps(self, obj): + @classmethod + def dumps(cls, obj): return json.dumps(obj, default=to_json, ensure_ascii=False) def from_db_value(self, value, expression, connection, context): @@ -85,8 +87,9 @@ class DictField(models.TextField): class TupleField(DictField): _type = (tuple, list) + @classmethod def loads(self, value): - value = DictField.loads(self, value) + value = DictField.loads(value) if isinstance(value, list): value = tuple(value) return value