0x2620
/
pandora
2
1
Fork 1
Code Issues 1.3k Pull Requests Releases Wiki Activity

better findMedia

This commit is contained in:
j 2016-08-30 18:55:30 +02:00
parent fc6cc93d26
commit cf3161d4bd
4 changed files with 145 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
pandora/archive/managers.py Normal file
View File

@ -0,0 +1,120 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from django.db.models import Q, Manager
from oxdjango.managers import get_operator
from oxdjango.query import QuerySet
from oxdjango.fields import DictField
keymap = {
'id': 'item__public_id',
}
default_key = 'oshash'
def parseCondition(condition, user):
'''
'''
k = condition.get('key', default_key)
k = keymap.get(k, k)
if not k:
k = default_key
v = condition.get('value', '')
op = condition.get('operator')
if not op:
op = '='
if op.startswith('!'):
op = op[1:]
exclude = True
else:
exclude = False
if isinstance(v, bool):
key = k
elif k == 'url':
key = 'info' + get_operator('=', 'istr')
v = DictField.dumps({'url': v})[1:-1]
else:
key = k + get_operator(op, 'istr')
key = str(key)
if exclude:
q = ~Q(**{key: v})
else:
q = Q(**{key: v})
return q
def parseConditions(conditions, operator, user):
'''
conditions: [
{
value: "war"
}
{
key: "year",
value: "1970-1980,
operator: "!="
},
{
key: "country",
value: "f",
operator: "^"
}
],
operator: "&"
'''
conn = []
for condition in conditions:
if 'conditions' in condition:
q = parseConditions(condition['conditions'],
condition.get('operator', '&'), user)
if q:
conn.append(q)
pass
else:
conn.append(parseCondition(condition, user))
if conn:
q = conn[0]
for c in conn[1:]:
if operator == '|':
q = q | c
else:
q = q & c
return q
return None
class FileManager(Manager):
def get_query_set(self):
return QuerySet(self.model)
def find(self, data, user):
'''
query: {
conditions: [
{
value: "war"
}
{
key: "year",
value: "1970-1980,
operator: "!="
},
{
key: "country",
value: "f",
operator: "^"
}
],
operator: "&"
}
'''
# join query with operator
qs = self.get_query_set()
query = data.get('query', {})
conditions = parseConditions(query.get('conditions', []),
query.get('operator', '&'),
user)
if conditions:
qs = qs.filter(conditions)
return qs

3
pandora/archive/models.py
View File

@ -25,6 +25,7 @@ from taskqueue.models import Task
from .chunk import save_chunk
from . import extract
from . import managers
if not PY2:
unicode = str
@ -100,6 +101,8 @@ class File(models.Model):
data = models.FileField(null=True, blank=True,
upload_to=data_path)
objects = managers.FileManager()
def __unicode__(self):
return self.path

54
pandora/archive/views.py
View File

@ -605,6 +605,17 @@ def _order_query(qs, sort, prefix=''):
qs = qs.order_by(*order_by)
return qs
def parse_query(data, user):
query = {}
query['range'] = [0, 100]
query['sort'] = [{'key': 'path', 'operator': '+'}]
for key in ('sort', 'keys', 'group', 'range', 'position', 'positions'):
if key in data:
query[key] = data[key]
if [r for r in query['range'] if not isinstance(r, int)]:
query['range'] = [0, 0]
query['qs'] = models.File.objects.find(data, user)
return query
def findMedia(request, data):
'''
@ -625,40 +636,9 @@ def findMedia(request, data):
response = json_response({})
if 'group' in query:
if 'sort' in query:
if len(query['sort']) == 1 and query['sort'][0]['key'] == 'items':
if query['group'] == "year":
order_by = query['sort'][0]['operator'] == '-' and 'items' or '-items'
else:
order_by = query['sort'][0]['operator'] == '-' and '-items' or 'items'
if query['group'] != "keyword":
order_by = (order_by, 'sortvalue')
else:
order_by = (order_by,)
else:
order_by = query['sort'][0]['operator'] == '-' and '-sortvalue' or 'sortvalue'
order_by = (order_by, 'items')
else:
order_by = ('-sortvalue', 'items')
response['data']['items'] = []
items = 'items'
item_qs = query['qs']
qs = models.Facet.objects.filter(key=query['group']).filter(item__id__in=item_qs)
qs = qs.values('value').annotate(items=Count('id')).order_by(*order_by)
if 'positions' in query:
response['data']['positions'] = {}
ids = [j['value'] for j in qs]
response['data']['positions'] = utils.get_positions(ids, query['positions'])
elif 'range' in data:
qs = qs[query['range'][0]:query['range'][1]]
response['data']['items'] = [{'path': i['value'], 'items': i[items]} for i in qs]
else:
response['data']['items'] = qs.count()
print('findMedia does not support group query')
elif 'positions' in query:
qs = models.File.objects.filter(item__in=query['qs'])
qs = _order_query(qs, query['sort'])
qs = _order_query(query['qs'], query['sort'])
response['data']['positions'] = {}
ids = list(qs.values_list('oshash', flat=True))
@ -666,21 +646,19 @@ def findMedia(request, data):
elif 'keys' in query:
response['data']['items'] = []
qs = models.File.objects.filter(item__in=query['qs'])
qs = _order_query(qs, query['sort'])
qs = _order_query(query['qs'], query['sort'])
qs = qs.select_related()
keys = query['keys']
qs = qs[query['range'][0]:query['range'][1]]
response['data']['items'] = [f.json(keys) for f in qs]
else: # otherwise stats
items = query['qs']
files = models.File.objects.filter(item__in=query['qs'])
files = query['qs']
response['data']['items'] = files.count()
return render_to_json_response(response)
actions.register(findMedia)
def parsePath(request, data): # parse path and return info
def parsePath(request, data): # parse path and return info
'''
Parses a path
takes {

9
pandora/oxdjango/fields.py
View File

@ -48,10 +48,12 @@ def from_json(json_object):
class DictField(models.TextField):
_type = dict
def loads(self, value):
@classmethod
def loads(cls, value):
return json.loads(value, object_hook=from_json)
def dumps(self, obj):
@classmethod
def dumps(cls, obj):
return json.dumps(obj, default=to_json, ensure_ascii=False)
def from_db_value(self, value, expression, connection, context):
@ -85,8 +87,9 @@ class DictField(models.TextField):
class TupleField(DictField):
_type = (tuple, list)
@classmethod
def loads(self, value):
value = DictField.loads(self, value)
value = DictField.loads(value)
if isinstance(value, list):
value = tuple(value)
return value