better findMedia
This commit is contained in:
parent
fc6cc93d26
commit
cf3161d4bd
4 changed files with 145 additions and 41 deletions
120
pandora/archive/managers.py
Normal file
120
pandora/archive/managers.py
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# vi:si:et:sw=4:sts=4:ts=4
|
||||
|
||||
from django.db.models import Q, Manager
|
||||
|
||||
from oxdjango.managers import get_operator
|
||||
from oxdjango.query import QuerySet
|
||||
from oxdjango.fields import DictField
|
||||
|
||||
keymap = {
|
||||
'id': 'item__public_id',
|
||||
}
|
||||
default_key = 'oshash'
|
||||
|
||||
def parseCondition(condition, user):
|
||||
'''
|
||||
'''
|
||||
k = condition.get('key', default_key)
|
||||
k = keymap.get(k, k)
|
||||
if not k:
|
||||
k = default_key
|
||||
v = condition.get('value', '')
|
||||
op = condition.get('operator')
|
||||
if not op:
|
||||
op = '='
|
||||
if op.startswith('!'):
|
||||
op = op[1:]
|
||||
exclude = True
|
||||
else:
|
||||
exclude = False
|
||||
if isinstance(v, bool):
|
||||
key = k
|
||||
elif k == 'url':
|
||||
key = 'info' + get_operator('=', 'istr')
|
||||
v = DictField.dumps({'url': v})[1:-1]
|
||||
else:
|
||||
key = k + get_operator(op, 'istr')
|
||||
key = str(key)
|
||||
if exclude:
|
||||
q = ~Q(**{key: v})
|
||||
else:
|
||||
q = Q(**{key: v})
|
||||
return q
|
||||
|
||||
def parseConditions(conditions, operator, user):
|
||||
'''
|
||||
conditions: [
|
||||
{
|
||||
value: "war"
|
||||
}
|
||||
{
|
||||
key: "year",
|
||||
value: "1970-1980,
|
||||
operator: "!="
|
||||
},
|
||||
{
|
||||
key: "country",
|
||||
value: "f",
|
||||
operator: "^"
|
||||
}
|
||||
],
|
||||
operator: "&"
|
||||
'''
|
||||
conn = []
|
||||
for condition in conditions:
|
||||
if 'conditions' in condition:
|
||||
q = parseConditions(condition['conditions'],
|
||||
condition.get('operator', '&'), user)
|
||||
if q:
|
||||
conn.append(q)
|
||||
pass
|
||||
else:
|
||||
conn.append(parseCondition(condition, user))
|
||||
if conn:
|
||||
q = conn[0]
|
||||
for c in conn[1:]:
|
||||
if operator == '|':
|
||||
q = q | c
|
||||
else:
|
||||
q = q & c
|
||||
return q
|
||||
return None
|
||||
|
||||
|
||||
class FileManager(Manager):
|
||||
|
||||
def get_query_set(self):
|
||||
return QuerySet(self.model)
|
||||
|
||||
def find(self, data, user):
|
||||
'''
|
||||
query: {
|
||||
conditions: [
|
||||
{
|
||||
value: "war"
|
||||
}
|
||||
{
|
||||
key: "year",
|
||||
value: "1970-1980,
|
||||
operator: "!="
|
||||
},
|
||||
{
|
||||
key: "country",
|
||||
value: "f",
|
||||
operator: "^"
|
||||
}
|
||||
],
|
||||
operator: "&"
|
||||
}
|
||||
'''
|
||||
|
||||
# join query with operator
|
||||
qs = self.get_query_set()
|
||||
query = data.get('query', {})
|
||||
conditions = parseConditions(query.get('conditions', []),
|
||||
query.get('operator', '&'),
|
||||
user)
|
||||
if conditions:
|
||||
qs = qs.filter(conditions)
|
||||
return qs
|
||||
|
|
@ -25,6 +25,7 @@ from taskqueue.models import Task
|
|||
|
||||
from .chunk import save_chunk
|
||||
from . import extract
|
||||
from . import managers
|
||||
|
||||
if not PY2:
|
||||
unicode = str
|
||||
|
|
@ -100,6 +101,8 @@ class File(models.Model):
|
|||
data = models.FileField(null=True, blank=True,
|
||||
upload_to=data_path)
|
||||
|
||||
objects = managers.FileManager()
|
||||
|
||||
def __unicode__(self):
|
||||
return self.path
|
||||
|
||||
|
|
|
|||
|
|
@ -605,6 +605,17 @@ def _order_query(qs, sort, prefix=''):
|
|||
qs = qs.order_by(*order_by)
|
||||
return qs
|
||||
|
||||
def parse_query(data, user):
|
||||
query = {}
|
||||
query['range'] = [0, 100]
|
||||
query['sort'] = [{'key': 'path', 'operator': '+'}]
|
||||
for key in ('sort', 'keys', 'group', 'range', 'position', 'positions'):
|
||||
if key in data:
|
||||
query[key] = data[key]
|
||||
if [r for r in query['range'] if not isinstance(r, int)]:
|
||||
query['range'] = [0, 0]
|
||||
query['qs'] = models.File.objects.find(data, user)
|
||||
return query
|
||||
|
||||
def findMedia(request, data):
|
||||
'''
|
||||
|
|
@ -625,40 +636,9 @@ def findMedia(request, data):
|
|||
|
||||
response = json_response({})
|
||||
if 'group' in query:
|
||||
if 'sort' in query:
|
||||
if len(query['sort']) == 1 and query['sort'][0]['key'] == 'items':
|
||||
if query['group'] == "year":
|
||||
order_by = query['sort'][0]['operator'] == '-' and 'items' or '-items'
|
||||
else:
|
||||
order_by = query['sort'][0]['operator'] == '-' and '-items' or 'items'
|
||||
if query['group'] != "keyword":
|
||||
order_by = (order_by, 'sortvalue')
|
||||
else:
|
||||
order_by = (order_by,)
|
||||
else:
|
||||
order_by = query['sort'][0]['operator'] == '-' and '-sortvalue' or 'sortvalue'
|
||||
order_by = (order_by, 'items')
|
||||
else:
|
||||
order_by = ('-sortvalue', 'items')
|
||||
response['data']['items'] = []
|
||||
items = 'items'
|
||||
item_qs = query['qs']
|
||||
qs = models.Facet.objects.filter(key=query['group']).filter(item__id__in=item_qs)
|
||||
qs = qs.values('value').annotate(items=Count('id')).order_by(*order_by)
|
||||
|
||||
if 'positions' in query:
|
||||
response['data']['positions'] = {}
|
||||
ids = [j['value'] for j in qs]
|
||||
response['data']['positions'] = utils.get_positions(ids, query['positions'])
|
||||
|
||||
elif 'range' in data:
|
||||
qs = qs[query['range'][0]:query['range'][1]]
|
||||
response['data']['items'] = [{'path': i['value'], 'items': i[items]} for i in qs]
|
||||
else:
|
||||
response['data']['items'] = qs.count()
|
||||
print('findMedia does not support group query')
|
||||
elif 'positions' in query:
|
||||
qs = models.File.objects.filter(item__in=query['qs'])
|
||||
qs = _order_query(qs, query['sort'])
|
||||
qs = _order_query(query['qs'], query['sort'])
|
||||
|
||||
response['data']['positions'] = {}
|
||||
ids = list(qs.values_list('oshash', flat=True))
|
||||
|
|
@ -666,21 +646,19 @@ def findMedia(request, data):
|
|||
|
||||
elif 'keys' in query:
|
||||
response['data']['items'] = []
|
||||
qs = models.File.objects.filter(item__in=query['qs'])
|
||||
qs = _order_query(qs, query['sort'])
|
||||
qs = _order_query(query['qs'], query['sort'])
|
||||
qs = qs.select_related()
|
||||
keys = query['keys']
|
||||
qs = qs[query['range'][0]:query['range'][1]]
|
||||
response['data']['items'] = [f.json(keys) for f in qs]
|
||||
else: # otherwise stats
|
||||
items = query['qs']
|
||||
files = models.File.objects.filter(item__in=query['qs'])
|
||||
files = query['qs']
|
||||
response['data']['items'] = files.count()
|
||||
return render_to_json_response(response)
|
||||
|
||||
actions.register(findMedia)
|
||||
|
||||
def parsePath(request, data): # parse path and return info
|
||||
def parsePath(request, data): # parse path and return info
|
||||
'''
|
||||
Parses a path
|
||||
takes {
|
||||
|
|
|
|||
|
|
@ -48,10 +48,12 @@ def from_json(json_object):
|
|||
class DictField(models.TextField):
|
||||
_type = dict
|
||||
|
||||
def loads(self, value):
|
||||
@classmethod
|
||||
def loads(cls, value):
|
||||
return json.loads(value, object_hook=from_json)
|
||||
|
||||
def dumps(self, obj):
|
||||
@classmethod
|
||||
def dumps(cls, obj):
|
||||
return json.dumps(obj, default=to_json, ensure_ascii=False)
|
||||
|
||||
def from_db_value(self, value, expression, connection, context):
|
||||
|
|
@ -85,8 +87,9 @@ class DictField(models.TextField):
|
|||
class TupleField(DictField):
|
||||
_type = (tuple, list)
|
||||
|
||||
@classmethod
|
||||
def loads(self, value):
|
||||
value = DictField.loads(self, value)
|
||||
value = DictField.loads(value)
|
||||
if isinstance(value, list):
|
||||
value = tuple(value)
|
||||
return value
|
||||
|
|
|
|||
Loading…
Reference in a new issue