better findMedia
This commit is contained in:
parent
fc6cc93d26
commit
cf3161d4bd
4 changed files with 145 additions and 41 deletions
120
pandora/archive/managers.py
Normal file
120
pandora/archive/managers.py
Normal file
|
@ -0,0 +1,120 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# vi:si:et:sw=4:sts=4:ts=4
|
||||||
|
|
||||||
|
from django.db.models import Q, Manager
|
||||||
|
|
||||||
|
from oxdjango.managers import get_operator
|
||||||
|
from oxdjango.query import QuerySet
|
||||||
|
from oxdjango.fields import DictField
|
||||||
|
|
||||||
|
keymap = {
|
||||||
|
'id': 'item__public_id',
|
||||||
|
}
|
||||||
|
default_key = 'oshash'
|
||||||
|
|
||||||
|
def parseCondition(condition, user):
|
||||||
|
'''
|
||||||
|
'''
|
||||||
|
k = condition.get('key', default_key)
|
||||||
|
k = keymap.get(k, k)
|
||||||
|
if not k:
|
||||||
|
k = default_key
|
||||||
|
v = condition.get('value', '')
|
||||||
|
op = condition.get('operator')
|
||||||
|
if not op:
|
||||||
|
op = '='
|
||||||
|
if op.startswith('!'):
|
||||||
|
op = op[1:]
|
||||||
|
exclude = True
|
||||||
|
else:
|
||||||
|
exclude = False
|
||||||
|
if isinstance(v, bool):
|
||||||
|
key = k
|
||||||
|
elif k == 'url':
|
||||||
|
key = 'info' + get_operator('=', 'istr')
|
||||||
|
v = DictField.dumps({'url': v})[1:-1]
|
||||||
|
else:
|
||||||
|
key = k + get_operator(op, 'istr')
|
||||||
|
key = str(key)
|
||||||
|
if exclude:
|
||||||
|
q = ~Q(**{key: v})
|
||||||
|
else:
|
||||||
|
q = Q(**{key: v})
|
||||||
|
return q
|
||||||
|
|
||||||
|
def parseConditions(conditions, operator, user):
|
||||||
|
'''
|
||||||
|
conditions: [
|
||||||
|
{
|
||||||
|
value: "war"
|
||||||
|
}
|
||||||
|
{
|
||||||
|
key: "year",
|
||||||
|
value: "1970-1980,
|
||||||
|
operator: "!="
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: "country",
|
||||||
|
value: "f",
|
||||||
|
operator: "^"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
operator: "&"
|
||||||
|
'''
|
||||||
|
conn = []
|
||||||
|
for condition in conditions:
|
||||||
|
if 'conditions' in condition:
|
||||||
|
q = parseConditions(condition['conditions'],
|
||||||
|
condition.get('operator', '&'), user)
|
||||||
|
if q:
|
||||||
|
conn.append(q)
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
conn.append(parseCondition(condition, user))
|
||||||
|
if conn:
|
||||||
|
q = conn[0]
|
||||||
|
for c in conn[1:]:
|
||||||
|
if operator == '|':
|
||||||
|
q = q | c
|
||||||
|
else:
|
||||||
|
q = q & c
|
||||||
|
return q
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class FileManager(Manager):
|
||||||
|
|
||||||
|
def get_query_set(self):
|
||||||
|
return QuerySet(self.model)
|
||||||
|
|
||||||
|
def find(self, data, user):
|
||||||
|
'''
|
||||||
|
query: {
|
||||||
|
conditions: [
|
||||||
|
{
|
||||||
|
value: "war"
|
||||||
|
}
|
||||||
|
{
|
||||||
|
key: "year",
|
||||||
|
value: "1970-1980,
|
||||||
|
operator: "!="
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: "country",
|
||||||
|
value: "f",
|
||||||
|
operator: "^"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
operator: "&"
|
||||||
|
}
|
||||||
|
'''
|
||||||
|
|
||||||
|
# join query with operator
|
||||||
|
qs = self.get_query_set()
|
||||||
|
query = data.get('query', {})
|
||||||
|
conditions = parseConditions(query.get('conditions', []),
|
||||||
|
query.get('operator', '&'),
|
||||||
|
user)
|
||||||
|
if conditions:
|
||||||
|
qs = qs.filter(conditions)
|
||||||
|
return qs
|
|
@ -25,6 +25,7 @@ from taskqueue.models import Task
|
||||||
|
|
||||||
from .chunk import save_chunk
|
from .chunk import save_chunk
|
||||||
from . import extract
|
from . import extract
|
||||||
|
from . import managers
|
||||||
|
|
||||||
if not PY2:
|
if not PY2:
|
||||||
unicode = str
|
unicode = str
|
||||||
|
@ -100,6 +101,8 @@ class File(models.Model):
|
||||||
data = models.FileField(null=True, blank=True,
|
data = models.FileField(null=True, blank=True,
|
||||||
upload_to=data_path)
|
upload_to=data_path)
|
||||||
|
|
||||||
|
objects = managers.FileManager()
|
||||||
|
|
||||||
def __unicode__(self):
|
def __unicode__(self):
|
||||||
return self.path
|
return self.path
|
||||||
|
|
||||||
|
|
|
@ -605,6 +605,17 @@ def _order_query(qs, sort, prefix=''):
|
||||||
qs = qs.order_by(*order_by)
|
qs = qs.order_by(*order_by)
|
||||||
return qs
|
return qs
|
||||||
|
|
||||||
|
def parse_query(data, user):
|
||||||
|
query = {}
|
||||||
|
query['range'] = [0, 100]
|
||||||
|
query['sort'] = [{'key': 'path', 'operator': '+'}]
|
||||||
|
for key in ('sort', 'keys', 'group', 'range', 'position', 'positions'):
|
||||||
|
if key in data:
|
||||||
|
query[key] = data[key]
|
||||||
|
if [r for r in query['range'] if not isinstance(r, int)]:
|
||||||
|
query['range'] = [0, 0]
|
||||||
|
query['qs'] = models.File.objects.find(data, user)
|
||||||
|
return query
|
||||||
|
|
||||||
def findMedia(request, data):
|
def findMedia(request, data):
|
||||||
'''
|
'''
|
||||||
|
@ -625,40 +636,9 @@ def findMedia(request, data):
|
||||||
|
|
||||||
response = json_response({})
|
response = json_response({})
|
||||||
if 'group' in query:
|
if 'group' in query:
|
||||||
if 'sort' in query:
|
print('findMedia does not support group query')
|
||||||
if len(query['sort']) == 1 and query['sort'][0]['key'] == 'items':
|
|
||||||
if query['group'] == "year":
|
|
||||||
order_by = query['sort'][0]['operator'] == '-' and 'items' or '-items'
|
|
||||||
else:
|
|
||||||
order_by = query['sort'][0]['operator'] == '-' and '-items' or 'items'
|
|
||||||
if query['group'] != "keyword":
|
|
||||||
order_by = (order_by, 'sortvalue')
|
|
||||||
else:
|
|
||||||
order_by = (order_by,)
|
|
||||||
else:
|
|
||||||
order_by = query['sort'][0]['operator'] == '-' and '-sortvalue' or 'sortvalue'
|
|
||||||
order_by = (order_by, 'items')
|
|
||||||
else:
|
|
||||||
order_by = ('-sortvalue', 'items')
|
|
||||||
response['data']['items'] = []
|
|
||||||
items = 'items'
|
|
||||||
item_qs = query['qs']
|
|
||||||
qs = models.Facet.objects.filter(key=query['group']).filter(item__id__in=item_qs)
|
|
||||||
qs = qs.values('value').annotate(items=Count('id')).order_by(*order_by)
|
|
||||||
|
|
||||||
if 'positions' in query:
|
|
||||||
response['data']['positions'] = {}
|
|
||||||
ids = [j['value'] for j in qs]
|
|
||||||
response['data']['positions'] = utils.get_positions(ids, query['positions'])
|
|
||||||
|
|
||||||
elif 'range' in data:
|
|
||||||
qs = qs[query['range'][0]:query['range'][1]]
|
|
||||||
response['data']['items'] = [{'path': i['value'], 'items': i[items]} for i in qs]
|
|
||||||
else:
|
|
||||||
response['data']['items'] = qs.count()
|
|
||||||
elif 'positions' in query:
|
elif 'positions' in query:
|
||||||
qs = models.File.objects.filter(item__in=query['qs'])
|
qs = _order_query(query['qs'], query['sort'])
|
||||||
qs = _order_query(qs, query['sort'])
|
|
||||||
|
|
||||||
response['data']['positions'] = {}
|
response['data']['positions'] = {}
|
||||||
ids = list(qs.values_list('oshash', flat=True))
|
ids = list(qs.values_list('oshash', flat=True))
|
||||||
|
@ -666,15 +646,13 @@ def findMedia(request, data):
|
||||||
|
|
||||||
elif 'keys' in query:
|
elif 'keys' in query:
|
||||||
response['data']['items'] = []
|
response['data']['items'] = []
|
||||||
qs = models.File.objects.filter(item__in=query['qs'])
|
qs = _order_query(query['qs'], query['sort'])
|
||||||
qs = _order_query(qs, query['sort'])
|
|
||||||
qs = qs.select_related()
|
qs = qs.select_related()
|
||||||
keys = query['keys']
|
keys = query['keys']
|
||||||
qs = qs[query['range'][0]:query['range'][1]]
|
qs = qs[query['range'][0]:query['range'][1]]
|
||||||
response['data']['items'] = [f.json(keys) for f in qs]
|
response['data']['items'] = [f.json(keys) for f in qs]
|
||||||
else: # otherwise stats
|
else: # otherwise stats
|
||||||
items = query['qs']
|
files = query['qs']
|
||||||
files = models.File.objects.filter(item__in=query['qs'])
|
|
||||||
response['data']['items'] = files.count()
|
response['data']['items'] = files.count()
|
||||||
return render_to_json_response(response)
|
return render_to_json_response(response)
|
||||||
|
|
||||||
|
|
|
@ -48,10 +48,12 @@ def from_json(json_object):
|
||||||
class DictField(models.TextField):
|
class DictField(models.TextField):
|
||||||
_type = dict
|
_type = dict
|
||||||
|
|
||||||
def loads(self, value):
|
@classmethod
|
||||||
|
def loads(cls, value):
|
||||||
return json.loads(value, object_hook=from_json)
|
return json.loads(value, object_hook=from_json)
|
||||||
|
|
||||||
def dumps(self, obj):
|
@classmethod
|
||||||
|
def dumps(cls, obj):
|
||||||
return json.dumps(obj, default=to_json, ensure_ascii=False)
|
return json.dumps(obj, default=to_json, ensure_ascii=False)
|
||||||
|
|
||||||
def from_db_value(self, value, expression, connection, context):
|
def from_db_value(self, value, expression, connection, context):
|
||||||
|
@ -85,8 +87,9 @@ class DictField(models.TextField):
|
||||||
class TupleField(DictField):
|
class TupleField(DictField):
|
||||||
_type = (tuple, list)
|
_type = (tuple, list)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
def loads(self, value):
|
def loads(self, value):
|
||||||
value = DictField.loads(self, value)
|
value = DictField.loads(value)
|
||||||
if isinstance(value, list):
|
if isinstance(value, list):
|
||||||
value = tuple(value)
|
value = tuple(value)
|
||||||
return value
|
return value
|
||||||
|
|
Loading…
Reference in a new issue