better findMedia

pandora/archive/managers.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+
+from django.db.models import Q, Manager
+
+from oxdjango.managers import get_operator
+from oxdjango.query import QuerySet
+from oxdjango.fields import DictField
+
+keymap = {
+    'id': 'item__public_id',
+}
+default_key = 'oshash'
+
+def parseCondition(condition, user):
+    '''
+    '''
+    k = condition.get('key', default_key)
+    k = keymap.get(k, k)
+    if not k:
+        k = default_key
+    v = condition.get('value', '')
+    op = condition.get('operator')
+    if not op:
+        op = '='
+    if op.startswith('!'):
+        op = op[1:]
+        exclude = True
+    else:
+        exclude = False
+    if isinstance(v, bool):
+        key = k
+    elif k == 'url':
+        key = 'info' + get_operator('=', 'istr')
+        v = DictField.dumps({'url': v})[1:-1]
+    else:
+        key = k + get_operator(op, 'istr')
+    key = str(key)
+    if exclude:
+        q = ~Q(**{key: v})
+    else:
+        q = Q(**{key: v})
+    return q
+
+def parseConditions(conditions, operator, user):
+    '''
+    conditions: [
+        {
+            value: "war"
+        }
+        {
+            key: "year",
+            value: "1970-1980,
+            operator: "!="
+        },
+        {
+            key: "country",
+            value: "f",
+            operator: "^"
+        }
+    ],
+    operator: "&"
+    '''
+    conn = []
+    for condition in conditions:
+        if 'conditions' in condition:
+            q = parseConditions(condition['conditions'],
+                                condition.get('operator', '&'), user)
+            if q:
+                conn.append(q)
+            pass
+        else:
+            conn.append(parseCondition(condition, user))
+    if conn:
+        q = conn[0]
+        for c in conn[1:]:
+            if operator == '|':
+                q = q | c
+            else:
+                q = q & c
+        return q
+    return None
+
+
+class FileManager(Manager):
+
+    def get_query_set(self):
+        return QuerySet(self.model)
+
+    def find(self, data, user):
+        '''
+            query: {
+                conditions: [
+                    {
+                        value: "war"
+                    }
+                    {
+                        key: "year",
+                        value: "1970-1980,
+                        operator: "!="
+                    },
+                    {
+                        key: "country",
+                        value: "f",
+                        operator: "^"
+                    }
+                ],
+                operator: "&"
+            }
+        '''
+
+        # join query with operator
+        qs = self.get_query_set()
+        query = data.get('query', {})
+        conditions = parseConditions(query.get('conditions', []),
+                                     query.get('operator', '&'),
+                                     user)
+        if conditions:
+            qs = qs.filter(conditions)
+        return qs

pandora/archive/models.py

@@ -25,6 +25,7 @@ from taskqueue.models import Task
 
 from .chunk import save_chunk
 from . import extract
+from . import managers
 
 if not PY2:
     unicode = str
@@ -100,6 +101,8 @@ class File(models.Model):
     data = models.FileField(null=True, blank=True,
                             upload_to=data_path)
 
+    objects = managers.FileManager()
+
     def __unicode__(self):
         return self.path
 

pandora/archive/views.py

@@ -605,6 +605,17 @@ def _order_query(qs, sort, prefix=''):
         qs = qs.order_by(*order_by)
     return qs
 
+def parse_query(data, user):
+    query = {}
+    query['range'] = [0, 100]
+    query['sort'] = [{'key': 'path', 'operator': '+'}]
+    for key in ('sort', 'keys', 'group', 'range', 'position', 'positions'):
+        if key in data:
+            query[key] = data[key]
+    if [r for r in query['range'] if not isinstance(r, int)]:
+        query['range'] = [0, 0]
+    query['qs'] = models.File.objects.find(data, user)
+    return query
 
 def findMedia(request, data):
     '''
@@ -625,40 +636,9 @@ def findMedia(request, data):
 
     response = json_response({})
     if 'group' in query:
-        if 'sort' in query:
+        print('findMedia does not support group query')
-            if len(query['sort']) == 1 and query['sort'][0]['key'] == 'items':
-                if query['group'] == "year":
-                    order_by = query['sort'][0]['operator'] == '-' and 'items' or '-items'
-                else:
-                    order_by = query['sort'][0]['operator'] == '-' and '-items' or 'items'
-                if query['group'] != "keyword":
-                    order_by = (order_by, 'sortvalue')
-                else:
-                    order_by = (order_by,)
-            else:
-                order_by = query['sort'][0]['operator'] == '-' and '-sortvalue' or 'sortvalue'
-                order_by = (order_by, 'items')
-        else:
-            order_by = ('-sortvalue', 'items')
-        response['data']['items'] = []
-        items = 'items'
-        item_qs = query['qs']
-        qs = models.Facet.objects.filter(key=query['group']).filter(item__id__in=item_qs)
-        qs = qs.values('value').annotate(items=Count('id')).order_by(*order_by)
-
-        if 'positions' in query:
-            response['data']['positions'] = {}
-            ids = [j['value'] for j in qs]
-            response['data']['positions'] = utils.get_positions(ids, query['positions'])
-
-        elif 'range' in data:
-            qs = qs[query['range'][0]:query['range'][1]]
-            response['data']['items'] = [{'path': i['value'], 'items': i[items]} for i in qs]
-        else:
-            response['data']['items'] = qs.count()
     elif 'positions' in query:
-        qs = models.File.objects.filter(item__in=query['qs'])
+        qs = _order_query(query['qs'], query['sort'])
-        qs = _order_query(qs, query['sort'])
 
         response['data']['positions'] = {}
         ids = list(qs.values_list('oshash', flat=True))
@@ -666,15 +646,13 @@ def findMedia(request, data):
 
     elif 'keys' in query:
         response['data']['items'] = []
-        qs = models.File.objects.filter(item__in=query['qs'])
+        qs = _order_query(query['qs'], query['sort'])
-        qs = _order_query(qs, query['sort'])
         qs = qs.select_related()
         keys = query['keys']
         qs = qs[query['range'][0]:query['range'][1]]
         response['data']['items'] = [f.json(keys) for f in qs]
     else:  # otherwise stats
-        items = query['qs']
+        files = query['qs']
-        files = models.File.objects.filter(item__in=query['qs'])
         response['data']['items'] = files.count()
     return render_to_json_response(response)
 

pandora/oxdjango/fields.py

@@ -48,10 +48,12 @@ def from_json(json_object):
 class DictField(models.TextField):
     _type = dict
 
-    def loads(self, value):
+    @classmethod
+    def loads(cls, value):
         return json.loads(value, object_hook=from_json)
 
-    def dumps(self, obj):
+    @classmethod
+    def dumps(cls, obj):
         return json.dumps(obj, default=to_json, ensure_ascii=False)
 
     def from_db_value(self, value, expression, connection, context):
@@ -85,8 +87,9 @@ class DictField(models.TextField):
 class TupleField(DictField):
     _type = (tuple, list)
 
+    @classmethod
     def loads(self, value):
-        value = DictField.loads(self, value)
+        value = DictField.loads(value)
         if isinstance(value, list):
             value = tuple(value)
         return value