add entities backend

pandora/config.0xdb.jsonc

@@ -24,6 +24,7 @@
         "canAddItems": {"staff": true, "admin": true},
         "canDownloadVideo": {"guest": -1, "member": -1, "friend": -1, "staff": -1, "admin": -1},
         "canEditAnnotations": {"staff": true, "admin": true},
+        "canEditEntities": {"staff": true, "admin": true},
         "canEditDocuments": {"staff": true, "admin": true},
         "canEditEvents": {"staff": true, "admin": true},
         "canEditFeaturedEdits": {"staff": true, "admin": true},

pandora/config.indiancinema.jsonc

@@ -25,6 +25,7 @@
         "canAddItems": {"researcher": true, "staff": true, "admin": true},
         "canDownloadVideo": {"guest": -1, "member": -1, "researcher": 3, "staff": 3, "admin": 3},
         "canEditAnnotations": {"staff": true, "admin": true},
+        "canEditEntities": {"staff": true, "admin": true},
         "canEditDocuments": {"researcher": true, "staff": true, "admin": true},
         "canEditEvents": {"researcher": true, "staff": true, "admin": true},
         "canEditFeaturedEdits": {"researcher": true, "staff": true, "admin": true},

pandora/config.padma.jsonc

@@ -24,6 +24,7 @@
         "canAddItems": {"member": true, "staff": true, "admin": true},
         "canDownloadVideo": {"guest": 0, "member": 0, "staff": 4, "admin": 4},
         "canEditAnnotations": {"staff": true, "admin": true},
+        "canEditEntities": {"staff": true, "admin": true},
         "canEditDocuments": {"staff": true, "admin": true},
         "canEditEvents": {"staff": true, "admin": true},
         "canEditFeaturedEdits": {"staff": true, "admin": true},

pandora/config.pandora.jsonc

@@ -25,6 +25,7 @@
         "canDownloadVideo": {"guest": 1, "member": 1, "staff": 4, "admin": 4},
         "canEditAnnotations": {"staff": true, "admin": true},
         "canEditDocuments": {"staff": true, "admin": true},
+        "canEditEntities": {"staff": true, "admin": true},
         "canEditEvents": {"staff": true, "admin": true},
         "canEditFeaturedEdits": {"staff": true, "admin": true},
         "canEditFeaturedLists": {"staff": true, "admin": true},

pandora/entity/managers.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+from django.db.models import Q, Manager
+
+import ox
+from ox.django.query import QuerySet
+
+def parseCondition(condition, user, item=None):
+    '''
+    '''
+    k = condition.get('key', 'name')
+    k = {
+        'user': 'user__username',
+    }.get(k, k)
+
+    v = condition['value']
+    op = condition.get('operator')
+    if not op:
+        op = '='
+    if op.startswith('!'):
+        op = op[1:]
+        exclude = True
+    else:
+        exclude = False
+    if k == 'id':
+        v = ox.fromAZ(v)
+        return Q(**{k: v})
+    if isinstance(v, bool): #featured and public flag
+        key = k
+    else:
+        key = "%s%s" % (k, {
+            '==': '__iexact',
+            '^': '__istartswith',
+            '$': '__iendswith',
+        }.get(op, '__icontains'))
+    key = str(key)
+    if exclude:
+        q = ~Q(**{key: v})
+    else:
+        q = Q(**{key: v})
+    return q
+
+def parseConditions(conditions, operator, user, item=None):
+    '''
+    conditions: [
+        {
+            value: "war"
+        }
+        {
+            key: "year",
+            value: "1970-1980,
+            operator: "!="
+        },
+        {
+            key: "country",
+            value: "f",
+            operator: "^"
+        }
+    ],
+    operator: "&"
+    '''
+    conn = []
+    for condition in conditions:
+        if 'conditions' in condition:
+            q = parseConditions(condition['conditions'],
+                             condition.get('operator', '&'), user, item)
+            if q:
+                conn.append(q)
+            pass
+        else:
+            conn.append(parseCondition(condition, user, item))
+    if conn:
+        q = conn[0]
+        for c in conn[1:]:
+            if operator == '|':
+                q = q | c
+            else:
+                q = q & c
+        return q
+    return None
+
+
+class EntityManager(Manager):
+
+    def get_query_set(self):
+        return QuerySet(self.model)
+
+    def find(self, data, user, item=None):
+        #join query with operator
+        qs = self.get_query_set()
+        conditions = parseConditions(data['query'].get('conditions', []),
+                                     data['query'].get('operator', '&'),
+                                     user, item)
+        if conditions:
+            qs = qs.filter(conditions)
+
+        return qs
+

pandora/entity/migrations/0001_initial.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'Entity'
+        db.create_table('entity_entity', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+            ('modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)),
+            ('type', self.gf('django.db.models.fields.CharField')(max_length=255)),
+            ('name', self.gf('django.db.models.fields.CharField')(max_length=255)),
+            ('alternativeNames', self.gf('ox.django.fields.TupleField')(default=[])),
+            ('data', self.gf('ox.django.fields.DictField')(default={})),
+            ('matches', self.gf('django.db.models.fields.IntegerField')(default=0)),
+            ('name_sort', self.gf('django.db.models.fields.CharField')(max_length=255, null=True)),
+            ('name_find', self.gf('django.db.models.fields.TextField')(default='')),
+        ))
+        db.send_create_signal('entity', ['Entity'])
+
+        # Adding unique constraint on 'Entity', fields ['type', 'name']
+        db.create_unique('entity_entity', ['type', 'name'])
+
+
+    def backwards(self, orm):
+        # Removing unique constraint on 'Entity', fields ['type', 'name']
+        db.delete_unique('entity_entity', ['type', 'name'])
+
+        # Deleting model 'Entity'
+        db.delete_table('entity_entity')
+
+
+    models = {
+        'entity.entity': {
+            'Meta': {'unique_together': "(('type', 'name'),)", 'object_name': 'Entity'},
+            'alternativeNames': ('ox.django.fields.TupleField', [], {'default': '[]'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'data': ('ox.django.fields.DictField', [], {'default': '{}'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'matches': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'name_find': ('django.db.models.fields.TextField', [], {'default': "''"}),
+            'name_sort': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}),
+            'type': ('django.db.models.fields.CharField', [], {'max_length': '255'})
+        }
+    }
+
+    complete_apps = ['entity']

pandora/entity/models.py

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+from __future__ import division, with_statement
+import os
+import re
+from glob import glob
+from urllib import quote, unquote
+
+from django.db import models
+from django.db.models import Max
+from django.contrib.auth.models import User
+from django.db.models.signals import pre_delete
+
+import ox
+from ox.django import fields
+
+import managers
+
+
+class Entity(models.Model):
+
+    class Meta:
+        unique_together = ("type", "name")
+
+    created = models.DateTimeField(auto_now_add=True)
+    modified = models.DateTimeField(auto_now=True)
+
+    type = models.CharField(max_length=255)
+    name = models.CharField(max_length=255)
+    alternativeNames = fields.TupleField(default=[])
+
+    data = fields.DictField(default={}, editable=False)
+    matches = models.IntegerField(default=0)
+
+    objects = managers.EntityManager()
+
+    name_sort = models.CharField(max_length=255, null=True)
+    name_find = models.TextField(default='', editable=True)
+
+
+    def save(self, *args, **kwargs):
+        self.name_sort = ox.sort_string(self.name or u'')[:255].lower()
+        self.name_find = '||' + self.name + '||'.join(self.alternativeNames) + '||'
+        super(Entity, self).save(*args, **kwargs)
+        #self.update_matches()
+
+    def __unicode__(self):
+        return self.get_id()
+
+    @classmethod
+    def get(cls, id):
+        return cls.objects.get(pk=ox.fromAZ(id))
+
+    @classmethod
+    def get_or_create(model, name):
+        qs = model.objects.filter(name_find__icontains=u'|%s|'%name)
+        if qs.count() == 0:
+            instance = model(name=name)
+            instance.save()
+        else:
+            instance = qs[0]
+        return instance
+
+    def get_absolute_url(self):
+        return ('/entities/%s' % quote(self.get_id())).replace('%3A', ':')
+
+    def get_id(self):
+        return ox.toAZ(self.id)
+
+    def editable(self, user, item=None):
+        if not user or user.is_anonymous():
+            return False
+        if user.is_staff or \
+           user.get_profile().capability('canEditEntities') == True or \
+           (item and item.editable(user)):
+            return True
+        return False
+
+    def edit(self, data):
+        for key in data:
+            if key == 'name':
+                data['name'] = re.sub(' \[\d+\]$', '', data['name']).strip()
+                if not data['name']:
+                    data['name'] = "Untitled"
+                name = data['name']
+                num = 1
+                while Entity.objects.filter(name=name, type=self.type).exclude(id=self.id).count()>0:
+                    num += 1
+                    name = data['name'] + ' [%d]' % num
+                self.name = name
+            elif key == 'type':
+                self.type = data[key]
+            elif key == 'alternativeNames':
+                self.alternativeNames = tuple([ox.escape_html(v) for v in data[key]])
+            else:
+                #FIXME: more data validation
+                if isinstance(data[key], basestring):
+                    self.data[key] = ox.sanitize_html(data[key])
+                else:
+                    self.data[key] = data[key]
+
+    def json(self, keys=None, user=None):
+        if not keys:
+            keys=[
+                'editable',
+                'id',
+                'type',
+                'name',
+                'alternativeNames',
+            ] + self.data.keys()
+        response = {}
+        for key in keys:
+            if key == 'id':
+                response[key] = self.get_id()
+            elif key == 'editable':
+                response[key] = self.editable(user)
+            elif key in ('name', 'alternativeNames', 'type'):
+                response[key] = getattr(self, key)
+            elif key in self.data:
+                response[key] = self.data[key]
+        return response
+
+    
+    def update_matches(self):
+        import annotation.models
+        import item.models
+        import text.models
+        urls = [self.get_absolute_url()]
+        url = unquote(urls[0])
+        if url != urls[0]:
+            urls.append(url)
+        matches = self.items.count()
+        for url in urls:
+            matches += annotation.models.Annotation.objects.filter(value__contains=url).count()
+            matches += item.models.Item.objects.filter(data__contains=url).count()
+            matches += text.models.Text.objects.filter(text__contains=url).count()
+        if matches != self.matches:
+            Entity.objects.filter(id=self.id).update(matches=matches)
+            self.matches = matches
+

pandora/entity/views.py

@@ -0,0 +1,215 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+from __future__ import division
+
+import ox
+from ox.utils import json
+from ox.django.api import actions
+from ox.django.decorators import login_required_json
+from ox.django.http import HttpFileResponse
+from ox.django.shortcuts import render_to_json_response, get_object_or_404_json, json_response, HttpErrorJson
+from django import forms
+from django.db.models import Sum
+
+from item import utils
+from item.models import Item
+from itemlist.models import List
+
+import models
+
+def get_entity_or_404_json(id):
+    try:
+        return models.Entity.get(id)
+    except models.Entity.DoesNotExist:
+        response = {'status': {'code': 404,
+                               'text': 'Entity not found'}}
+        raise HttpErrorJson(response)
+
+@login_required_json
+def addEntity(request, data):
+    '''
+        add entity
+        takes {
+            type:
+            name:
+            alternativeNames
+        }
+        returns {
+        }
+    '''
+    existing_names = []
+    exists = False
+    names = [data['name']] + data.get('alternativeNames', [])
+    for name in names:
+        name = ox.decode_html(name)
+        if models.Entity.objects.filter(type=data['type'],
+                name_find__icontains=u'|%s|'%name).count() != 0:
+            exists = True
+            existing_names.append(name)
+    if not exists:
+        data['name'] = ox.escape_html(data['name'])
+        entity = models.Entity(name=data['name'])
+        for key in ('type', 'alternativeNames'):
+            if key in data and data[key]:
+                value = data[key]
+                if isinstance(value, basestring):
+                    value = ox.escape_html(value)
+                if key == 'alternativeNames':
+                    value = tuple([ox.escape_html(v) for v in value])
+                setattr(entity, key, value)
+        entity.matches = 0
+        entity.save()
+        response = json_response(status=200, text='created')
+        response['data'] = entity.json()
+    else:
+        response = json_response(status=409, text='name exists')
+        response['data']['names'] = existing_names
+    return render_to_json_response(response)
+actions.register(addEntity, cache=False)
+
+@login_required_json
+def editEntity(request, data):
+    '''
+        takes {
+            id: string
+            name: string
+            description: string
+            item(optional): edit descriptoin per item
+        }
+        returns {
+            id:
+            ...
+        }
+    '''
+    response = json_response()
+    entity = get_entity_or_404_json(data['id'])
+    if entity.editable(request.user):
+        entity.edit(data)
+        entity.save()
+        response['data'] = entity.json(user=request.user)
+    else:
+        response = json_response(status=403, text='permission denied')
+    return render_to_json_response(response)
+actions.register(editEntity, cache=False)
+
+
+def _order_query(qs, sort, item=None):
+    order_by = []
+    for e in sort:
+        operator = e['operator']
+        if operator != '-':
+            operator = ''
+        key = {
+            'name': 'name_sort',
+        }.get(e['key'], e['key'])
+        order = '%s%s' % (operator, key)
+        order_by.append(order)
+    if order_by:
+        qs = qs.order_by(*order_by, nulls_last=True)
+    qs = qs.distinct()
+    return qs
+
+def parse_query(data, user):
+    query = {}
+    query['range'] = [0, 100]
+    query['sort'] = [{'key':'name', 'operator':'+'}, {'key':'type', 'operator':'+'}]
+    for key in ('keys', 'group', 'range', 'position', 'positions', 'sort'):
+        if key in data:
+            query[key] = data[key]
+    query['qs'] = models.Entity.objects.find(data, user).exclude(name='')
+    return query
+
+
+def findEntities(request, data):
+    '''
+        takes {
+            query: {
+                conditions: [
+                    {
+                        key: 'name',
+                        value: 'something',
+                        operator: '='
+                    }
+                ]
+                operator: ","
+            },
+            sort: [{key: 'name', operator: '+'}],
+            range: [0, 100]
+            keys: []
+        }
+
+        possible query keys:
+            name, type
+
+        possible keys:
+            name, type, alternativeNames
+
+        }
+        returns {
+            items: [object]
+        }
+    '''
+    query = parse_query(data, request.user)
+
+    #order
+    qs = _order_query(query['qs'], query['sort'])
+    response = json_response()
+    if 'keys' in data:
+        qs = qs[query['range'][0]:query['range'][1]]
+
+        response['data']['items'] = [l.json(data['keys'], request.user) for l in qs]
+    elif 'position' in data:
+        #FIXME: actually implement position requests
+        response['data']['position'] = 0
+    elif 'positions' in data:
+        ids = [i.get_id() for i in qs]
+        response['data']['positions'] = utils.get_positions(ids, query['positions'])
+    else:
+        response['data']['items'] = qs.count()
+    return render_to_json_response(response)
+actions.register(findEntities)
+
+def getEntity(request, data):
+    '''
+        takes {
+            id: string,
+            keys: [string]
+        }
+        returns {
+            key: value
+        }
+    '''
+    response = json_response({})
+    data['keys'] = data.get('keys', [])
+    entity = get_entity_or_404_json(data['id'])
+    response['data'] = entity.json(keys=data['keys'], user=request.user)
+    return render_to_json_response(response)
+actions.register(getEntity)
+
+@login_required_json
+def removeEntity(request, data):
+    '''
+        takes {
+            id: string,
+            or
+            ids: [string]
+        }
+        returns {
+        }
+    '''
+    response = json_response()
+
+    if 'ids' in data:
+        ids = data['ids']
+    else:
+        ids = [data['id']]
+    for id in ids:
+        entity = get_entity_or_404_json(id)
+        if entity.editable(request.user):
+            entity.delete()
+        else:
+            response = json_response(status=403, text='not allowed')
+            break
+    return render_to_json_response(response)
+actions.register(removeEntity, cache=False)
+

pandora/settings.py

@@ -128,6 +128,7 @@ INSTALLED_APPS = (
     'urlalias',
     'tv',
     'document',
+    'entity',
 )
 
 # Log errors into db