From 80ce81ac670d3566a2d540807d3455d611534b90 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Tue, 18 Nov 2014 18:16:28 +0000 Subject: [PATCH] add entities backend --- pandora/config.0xdb.jsonc | 1 + pandora/config.indiancinema.jsonc | 1 + pandora/config.padma.jsonc | 1 + pandora/config.pandora.jsonc | 1 + pandora/entity/__init__.py | 0 pandora/entity/managers.py | 98 ++++++++++ pandora/entity/migrations/0001_initial.py | 54 ++++++ pandora/entity/migrations/__init__.py | 0 pandora/entity/models.py | 140 ++++++++++++++ pandora/entity/views.py | 215 ++++++++++++++++++++++ pandora/settings.py | 1 + 11 files changed, 512 insertions(+) create mode 100644 pandora/entity/__init__.py create mode 100644 pandora/entity/managers.py create mode 100644 pandora/entity/migrations/0001_initial.py create mode 100644 pandora/entity/migrations/__init__.py create mode 100644 pandora/entity/models.py create mode 100644 pandora/entity/views.py diff --git a/pandora/config.0xdb.jsonc b/pandora/config.0xdb.jsonc index b210226c..ed408d4f 100644 --- a/pandora/config.0xdb.jsonc +++ b/pandora/config.0xdb.jsonc @@ -24,6 +24,7 @@ "canAddItems": {"staff": true, "admin": true}, "canDownloadVideo": {"guest": -1, "member": -1, "friend": -1, "staff": -1, "admin": -1}, "canEditAnnotations": {"staff": true, "admin": true}, + "canEditEntities": {"staff": true, "admin": true}, "canEditDocuments": {"staff": true, "admin": true}, "canEditEvents": {"staff": true, "admin": true}, "canEditFeaturedEdits": {"staff": true, "admin": true}, diff --git a/pandora/config.indiancinema.jsonc b/pandora/config.indiancinema.jsonc index dafd55c1..33dd755a 100644 --- a/pandora/config.indiancinema.jsonc +++ b/pandora/config.indiancinema.jsonc @@ -25,6 +25,7 @@ "canAddItems": {"researcher": true, "staff": true, "admin": true}, "canDownloadVideo": {"guest": -1, "member": -1, "researcher": 3, "staff": 3, "admin": 3}, "canEditAnnotations": {"staff": true, "admin": true}, + "canEditEntities": {"staff": true, "admin": true}, "canEditDocuments": {"researcher": true, "staff": true, "admin": true}, "canEditEvents": {"researcher": true, "staff": true, "admin": true}, "canEditFeaturedEdits": {"researcher": true, "staff": true, "admin": true}, diff --git a/pandora/config.padma.jsonc b/pandora/config.padma.jsonc index 589e6a06..6062c82f 100644 --- a/pandora/config.padma.jsonc +++ b/pandora/config.padma.jsonc @@ -24,6 +24,7 @@ "canAddItems": {"member": true, "staff": true, "admin": true}, "canDownloadVideo": {"guest": 0, "member": 0, "staff": 4, "admin": 4}, "canEditAnnotations": {"staff": true, "admin": true}, + "canEditEntities": {"staff": true, "admin": true}, "canEditDocuments": {"staff": true, "admin": true}, "canEditEvents": {"staff": true, "admin": true}, "canEditFeaturedEdits": {"staff": true, "admin": true}, diff --git a/pandora/config.pandora.jsonc b/pandora/config.pandora.jsonc index e2f92d48..f4760793 100644 --- a/pandora/config.pandora.jsonc +++ b/pandora/config.pandora.jsonc @@ -25,6 +25,7 @@ "canDownloadVideo": {"guest": 1, "member": 1, "staff": 4, "admin": 4}, "canEditAnnotations": {"staff": true, "admin": true}, "canEditDocuments": {"staff": true, "admin": true}, + "canEditEntities": {"staff": true, "admin": true}, "canEditEvents": {"staff": true, "admin": true}, "canEditFeaturedEdits": {"staff": true, "admin": true}, "canEditFeaturedLists": {"staff": true, "admin": true}, diff --git a/pandora/entity/__init__.py b/pandora/entity/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pandora/entity/managers.py b/pandora/entity/managers.py new file mode 100644 index 00000000..a03d5d85 --- /dev/null +++ b/pandora/entity/managers.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from django.db.models import Q, Manager + +import ox +from ox.django.query import QuerySet + +def parseCondition(condition, user, item=None): + ''' + ''' + k = condition.get('key', 'name') + k = { + 'user': 'user__username', + }.get(k, k) + + v = condition['value'] + op = condition.get('operator') + if not op: + op = '=' + if op.startswith('!'): + op = op[1:] + exclude = True + else: + exclude = False + if k == 'id': + v = ox.fromAZ(v) + return Q(**{k: v}) + if isinstance(v, bool): #featured and public flag + key = k + else: + key = "%s%s" % (k, { + '==': '__iexact', + '^': '__istartswith', + '$': '__iendswith', + }.get(op, '__icontains')) + key = str(key) + if exclude: + q = ~Q(**{key: v}) + else: + q = Q(**{key: v}) + return q + +def parseConditions(conditions, operator, user, item=None): + ''' + conditions: [ + { + value: "war" + } + { + key: "year", + value: "1970-1980, + operator: "!=" + }, + { + key: "country", + value: "f", + operator: "^" + } + ], + operator: "&" + ''' + conn = [] + for condition in conditions: + if 'conditions' in condition: + q = parseConditions(condition['conditions'], + condition.get('operator', '&'), user, item) + if q: + conn.append(q) + pass + else: + conn.append(parseCondition(condition, user, item)) + if conn: + q = conn[0] + for c in conn[1:]: + if operator == '|': + q = q | c + else: + q = q & c + return q + return None + + +class EntityManager(Manager): + + def get_query_set(self): + return QuerySet(self.model) + + def find(self, data, user, item=None): + #join query with operator + qs = self.get_query_set() + conditions = parseConditions(data['query'].get('conditions', []), + data['query'].get('operator', '&'), + user, item) + if conditions: + qs = qs.filter(conditions) + + return qs + diff --git a/pandora/entity/migrations/0001_initial.py b/pandora/entity/migrations/0001_initial.py new file mode 100644 index 00000000..38e90354 --- /dev/null +++ b/pandora/entity/migrations/0001_initial.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'Entity' + db.create_table('entity_entity', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), + ('modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)), + ('type', self.gf('django.db.models.fields.CharField')(max_length=255)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=255)), + ('alternativeNames', self.gf('ox.django.fields.TupleField')(default=[])), + ('data', self.gf('ox.django.fields.DictField')(default={})), + ('matches', self.gf('django.db.models.fields.IntegerField')(default=0)), + ('name_sort', self.gf('django.db.models.fields.CharField')(max_length=255, null=True)), + ('name_find', self.gf('django.db.models.fields.TextField')(default='')), + )) + db.send_create_signal('entity', ['Entity']) + + # Adding unique constraint on 'Entity', fields ['type', 'name'] + db.create_unique('entity_entity', ['type', 'name']) + + + def backwards(self, orm): + # Removing unique constraint on 'Entity', fields ['type', 'name'] + db.delete_unique('entity_entity', ['type', 'name']) + + # Deleting model 'Entity' + db.delete_table('entity_entity') + + + models = { + 'entity.entity': { + 'Meta': {'unique_together': "(('type', 'name'),)", 'object_name': 'Entity'}, + 'alternativeNames': ('ox.django.fields.TupleField', [], {'default': '[]'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'data': ('ox.django.fields.DictField', [], {'default': '{}'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'matches': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'name_find': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'name_sort': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), + 'type': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + } + } + + complete_apps = ['entity'] \ No newline at end of file diff --git a/pandora/entity/migrations/__init__.py b/pandora/entity/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pandora/entity/models.py b/pandora/entity/models.py new file mode 100644 index 00000000..961f20a5 --- /dev/null +++ b/pandora/entity/models.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from __future__ import division, with_statement +import os +import re +from glob import glob +from urllib import quote, unquote + +from django.db import models +from django.db.models import Max +from django.contrib.auth.models import User +from django.db.models.signals import pre_delete + +import ox +from ox.django import fields + +import managers + + +class Entity(models.Model): + + class Meta: + unique_together = ("type", "name") + + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + + type = models.CharField(max_length=255) + name = models.CharField(max_length=255) + alternativeNames = fields.TupleField(default=[]) + + data = fields.DictField(default={}, editable=False) + matches = models.IntegerField(default=0) + + objects = managers.EntityManager() + + name_sort = models.CharField(max_length=255, null=True) + name_find = models.TextField(default='', editable=True) + + + def save(self, *args, **kwargs): + self.name_sort = ox.sort_string(self.name or u'')[:255].lower() + self.name_find = '||' + self.name + '||'.join(self.alternativeNames) + '||' + super(Entity, self).save(*args, **kwargs) + #self.update_matches() + + def __unicode__(self): + return self.get_id() + + @classmethod + def get(cls, id): + return cls.objects.get(pk=ox.fromAZ(id)) + + @classmethod + def get_or_create(model, name): + qs = model.objects.filter(name_find__icontains=u'|%s|'%name) + if qs.count() == 0: + instance = model(name=name) + instance.save() + else: + instance = qs[0] + return instance + + def get_absolute_url(self): + return ('/entities/%s' % quote(self.get_id())).replace('%3A', ':') + + def get_id(self): + return ox.toAZ(self.id) + + def editable(self, user, item=None): + if not user or user.is_anonymous(): + return False + if user.is_staff or \ + user.get_profile().capability('canEditEntities') == True or \ + (item and item.editable(user)): + return True + return False + + def edit(self, data): + for key in data: + if key == 'name': + data['name'] = re.sub(' \[\d+\]$', '', data['name']).strip() + if not data['name']: + data['name'] = "Untitled" + name = data['name'] + num = 1 + while Entity.objects.filter(name=name, type=self.type).exclude(id=self.id).count()>0: + num += 1 + name = data['name'] + ' [%d]' % num + self.name = name + elif key == 'type': + self.type = data[key] + elif key == 'alternativeNames': + self.alternativeNames = tuple([ox.escape_html(v) for v in data[key]]) + else: + #FIXME: more data validation + if isinstance(data[key], basestring): + self.data[key] = ox.sanitize_html(data[key]) + else: + self.data[key] = data[key] + + def json(self, keys=None, user=None): + if not keys: + keys=[ + 'editable', + 'id', + 'type', + 'name', + 'alternativeNames', + ] + self.data.keys() + response = {} + for key in keys: + if key == 'id': + response[key] = self.get_id() + elif key == 'editable': + response[key] = self.editable(user) + elif key in ('name', 'alternativeNames', 'type'): + response[key] = getattr(self, key) + elif key in self.data: + response[key] = self.data[key] + return response + + + def update_matches(self): + import annotation.models + import item.models + import text.models + urls = [self.get_absolute_url()] + url = unquote(urls[0]) + if url != urls[0]: + urls.append(url) + matches = self.items.count() + for url in urls: + matches += annotation.models.Annotation.objects.filter(value__contains=url).count() + matches += item.models.Item.objects.filter(data__contains=url).count() + matches += text.models.Text.objects.filter(text__contains=url).count() + if matches != self.matches: + Entity.objects.filter(id=self.id).update(matches=matches) + self.matches = matches + diff --git a/pandora/entity/views.py b/pandora/entity/views.py new file mode 100644 index 00000000..9ad9d9e6 --- /dev/null +++ b/pandora/entity/views.py @@ -0,0 +1,215 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from __future__ import division + +import ox +from ox.utils import json +from ox.django.api import actions +from ox.django.decorators import login_required_json +from ox.django.http import HttpFileResponse +from ox.django.shortcuts import render_to_json_response, get_object_or_404_json, json_response, HttpErrorJson +from django import forms +from django.db.models import Sum + +from item import utils +from item.models import Item +from itemlist.models import List + +import models + +def get_entity_or_404_json(id): + try: + return models.Entity.get(id) + except models.Entity.DoesNotExist: + response = {'status': {'code': 404, + 'text': 'Entity not found'}} + raise HttpErrorJson(response) + +@login_required_json +def addEntity(request, data): + ''' + add entity + takes { + type: + name: + alternativeNames + } + returns { + } + ''' + existing_names = [] + exists = False + names = [data['name']] + data.get('alternativeNames', []) + for name in names: + name = ox.decode_html(name) + if models.Entity.objects.filter(type=data['type'], + name_find__icontains=u'|%s|'%name).count() != 0: + exists = True + existing_names.append(name) + if not exists: + data['name'] = ox.escape_html(data['name']) + entity = models.Entity(name=data['name']) + for key in ('type', 'alternativeNames'): + if key in data and data[key]: + value = data[key] + if isinstance(value, basestring): + value = ox.escape_html(value) + if key == 'alternativeNames': + value = tuple([ox.escape_html(v) for v in value]) + setattr(entity, key, value) + entity.matches = 0 + entity.save() + response = json_response(status=200, text='created') + response['data'] = entity.json() + else: + response = json_response(status=409, text='name exists') + response['data']['names'] = existing_names + return render_to_json_response(response) +actions.register(addEntity, cache=False) + +@login_required_json +def editEntity(request, data): + ''' + takes { + id: string + name: string + description: string + item(optional): edit descriptoin per item + } + returns { + id: + ... + } + ''' + response = json_response() + entity = get_entity_or_404_json(data['id']) + if entity.editable(request.user): + entity.edit(data) + entity.save() + response['data'] = entity.json(user=request.user) + else: + response = json_response(status=403, text='permission denied') + return render_to_json_response(response) +actions.register(editEntity, cache=False) + + +def _order_query(qs, sort, item=None): + order_by = [] + for e in sort: + operator = e['operator'] + if operator != '-': + operator = '' + key = { + 'name': 'name_sort', + }.get(e['key'], e['key']) + order = '%s%s' % (operator, key) + order_by.append(order) + if order_by: + qs = qs.order_by(*order_by, nulls_last=True) + qs = qs.distinct() + return qs + +def parse_query(data, user): + query = {} + query['range'] = [0, 100] + query['sort'] = [{'key':'name', 'operator':'+'}, {'key':'type', 'operator':'+'}] + for key in ('keys', 'group', 'range', 'position', 'positions', 'sort'): + if key in data: + query[key] = data[key] + query['qs'] = models.Entity.objects.find(data, user).exclude(name='') + return query + + +def findEntities(request, data): + ''' + takes { + query: { + conditions: [ + { + key: 'name', + value: 'something', + operator: '=' + } + ] + operator: "," + }, + sort: [{key: 'name', operator: '+'}], + range: [0, 100] + keys: [] + } + + possible query keys: + name, type + + possible keys: + name, type, alternativeNames + + } + returns { + items: [object] + } + ''' + query = parse_query(data, request.user) + + #order + qs = _order_query(query['qs'], query['sort']) + response = json_response() + if 'keys' in data: + qs = qs[query['range'][0]:query['range'][1]] + + response['data']['items'] = [l.json(data['keys'], request.user) for l in qs] + elif 'position' in data: + #FIXME: actually implement position requests + response['data']['position'] = 0 + elif 'positions' in data: + ids = [i.get_id() for i in qs] + response['data']['positions'] = utils.get_positions(ids, query['positions']) + else: + response['data']['items'] = qs.count() + return render_to_json_response(response) +actions.register(findEntities) + +def getEntity(request, data): + ''' + takes { + id: string, + keys: [string] + } + returns { + key: value + } + ''' + response = json_response({}) + data['keys'] = data.get('keys', []) + entity = get_entity_or_404_json(data['id']) + response['data'] = entity.json(keys=data['keys'], user=request.user) + return render_to_json_response(response) +actions.register(getEntity) + +@login_required_json +def removeEntity(request, data): + ''' + takes { + id: string, + or + ids: [string] + } + returns { + } + ''' + response = json_response() + + if 'ids' in data: + ids = data['ids'] + else: + ids = [data['id']] + for id in ids: + entity = get_entity_or_404_json(id) + if entity.editable(request.user): + entity.delete() + else: + response = json_response(status=403, text='not allowed') + break + return render_to_json_response(response) +actions.register(removeEntity, cache=False) + diff --git a/pandora/settings.py b/pandora/settings.py index 00af0ac6..0ad4af0f 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -128,6 +128,7 @@ INSTALLED_APPS = ( 'urlalias', 'tv', 'document', + 'entity', ) # Log errors into db