From ccfaa7db4a078a3ddda4ea40d05686d30cbbf5c3 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Fri, 13 Feb 2015 11:06:09 +0000 Subject: [PATCH] add entity/document relation --- pandora/annotation/models.py | 2 +- pandora/document/models.py | 10 +- pandora/document/views.py | 38 ++++- pandora/entity/migrations/0003_documents.py | 155 ++++++++++++++++++++ pandora/entity/models.py | 34 +++++ pandora/item/models.py | 3 +- 6 files changed, 235 insertions(+), 7 deletions(-) create mode 100644 pandora/entity/migrations/0003_documents.py diff --git a/pandora/annotation/models.py b/pandora/annotation/models.py index cfd2f35c..1a00d723 100644 --- a/pandora/annotation/models.py +++ b/pandora/annotation/models.py @@ -13,7 +13,6 @@ from django.db.models.signals import pre_delete import ox from clip.models import Clip -from entity.models import Entity from item.utils import sort_string, get_by_key import managers @@ -205,6 +204,7 @@ class Annotation(models.Model): e.delete() def json(self, layer=False, keys=None, user=None): + from entity.models import Entity j = { 'user': self.user.username, } diff --git a/pandora/document/models.py b/pandora/document/models.py index 5a014385..a3d4d7ef 100644 --- a/pandora/document/models.py +++ b/pandora/document/models.py @@ -144,12 +144,13 @@ class Document(models.Model): 'description', 'dimensions', 'editable', + 'entities', + 'extension', 'id', 'name', - 'extension', 'oshash', - 'size', 'ratio', + 'size', 'user', ] response = {} @@ -162,6 +163,9 @@ class Document(models.Model): response[key] = self.editable(user) elif key == 'user': response[key] = self.user.username + elif key == 'entities': + response[key] = [e.json(['id', 'type', 'name']) + for e in self.entities.all().order_by('documentproperties__index')] elif hasattr(self, _map.get(key, key)): response[key] = getattr(self, _map.get(key,key)) or '' if item: @@ -280,7 +284,7 @@ class Document(models.Model): url = unquote(urls[0]) if url != urls[0]: urls.append(url) - matches = self.items.count() + matches = self.items.count() + self.entities.count() for url in urls: matches += annotation.models.Annotation.objects.filter(value__contains=url).count() matches += item.models.Item.objects.filter(data__contains=url).count() diff --git a/pandora/document/views.py b/pandora/document/views.py index b21433b7..485418e8 100644 --- a/pandora/document/views.py +++ b/pandora/document/views.py @@ -2,6 +2,7 @@ # vi:si:et:sw=4:sts=4:ts=4 from __future__ import division +import ox from ox.utils import json from ox.django.api import actions from ox.django.decorators import login_required_json @@ -13,6 +14,7 @@ from django.db.models import Sum from item import utils from item.models import Item from itemlist.models import List +from entity.models import Entity from archive.chunk import process_chunk from changelog.models import add_changelog @@ -31,9 +33,11 @@ def addDocument(request, data): ''' Adds one or more documents to one or more items takes { - item: string or [string], // one or more item ids + item: string or [string], // one or more item ids (optional) + entity: string or [string], // one or more entity ids (optional) id: string or [string] // one or more document ids } + notes: either `item` or `entity` must be provided returns {} see: editDocument, findDocument, getDocument, removeDocument, sortDocuments ''' @@ -59,6 +63,25 @@ def addDocument(request, data): document = models.Document.get(id) document.add(item) add_changelog(request, data, data['item']) + elif 'entity' in data: + if isinstance(data['entity'], basestring): + entity = Entity.get(data['entity']) + if entity.editable(request.user): + for id in ids: + document = models.Document.get(id) + entity.add(document) + add_changelog(request, data, entity.get_id()) + else: + response = json_response(status=403, text='permission denied') + else: + for entity in Entity.objects.filter(id__in=map(ox.fromAZ, data['entity'])): + if entity.editable(request.user): + for id in ids: + document = models.Document.get(id) + entity.add(document) + add_changelog(request, data, data['entity']) + else: + response = json_response(status=500, text='invalid request') return render_to_json_response(response) actions.register(addDocument, cache=False) @@ -207,13 +230,15 @@ actions.register(getDocument) @login_required_json def removeDocument(request, data): ''' - Removes one or more documents, either from an item or from the database + Removes one or more documents, either from an item/entity or from the database takes { id or ids: string or [string], // one or more document ids item: string // item id (optional) + entity: string // entity id (optional) } returns {} notes: If `item` is present, this removes the documents from that item. + If `entity` is present, this removes the documents from that entity. Otherwise, it removes the documents from the database. see: addDocument, editDocument, findDocument, getDocument, sortDocuments ''' @@ -224,6 +249,7 @@ def removeDocument(request, data): else: ids = [data['id']] item = 'item' in data and Item.objects.get(public_id=data['item']) or None + entity = 'entity' in data and Entity.objects.get(id=ox.fromAZ(data['entity'])) or None if item: if item.editable(request.user): add_changelog(request, data, item.public_id) @@ -232,6 +258,14 @@ def removeDocument(request, data): document.remove(item) else: response = json_response(status=403, text='not allowed') + elif entity: + if entity.editable(request.user): + add_changelog(request, data, entity.get_id()) + for id in ids: + document = models.Document.get(id) + entity.remove(document) + else: + response = json_response(status=403, text='not allowed') else: add_changelog(request, data, ids) for id in ids: diff --git a/pandora/entity/migrations/0003_documents.py b/pandora/entity/migrations/0003_documents.py new file mode 100644 index 00000000..16ef14a1 --- /dev/null +++ b/pandora/entity/migrations/0003_documents.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'DocumentProperties' + db.create_table('entity_documentproperties', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), + ('modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)), + ('document', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['document.Document'])), + ('entity', self.gf('django.db.models.fields.related.ForeignKey')(related_name='documentproperties', to=orm['entity.Entity'])), + ('index', self.gf('django.db.models.fields.IntegerField')(default=0)), + )) + db.send_create_signal('entity', ['DocumentProperties']) + + # Adding unique constraint on 'DocumentProperties', fields ['entity', 'document'] + db.create_unique('entity_documentproperties', ['entity_id', 'document_id']) + + + def backwards(self, orm): + # Removing unique constraint on 'DocumentProperties', fields ['entity', 'document'] + db.delete_unique('entity_documentproperties', ['entity_id', 'document_id']) + + # Deleting model 'DocumentProperties' + db.delete_table('entity_documentproperties') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'document.document': { + 'Meta': {'unique_together': "(('user', 'name', 'extension'),)", 'object_name': 'Document'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'description_sort': ('django.db.models.fields.CharField', [], {'max_length': '512', 'null': 'True'}), + 'dimensions_sort': ('django.db.models.fields.CharField', [], {'max_length': '512'}), + 'extension': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'file': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'height': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'items': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'documents'", 'symmetrical': 'False', 'through': "orm['document.ItemProperties']", 'to': "orm['item.Item']"}), + 'matches': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'name_sort': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), + 'oshash': ('django.db.models.fields.CharField', [], {'max_length': '16', 'unique': 'True', 'null': 'True'}), + 'pages': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'ratio': ('django.db.models.fields.FloatField', [], {'default': '1'}), + 'size': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'uploading': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'files'", 'to': "orm['auth.User']"}), + 'width': ('django.db.models.fields.IntegerField', [], {'default': '-1'}) + }, + 'document.itemproperties': { + 'Meta': {'unique_together': "(('item', 'document'),)", 'object_name': 'ItemProperties'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'description_sort': ('django.db.models.fields.CharField', [], {'max_length': '512', 'null': 'True'}), + 'document': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'descriptions'", 'to': "orm['document.Document']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'index': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'item': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['item.Item']"}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}) + }, + 'entity.documentproperties': { + 'Meta': {'unique_together': "(('entity', 'document'),)", 'object_name': 'DocumentProperties'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'document': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['document.Document']"}), + 'entity': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'documentproperties'", 'to': "orm['entity.Entity']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'index': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}) + }, + 'entity.entity': { + 'Meta': {'unique_together': "(('type', 'name'),)", 'object_name': 'Entity'}, + 'alternativeNames': ('ox.django.fields.TupleField', [], {'default': '[]'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'data': ('ox.django.fields.DictField', [], {'default': '{}'}), + 'documents': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'entities'", 'symmetrical': 'False', 'through': "orm['entity.DocumentProperties']", 'to': "orm['document.Document']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'matches': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'name_find': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'name_sort': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), + 'type': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'entities'", 'null': 'True', 'to': "orm['auth.User']"}) + }, + 'item.item': { + 'Meta': {'object_name': 'Item'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'data': ('ox.django.fields.DictField', [], {'default': '{}'}), + 'external_data': ('ox.django.fields.DictField', [], {'default': '{}'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'items'", 'blank': 'True', 'to': "orm['auth.Group']"}), + 'icon': ('django.db.models.fields.files.ImageField', [], {'default': 'None', 'max_length': '100', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'json': ('ox.django.fields.DictField', [], {'default': '{}'}), + 'level': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'oxdbId': ('django.db.models.fields.CharField', [], {'max_length': '42', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'poster': ('django.db.models.fields.files.ImageField', [], {'default': 'None', 'max_length': '100', 'blank': 'True'}), + 'poster_frame': ('django.db.models.fields.FloatField', [], {'default': '-1'}), + 'poster_height': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'poster_source': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'poster_width': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'public_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'blank': 'True'}), + 'rendered': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'stream_aspect': ('django.db.models.fields.FloatField', [], {'default': '1.3333333333333333'}), + 'stream_info': ('ox.django.fields.DictField', [], {'default': '{}'}), + 'torrent': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '1000', 'blank': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'items'", 'null': 'True', 'to': "orm['auth.User']"}) + } + } + + complete_apps = ['entity'] \ No newline at end of file diff --git a/pandora/entity/models.py b/pandora/entity/models.py index 3bcce31e..cd6bf161 100644 --- a/pandora/entity/models.py +++ b/pandora/entity/models.py @@ -17,6 +17,8 @@ from ox.django import fields from person.models import get_name_sort from item.utils import get_by_id +from document.models import Document + import managers @@ -42,6 +44,7 @@ class Entity(models.Model): name_sort = models.CharField(max_length=255, null=True) name_find = models.TextField(default='', editable=True) + documents = models.ManyToManyField(Document, through='DocumentProperties', related_name='entities') def save(self, *args, **kwargs): entity = get_by_id(settings.CONFIG['entities'], self.type) @@ -82,6 +85,16 @@ class Entity(models.Model): def get_id(self): return ox.toAZ(self.id) + def add(self, document): + p, created = DocumentProperties.objects.get_or_create(document=document, entity=self) + if created: + p.index = DocumentProperties.objects.filter(entity=self).aggregate(Max('index'))['index__max'] + 1 + p.save() + p.document.update_matches() + + def remove(self, document): + DocumentProperties.objects.filter(document=document, entity=self).delete() + def editable(self, user, item=None): if not user or user.is_anonymous(): return False @@ -124,6 +137,7 @@ class Entity(models.Model): 'sortName', 'type', 'user', + 'documents', ] + self.data.keys() response = {} for key in keys: @@ -137,6 +151,9 @@ class Entity(models.Model): response[key] = getattr(self, key) elif key == 'sortName': response[key] = self.name_sort + elif key == 'documents': + response[key] = [ox.toAZ(d['id']) + for d in self.documents.all().values('id').order_by('documentproperties__index')] elif key in self.data: response[key] = self.data[key] return response @@ -165,3 +182,20 @@ class Entity(models.Model): Entity.objects.filter(id=self.id).update(matches=matches) self.matches = matches + +class DocumentProperties(models.Model): + + class Meta: + unique_together = ("entity", "document") + + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + + document = models.ForeignKey(Document) + entity = models.ForeignKey(Entity, related_name='documentproperties') + index = models.IntegerField(default=0) + + + def save(self, *args, **kwargs): + + super(DocumentProperties, self).save(*args, **kwargs) diff --git a/pandora/item/models.py b/pandora/item/models.py index 1b1322ee..ee42ba0c 100644 --- a/pandora/item/models.py +++ b/pandora/item/models.py @@ -35,7 +35,6 @@ from data_api import external_data from annotation.models import Annotation from archive import extract from clip.models import Clip -from entity.models import Entity from person.models import get_name_sort from sequence.tasks import get_sequences from title.models import get_title_sort @@ -1003,6 +1002,7 @@ class Item(models.Model): s.save() def update_layer_facet(self, key): + from entity.models import Entity current_values = [a['value'] for a in self.annotations.filter(layer=key).distinct().values('value')] layer = utils.get_by_id(settings.CONFIG['layers'], key) @@ -1674,6 +1674,7 @@ for key in filter(lambda k: k.get('sort', False) or k['type'] in ('integer', 'ti 'float': (models.FloatField, dict(null=True, blank=True, db_index=True)), 'date': (models.DateTimeField, dict(null=True, blank=True, db_index=True)) }[{ + 'layer': 'char', 'string': 'char', 'title': 'char', 'person': 'char',