From a869c7a89aee9849fc748105aff359685b0cc14a Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Fri, 17 Apr 2015 11:17:12 +0100 Subject: [PATCH] extend findEntities to support entity keys and * --- pandora/entity/managers.py | 19 +- pandora/entity/migrations/0004_find.py | 163 ++++++++++++++++++ pandora/entity/models.py | 38 ++++ .../item/management/commands/sqlfindindex.py | 22 +-- static/js/entitiesDialog.js | 2 +- 5 files changed, 229 insertions(+), 15 deletions(-) create mode 100644 pandora/entity/migrations/0004_find.py diff --git a/pandora/entity/managers.py b/pandora/entity/managers.py index a03d5d85..e3151940 100644 --- a/pandora/entity/managers.py +++ b/pandora/entity/managers.py @@ -11,10 +11,12 @@ def parseCondition(condition, user, item=None): k = condition.get('key', 'name') k = { 'user': 'user__username', + 'name': 'name_find' }.get(k, k) v = condition['value'] op = condition.get('operator') + find_key = None if not op: op = '=' if op.startswith('!'): @@ -28,16 +30,27 @@ def parseCondition(condition, user, item=None): if isinstance(v, bool): #featured and public flag key = k else: + if k == '*': + k = 'find__value' + elif k not in ('name_find', 'id', 'user__username', 'type'): + find_key = k + k = 'find__value' key = "%s%s" % (k, { '==': '__iexact', '^': '__istartswith', '$': '__iendswith', }.get(op, '__icontains')) key = str(key) - if exclude: - q = ~Q(**{key: v}) + if find_key: + if exclude: + q = Q(**{'find__key': find_key, key: v}) + else: + q = Q(**{'find__key': find_key, key: v}) else: - q = Q(**{key: v}) + if exclude: + q = ~Q(**{key: v}) + else: + q = Q(**{key: v}) return q def parseConditions(conditions, operator, user, item=None): diff --git a/pandora/entity/migrations/0004_find.py b/pandora/entity/migrations/0004_find.py new file mode 100644 index 00000000..f7cb8771 --- /dev/null +++ b/pandora/entity/migrations/0004_find.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'Find' + db.create_table('entity_find', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('entity', self.gf('django.db.models.fields.related.ForeignKey')(related_name='find', to=orm['entity.Entity'])), + ('key', self.gf('django.db.models.fields.CharField')(max_length=200, db_index=True)), + ('value', self.gf('django.db.models.fields.TextField')(db_index=True, blank=True)), + )) + db.send_create_signal('entity', ['Find']) + + # Adding unique constraint on 'Find', fields ['entity', 'key'] + db.create_unique('entity_find', ['entity_id', 'key']) + + # Update existing entites + for e in orm['entity.Entity'].objects.all(): + e.update_find() + + def backwards(self, orm): + # Removing unique constraint on 'Find', fields ['entity', 'key'] + db.delete_unique('entity_find', ['entity_id', 'key']) + + # Deleting model 'Find' + db.delete_table('entity_find') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'document.document': { + 'Meta': {'unique_together': "(('user', 'name', 'extension'),)", 'object_name': 'Document'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'description_sort': ('django.db.models.fields.CharField', [], {'max_length': '512', 'null': 'True'}), + 'dimensions_sort': ('django.db.models.fields.CharField', [], {'max_length': '512'}), + 'extension': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'file': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'height': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'items': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'documents'", 'symmetrical': 'False', 'through': "orm['document.ItemProperties']", 'to': "orm['item.Item']"}), + 'matches': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'name_sort': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), + 'oshash': ('django.db.models.fields.CharField', [], {'max_length': '16', 'unique': 'True', 'null': 'True'}), + 'pages': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'ratio': ('django.db.models.fields.FloatField', [], {'default': '1'}), + 'size': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'uploading': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'files'", 'to': "orm['auth.User']"}), + 'width': ('django.db.models.fields.IntegerField', [], {'default': '-1'}) + }, + 'document.itemproperties': { + 'Meta': {'unique_together': "(('item', 'document'),)", 'object_name': 'ItemProperties'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'description_sort': ('django.db.models.fields.CharField', [], {'max_length': '512', 'null': 'True'}), + 'document': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'descriptions'", 'to': "orm['document.Document']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'index': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'item': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['item.Item']"}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}) + }, + 'entity.documentproperties': { + 'Meta': {'unique_together': "(('entity', 'document'),)", 'object_name': 'DocumentProperties'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'document': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['document.Document']"}), + 'entity': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'documentproperties'", 'to': "orm['entity.Entity']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'index': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}) + }, + 'entity.entity': { + 'Meta': {'unique_together': "(('type', 'name'),)", 'object_name': 'Entity'}, + 'alternativeNames': ('ox.django.fields.TupleField', [], {'default': '[]'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'data': ('ox.django.fields.DictField', [], {'default': '{}'}), + 'documents': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'entities'", 'symmetrical': 'False', 'through': "orm['entity.DocumentProperties']", 'to': "orm['document.Document']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'matches': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'name_find': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'name_sort': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}), + 'type': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'entities'", 'null': 'True', 'to': "orm['auth.User']"}) + }, + 'entity.find': { + 'Meta': {'unique_together': "(('entity', 'key'),)", 'object_name': 'Find'}, + 'entity': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'find'", 'to': "orm['entity.Entity']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '200', 'db_index': 'True'}), + 'value': ('django.db.models.fields.TextField', [], {'db_index': 'True', 'blank': 'True'}) + }, + 'item.item': { + 'Meta': {'object_name': 'Item'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'data': ('ox.django.fields.DictField', [], {'default': '{}'}), + 'external_data': ('ox.django.fields.DictField', [], {'default': '{}'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'items'", 'blank': 'True', 'to': "orm['auth.Group']"}), + 'icon': ('django.db.models.fields.files.ImageField', [], {'default': 'None', 'max_length': '100', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'json': ('ox.django.fields.DictField', [], {'default': '{}'}), + 'level': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'oxdbId': ('django.db.models.fields.CharField', [], {'max_length': '42', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'poster': ('django.db.models.fields.files.ImageField', [], {'default': 'None', 'max_length': '100', 'blank': 'True'}), + 'poster_frame': ('django.db.models.fields.FloatField', [], {'default': '-1'}), + 'poster_height': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'poster_source': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'poster_width': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'public_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'blank': 'True'}), + 'rendered': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'stream_aspect': ('django.db.models.fields.FloatField', [], {'default': '1.3333333333333333'}), + 'stream_info': ('ox.django.fields.DictField', [], {'default': '{}'}), + 'torrent': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '1000', 'blank': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'items'", 'null': 'True', 'to': "orm['auth.User']"}) + } + } + + complete_apps = ['entity'] diff --git a/pandora/entity/models.py b/pandora/entity/models.py index 3642561e..e44ef67a 100644 --- a/pandora/entity/models.py +++ b/pandora/entity/models.py @@ -5,6 +5,7 @@ import os import re from glob import glob from urllib import quote, unquote +import unicodedata from django.db import models, transaction from django.db.models import Max @@ -58,6 +59,7 @@ class Entity(models.Model): super(Entity, self).save(*args, **kwargs) self.update_matches() self.update_annotations() + self.update_find() def __unicode__(self): return self.get_id() @@ -162,6 +164,30 @@ class Entity(models.Model): def annotation_value(self): #return u'%s' % (self.get_id(), ox.escape_html(self.name)) return ox.escape_html(self.name) + + def update_find(self): + + def save(key, value): + if value not in ('', None): + f, created = Find.objects.get_or_create(entity=self, key=key) + if isinstance(value, bool): + value = value and 'true' or 'false' + if isinstance(value, basestring): + value = ox.decode_html(ox.strip_tags(value.strip())) + value = unicodedata.normalize('NFKD', value).lower() + f.value = value + f.save() + else: + Find.objects.filter(entity=self, key=key).delete() + + with transaction.commit_on_success(): + entity = get_by_id(settings.CONFIG['entities'], self.type) + for key in entity['keys']: + value = self.data.get(key['id']) + if isinstance(value, list): + value = u'\n'.join(value) + save(key['id'], value) + def update_matches(self): import annotation.models @@ -208,3 +234,15 @@ class DocumentProperties(models.Model): def save(self, *args, **kwargs): super(DocumentProperties, self).save(*args, **kwargs) + +class Find(models.Model): + + class Meta: + unique_together = ("entity", "key") + + entity = models.ForeignKey('Entity', related_name='find', db_index=True) + key = models.CharField(max_length=200, db_index=True) + value = models.TextField(blank=True, db_index=settings.DB_GIN_TRGM) + + def __unicode__(self): + return u"%s=%s" % (self.key, self.value) diff --git a/pandora/item/management/commands/sqlfindindex.py b/pandora/item/management/commands/sqlfindindex.py index 3d4d8f76..7312c21c 100644 --- a/pandora/item/management/commands/sqlfindindex.py +++ b/pandora/item/management/commands/sqlfindindex.py @@ -31,15 +31,15 @@ class Command(BaseCommand): cursor.execute(sql) if settings.DB_GIN_TRGM: - table_name = models.ItemFind._meta.db_table - indexes = connection.introspection.get_indexes(cursor, table_name) - name = 'value' - if name not in indexes: - create_table("%s_%s_idx"%(table_name, name), table_name, name) - table_name = models.Clip._meta.db_table - cursor = connection.cursor() - indexes = connection.introspection.get_indexes(cursor, table_name) - name = 'findvalue' - if name not in indexes: - create_table("%s_%s_idx"%(table_name, name), table_name, name) + import entity.models + for table_name, name in ( + (models.ItemFind._meta.db_table, 'value'), # Item Find + (models.Clip._meta.db_table, 'findvalue'), # Clip Find + (entity.models.Find._meta.db_table, 'value'),# Entity Find + ): + cursor = connection.cursor() + indexes = connection.introspection.get_indexes(cursor, table_name) + name = 'value' + if name not in indexes: + create_table("%s_%s_idx"%(table_name, name), table_name, name) transaction.commit_unless_managed() diff --git a/static/js/entitiesDialog.js b/static/js/entitiesDialog.js index f93be773..93a3e176 100644 --- a/static/js/entitiesDialog.js +++ b/static/js/entitiesDialog.js @@ -51,7 +51,7 @@ pandora.ui.entitiesDialog = function(options) { $list.options({ query: { conditions: data.value ? [ - {key: 'name', operator: '=', value: data.value} + {key: '*', operator: '=', value: data.value} ] : [], operator: '&' }