extend findEntities to support entity keys and *

pandora/entity/managers.py

@@ -11,10 +11,12 @@ def parseCondition(condition, user, item=None):
     k = condition.get('key', 'name')
     k = {
         'user': 'user__username',
+        'name': 'name_find'
     }.get(k, k)
 
     v = condition['value']
     op = condition.get('operator')
+    find_key = None
     if not op:
         op = '='
     if op.startswith('!'):
@@ -28,16 +30,27 @@ def parseCondition(condition, user, item=None):
     if isinstance(v, bool): #featured and public flag
         key = k
     else:
+        if k == '*':
+            k = 'find__value'
+        elif k not in ('name_find', 'id', 'user__username', 'type'):
+            find_key = k
+            k = 'find__value'
         key = "%s%s" % (k, {
             '==': '__iexact',
             '^': '__istartswith',
             '$': '__iendswith',
         }.get(op, '__icontains'))
     key = str(key)
-    if exclude:
+    if find_key:
-        q = ~Q(**{key: v})
+        if exclude:
+            q = Q(**{'find__key': find_key, key: v})
+        else:
+            q = Q(**{'find__key': find_key, key: v})
     else:
-        q = Q(**{key: v})
+        if exclude:
+            q = ~Q(**{key: v})
+        else:
+            q = Q(**{key: v})
     return q
 
 def parseConditions(conditions, operator, user, item=None):

pandora/entity/migrations/0004_find.py

@@ -0,0 +1,163 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'Find'
+        db.create_table('entity_find', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('entity', self.gf('django.db.models.fields.related.ForeignKey')(related_name='find', to=orm['entity.Entity'])),
+            ('key', self.gf('django.db.models.fields.CharField')(max_length=200, db_index=True)),
+            ('value', self.gf('django.db.models.fields.TextField')(db_index=True, blank=True)),
+        ))
+        db.send_create_signal('entity', ['Find'])
+
+        # Adding unique constraint on 'Find', fields ['entity', 'key']
+        db.create_unique('entity_find', ['entity_id', 'key'])
+
+        # Update existing entites
+        for e in orm['entity.Entity'].objects.all():
+            e.update_find()
+
+    def backwards(self, orm):
+        # Removing unique constraint on 'Find', fields ['entity', 'key']
+        db.delete_unique('entity_find', ['entity_id', 'key'])
+
+        # Deleting model 'Find'
+        db.delete_table('entity_find')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'document.document': {
+            'Meta': {'unique_together': "(('user', 'name', 'extension'),)", 'object_name': 'Document'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'default': "''"}),
+            'description_sort': ('django.db.models.fields.CharField', [], {'max_length': '512', 'null': 'True'}),
+            'dimensions_sort': ('django.db.models.fields.CharField', [], {'max_length': '512'}),
+            'extension': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'file': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'height': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'items': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'documents'", 'symmetrical': 'False', 'through': "orm['document.ItemProperties']", 'to': "orm['item.Item']"}),
+            'matches': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'name_sort': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}),
+            'oshash': ('django.db.models.fields.CharField', [], {'max_length': '16', 'unique': 'True', 'null': 'True'}),
+            'pages': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'ratio': ('django.db.models.fields.FloatField', [], {'default': '1'}),
+            'size': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'uploading': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'files'", 'to': "orm['auth.User']"}),
+            'width': ('django.db.models.fields.IntegerField', [], {'default': '-1'})
+        },
+        'document.itemproperties': {
+            'Meta': {'unique_together': "(('item', 'document'),)", 'object_name': 'ItemProperties'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'default': "''"}),
+            'description_sort': ('django.db.models.fields.CharField', [], {'max_length': '512', 'null': 'True'}),
+            'document': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'descriptions'", 'to': "orm['document.Document']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'index': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'item': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['item.Item']"}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        },
+        'entity.documentproperties': {
+            'Meta': {'unique_together': "(('entity', 'document'),)", 'object_name': 'DocumentProperties'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'document': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['document.Document']"}),
+            'entity': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'documentproperties'", 'to': "orm['entity.Entity']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'index': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        },
+        'entity.entity': {
+            'Meta': {'unique_together': "(('type', 'name'),)", 'object_name': 'Entity'},
+            'alternativeNames': ('ox.django.fields.TupleField', [], {'default': '[]'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'data': ('ox.django.fields.DictField', [], {'default': '{}'}),
+            'documents': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'entities'", 'symmetrical': 'False', 'through': "orm['entity.DocumentProperties']", 'to': "orm['document.Document']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'matches': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'name_find': ('django.db.models.fields.TextField', [], {'default': "''"}),
+            'name_sort': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}),
+            'type': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'entities'", 'null': 'True', 'to': "orm['auth.User']"})
+        },
+        'entity.find': {
+            'Meta': {'unique_together': "(('entity', 'key'),)", 'object_name': 'Find'},
+            'entity': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'find'", 'to': "orm['entity.Entity']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'key': ('django.db.models.fields.CharField', [], {'max_length': '200', 'db_index': 'True'}),
+            'value': ('django.db.models.fields.TextField', [], {'db_index': 'True', 'blank': 'True'})
+        },
+        'item.item': {
+            'Meta': {'object_name': 'Item'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'data': ('ox.django.fields.DictField', [], {'default': '{}'}),
+            'external_data': ('ox.django.fields.DictField', [], {'default': '{}'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'items'", 'blank': 'True', 'to': "orm['auth.Group']"}),
+            'icon': ('django.db.models.fields.files.ImageField', [], {'default': 'None', 'max_length': '100', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'json': ('ox.django.fields.DictField', [], {'default': '{}'}),
+            'level': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'oxdbId': ('django.db.models.fields.CharField', [], {'max_length': '42', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'poster': ('django.db.models.fields.files.ImageField', [], {'default': 'None', 'max_length': '100', 'blank': 'True'}),
+            'poster_frame': ('django.db.models.fields.FloatField', [], {'default': '-1'}),
+            'poster_height': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'poster_source': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'poster_width': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'public_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'blank': 'True'}),
+            'rendered': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'stream_aspect': ('django.db.models.fields.FloatField', [], {'default': '1.3333333333333333'}),
+            'stream_info': ('ox.django.fields.DictField', [], {'default': '{}'}),
+            'torrent': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '1000', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'items'", 'null': 'True', 'to': "orm['auth.User']"})
+        }
+    }
+
+    complete_apps = ['entity']

pandora/entity/models.py

@@ -5,6 +5,7 @@ import os
 import re
 from glob import glob
 from urllib import quote, unquote
+import unicodedata
 
 from django.db import models, transaction
 from django.db.models import Max
@@ -58,6 +59,7 @@ class Entity(models.Model):
         super(Entity, self).save(*args, **kwargs)
         self.update_matches()
         self.update_annotations()
+        self.update_find()
 
     def __unicode__(self):
         return self.get_id()
@@ -162,6 +164,30 @@ class Entity(models.Model):
     def annotation_value(self):
         #return u'<a href="/entities/%s">%s</a>' % (self.get_id(), ox.escape_html(self.name))
         return ox.escape_html(self.name)
+
+    def update_find(self):
+
+        def save(key, value):
+            if value not in ('', None):
+                f, created = Find.objects.get_or_create(entity=self, key=key)
+                if isinstance(value, bool):
+                    value = value and 'true' or 'false'
+                if isinstance(value, basestring):
+                    value = ox.decode_html(ox.strip_tags(value.strip()))
+                    value = unicodedata.normalize('NFKD', value).lower()
+                f.value = value
+                f.save()
+            else:
+                Find.objects.filter(entity=self, key=key).delete()
+
+        with transaction.commit_on_success():
+            entity = get_by_id(settings.CONFIG['entities'], self.type)
+            for key in entity['keys']:
+                value = self.data.get(key['id'])
+                if isinstance(value, list):
+                    value = u'\n'.join(value)
+                save(key['id'], value)
+
     
     def update_matches(self):
         import annotation.models
@@ -208,3 +234,15 @@ class DocumentProperties(models.Model):
     def save(self, *args, **kwargs):
 
         super(DocumentProperties, self).save(*args, **kwargs)
+
+class Find(models.Model):
+
+    class Meta:
+        unique_together = ("entity", "key")
+
+    entity = models.ForeignKey('Entity', related_name='find', db_index=True)
+    key = models.CharField(max_length=200, db_index=True)
+    value = models.TextField(blank=True, db_index=settings.DB_GIN_TRGM)
+
+    def __unicode__(self):
+        return u"%s=%s" % (self.key, self.value)

pandora/item/management/commands/sqlfindindex.py

@@ -31,15 +31,15 @@ class Command(BaseCommand):
             cursor.execute(sql)
 
         if settings.DB_GIN_TRGM:
-            table_name = models.ItemFind._meta.db_table
+            import entity.models
-            indexes = connection.introspection.get_indexes(cursor, table_name)
+            for table_name, name in (
-            name = 'value'
+                (models.ItemFind._meta.db_table, 'value'),   # Item Find
-            if name not in indexes:
+                (models.Clip._meta.db_table, 'findvalue'),   # Clip Find
-                create_table("%s_%s_idx"%(table_name, name), table_name, name)
+                (entity.models.Find._meta.db_table, 'value'),# Entity Find
-            table_name = models.Clip._meta.db_table
+            ):
-            cursor = connection.cursor()
+                cursor = connection.cursor()
-            indexes = connection.introspection.get_indexes(cursor, table_name)
+                indexes = connection.introspection.get_indexes(cursor, table_name)
-            name = 'findvalue'
+                name = 'value'
-            if name not in indexes:
+                if name not in indexes:
-                create_table("%s_%s_idx"%(table_name, name), table_name, name)
+                    create_table("%s_%s_idx"%(table_name, name), table_name, name)
             transaction.commit_unless_managed()

static/js/entitiesDialog.js

@@ -51,7 +51,7 @@ pandora.ui.entitiesDialog = function(options) {
                 $list.options({
                     query: {
                         conditions: data.value ? [
-                            {key: 'name', operator: '=', value: data.value}
+                            {key: '*', operator: '=', value: data.value}
                         ] : [],
                         operator: '&'
                     }