Port EntityLink backend

pandora/entity/managers.py

@@ -61,6 +61,31 @@ def buildCondition(k, op, v):
     key = str(key)
     if find_key:
         return Q(**{'find__key': find_key, key: v})
+        q = Q(**{'find__key': find_key, key: v})
+
+        # TODO: support more elaborate predicates on link targets?
+        # Straw man syntax:
+        #   // entities with 'ABC' in friends:
+        #   {key: 'link:friends', operator: '==', value: 'ABC'}
+        #   // entities whose friends have 'science' in their description
+        #   {key: 'link:friends', operator: '~', value: {
+        #       key: 'description',
+        #       operator: '=',
+        #       value: 'science'
+        #   }}
+        #   // entities listed in PQR's friends
+        #   {key: 'backlink:friends', operator: '==', value: 'PQR'}
+        # It's a bit tricky without special syntax because at this point we
+        # don't know what the matching entity type is.
+        if op == '==':
+            try:
+                v_ = ox.fromAZ(v)
+            except:
+                pass
+            else:
+                q |= Q(links__key=find_key, links__target=v_)
+
+        return q
     else:
         return Q(**{key: v})
 
@@ -118,7 +143,7 @@ class EntityManager(Manager):
                                      query.get('operator', '&'),
                                      user, item)
         if conditions:
-            qs = qs.filter(conditions)
+            qs = qs.filter(conditions).distinct()
 
         return qs
 

pandora/entity/migrations/0006_auto_20180918_0903.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.13 on 2018-09-18 09:03
+from __future__ import unicode_literals
+
+import django.core.serializers.json
+from django.db import migrations, models
+import django.db.models.deletion
+import oxdjango.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('entity', '0005_jsonfield'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Link',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('key', models.CharField(max_length=200)),
+            ],
+        ),
+        migrations.AddField(
+            model_name='link',
+            name='source',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='links', to='entity.Entity'),
+        ),
+        migrations.AddField(
+            model_name='link',
+            name='target',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='backlinks', to='entity.Entity'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='link',
+            unique_together=set([('source', 'key', 'target')]),
+        ),
+    ]

pandora/entity/models.py

@@ -30,6 +30,10 @@ User = get_user_model()
 
 @python_2_unicode_compatible
 class Entity(models.Model):
+    class ValueError(ValueError):
+        '''Raised if a field name or value is invalid (based on the "entities"
+        key in config.jsonc)'''
+        pass
 
     class Meta:
         unique_together = ("type", "name")
@@ -54,7 +58,7 @@ class Entity(models.Model):
     documents = models.ManyToManyField(Document, through='DocumentProperties', related_name='entities')
 
     def save(self, *args, **kwargs):
-        entity = get_by_id(settings.CONFIG['entities'], self.type)
+        entity = self.get_entity(self.type)
         if entity.get('sortType') == 'person' and self.name:
             if isinstance(self.name, bytes):
                 self.name = self.name.decode('utf-8')
@@ -75,6 +79,13 @@ class Entity(models.Model):
     def get(cls, id):
         return cls.objects.get(pk=ox.fromAZ(id))
 
+    @classmethod
+    def get_entity(cls, type_):
+        e = get_by_id(settings.CONFIG['entities'], type_)
+        if e is None:
+            raise cls.ValueError('Unknown entity type {!r}'.format(type_))
+        return e
+
     @classmethod
     def get_by_name(cls, name, type):
         return cls.objects.get(name_find__contains=u'|%s|' % name.lower(), type=type)
@@ -115,7 +126,14 @@ class Entity(models.Model):
         return False
 
     def edit(self, data):
-        for key in data:
+        if 'type' in data:
+            entity = self.get_entity(data['type'])
+            self.type = data['type']
+        else:
+            entity = self.get_entity(self.type)
+
+        config_keys = {k['id']: k for k in entity['keys']}
+        for key, value in data.items():
             if key == 'name':
                 data['name'] = re.sub(' \[\d+\]$', '', data['name']).strip()
                 if not data['name']:
@@ -127,7 +145,7 @@ class Entity(models.Model):
                     name = data['name'] + ' [%d]' % n
                 self.name = name
             elif key == 'type':
-                self.type = data[key]
+                pass
             elif key == 'alternativeNames':
                 used_names = [self.name.lower()]
                 names = []
@@ -143,6 +161,34 @@ class Entity(models.Model):
                     names.append(name)
                     used_names.append(name.lower())
                 self.alternativeNames = tuple(ox.escape_html(n) for n in names)
+            elif key not in config_keys:
+                raise self.ValueError('Unknown key "{}" for entity type "{}"'.format(key, self.type))
+            elif config_keys[key]['type'] == ["entity"]:
+                n = config_keys[key].get('max')
+                if n is not None:
+                    value = value[:n]
+
+                es = []
+                for d in value:
+                    if not isinstance(d, dict):
+                        raise self.ValueError('"{}" should be [object]'.format(key))
+
+                    try:
+                        if 'id' in d:
+                            es.append(Entity.get(d['id']))
+                        elif 'name' in d and 'type' in d:
+                            es.append(Entity.get_by_name(d['name'], d['type']))
+                        else:
+                            raise self.ValueError('"{}" elements should have either "id" or both "name" and "type"'.format(key))
+                    except Entity.DoesNotExist:
+                        pass  # consistent with addDocument when "entity" is a list of IDs
+
+                for e in es:
+                    Link.objects.get_or_create(source=self, key=key, target=e)
+
+                Link.objects.filter(source=self, key=key) \
+                    .exclude(target__in=es) \
+                    .delete()
             else:
                 #FIXME: more data validation
                 if isinstance(data[key], string_types):
@@ -150,7 +196,29 @@ class Entity(models.Model):
                 else:
                     self.data[key] = data[key]
 
+    def _expand_links(self, keys, back=False):
+        response = {}
+
+        # Not applying any filters here to allow .prefetch_related()
+        # on sets of entities.
+        qs = (self.backlinks if back else self.links).all()
+        for link in qs:
+            if link.key in keys:
+                other = link.source if back else link.target
+                j = other.json(keys=['id', 'type', 'name', 'sortName'])
+                key = '-' + link.key if back else link.key
+                response.setdefault(key, []).append(j)
+
+        for k in response:
+            response[k].sort(key=lambda j: j['sortName'])
+
+        return response
+
+
     def json(self, keys=None, user=None):
+        entity = self.get_entity(self.type)
+        config_keys = {k['id']: k for k in entity['keys']}
+
         if not keys:
             keys = [
                 'alternativeNames',
@@ -161,8 +229,19 @@ class Entity(models.Model):
                 'type',
                 'user',
                 'documents',
-            ] + list(self.data)
+            ] + list(config_keys)
+
         response = {}
+
+        link_keys = {
+            id
+            for id, k in config_keys.items()
+            if id in keys and k['type'] in ('entity', ['entity'])
+        }
+        if link_keys:
+            response.update(self._expand_links(link_keys))
+            response.update(self._expand_links(link_keys, back=True))
+
         for key in keys:
             if key == 'id':
                 response[key] = self.get_id()
@@ -182,6 +261,8 @@ class Entity(models.Model):
                     sort_key = 'document__created'
                 response[key] = [ox.toAZ(id_)
                     for id_, in self.documentproperties.order_by(sort_key).values_list('document_id')]
+            elif key in link_keys:
+                pass # expanded above
             elif key in self.data:
                 response[key] = self.data[key]
         return response
@@ -205,9 +286,7 @@ class Entity(models.Model):
             else:
                 Find.objects.filter(entity=self, key=key).delete()
 
-        entity = get_by_id(settings.CONFIG['entities'], self.type)
+        entity = self.get_entity(self.type)
-        if not entity:
-            return
         with transaction.atomic():
             ids = ['name']
             for key in entity['keys']:
@@ -232,6 +311,9 @@ class Entity(models.Model):
             matches = annotation.models.Annotation.objects.filter(layer__in=entity_layers, value=self.get_id()).count()
         else:
             matches = 0
+
+        matches += Link.objects.filter(target=self).count()
+
         for url in urls:
             matches += annotation.models.Annotation.objects.filter(value__contains=url).count()
             matches += item.models.Item.objects.filter(data__contains=url).count()
@@ -295,3 +377,18 @@ class Find(models.Model):
 
     def __str__(self):
         return u"%s=%s" % (self.key, self.value)
+
+@python_2_unicode_compatible
+class Link(models.Model):
+    '''Models entity fields of type "entity".'''
+
+    class Meta:
+        unique_together = ("source", "key", "target")
+
+    source = models.ForeignKey(Entity, related_name='links')
+    key = models.CharField(max_length=200)
+    target = models.ForeignKey(Entity, related_name='backlinks')
+
+    def __str__(self):
+        return u"%s-[%s]->%s" % (self.source, self.key, self.target)
+

pandora/entity/views.py

@@ -29,6 +29,14 @@ def get_entity_or_404_json(id):
                                'text': 'Entity not found'}}
         raise HttpErrorJson(response)
 
+def get_type_or_400_json(type_):
+    try:
+        return models.Entity.get_entity(type_)
+    except models.Entity.ValueError as e:
+        raise HttpErrorJson(json_response(
+            status=400,
+            text=e.message))
+
 @login_required_json
 def addEntity(request, data):
     '''
@@ -45,9 +53,7 @@ def addEntity(request, data):
     existing_names = []
     exists = False
 
-    if not utils.get_by_id(settings.CONFIG['entities'], data['type']):
+    get_type_or_400_json(data.get('type'))
-        response = json_response(status=500, text='unknown entity type')
-        return render_to_json_response(response)
 
     if 'name' in data:
         names = [data['name']] + data.get('alternativeNames', [])
@@ -113,7 +119,7 @@ def autocompleteEntities(request, data):
         data['range'] = [0, 10]
     op = data.get('operator', '=')
 
-    entity = utils.get_by_id(settings.CONFIG['entities'], data['key'])
+    entity = get_type_or_400_json(data['key'])
     order_by = entity.get('autocompleteSort', False)
     if order_by:
         for o in order_by:
@@ -157,6 +163,17 @@ def editEntity(request, data):
         id: string, // entity id
         name: string, // entity name
         description: string, // entity description
+        friends: [ // key with type "entity"
+            {
+                // either:
+                id: string, // related entity ID
+
+                // or:
+                type: string, // related entity type, and
+                name: string // related entity name
+            }
+            ... // more entities
+        ]
         ... // more properties, as defined in config
     }
     returns {
@@ -168,8 +185,11 @@ def editEntity(request, data):
     response = json_response()
     entity = get_entity_or_404_json(data['id'])
     if entity.editable(request.user):
+        try:
             entity.edit(data)
             entity.save()
+        except models.Entity.ValueError as e:
+            raise HttpErrorJson(json_response(status=400, text=e.message))
         response['data'] = entity.json(user=request.user)
         add_changelog(request, data)
     else:
@@ -228,6 +248,8 @@ def findEntities(request, data):
     response = json_response()
     if 'keys' in data:
         qs = qs[query['range'][0]:query['range'][1]]
+        qs = qs.prefetch_related('links', 'links__target')
+        qs = qs.prefetch_related('backlinks', 'backlinks__source')
 
         response['data']['items'] = [l.json(data['keys'], request.user) for l in qs]
     elif 'position' in data: