Port EntityLink backend

pandora/annotation/models.py

@@ -292,6 +292,13 @@ class Annotation(models.Model):
     def _get_entity_json(self, user=None, entity_cache=None):
         """When serializing many annotations pointing to the same entity, it is expensive to
         repeatedly look up and serialize the same entity.
+
+        TODO: if Entity were a (nullable) foreign key of Annotation, we could just:
+
+            prefetch_related('entity', 'entity__user', 'entity__documents')
+
+        before serializing the annotations, which would make self.entity.json(user=user) cheap and
+        all this unnecessary.
         """
         from entity.models import Entity
 

pandora/entity/managers.py

@@ -61,6 +61,31 @@ def buildCondition(k, op, v):
     key = str(key)
     if find_key:
         return Q(**{'find__key': find_key, key: v})
+        q = Q(**{'find__key': find_key, key: v})
+
+        # TODO: support more elaborate predicates on link targets?
+        # Straw man syntax:
+        #   // entities with 'ABC' in friends:
+        #   {key: 'link:friends', operator: '==', value: 'ABC'}
+        #   // entities whose friends have 'science' in their description
+        #   {key: 'link:friends', operator: '~', value: {
+        #       key: 'description',
+        #       operator: '=',
+        #       value: 'science'
+        #   }}
+        #   // entities listed in PQR's friends
+        #   {key: 'backlink:friends', operator: '==', value: 'PQR'}
+        # It's a bit tricky without special syntax because at this point we
+        # don't know what the matching entity type is.
+        if op == '==':
+            try:
+                v_ = ox.fromAZ(v)
+            except:
+                pass
+            else:
+                q |= Q(links__key=find_key, links__target=v_)
+
+        return q
     else:
         return Q(**{key: v})
 
@@ -118,7 +143,7 @@ class EntityManager(Manager):
                                      query.get('operator', '&'),
                                      user, item)
         if conditions:
-            qs = qs.filter(conditions)
+            qs = qs.filter(conditions).distinct()
 
         return qs
 

pandora/entity/migrations/0006_auto_20180918_0903.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.13 on 2018-09-18 09:03
+from __future__ import unicode_literals
+
+import django.core.serializers.json
+from django.db import migrations, models
+import django.db.models.deletion
+import oxdjango.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('entity', '0005_jsonfield'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Link',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('key', models.CharField(max_length=200)),
+            ],
+        ),
+        migrations.AddField(
+            model_name='link',
+            name='source',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='links', to='entity.Entity'),
+        ),
+        migrations.AddField(
+            model_name='link',
+            name='target',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='backlinks', to='entity.Entity'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='link',
+            unique_together=set([('source', 'key', 'target')]),
+        ),
+    ]

pandora/entity/models.py

@@ -30,6 +30,10 @@ User = get_user_model()
 
 @python_2_unicode_compatible
 class Entity(models.Model):
+    class ValueError(ValueError):
+        '''Raised if a field name or value is invalid (based on the "entities"
+        key in config.jsonc)'''
+        pass
 
     class Meta:
         unique_together = ("type", "name")
@@ -54,7 +58,7 @@ class Entity(models.Model):
     documents = models.ManyToManyField(Document, through='DocumentProperties', related_name='entities')
 
     def save(self, *args, **kwargs):
-        entity = get_by_id(settings.CONFIG['entities'], self.type)
+        entity = self.get_entity(self.type)
         if entity.get('sortType') == 'person' and self.name:
             if isinstance(self.name, bytes):
                 self.name = self.name.decode('utf-8')
@@ -75,6 +79,13 @@ class Entity(models.Model):
     def get(cls, id):
         return cls.objects.get(pk=ox.fromAZ(id))
 
+    @classmethod
+    def get_entity(cls, type_):
+        e = get_by_id(settings.CONFIG['entities'], type_)
+        if e is None:
+            raise cls.ValueError('Unknown entity type {!r}'.format(type_))
+        return e
+
     @classmethod
     def get_by_name(cls, name, type):
         return cls.objects.get(name_find__contains=u'|%s|' % name.lower(), type=type)
@@ -115,7 +126,14 @@ class Entity(models.Model):
         return False
 
     def edit(self, data):
-        for key in data:
+        if 'type' in data:
+            entity = self.get_entity(data['type'])
+            self.type = data['type']
+        else:
+            entity = self.get_entity(self.type)
+
+        config_keys = {k['id']: k for k in entity['keys']}
+        for key, value in data.items():
             if key == 'name':
                 data['name'] = re.sub(' \[\d+\]$', '', data['name']).strip()
                 if not data['name']:
@@ -127,7 +145,7 @@ class Entity(models.Model):
                     name = data['name'] + ' [%d]' % n
                 self.name = name
             elif key == 'type':
-                self.type = data[key]
+                pass
             elif key == 'alternativeNames':
                 used_names = [self.name.lower()]
                 names = []
@@ -143,6 +161,34 @@ class Entity(models.Model):
                     names.append(name)
                     used_names.append(name.lower())
                 self.alternativeNames = tuple(ox.escape_html(n) for n in names)
+            elif key not in config_keys:
+                raise self.ValueError('Unknown key "{}" for entity type "{}"'.format(key, self.type))
+            elif config_keys[key]['type'] == ["entity"]:
+                n = config_keys[key].get('max')
+                if n is not None:
+                    value = value[:n]
+
+                es = []
+                for d in value:
+                    if not isinstance(d, dict):
+                        raise self.ValueError('"{}" should be [object]'.format(key))
+
+                    try:
+                        if 'id' in d:
+                            es.append(Entity.get(d['id']))
+                        elif 'name' in d and 'type' in d:
+                            es.append(Entity.get_by_name(d['name'], d['type']))
+                        else:
+                            raise self.ValueError('"{}" elements should have either "id" or both "name" and "type"'.format(key))
+                    except Entity.DoesNotExist:
+                        pass  # consistent with addDocument when "entity" is a list of IDs
+
+                for e in es:
+                    Link.objects.get_or_create(source=self, key=key, target=e)
+
+                Link.objects.filter(source=self, key=key) \
+                    .exclude(target__in=es) \
+                    .delete()
             else:
                 #FIXME: more data validation
                 if isinstance(data[key], string_types):
@@ -150,7 +196,29 @@ class Entity(models.Model):
                 else:
                     self.data[key] = data[key]
 
+    def _expand_links(self, keys, back=False):
+        response = {}
+
+        # Not applying any filters here to allow .prefetch_related()
+        # on sets of entities.
+        qs = (self.backlinks if back else self.links).all()
+        for link in qs:
+            if link.key in keys:
+                other = link.source if back else link.target
+                j = other.json(keys=['id', 'type', 'name', 'sortName'])
+                key = '-' + link.key if back else link.key
+                response.setdefault(key, []).append(j)
+
+        for k in response:
+            response[k].sort(key=lambda j: j['sortName'])
+
+        return response
+
+
     def json(self, keys=None, user=None):
+        entity = self.get_entity(self.type)
+        config_keys = {k['id']: k for k in entity['keys']}
+
         if not keys:
             keys = [
                 'alternativeNames',
@@ -161,8 +229,19 @@ class Entity(models.Model):
                 'type',
                 'user',
                 'documents',
-            ] + list(self.data)
+            ] + list(config_keys)
+
         response = {}
+
+        link_keys = {
+            id
+            for id, k in config_keys.items()
+            if id in keys and k['type'] in ('entity', ['entity'])
+        }
+        if link_keys:
+            response.update(self._expand_links(link_keys))
+            response.update(self._expand_links(link_keys, back=True))
+
         for key in keys:
             if key == 'id':
                 response[key] = self.get_id()
@@ -182,6 +261,8 @@ class Entity(models.Model):
                     sort_key = 'document__created'
                 response[key] = [ox.toAZ(id_)
                     for id_, in self.documentproperties.order_by(sort_key).values_list('document_id')]
+            elif key in link_keys:
+                pass # expanded above
             elif key in self.data:
                 response[key] = self.data[key]
         return response
@@ -205,9 +286,7 @@ class Entity(models.Model):
             else:
                 Find.objects.filter(entity=self, key=key).delete()
 
-        entity = get_by_id(settings.CONFIG['entities'], self.type)
+        entity = self.get_entity(self.type)
-        if not entity:
-            return
         with transaction.atomic():
             ids = ['name']
             for key in entity['keys']:
@@ -232,6 +311,9 @@ class Entity(models.Model):
             matches = annotation.models.Annotation.objects.filter(layer__in=entity_layers, value=self.get_id()).count()
         else:
             matches = 0
+
+        matches += Link.objects.filter(target=self).count()
+
         for url in urls:
             matches += annotation.models.Annotation.objects.filter(value__contains=url).count()
             matches += item.models.Item.objects.filter(data__contains=url).count()
@@ -295,3 +377,18 @@ class Find(models.Model):
 
     def __str__(self):
         return u"%s=%s" % (self.key, self.value)
+
+@python_2_unicode_compatible
+class Link(models.Model):
+    '''Models entity fields of type "entity".'''
+
+    class Meta:
+        unique_together = ("source", "key", "target")
+
+    source = models.ForeignKey(Entity, related_name='links')
+    key = models.CharField(max_length=200)
+    target = models.ForeignKey(Entity, related_name='backlinks')
+
+    def __str__(self):
+        return u"%s-[%s]->%s" % (self.source, self.key, self.target)
+

pandora/entity/views.py

@@ -29,6 +29,14 @@ def get_entity_or_404_json(id):
                                'text': 'Entity not found'}}
         raise HttpErrorJson(response)
 
+def get_type_or_400_json(type_):
+    try:
+        return models.Entity.get_entity(type_)
+    except models.Entity.ValueError as e:
+        raise HttpErrorJson(json_response(
+            status=400,
+            text=e.message))
+
 @login_required_json
 def addEntity(request, data):
     '''
@@ -45,9 +53,7 @@ def addEntity(request, data):
     existing_names = []
     exists = False
 
-    if not utils.get_by_id(settings.CONFIG['entities'], data['type']):
+    get_type_or_400_json(data.get('type'))
-        response = json_response(status=500, text='unknown entity type')
-        return render_to_json_response(response)
 
     if 'name' in data:
         names = [data['name']] + data.get('alternativeNames', [])
@@ -113,7 +119,7 @@ def autocompleteEntities(request, data):
         data['range'] = [0, 10]
     op = data.get('operator', '=')
 
-    entity = utils.get_by_id(settings.CONFIG['entities'], data['key'])
+    entity = get_type_or_400_json(data['key'])
     order_by = entity.get('autocompleteSort', False)
     if order_by:
         for o in order_by:
@@ -157,6 +163,17 @@ def editEntity(request, data):
         id: string, // entity id
         name: string, // entity name
         description: string, // entity description
+        friends: [ // key with type "entity"
+            {
+                // either:
+                id: string, // related entity ID
+
+                // or:
+                type: string, // related entity type, and
+                name: string // related entity name
+            }
+            ... // more entities
+        ]
         ... // more properties, as defined in config
     }
     returns {
@@ -168,8 +185,11 @@ def editEntity(request, data):
     response = json_response()
     entity = get_entity_or_404_json(data['id'])
     if entity.editable(request.user):
+        try:
             entity.edit(data)
             entity.save()
+        except models.Entity.ValueError as e:
+            raise HttpErrorJson(json_response(status=400, text=e.message))
         response['data'] = entity.json(user=request.user)
         add_changelog(request, data)
     else:
@@ -228,6 +248,8 @@ def findEntities(request, data):
     response = json_response()
     if 'keys' in data:
         qs = qs[query['range'][0]:query['range'][1]]
+        qs = qs.prefetch_related('links', 'links__target')
+        qs = qs.prefetch_related('backlinks', 'backlinks__source')
 
         response['data']['items'] = [l.json(data['keys'], request.user) for l in qs]
     elif 'position' in data:

static/js/entitiesDialog.js

@@ -1,5 +1,82 @@
 'use strict';
 
+(function() {
+
+// cribbed from documentsPanel.js, TODO: refactor
+var entitiesInput = function(width, defaultType, max) {
+    var labelWidth = 80;
+
+    if (!defaultType) {
+        defaultType = pandora.site.entities[0].id;
+    }
+
+    return Ox.ArrayInput({
+            input: {
+                get: function(width) {
+                    var $input = Ox.FormElementGroup({
+                        elements: [
+                            Ox.Select({
+                                items: pandora.site.entities.map(function(entity) {
+                                    return {
+                                        id: entity.id,
+                                        title: entity.title
+                                    };
+                                }),
+                                overlap: 'right',
+                                width: labelWidth
+                            })
+                            .bindEvent({
+                                change: function() {
+                                    var v = $input.value();
+                                    $input.value({type: v.type, name: ''});
+                                    $input.options('elements')[1].focusInput();
+                                }
+                            }),
+                            Ox.Input({
+                                autocomplete: function(value, callback) {
+                                    pandora.api.autocompleteEntities({
+                                        key: $input.value().type,
+                                        operator: '=',
+                                        range: [0, 10],
+                                        value: value
+                                    }, function(result) {
+                                        callback(result.data.items);
+                                    });
+                                },
+                                autocompleteReplace: true,
+                                autocompleteSelect: true,
+                                autocompleteSelectSubmit: true,
+                                width: width - labelWidth
+                            })
+                        ],
+                        width: width,
+                        split: function(value) {
+                            return [value.type || defaultType, value.name || ''];
+                        },
+                        join: function(value) {
+                            return {type: value[0], name: value[1]};
+                        }
+                    });
+                    return $input;
+                },
+                getEmpty: function(value) {
+                    var type = (value && value[0]) || defaultType;
+                    return {type: type, name: ''};
+                },
+                isEmpty: function(value) {
+                    return !value.name;
+                },
+                setWidth: function($input, width) {
+                    $input.options('elements')[1].options({
+                        width: width - labelWidth
+                    });
+                }
+            },
+            width: width,
+            max: max || 0,
+        });
+}
+
 pandora.ui.entitiesDialog = function(options) {
 
     var dialogHeight = Math.round((window.innerHeight - 48) * 0.9),
@@ -372,6 +449,7 @@ pandora.ui.entitiesDialog = function(options) {
             $form.empty()
             keys.forEach(function(key, index) {
                 var defaultValue = void 0,
+                    value = result.data[key.id],
                     $label = Ox.Label({
                             title: Ox._(key.title),
                             width: width
@@ -397,15 +475,19 @@ pandora.ui.entitiesDialog = function(options) {
                         height: width,
                         type: 'textarea'
                     });
+                } else if (key.type[0] === 'entity') {
+                    $input = entitiesInput(width, type, key.max || 0);
+                    defaultValue = [];
+
+                    $input.bindEvent({
+                        submit: function(data) {
+                            $input.triggerEvent('change', data);
                         }
-                $input.options({
-                        disabled: key.id == 'id',
-                        value: result.data[key.id] || defaultValue,
-                        width: width
                     })
-                    .css({margin: '4px'})
+                }
-                    .bindEvent({
+
-                        change: function(data) {
+                var change = function(data, eventName) {
+                    console.log(eventName, data);
                     pandora.api.editEntity(Ox.extend({
                         id: id
                     }, key.id, data.value), function(result) {
@@ -419,7 +501,16 @@ pandora.ui.entitiesDialog = function(options) {
                         }
                         renderEntity();
                     });
-                        }
+                };
+
+                $input.options({
+                        disabled: key.id == 'id',
+                        value: value || defaultValue,
+                        width: width
+                    })
+                    .css({margin: '4px'})
+                    .bindEvent({
+                        change: change,
                     })
                     .appendTo($form);
                 $labels.push($label);
@@ -467,3 +558,4 @@ pandora.ui.entitiesDialog = function(options) {
     return that;
 
 };
+}());