From 00746ed7d900d3dee4d6900d2d5a0b0ebd9cd18c Mon Sep 17 00:00:00 2001 From: Will Thompson Date: Mon, 25 Apr 2016 17:52:58 +0100 Subject: [PATCH 1/3] Entity link UI --- static/js/entitiesDialog.js | 124 +++++++++++++++++++++++++++++++----- 1 file changed, 108 insertions(+), 16 deletions(-) diff --git a/static/js/entitiesDialog.js b/static/js/entitiesDialog.js index e67c49c5..1829ed42 100644 --- a/static/js/entitiesDialog.js +++ b/static/js/entitiesDialog.js @@ -1,5 +1,82 @@ 'use strict'; +(function() { + +// cribbed from documentsPanel.js, TODO: refactor +var entitiesInput = function(width, defaultType, max) { + var labelWidth = 80; + + if (!defaultType) { + defaultType = pandora.site.entities[0].id; + } + + return Ox.ArrayInput({ + input: { + get: function(width) { + var $input = Ox.FormElementGroup({ + elements: [ + Ox.Select({ + items: pandora.site.entities.map(function(entity) { + return { + id: entity.id, + title: entity.title + }; + }), + overlap: 'right', + width: labelWidth + }) + .bindEvent({ + change: function() { + var v = $input.value(); + $input.value({type: v.type, name: ''}); + $input.options('elements')[1].focusInput(); + } + }), + Ox.Input({ + autocomplete: function(value, callback) { + pandora.api.autocompleteEntities({ + key: $input.value().type, + operator: '=', + range: [0, 10], + value: value + }, function(result) { + callback(result.data.items); + }); + }, + autocompleteReplace: true, + autocompleteSelect: true, + autocompleteSelectSubmit: true, + width: width - labelWidth + }) + ], + width: width, + split: function(value) { + return [value.type || defaultType, value.name || '']; + }, + join: function(value) { + return {type: value[0], name: value[1]}; + } + }); + return $input; + }, + getEmpty: function(value) { + var type = (value && value[0]) || defaultType; + return {type: type, name: ''}; + }, + isEmpty: function(value) { + return !value.name; + }, + setWidth: function($input, width) { + $input.options('elements')[1].options({ + width: width - labelWidth + }); + } + }, + width: width, + max: max || 0, + }); +} + pandora.ui.entitiesDialog = function(options) { var dialogHeight = Math.round((window.innerHeight - 48) * 0.9), @@ -372,6 +449,7 @@ pandora.ui.entitiesDialog = function(options) { $form.empty() keys.forEach(function(key, index) { var defaultValue = void 0, + value = result.data[key.id], $label = Ox.Label({ title: Ox._(key.title), width: width @@ -397,29 +475,42 @@ pandora.ui.entitiesDialog = function(options) { height: width, type: 'textarea' }); + } else if (key.type[0] === 'entity') { + $input = entitiesInput(width, type, key.max || 0); + defaultValue = []; + + $input.bindEvent({ + submit: function(data) { + $input.triggerEvent('change', data); + } + }) } + + var change = function(data, eventName) { + console.log(eventName, data); + pandora.api.editEntity(Ox.extend({ + id: id + }, key.id, data.value), function(result) { + Ox.Request.clearCache('findEntities'); + Ox.Request.clearCache('getEntity'); + if (key.id == 'name') { + $input.value(result.data.name); + $list.reloadList(true); + } else if (key.id == 'alternativeNames') { + $input.value(result.data.alternativeNames); + } + renderEntity(); + }); + }; + $input.options({ disabled: key.id == 'id', - value: result.data[key.id] || defaultValue, + value: value || defaultValue, width: width }) .css({margin: '4px'}) .bindEvent({ - change: function(data) { - pandora.api.editEntity(Ox.extend({ - id: id - }, key.id, data.value), function(result) { - Ox.Request.clearCache('findEntities'); - Ox.Request.clearCache('getEntity'); - if (key.id == 'name') { - $input.value(result.data.name); - $list.reloadList(true); - } else if (key.id == 'alternativeNames') { - $input.value(result.data.alternativeNames); - } - renderEntity(); - }); - } + change: change, }) .appendTo($form); $labels.push($label); @@ -467,3 +558,4 @@ pandora.ui.entitiesDialog = function(options) { return that; }; +}()); From 1e5d7e99b362087fc7e8a476fac8dc98377f172c Mon Sep 17 00:00:00 2001 From: j Date: Tue, 18 Sep 2018 11:05:58 +0200 Subject: [PATCH 2/3] merge comment --- pandora/annotation/models.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pandora/annotation/models.py b/pandora/annotation/models.py index 1327f3bd..0e27f848 100644 --- a/pandora/annotation/models.py +++ b/pandora/annotation/models.py @@ -292,6 +292,13 @@ class Annotation(models.Model): def _get_entity_json(self, user=None, entity_cache=None): """When serializing many annotations pointing to the same entity, it is expensive to repeatedly look up and serialize the same entity. + + TODO: if Entity were a (nullable) foreign key of Annotation, we could just: + + prefetch_related('entity', 'entity__user', 'entity__documents') + + before serializing the annotations, which would make self.entity.json(user=user) cheap and + all this unnecessary. """ from entity.models import Entity From 06e89c264b8988edfcd24863120868063c92b70a Mon Sep 17 00:00:00 2001 From: j Date: Tue, 18 Sep 2018 11:11:19 +0200 Subject: [PATCH 3/3] Port EntityLink backend --- pandora/entity/managers.py | 27 ++++- .../migrations/0006_auto_20180918_0903.py | 39 ++++++ pandora/entity/models.py | 111 ++++++++++++++++-- pandora/entity/views.py | 34 +++++- 4 files changed, 197 insertions(+), 14 deletions(-) create mode 100644 pandora/entity/migrations/0006_auto_20180918_0903.py diff --git a/pandora/entity/managers.py b/pandora/entity/managers.py index 04981336..3eceb7b7 100644 --- a/pandora/entity/managers.py +++ b/pandora/entity/managers.py @@ -61,6 +61,31 @@ def buildCondition(k, op, v): key = str(key) if find_key: return Q(**{'find__key': find_key, key: v}) + q = Q(**{'find__key': find_key, key: v}) + + # TODO: support more elaborate predicates on link targets? + # Straw man syntax: + # // entities with 'ABC' in friends: + # {key: 'link:friends', operator: '==', value: 'ABC'} + # // entities whose friends have 'science' in their description + # {key: 'link:friends', operator: '~', value: { + # key: 'description', + # operator: '=', + # value: 'science' + # }} + # // entities listed in PQR's friends + # {key: 'backlink:friends', operator: '==', value: 'PQR'} + # It's a bit tricky without special syntax because at this point we + # don't know what the matching entity type is. + if op == '==': + try: + v_ = ox.fromAZ(v) + except: + pass + else: + q |= Q(links__key=find_key, links__target=v_) + + return q else: return Q(**{key: v}) @@ -118,7 +143,7 @@ class EntityManager(Manager): query.get('operator', '&'), user, item) if conditions: - qs = qs.filter(conditions) + qs = qs.filter(conditions).distinct() return qs diff --git a/pandora/entity/migrations/0006_auto_20180918_0903.py b/pandora/entity/migrations/0006_auto_20180918_0903.py new file mode 100644 index 00000000..3b8b84af --- /dev/null +++ b/pandora/entity/migrations/0006_auto_20180918_0903.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.13 on 2018-09-18 09:03 +from __future__ import unicode_literals + +import django.core.serializers.json +from django.db import migrations, models +import django.db.models.deletion +import oxdjango.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('entity', '0005_jsonfield'), + ] + + operations = [ + migrations.CreateModel( + name='Link', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('key', models.CharField(max_length=200)), + ], + ), + migrations.AddField( + model_name='link', + name='source', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='links', to='entity.Entity'), + ), + migrations.AddField( + model_name='link', + name='target', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='backlinks', to='entity.Entity'), + ), + migrations.AlterUniqueTogether( + name='link', + unique_together=set([('source', 'key', 'target')]), + ), + ] diff --git a/pandora/entity/models.py b/pandora/entity/models.py index 80ab9b5e..bb58cc53 100644 --- a/pandora/entity/models.py +++ b/pandora/entity/models.py @@ -30,6 +30,10 @@ User = get_user_model() @python_2_unicode_compatible class Entity(models.Model): + class ValueError(ValueError): + '''Raised if a field name or value is invalid (based on the "entities" + key in config.jsonc)''' + pass class Meta: unique_together = ("type", "name") @@ -54,7 +58,7 @@ class Entity(models.Model): documents = models.ManyToManyField(Document, through='DocumentProperties', related_name='entities') def save(self, *args, **kwargs): - entity = get_by_id(settings.CONFIG['entities'], self.type) + entity = self.get_entity(self.type) if entity.get('sortType') == 'person' and self.name: if isinstance(self.name, bytes): self.name = self.name.decode('utf-8') @@ -75,6 +79,13 @@ class Entity(models.Model): def get(cls, id): return cls.objects.get(pk=ox.fromAZ(id)) + @classmethod + def get_entity(cls, type_): + e = get_by_id(settings.CONFIG['entities'], type_) + if e is None: + raise cls.ValueError('Unknown entity type {!r}'.format(type_)) + return e + @classmethod def get_by_name(cls, name, type): return cls.objects.get(name_find__contains=u'|%s|' % name.lower(), type=type) @@ -115,7 +126,14 @@ class Entity(models.Model): return False def edit(self, data): - for key in data: + if 'type' in data: + entity = self.get_entity(data['type']) + self.type = data['type'] + else: + entity = self.get_entity(self.type) + + config_keys = {k['id']: k for k in entity['keys']} + for key, value in data.items(): if key == 'name': data['name'] = re.sub(' \[\d+\]$', '', data['name']).strip() if not data['name']: @@ -127,7 +145,7 @@ class Entity(models.Model): name = data['name'] + ' [%d]' % n self.name = name elif key == 'type': - self.type = data[key] + pass elif key == 'alternativeNames': used_names = [self.name.lower()] names = [] @@ -143,6 +161,34 @@ class Entity(models.Model): names.append(name) used_names.append(name.lower()) self.alternativeNames = tuple(ox.escape_html(n) for n in names) + elif key not in config_keys: + raise self.ValueError('Unknown key "{}" for entity type "{}"'.format(key, self.type)) + elif config_keys[key]['type'] == ["entity"]: + n = config_keys[key].get('max') + if n is not None: + value = value[:n] + + es = [] + for d in value: + if not isinstance(d, dict): + raise self.ValueError('"{}" should be [object]'.format(key)) + + try: + if 'id' in d: + es.append(Entity.get(d['id'])) + elif 'name' in d and 'type' in d: + es.append(Entity.get_by_name(d['name'], d['type'])) + else: + raise self.ValueError('"{}" elements should have either "id" or both "name" and "type"'.format(key)) + except Entity.DoesNotExist: + pass # consistent with addDocument when "entity" is a list of IDs + + for e in es: + Link.objects.get_or_create(source=self, key=key, target=e) + + Link.objects.filter(source=self, key=key) \ + .exclude(target__in=es) \ + .delete() else: #FIXME: more data validation if isinstance(data[key], string_types): @@ -150,7 +196,29 @@ class Entity(models.Model): else: self.data[key] = data[key] + def _expand_links(self, keys, back=False): + response = {} + + # Not applying any filters here to allow .prefetch_related() + # on sets of entities. + qs = (self.backlinks if back else self.links).all() + for link in qs: + if link.key in keys: + other = link.source if back else link.target + j = other.json(keys=['id', 'type', 'name', 'sortName']) + key = '-' + link.key if back else link.key + response.setdefault(key, []).append(j) + + for k in response: + response[k].sort(key=lambda j: j['sortName']) + + return response + + def json(self, keys=None, user=None): + entity = self.get_entity(self.type) + config_keys = {k['id']: k for k in entity['keys']} + if not keys: keys = [ 'alternativeNames', @@ -161,8 +229,19 @@ class Entity(models.Model): 'type', 'user', 'documents', - ] + list(self.data) + ] + list(config_keys) + response = {} + + link_keys = { + id + for id, k in config_keys.items() + if id in keys and k['type'] in ('entity', ['entity']) + } + if link_keys: + response.update(self._expand_links(link_keys)) + response.update(self._expand_links(link_keys, back=True)) + for key in keys: if key == 'id': response[key] = self.get_id() @@ -182,6 +261,8 @@ class Entity(models.Model): sort_key = 'document__created' response[key] = [ox.toAZ(id_) for id_, in self.documentproperties.order_by(sort_key).values_list('document_id')] + elif key in link_keys: + pass # expanded above elif key in self.data: response[key] = self.data[key] return response @@ -205,9 +286,7 @@ class Entity(models.Model): else: Find.objects.filter(entity=self, key=key).delete() - entity = get_by_id(settings.CONFIG['entities'], self.type) - if not entity: - return + entity = self.get_entity(self.type) with transaction.atomic(): ids = ['name'] for key in entity['keys']: @@ -232,6 +311,9 @@ class Entity(models.Model): matches = annotation.models.Annotation.objects.filter(layer__in=entity_layers, value=self.get_id()).count() else: matches = 0 + + matches += Link.objects.filter(target=self).count() + for url in urls: matches += annotation.models.Annotation.objects.filter(value__contains=url).count() matches += item.models.Item.objects.filter(data__contains=url).count() @@ -295,3 +377,18 @@ class Find(models.Model): def __str__(self): return u"%s=%s" % (self.key, self.value) + +@python_2_unicode_compatible +class Link(models.Model): + '''Models entity fields of type "entity".''' + + class Meta: + unique_together = ("source", "key", "target") + + source = models.ForeignKey(Entity, related_name='links') + key = models.CharField(max_length=200) + target = models.ForeignKey(Entity, related_name='backlinks') + + def __str__(self): + return u"%s-[%s]->%s" % (self.source, self.key, self.target) + diff --git a/pandora/entity/views.py b/pandora/entity/views.py index 2d562a89..a42ec38d 100644 --- a/pandora/entity/views.py +++ b/pandora/entity/views.py @@ -29,6 +29,14 @@ def get_entity_or_404_json(id): 'text': 'Entity not found'}} raise HttpErrorJson(response) +def get_type_or_400_json(type_): + try: + return models.Entity.get_entity(type_) + except models.Entity.ValueError as e: + raise HttpErrorJson(json_response( + status=400, + text=e.message)) + @login_required_json def addEntity(request, data): ''' @@ -45,9 +53,7 @@ def addEntity(request, data): existing_names = [] exists = False - if not utils.get_by_id(settings.CONFIG['entities'], data['type']): - response = json_response(status=500, text='unknown entity type') - return render_to_json_response(response) + get_type_or_400_json(data.get('type')) if 'name' in data: names = [data['name']] + data.get('alternativeNames', []) @@ -113,7 +119,7 @@ def autocompleteEntities(request, data): data['range'] = [0, 10] op = data.get('operator', '=') - entity = utils.get_by_id(settings.CONFIG['entities'], data['key']) + entity = get_type_or_400_json(data['key']) order_by = entity.get('autocompleteSort', False) if order_by: for o in order_by: @@ -157,6 +163,17 @@ def editEntity(request, data): id: string, // entity id name: string, // entity name description: string, // entity description + friends: [ // key with type "entity" + { + // either: + id: string, // related entity ID + + // or: + type: string, // related entity type, and + name: string // related entity name + } + ... // more entities + ] ... // more properties, as defined in config } returns { @@ -168,8 +185,11 @@ def editEntity(request, data): response = json_response() entity = get_entity_or_404_json(data['id']) if entity.editable(request.user): - entity.edit(data) - entity.save() + try: + entity.edit(data) + entity.save() + except models.Entity.ValueError as e: + raise HttpErrorJson(json_response(status=400, text=e.message)) response['data'] = entity.json(user=request.user) add_changelog(request, data) else: @@ -228,6 +248,8 @@ def findEntities(request, data): response = json_response() if 'keys' in data: qs = qs[query['range'][0]:query['range'][1]] + qs = qs.prefetch_related('links', 'links__target') + qs = qs.prefetch_related('backlinks', 'backlinks__source') response['data']['items'] = [l.json(data['keys'], request.user) for l in qs] elif 'position' in data: