From 06e89c264b8988edfcd24863120868063c92b70a Mon Sep 17 00:00:00 2001 From: j Date: Tue, 18 Sep 2018 11:11:19 +0200 Subject: [PATCH] Port EntityLink backend --- pandora/entity/managers.py | 27 ++++- .../migrations/0006_auto_20180918_0903.py | 39 ++++++ pandora/entity/models.py | 111 ++++++++++++++++-- pandora/entity/views.py | 34 +++++- 4 files changed, 197 insertions(+), 14 deletions(-) create mode 100644 pandora/entity/migrations/0006_auto_20180918_0903.py diff --git a/pandora/entity/managers.py b/pandora/entity/managers.py index 049813366..3eceb7b7b 100644 --- a/pandora/entity/managers.py +++ b/pandora/entity/managers.py @@ -61,6 +61,31 @@ def buildCondition(k, op, v): key = str(key) if find_key: return Q(**{'find__key': find_key, key: v}) + q = Q(**{'find__key': find_key, key: v}) + + # TODO: support more elaborate predicates on link targets? + # Straw man syntax: + # // entities with 'ABC' in friends: + # {key: 'link:friends', operator: '==', value: 'ABC'} + # // entities whose friends have 'science' in their description + # {key: 'link:friends', operator: '~', value: { + # key: 'description', + # operator: '=', + # value: 'science' + # }} + # // entities listed in PQR's friends + # {key: 'backlink:friends', operator: '==', value: 'PQR'} + # It's a bit tricky without special syntax because at this point we + # don't know what the matching entity type is. + if op == '==': + try: + v_ = ox.fromAZ(v) + except: + pass + else: + q |= Q(links__key=find_key, links__target=v_) + + return q else: return Q(**{key: v}) @@ -118,7 +143,7 @@ class EntityManager(Manager): query.get('operator', '&'), user, item) if conditions: - qs = qs.filter(conditions) + qs = qs.filter(conditions).distinct() return qs diff --git a/pandora/entity/migrations/0006_auto_20180918_0903.py b/pandora/entity/migrations/0006_auto_20180918_0903.py new file mode 100644 index 000000000..3b8b84af4 --- /dev/null +++ b/pandora/entity/migrations/0006_auto_20180918_0903.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.13 on 2018-09-18 09:03 +from __future__ import unicode_literals + +import django.core.serializers.json +from django.db import migrations, models +import django.db.models.deletion +import oxdjango.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('entity', '0005_jsonfield'), + ] + + operations = [ + migrations.CreateModel( + name='Link', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('key', models.CharField(max_length=200)), + ], + ), + migrations.AddField( + model_name='link', + name='source', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='links', to='entity.Entity'), + ), + migrations.AddField( + model_name='link', + name='target', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='backlinks', to='entity.Entity'), + ), + migrations.AlterUniqueTogether( + name='link', + unique_together=set([('source', 'key', 'target')]), + ), + ] diff --git a/pandora/entity/models.py b/pandora/entity/models.py index 80ab9b5e7..bb58cc536 100644 --- a/pandora/entity/models.py +++ b/pandora/entity/models.py @@ -30,6 +30,10 @@ User = get_user_model() @python_2_unicode_compatible class Entity(models.Model): + class ValueError(ValueError): + '''Raised if a field name or value is invalid (based on the "entities" + key in config.jsonc)''' + pass class Meta: unique_together = ("type", "name") @@ -54,7 +58,7 @@ class Entity(models.Model): documents = models.ManyToManyField(Document, through='DocumentProperties', related_name='entities') def save(self, *args, **kwargs): - entity = get_by_id(settings.CONFIG['entities'], self.type) + entity = self.get_entity(self.type) if entity.get('sortType') == 'person' and self.name: if isinstance(self.name, bytes): self.name = self.name.decode('utf-8') @@ -75,6 +79,13 @@ class Entity(models.Model): def get(cls, id): return cls.objects.get(pk=ox.fromAZ(id)) + @classmethod + def get_entity(cls, type_): + e = get_by_id(settings.CONFIG['entities'], type_) + if e is None: + raise cls.ValueError('Unknown entity type {!r}'.format(type_)) + return e + @classmethod def get_by_name(cls, name, type): return cls.objects.get(name_find__contains=u'|%s|' % name.lower(), type=type) @@ -115,7 +126,14 @@ class Entity(models.Model): return False def edit(self, data): - for key in data: + if 'type' in data: + entity = self.get_entity(data['type']) + self.type = data['type'] + else: + entity = self.get_entity(self.type) + + config_keys = {k['id']: k for k in entity['keys']} + for key, value in data.items(): if key == 'name': data['name'] = re.sub(' \[\d+\]$', '', data['name']).strip() if not data['name']: @@ -127,7 +145,7 @@ class Entity(models.Model): name = data['name'] + ' [%d]' % n self.name = name elif key == 'type': - self.type = data[key] + pass elif key == 'alternativeNames': used_names = [self.name.lower()] names = [] @@ -143,6 +161,34 @@ class Entity(models.Model): names.append(name) used_names.append(name.lower()) self.alternativeNames = tuple(ox.escape_html(n) for n in names) + elif key not in config_keys: + raise self.ValueError('Unknown key "{}" for entity type "{}"'.format(key, self.type)) + elif config_keys[key]['type'] == ["entity"]: + n = config_keys[key].get('max') + if n is not None: + value = value[:n] + + es = [] + for d in value: + if not isinstance(d, dict): + raise self.ValueError('"{}" should be [object]'.format(key)) + + try: + if 'id' in d: + es.append(Entity.get(d['id'])) + elif 'name' in d and 'type' in d: + es.append(Entity.get_by_name(d['name'], d['type'])) + else: + raise self.ValueError('"{}" elements should have either "id" or both "name" and "type"'.format(key)) + except Entity.DoesNotExist: + pass # consistent with addDocument when "entity" is a list of IDs + + for e in es: + Link.objects.get_or_create(source=self, key=key, target=e) + + Link.objects.filter(source=self, key=key) \ + .exclude(target__in=es) \ + .delete() else: #FIXME: more data validation if isinstance(data[key], string_types): @@ -150,7 +196,29 @@ class Entity(models.Model): else: self.data[key] = data[key] + def _expand_links(self, keys, back=False): + response = {} + + # Not applying any filters here to allow .prefetch_related() + # on sets of entities. + qs = (self.backlinks if back else self.links).all() + for link in qs: + if link.key in keys: + other = link.source if back else link.target + j = other.json(keys=['id', 'type', 'name', 'sortName']) + key = '-' + link.key if back else link.key + response.setdefault(key, []).append(j) + + for k in response: + response[k].sort(key=lambda j: j['sortName']) + + return response + + def json(self, keys=None, user=None): + entity = self.get_entity(self.type) + config_keys = {k['id']: k for k in entity['keys']} + if not keys: keys = [ 'alternativeNames', @@ -161,8 +229,19 @@ class Entity(models.Model): 'type', 'user', 'documents', - ] + list(self.data) + ] + list(config_keys) + response = {} + + link_keys = { + id + for id, k in config_keys.items() + if id in keys and k['type'] in ('entity', ['entity']) + } + if link_keys: + response.update(self._expand_links(link_keys)) + response.update(self._expand_links(link_keys, back=True)) + for key in keys: if key == 'id': response[key] = self.get_id() @@ -182,6 +261,8 @@ class Entity(models.Model): sort_key = 'document__created' response[key] = [ox.toAZ(id_) for id_, in self.documentproperties.order_by(sort_key).values_list('document_id')] + elif key in link_keys: + pass # expanded above elif key in self.data: response[key] = self.data[key] return response @@ -205,9 +286,7 @@ class Entity(models.Model): else: Find.objects.filter(entity=self, key=key).delete() - entity = get_by_id(settings.CONFIG['entities'], self.type) - if not entity: - return + entity = self.get_entity(self.type) with transaction.atomic(): ids = ['name'] for key in entity['keys']: @@ -232,6 +311,9 @@ class Entity(models.Model): matches = annotation.models.Annotation.objects.filter(layer__in=entity_layers, value=self.get_id()).count() else: matches = 0 + + matches += Link.objects.filter(target=self).count() + for url in urls: matches += annotation.models.Annotation.objects.filter(value__contains=url).count() matches += item.models.Item.objects.filter(data__contains=url).count() @@ -295,3 +377,18 @@ class Find(models.Model): def __str__(self): return u"%s=%s" % (self.key, self.value) + +@python_2_unicode_compatible +class Link(models.Model): + '''Models entity fields of type "entity".''' + + class Meta: + unique_together = ("source", "key", "target") + + source = models.ForeignKey(Entity, related_name='links') + key = models.CharField(max_length=200) + target = models.ForeignKey(Entity, related_name='backlinks') + + def __str__(self): + return u"%s-[%s]->%s" % (self.source, self.key, self.target) + diff --git a/pandora/entity/views.py b/pandora/entity/views.py index 2d562a89d..a42ec38dd 100644 --- a/pandora/entity/views.py +++ b/pandora/entity/views.py @@ -29,6 +29,14 @@ def get_entity_or_404_json(id): 'text': 'Entity not found'}} raise HttpErrorJson(response) +def get_type_or_400_json(type_): + try: + return models.Entity.get_entity(type_) + except models.Entity.ValueError as e: + raise HttpErrorJson(json_response( + status=400, + text=e.message)) + @login_required_json def addEntity(request, data): ''' @@ -45,9 +53,7 @@ def addEntity(request, data): existing_names = [] exists = False - if not utils.get_by_id(settings.CONFIG['entities'], data['type']): - response = json_response(status=500, text='unknown entity type') - return render_to_json_response(response) + get_type_or_400_json(data.get('type')) if 'name' in data: names = [data['name']] + data.get('alternativeNames', []) @@ -113,7 +119,7 @@ def autocompleteEntities(request, data): data['range'] = [0, 10] op = data.get('operator', '=') - entity = utils.get_by_id(settings.CONFIG['entities'], data['key']) + entity = get_type_or_400_json(data['key']) order_by = entity.get('autocompleteSort', False) if order_by: for o in order_by: @@ -157,6 +163,17 @@ def editEntity(request, data): id: string, // entity id name: string, // entity name description: string, // entity description + friends: [ // key with type "entity" + { + // either: + id: string, // related entity ID + + // or: + type: string, // related entity type, and + name: string // related entity name + } + ... // more entities + ] ... // more properties, as defined in config } returns { @@ -168,8 +185,11 @@ def editEntity(request, data): response = json_response() entity = get_entity_or_404_json(data['id']) if entity.editable(request.user): - entity.edit(data) - entity.save() + try: + entity.edit(data) + entity.save() + except models.Entity.ValueError as e: + raise HttpErrorJson(json_response(status=400, text=e.message)) response['data'] = entity.json(user=request.user) add_changelog(request, data) else: @@ -228,6 +248,8 @@ def findEntities(request, data): response = json_response() if 'keys' in data: qs = qs[query['range'][0]:query['range'][1]] + qs = qs.prefetch_related('links', 'links__target') + qs = qs.prefetch_related('backlinks', 'backlinks__source') response['data']['items'] = [l.json(data['keys'], request.user) for l in qs] elif 'position' in data: