Port EntityLink backend

This commit is contained in:
j 2018-09-18 11:11:19 +02:00
parent 1e5d7e99b3
commit 06e89c264b
4 changed files with 197 additions and 14 deletions

View file

@ -61,6 +61,31 @@ def buildCondition(k, op, v):
key = str(key) key = str(key)
if find_key: if find_key:
return Q(**{'find__key': find_key, key: v}) return Q(**{'find__key': find_key, key: v})
q = Q(**{'find__key': find_key, key: v})
# TODO: support more elaborate predicates on link targets?
# Straw man syntax:
# // entities with 'ABC' in friends:
# {key: 'link:friends', operator: '==', value: 'ABC'}
# // entities whose friends have 'science' in their description
# {key: 'link:friends', operator: '~', value: {
# key: 'description',
# operator: '=',
# value: 'science'
# }}
# // entities listed in PQR's friends
# {key: 'backlink:friends', operator: '==', value: 'PQR'}
# It's a bit tricky without special syntax because at this point we
# don't know what the matching entity type is.
if op == '==':
try:
v_ = ox.fromAZ(v)
except:
pass
else:
q |= Q(links__key=find_key, links__target=v_)
return q
else: else:
return Q(**{key: v}) return Q(**{key: v})
@ -118,7 +143,7 @@ class EntityManager(Manager):
query.get('operator', '&'), query.get('operator', '&'),
user, item) user, item)
if conditions: if conditions:
qs = qs.filter(conditions) qs = qs.filter(conditions).distinct()
return qs return qs

View file

@ -0,0 +1,39 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.13 on 2018-09-18 09:03
from __future__ import unicode_literals
import django.core.serializers.json
from django.db import migrations, models
import django.db.models.deletion
import oxdjango.fields
class Migration(migrations.Migration):
dependencies = [
('entity', '0005_jsonfield'),
]
operations = [
migrations.CreateModel(
name='Link',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('key', models.CharField(max_length=200)),
],
),
migrations.AddField(
model_name='link',
name='source',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='links', to='entity.Entity'),
),
migrations.AddField(
model_name='link',
name='target',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='backlinks', to='entity.Entity'),
),
migrations.AlterUniqueTogether(
name='link',
unique_together=set([('source', 'key', 'target')]),
),
]

View file

@ -30,6 +30,10 @@ User = get_user_model()
@python_2_unicode_compatible @python_2_unicode_compatible
class Entity(models.Model): class Entity(models.Model):
class ValueError(ValueError):
'''Raised if a field name or value is invalid (based on the "entities"
key in config.jsonc)'''
pass
class Meta: class Meta:
unique_together = ("type", "name") unique_together = ("type", "name")
@ -54,7 +58,7 @@ class Entity(models.Model):
documents = models.ManyToManyField(Document, through='DocumentProperties', related_name='entities') documents = models.ManyToManyField(Document, through='DocumentProperties', related_name='entities')
def save(self, *args, **kwargs): def save(self, *args, **kwargs):
entity = get_by_id(settings.CONFIG['entities'], self.type) entity = self.get_entity(self.type)
if entity.get('sortType') == 'person' and self.name: if entity.get('sortType') == 'person' and self.name:
if isinstance(self.name, bytes): if isinstance(self.name, bytes):
self.name = self.name.decode('utf-8') self.name = self.name.decode('utf-8')
@ -75,6 +79,13 @@ class Entity(models.Model):
def get(cls, id): def get(cls, id):
return cls.objects.get(pk=ox.fromAZ(id)) return cls.objects.get(pk=ox.fromAZ(id))
@classmethod
def get_entity(cls, type_):
e = get_by_id(settings.CONFIG['entities'], type_)
if e is None:
raise cls.ValueError('Unknown entity type {!r}'.format(type_))
return e
@classmethod @classmethod
def get_by_name(cls, name, type): def get_by_name(cls, name, type):
return cls.objects.get(name_find__contains=u'|%s|' % name.lower(), type=type) return cls.objects.get(name_find__contains=u'|%s|' % name.lower(), type=type)
@ -115,7 +126,14 @@ class Entity(models.Model):
return False return False
def edit(self, data): def edit(self, data):
for key in data: if 'type' in data:
entity = self.get_entity(data['type'])
self.type = data['type']
else:
entity = self.get_entity(self.type)
config_keys = {k['id']: k for k in entity['keys']}
for key, value in data.items():
if key == 'name': if key == 'name':
data['name'] = re.sub(' \[\d+\]$', '', data['name']).strip() data['name'] = re.sub(' \[\d+\]$', '', data['name']).strip()
if not data['name']: if not data['name']:
@ -127,7 +145,7 @@ class Entity(models.Model):
name = data['name'] + ' [%d]' % n name = data['name'] + ' [%d]' % n
self.name = name self.name = name
elif key == 'type': elif key == 'type':
self.type = data[key] pass
elif key == 'alternativeNames': elif key == 'alternativeNames':
used_names = [self.name.lower()] used_names = [self.name.lower()]
names = [] names = []
@ -143,6 +161,34 @@ class Entity(models.Model):
names.append(name) names.append(name)
used_names.append(name.lower()) used_names.append(name.lower())
self.alternativeNames = tuple(ox.escape_html(n) for n in names) self.alternativeNames = tuple(ox.escape_html(n) for n in names)
elif key not in config_keys:
raise self.ValueError('Unknown key "{}" for entity type "{}"'.format(key, self.type))
elif config_keys[key]['type'] == ["entity"]:
n = config_keys[key].get('max')
if n is not None:
value = value[:n]
es = []
for d in value:
if not isinstance(d, dict):
raise self.ValueError('"{}" should be [object]'.format(key))
try:
if 'id' in d:
es.append(Entity.get(d['id']))
elif 'name' in d and 'type' in d:
es.append(Entity.get_by_name(d['name'], d['type']))
else:
raise self.ValueError('"{}" elements should have either "id" or both "name" and "type"'.format(key))
except Entity.DoesNotExist:
pass # consistent with addDocument when "entity" is a list of IDs
for e in es:
Link.objects.get_or_create(source=self, key=key, target=e)
Link.objects.filter(source=self, key=key) \
.exclude(target__in=es) \
.delete()
else: else:
#FIXME: more data validation #FIXME: more data validation
if isinstance(data[key], string_types): if isinstance(data[key], string_types):
@ -150,7 +196,29 @@ class Entity(models.Model):
else: else:
self.data[key] = data[key] self.data[key] = data[key]
def _expand_links(self, keys, back=False):
response = {}
# Not applying any filters here to allow .prefetch_related()
# on sets of entities.
qs = (self.backlinks if back else self.links).all()
for link in qs:
if link.key in keys:
other = link.source if back else link.target
j = other.json(keys=['id', 'type', 'name', 'sortName'])
key = '-' + link.key if back else link.key
response.setdefault(key, []).append(j)
for k in response:
response[k].sort(key=lambda j: j['sortName'])
return response
def json(self, keys=None, user=None): def json(self, keys=None, user=None):
entity = self.get_entity(self.type)
config_keys = {k['id']: k for k in entity['keys']}
if not keys: if not keys:
keys = [ keys = [
'alternativeNames', 'alternativeNames',
@ -161,8 +229,19 @@ class Entity(models.Model):
'type', 'type',
'user', 'user',
'documents', 'documents',
] + list(self.data) ] + list(config_keys)
response = {} response = {}
link_keys = {
id
for id, k in config_keys.items()
if id in keys and k['type'] in ('entity', ['entity'])
}
if link_keys:
response.update(self._expand_links(link_keys))
response.update(self._expand_links(link_keys, back=True))
for key in keys: for key in keys:
if key == 'id': if key == 'id':
response[key] = self.get_id() response[key] = self.get_id()
@ -182,6 +261,8 @@ class Entity(models.Model):
sort_key = 'document__created' sort_key = 'document__created'
response[key] = [ox.toAZ(id_) response[key] = [ox.toAZ(id_)
for id_, in self.documentproperties.order_by(sort_key).values_list('document_id')] for id_, in self.documentproperties.order_by(sort_key).values_list('document_id')]
elif key in link_keys:
pass # expanded above
elif key in self.data: elif key in self.data:
response[key] = self.data[key] response[key] = self.data[key]
return response return response
@ -205,9 +286,7 @@ class Entity(models.Model):
else: else:
Find.objects.filter(entity=self, key=key).delete() Find.objects.filter(entity=self, key=key).delete()
entity = get_by_id(settings.CONFIG['entities'], self.type) entity = self.get_entity(self.type)
if not entity:
return
with transaction.atomic(): with transaction.atomic():
ids = ['name'] ids = ['name']
for key in entity['keys']: for key in entity['keys']:
@ -232,6 +311,9 @@ class Entity(models.Model):
matches = annotation.models.Annotation.objects.filter(layer__in=entity_layers, value=self.get_id()).count() matches = annotation.models.Annotation.objects.filter(layer__in=entity_layers, value=self.get_id()).count()
else: else:
matches = 0 matches = 0
matches += Link.objects.filter(target=self).count()
for url in urls: for url in urls:
matches += annotation.models.Annotation.objects.filter(value__contains=url).count() matches += annotation.models.Annotation.objects.filter(value__contains=url).count()
matches += item.models.Item.objects.filter(data__contains=url).count() matches += item.models.Item.objects.filter(data__contains=url).count()
@ -295,3 +377,18 @@ class Find(models.Model):
def __str__(self): def __str__(self):
return u"%s=%s" % (self.key, self.value) return u"%s=%s" % (self.key, self.value)
@python_2_unicode_compatible
class Link(models.Model):
'''Models entity fields of type "entity".'''
class Meta:
unique_together = ("source", "key", "target")
source = models.ForeignKey(Entity, related_name='links')
key = models.CharField(max_length=200)
target = models.ForeignKey(Entity, related_name='backlinks')
def __str__(self):
return u"%s-[%s]->%s" % (self.source, self.key, self.target)

View file

@ -29,6 +29,14 @@ def get_entity_or_404_json(id):
'text': 'Entity not found'}} 'text': 'Entity not found'}}
raise HttpErrorJson(response) raise HttpErrorJson(response)
def get_type_or_400_json(type_):
try:
return models.Entity.get_entity(type_)
except models.Entity.ValueError as e:
raise HttpErrorJson(json_response(
status=400,
text=e.message))
@login_required_json @login_required_json
def addEntity(request, data): def addEntity(request, data):
''' '''
@ -45,9 +53,7 @@ def addEntity(request, data):
existing_names = [] existing_names = []
exists = False exists = False
if not utils.get_by_id(settings.CONFIG['entities'], data['type']): get_type_or_400_json(data.get('type'))
response = json_response(status=500, text='unknown entity type')
return render_to_json_response(response)
if 'name' in data: if 'name' in data:
names = [data['name']] + data.get('alternativeNames', []) names = [data['name']] + data.get('alternativeNames', [])
@ -113,7 +119,7 @@ def autocompleteEntities(request, data):
data['range'] = [0, 10] data['range'] = [0, 10]
op = data.get('operator', '=') op = data.get('operator', '=')
entity = utils.get_by_id(settings.CONFIG['entities'], data['key']) entity = get_type_or_400_json(data['key'])
order_by = entity.get('autocompleteSort', False) order_by = entity.get('autocompleteSort', False)
if order_by: if order_by:
for o in order_by: for o in order_by:
@ -157,6 +163,17 @@ def editEntity(request, data):
id: string, // entity id id: string, // entity id
name: string, // entity name name: string, // entity name
description: string, // entity description description: string, // entity description
friends: [ // key with type "entity"
{
// either:
id: string, // related entity ID
// or:
type: string, // related entity type, and
name: string // related entity name
}
... // more entities
]
... // more properties, as defined in config ... // more properties, as defined in config
} }
returns { returns {
@ -168,8 +185,11 @@ def editEntity(request, data):
response = json_response() response = json_response()
entity = get_entity_or_404_json(data['id']) entity = get_entity_or_404_json(data['id'])
if entity.editable(request.user): if entity.editable(request.user):
entity.edit(data) try:
entity.save() entity.edit(data)
entity.save()
except models.Entity.ValueError as e:
raise HttpErrorJson(json_response(status=400, text=e.message))
response['data'] = entity.json(user=request.user) response['data'] = entity.json(user=request.user)
add_changelog(request, data) add_changelog(request, data)
else: else:
@ -228,6 +248,8 @@ def findEntities(request, data):
response = json_response() response = json_response()
if 'keys' in data: if 'keys' in data:
qs = qs[query['range'][0]:query['range'][1]] qs = qs[query['range'][0]:query['range'][1]]
qs = qs.prefetch_related('links', 'links__target')
qs = qs.prefetch_related('backlinks', 'backlinks__source')
response['data']['items'] = [l.json(data['keys'], request.user) for l in qs] response['data']['items'] = [l.json(data['keys'], request.user) for l in qs]
elif 'position' in data: elif 'position' in data: