0x2620/pandora
2
1
Fork 1
Code Issues 1.3k Pull requests Releases Wiki Activity

Port EntityLink backend

Browse source
This commit is contained in:
j 2018-09-18 11:11:19 +02:00
parent 1e5d7e99b3
commit 06e89c264b
4 changed files with 197 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
pandora/entity/managers.py
View file

@ -61,6 +61,31 @@ def buildCondition(k, op, v):
key = str(key)
if find_key:
return Q(**{'find__key': find_key, key: v})
q = Q(**{'find__key': find_key, key: v})
# TODO: support more elaborate predicates on link targets?
# Straw man syntax:
# // entities with 'ABC' in friends:
# {key: 'link:friends', operator: '==', value: 'ABC'}
# // entities whose friends have 'science' in their description
# {key: 'link:friends', operator: '~', value: {
# key: 'description',
# operator: '=',
# value: 'science'
# }}
# // entities listed in PQR's friends
# {key: 'backlink:friends', operator: '==', value: 'PQR'}
# It's a bit tricky without special syntax because at this point we
# don't know what the matching entity type is.
if op == '==':
try:
v_ = ox.fromAZ(v)
except:
pass
else:
q |= Q(links__key=find_key, links__target=v_)
return q
else:
return Q(**{key: v})
@ -118,7 +143,7 @@ class EntityManager(Manager):
query.get('operator', '&'),
user, item)
if conditions:
qs = qs.filter(conditions)
qs = qs.filter(conditions).distinct()
return qs

39
pandora/entity/migrations/0006_auto_20180918_0903.py Normal file
View file

@ -0,0 +1,39 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.13 on 2018-09-18 09:03
from __future__ import unicode_literals
import django.core.serializers.json
from django.db import migrations, models
import django.db.models.deletion
import oxdjango.fields
class Migration(migrations.Migration):
dependencies = [
('entity', '0005_jsonfield'),
]
operations = [
migrations.CreateModel(
name='Link',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('key', models.CharField(max_length=200)),
],
),
migrations.AddField(
model_name='link',
name='source',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='links', to='entity.Entity'),
),
migrations.AddField(
model_name='link',
name='target',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='backlinks', to='entity.Entity'),
),
migrations.AlterUniqueTogether(
name='link',
unique_together=set([('source', 'key', 'target')]),
),
]

111
pandora/entity/models.py
View file

@ -30,6 +30,10 @@ User = get_user_model()
@python_2_unicode_compatible
class Entity(models.Model):
class ValueError(ValueError):
'''Raised if a field name or value is invalid (based on the "entities"
key in config.jsonc)'''
pass
class Meta:
unique_together = ("type", "name")
@ -54,7 +58,7 @@ class Entity(models.Model):
documents = models.ManyToManyField(Document, through='DocumentProperties', related_name='entities')
def save(self, *args, **kwargs):
entity = get_by_id(settings.CONFIG['entities'], self.type)
entity = self.get_entity(self.type)
if entity.get('sortType') == 'person' and self.name:
if isinstance(self.name, bytes):
self.name = self.name.decode('utf-8')
@ -75,6 +79,13 @@ class Entity(models.Model):
def get(cls, id):
return cls.objects.get(pk=ox.fromAZ(id))
@classmethod
def get_entity(cls, type_):
e = get_by_id(settings.CONFIG['entities'], type_)
if e is None:
raise cls.ValueError('Unknown entity type {!r}'.format(type_))
return e
@classmethod
def get_by_name(cls, name, type):
return cls.objects.get(name_find__contains=u'|%s|' % name.lower(), type=type)
@ -115,7 +126,14 @@ class Entity(models.Model):
return False
def edit(self, data):
for key in data:
if 'type' in data:
entity = self.get_entity(data['type'])
self.type = data['type']
else:
entity = self.get_entity(self.type)
config_keys = {k['id']: k for k in entity['keys']}
for key, value in data.items():
if key == 'name':
data['name'] = re.sub(' \[\d+\]$', '', data['name']).strip()
if not data['name']:
@ -127,7 +145,7 @@ class Entity(models.Model):
name = data['name'] + ' [%d]' % n
self.name = name
elif key == 'type':
self.type = data[key]
pass
elif key == 'alternativeNames':
used_names = [self.name.lower()]
names = []
@ -143,6 +161,34 @@ class Entity(models.Model):
names.append(name)
used_names.append(name.lower())
self.alternativeNames = tuple(ox.escape_html(n) for n in names)
elif key not in config_keys:
raise self.ValueError('Unknown key "{}" for entity type "{}"'.format(key, self.type))
elif config_keys[key]['type'] == ["entity"]:
n = config_keys[key].get('max')
if n is not None:
value = value[:n]
es = []
for d in value:
if not isinstance(d, dict):
raise self.ValueError('"{}" should be [object]'.format(key))
try:
if 'id' in d:
es.append(Entity.get(d['id']))
elif 'name' in d and 'type' in d:
es.append(Entity.get_by_name(d['name'], d['type']))
else:
raise self.ValueError('"{}" elements should have either "id" or both "name" and "type"'.format(key))
except Entity.DoesNotExist:
pass # consistent with addDocument when "entity" is a list of IDs
for e in es:
Link.objects.get_or_create(source=self, key=key, target=e)
Link.objects.filter(source=self, key=key) \
.exclude(target__in=es) \
.delete()
else:
#FIXME: more data validation
if isinstance(data[key], string_types):
@ -150,7 +196,29 @@ class Entity(models.Model):
else:
self.data[key] = data[key]
def _expand_links(self, keys, back=False):
response = {}
# Not applying any filters here to allow .prefetch_related()
# on sets of entities.
qs = (self.backlinks if back else self.links).all()
for link in qs:
if link.key in keys:
other = link.source if back else link.target
j = other.json(keys=['id', 'type', 'name', 'sortName'])
key = '-' + link.key if back else link.key
response.setdefault(key, []).append(j)
for k in response:
response[k].sort(key=lambda j: j['sortName'])
return response
def json(self, keys=None, user=None):
entity = self.get_entity(self.type)
config_keys = {k['id']: k for k in entity['keys']}
if not keys:
keys = [
'alternativeNames',
@ -161,8 +229,19 @@ class Entity(models.Model):
'type',
'user',
'documents',
] + list(self.data)
] + list(config_keys)
response = {}
link_keys = {
id
for id, k in config_keys.items()
if id in keys and k['type'] in ('entity', ['entity'])
}
if link_keys:
response.update(self._expand_links(link_keys))
response.update(self._expand_links(link_keys, back=True))
for key in keys:
if key == 'id':
response[key] = self.get_id()
@ -182,6 +261,8 @@ class Entity(models.Model):
sort_key = 'document__created'
response[key] = [ox.toAZ(id_)
for id_, in self.documentproperties.order_by(sort_key).values_list('document_id')]
elif key in link_keys:
pass # expanded above
elif key in self.data:
response[key] = self.data[key]
return response
@ -205,9 +286,7 @@ class Entity(models.Model):
else:
Find.objects.filter(entity=self, key=key).delete()
entity = get_by_id(settings.CONFIG['entities'], self.type)
if not entity:
return
entity = self.get_entity(self.type)
with transaction.atomic():
ids = ['name']
for key in entity['keys']:
@ -232,6 +311,9 @@ class Entity(models.Model):
matches = annotation.models.Annotation.objects.filter(layer__in=entity_layers, value=self.get_id()).count()
else:
matches = 0
matches += Link.objects.filter(target=self).count()
for url in urls:
matches += annotation.models.Annotation.objects.filter(value__contains=url).count()
matches += item.models.Item.objects.filter(data__contains=url).count()
@ -295,3 +377,18 @@ class Find(models.Model):
def __str__(self):
return u"%s=%s" % (self.key, self.value)
@python_2_unicode_compatible
class Link(models.Model):
'''Models entity fields of type "entity".'''
class Meta:
unique_together = ("source", "key", "target")
source = models.ForeignKey(Entity, related_name='links')
key = models.CharField(max_length=200)
target = models.ForeignKey(Entity, related_name='backlinks')
def __str__(self):
return u"%s-[%s]->%s" % (self.source, self.key, self.target)

34
pandora/entity/views.py
View file

@ -29,6 +29,14 @@ def get_entity_or_404_json(id):
'text': 'Entity not found'}}
raise HttpErrorJson(response)
def get_type_or_400_json(type_):
try:
return models.Entity.get_entity(type_)
except models.Entity.ValueError as e:
raise HttpErrorJson(json_response(
status=400,
text=e.message))
@login_required_json
def addEntity(request, data):
'''
@ -45,9 +53,7 @@ def addEntity(request, data):
existing_names = []
exists = False
if not utils.get_by_id(settings.CONFIG['entities'], data['type']):
response = json_response(status=500, text='unknown entity type')
return render_to_json_response(response)
get_type_or_400_json(data.get('type'))
if 'name' in data:
names = [data['name']] + data.get('alternativeNames', [])
@ -113,7 +119,7 @@ def autocompleteEntities(request, data):
data['range'] = [0, 10]
op = data.get('operator', '=')
entity = utils.get_by_id(settings.CONFIG['entities'], data['key'])
entity = get_type_or_400_json(data['key'])
order_by = entity.get('autocompleteSort', False)
if order_by:
for o in order_by:
@ -157,6 +163,17 @@ def editEntity(request, data):
id: string, // entity id
name: string, // entity name
description: string, // entity description
friends: [ // key with type "entity"
{
// either:
id: string, // related entity ID
// or:
type: string, // related entity type, and
name: string // related entity name
}
... // more entities
]
... // more properties, as defined in config
}
returns {
@ -168,8 +185,11 @@ def editEntity(request, data):
response = json_response()
entity = get_entity_or_404_json(data['id'])
if entity.editable(request.user):
entity.edit(data)
entity.save()
try:
entity.edit(data)
entity.save()
except models.Entity.ValueError as e:
raise HttpErrorJson(json_response(status=400, text=e.message))
response['data'] = entity.json(user=request.user)
add_changelog(request, data)
else:
@ -228,6 +248,8 @@ def findEntities(request, data):
response = json_response()
if 'keys' in data:
qs = qs[query['range'][0]:query['range'][1]]
qs = qs.prefetch_related('links', 'links__target')
qs = qs.prefetch_related('backlinks', 'backlinks__source')
response['data']['items'] = [l.json(data['keys'], request.user) for l in qs]
elif 'position' in data: