manage names and titles

This commit is contained in:
j 2011-10-11 13:29:05 +02:00
commit 4948f0fa39
14 changed files with 1181 additions and 11 deletions

View file

@ -1,6 +1,8 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, with_statement
import unicodedata
import string
from django.db import models
from django.contrib.auth.models import User, Group
@ -10,9 +12,10 @@ import ox
from ox.django import fields
from annotation.models import Annotation
from item.models import Item, canonicalTitle
from item.models import Item
from item import utils
from person.models import get_name_sort
from title.models import get_title_sort
import managers
@ -63,7 +66,7 @@ class Event(models.Model):
if self.user == user or user.is_staff:
return True
return False
def get_matches(self):
q = Q(value__icontains=" " + self.name)|Q(value__startswith=self.name)
for name in self.alternativeNames:
@ -91,7 +94,7 @@ class Event(models.Model):
if self.type == 'person':
value = get_name_sort(value)
else:
value = utils.sort_title(canonicalTitle(value))
value = get_title_sort(value)
self.name_sort = utils.sort_string(value)
def save(self, *args, **kwargs):

View file

@ -23,7 +23,6 @@ from django.contrib.sites.models import Site
import ox
from ox.django import fields
from ox.normalize import canonicalTitle
import ox.web.imdb
import ox.image
@ -37,6 +36,7 @@ from annotation.models import Annotation, Layer
import archive.models
from person.models import get_name_sort
from title.models import get_title_sort
def get_item(info, user=None, async=False):
@ -618,8 +618,8 @@ class Item(models.Model):
if name not in base_keys:
if sort_type == 'title':
value = utils.sort_title(canonicalTitle(self.get(source, u'Untitled')))
value = utils.sort_string(value)
value = get_title_sort(self.get(source, u'Untitled'))
value = utils.sort_string(value)[:955]
set_value(s, name, value)
elif sort_type == 'person':
value = sortNames(self.get(source, []))

143
pandora/person/managers.py Normal file
View file

@ -0,0 +1,143 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from django.db.models import Q, Manager
from ox.django.query import QuerySet
import ox
def parseCondition(condition, user):
'''
condition: {
value: "war"
}
or
condition: {
key: "year",
value: "1970-1980,
operator: "!="
}
...
'''
k = condition.get('key', 'name')
k = {
'user': 'user__username',
}.get(k, k)
if not k:
k = 'name'
v = condition['value']
op = condition.get('operator')
if not op:
op = '='
if op.startswith('!'):
op = op[1:]
exclude = True
else:
exclude = False
if op == '-':
q = parseCondition({'key': k, 'value': v[0], 'operator': '>='}, user) \
& parseCondition({'key': k, 'value': v[1], 'operator': '<'}, user)
if exclude:
return ~q
else:
return q
if k == 'id':
v = ox.from26(v)
if isinstance(v, bool): #featured and public flag
key = k
elif k in ('lat', 'lng', 'area', 'south', 'west', 'north', 'east', 'matches', 'id'):
key = '%s%s' % (k, {
'>': '__gt',
'>=': '__gte',
'<': '__lt',
'<=': '__lte',
}.get(op,''))
else:
key = '%s%s' % (k, {
'==': '__iexact',
'^': '__istartswith',
'$': '__iendswith',
}.get(op,'__icontains'))
key = str(key)
if exclude:
q = ~Q(**{key: v})
else:
q = Q(**{key: v})
return q
def parseConditions(conditions, operator, user):
'''
conditions: [
{
value: "war"
}
{
key: "year",
value: "1970-1980,
operator: "!="
},
{
key: "country",
value: "f",
operator: "^"
}
],
operator: "&"
'''
conn = []
for condition in conditions:
if 'conditions' in condition:
q = parseConditions(condition['conditions'],
condition.get('operator', '&'), user)
if q:
conn.append(q)
pass
else:
conn.append(parseCondition(condition, user))
if conn:
q = conn[0]
for c in conn[1:]:
if operator == '|':
q = q | c
else:
q = q & c
return q
return None
class PersonManager(Manager):
def get_query_set(self):
return QuerySet(self.model)
def find(self, data, user):
'''
query: {
conditions: [
{
value: "war"
}
{
key: "year",
value: "1970-1980,
operator: "!="
},
{
key: "country",
value: "f",
operator: "^"
}
],
operator: "&"
}
'''
#join query with operator
qs = self.get_query_set()
query = data.get('query', {})
conditions = parseConditions(query.get('conditions', []),
query.get('operator', '&'),
user)
if conditions:
qs = qs.filter(conditions)
return qs

View file

@ -9,6 +9,7 @@ from django.db import models
from ox.django import fields
import ox
import managers
def get_name_sort(name):
person, created = Person.objects.get_or_create(name=name)
@ -18,6 +19,7 @@ def get_name_sort(name):
class Person(models.Model):
name = models.CharField(max_length=200, unique=True)
name_sort = models.CharField(max_length=200)
numberofnames = models.IntegerField(default=0)
#FIXME: how to deal with aliases
aliases = fields.TupleField(default=[])
@ -25,8 +27,7 @@ class Person(models.Model):
imdbId = models.CharField(max_length=7, blank=True)
wikipediaId = models.CharField(max_length=1000, blank=True)
class Meta:
ordering = ('name_sort', )
objects = managers.PersonManager()
def __unicode__(self):
return self.name
@ -34,6 +35,8 @@ class Person(models.Model):
def save(self, *args, **kwargs):
if not self.name_sort:
self.name_sort = ox.normalize.canonicalName(self.name)
self.name_sort = unicodedata.normalize('NFKD', self.name_sort)
self.numberofnames = len(self.name.split(' '))
super(Person, self).save(*args, **kwargs)
def get_or_create(model, name, imdbId=None):
@ -51,5 +54,18 @@ class Person(models.Model):
return o
get_or_create = classmethod(get_or_create)
def json(self):
return self.name
def get_id(self):
return ox.to26(self.id)
def json(self, keys=None, user=None):
j = {
'id': self.get_id(),
'name': self.name,
'nameSort': self.name_sort,
'numberofnames': self.numberofnames,
}
if keys:
for key in j.keys():
if key not in keys:
del j[key]
return j

View file

@ -1 +1,154 @@
# Create your views here.
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division
from django.db.models import Max, Min
import ox
from ox.utils import json
from ox.django.decorators import login_required_json, admin_required_json
from ox.django.shortcuts import render_to_json_response, get_object_or_404_json, json_response
from api.actions import actions
from item import utils
import models
@admin_required_json
def editName(request):
'''
param data {
'id': nameid,
'nameSort': ...
}
can contain any of the allowed keys for name
'''
data = json.loads(request.POST['data'])
person = get_object_or_404_json(models.Person, pk=ox.from26(data['id']))
response = json_response()
if 'nameSort' in data:
person.name_sort = utils.sort_string(data['nameSort'])
person.save()
response['data'] = person.json()
return render_to_json_response(response)
actions.register(editName, cache=False)
def parse_query(data, user):
query = {}
query['range'] = [0, 100]
query['sort'] = [{'key':'name', 'operator':'+'}]
for key in ('keys', 'group', 'list', 'range', 'sort', 'query'):
if key in data:
query[key] = data[key]
query['qs'] = models.Person.objects.find(query, user)
#if 'itemQuery' in data:
# item_query = models.Item.objects.find({'query': data['itemQuery']}, user)
# query['qs'] = query['qs'].filter(items__in=item_query)
return query
def order_query(qs, sort):
order_by = []
for e in sort:
operator = e['operator']
if operator != '-':
operator = ''
key = {
'name': 'name_sort',
'nameSort': 'name_sort',
}.get(e['key'], e['key'])
order = '%s%s' % (operator, key)
order_by.append(order)
if order_by:
qs = qs.order_by(*order_by, nulls_last=True)
return qs
def findNames(request):
'''
param data {
query: {
conditions: [
{
key: 'user',
value: 'something',
operator: '='
}
]
operator: ","
},
itemQuery: {
//see find request
},
sort: [{key: 'name', operator: '+'}],
range: [0, 100]
keys: []
}
possible query keys:
name, numberofnames
possible keys:
name, nameSort, numberofnames
return {
status: {
code: int,
text: string
},
data: {
items: [
{name:, user:, featured:, public...}
]
}
}
param data
{'query': query, 'sort': array, 'range': array}
query: query object, more on query syntax at
https://wiki.0x2620.org/wiki/pandora/QuerySyntax
sort: array of key, operator dics
[
{
key: "year",
operator: "-"
},
{
key: "director",
operator: ""
}
]
range: result range, array [from, to]
with keys, items is list of dicts with requested properties:
return {'status': {'code': int, 'text': string},
'data': {items: array}}
'''
data = json.loads(request.POST['data'])
response = json_response()
query = parse_query(data, request.user)
qs = order_query(query['qs'], query['sort'])
qs = qs.distinct()
if 'keys' in data:
qs = qs.select_related()
qs = qs[query['range'][0]:query['range'][1]]
response['data']['items'] = [p.json(data['keys'], request.user) for p in qs]
elif 'position' in query:
ids = [i.get_id() for i in qs]
data['conditions'] = data['conditions'] + {
'value': data['position'],
'key': query['sort'][0]['key'],
'operator': '^'
}
query = parse_query(data, request.user)
qs = order_query(query['qs'], query['sort'])
if qs.count() > 0:
response['data']['position'] = utils.get_positions(ids, [qs[0].itemId])[0]
elif 'positions' in data:
ids = [i.get_id() for i in qs]
response['data']['positions'] = utils.get_positions(ids, data['positions'])
else:
response['data']['items'] = qs.count()
return render_to_json_response(response)
actions.register(findNames)

View file

@ -130,6 +130,7 @@ INSTALLED_APPS = (
'item',
'itemlist',
'person',
'title',
'place',
'text',
'edit',

View file

143
pandora/title/managers.py Normal file
View file

@ -0,0 +1,143 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from django.db.models import Q, Manager
from ox.django.query import QuerySet
import ox
def parseCondition(condition, user):
'''
condition: {
value: "war"
}
or
condition: {
key: "year",
value: "1970-1980,
operator: "!="
}
...
'''
k = condition.get('key', 'title')
k = {
'user': 'user__usertitle',
}.get(k, k)
if not k:
k = 'title'
v = condition['value']
op = condition.get('operator')
if not op:
op = '='
if op.startswith('!'):
op = op[1:]
exclude = True
else:
exclude = False
if op == '-':
q = parseCondition({'key': k, 'value': v[0], 'operator': '>='}, user) \
& parseCondition({'key': k, 'value': v[1], 'operator': '<'}, user)
if exclude:
return ~q
else:
return q
if k == 'id':
v = ox.from26(v)
if isinstance(v, bool): #featured and public flag
key = k
elif k in ('lat', 'lng', 'area', 'south', 'west', 'north', 'east', 'matches', 'id'):
key = '%s%s' % (k, {
'>': '__gt',
'>=': '__gte',
'<': '__lt',
'<=': '__lte',
}.get(op,''))
else:
key = '%s%s' % (k, {
'==': '__iexact',
'^': '__istartswith',
'$': '__iendswith',
}.get(op,'__icontains'))
key = str(key)
if exclude:
q = ~Q(**{key: v})
else:
q = Q(**{key: v})
return q
def parseConditions(conditions, operator, user):
'''
conditions: [
{
value: "war"
}
{
key: "year",
value: "1970-1980,
operator: "!="
},
{
key: "country",
value: "f",
operator: "^"
}
],
operator: "&"
'''
conn = []
for condition in conditions:
if 'conditions' in condition:
q = parseConditions(condition['conditions'],
condition.get('operator', '&'), user)
if q:
conn.append(q)
pass
else:
conn.append(parseCondition(condition, user))
if conn:
q = conn[0]
for c in conn[1:]:
if operator == '|':
q = q | c
else:
q = q & c
return q
return None
class TitleManager(Manager):
def get_query_set(self):
return QuerySet(self.model)
def find(self, data, user):
'''
query: {
conditions: [
{
value: "war"
}
{
key: "year",
value: "1970-1980,
operator: "!="
},
{
key: "country",
value: "f",
operator: "^"
}
],
operator: "&"
}
'''
#join query with operator
qs = self.get_query_set()
query = data.get('query', {})
conditions = parseConditions(query.get('conditions', []),
query.get('operator', '&'),
user)
if conditions:
qs = qs.filter(conditions)
return qs

63
pandora/title/models.py Normal file
View file

@ -0,0 +1,63 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, with_statement
import unicodedata
from django.db import models
import ox
import managers
def get_title_sort(title):
title, created = Title.objects.get_or_create(title=title)
title_sort = unicodedata.normalize('NFKD', title.title_sort)
return title_sort
class Title(models.Model):
title = models.CharField(max_length=200, unique=True)
title_sort = models.CharField(max_length=200)
imdbId = models.CharField(max_length=7, blank=True)
objects = managers.TitleManager()
def __unicode__(self):
return self.title
def save(self, *args, **kwargs):
if not self.title_sort:
self.title_sort = ox.normalize.canonicalTitle(self.title)
self.title_sort = unicodedata.normalize('NFKD', self.title_sort)
super(Title, self).save(*args, **kwargs)
def get_or_create(model, title, imdbId=None):
if imdbId:
q = model.objects.filter(title=title, imdbId=imdbId)
else:
q = model.objects.all().filter(title=title)
if q.count() > 0:
o = q[0]
else:
o = model.objects.create(title=title)
if imdbId:
o.imdbId = imdbId
o.save()
return o
get_or_create = classmethod(get_or_create)
def get_id(self):
return ox.to26(self.id)
def json(self, keys=None, user=None):
j = {
'id': self.get_id(),
'title': self.title,
'titleSort': self.title_sort,
}
if keys:
for key in j.keys():
if key not in keys:
del j[key]
return j

154
pandora/title/views.py Normal file
View file

@ -0,0 +1,154 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division
from django.db.models import Max, Min
import ox
from ox.utils import json
from ox.django.decorators import admin_required_json
from ox.django.shortcuts import render_to_json_response, get_object_or_404_json, json_response
from api.actions import actions
from item import utils
import models
@admin_required_json
def editTitle(request):
'''
param data {
'id': titleid,
'titleSort': ...
}
can contain any of the allowed keys for title
'''
data = json.loads(request.POST['data'])
title = get_object_or_404_json(models.Title, pk=ox.from26(data['id']))
response = json_response()
if 'titleSort' in data:
title.title_sort = utils.sort_string(data['titleSort'])
title.save()
response['data'] = title.json()
return render_to_json_response(response)
actions.register(editTitle, cache=False)
def parse_query(data, user):
query = {}
query['range'] = [0, 100]
query['sort'] = [{'key':'title', 'operator':'+'}]
for key in ('keys', 'group', 'list', 'range', 'sort', 'query'):
if key in data:
query[key] = data[key]
query['qs'] = models.Title.objects.find(query, user)
#if 'itemQuery' in data:
# item_query = models.Item.objects.find({'query': data['itemQuery']}, user)
# query['qs'] = query['qs'].filter(items__in=item_query)
return query
def order_query(qs, sort):
order_by = []
for e in sort:
operator = e['operator']
if operator != '-':
operator = ''
key = {
'title': 'title_sort',
'titleSort': 'title_sort',
}.get(e['key'], e['key'])
order = '%s%s' % (operator, key)
order_by.append(order)
if order_by:
qs = qs.order_by(*order_by, nulls_last=True)
return qs
def findTitles(request):
'''
param data {
query: {
conditions: [
{
key: 'user',
value: 'something',
operator: '='
}
]
operator: ","
},
itemQuery: {
//see find request
},
sort: [{key: 'title', operator: '+'}],
range: [0, 100]
keys: []
}
possible query keys:
title, numberoftitles
possible keys:
title, titleSort, numberoftitles
return {
status: {
code: int,
text: string
},
data: {
items: [
{title:, user:, featured:, public...}
]
}
}
param data
{'query': query, 'sort': array, 'range': array}
query: query object, more on query syntax at
https://wiki.0x2620.org/wiki/pandora/QuerySyntax
sort: array of key, operator dics
[
{
key: "year",
operator: "-"
},
{
key: "director",
operator: ""
}
]
range: result range, array [from, to]
with keys, items is list of dicts with requested properties:
return {'status': {'code': int, 'text': string},
'data': {items: array}}
'''
data = json.loads(request.POST['data'])
response = json_response()
query = parse_query(data, request.user)
qs = order_query(query['qs'], query['sort'])
qs = qs.distinct()
if 'keys' in data:
qs = qs.select_related()
qs = qs[query['range'][0]:query['range'][1]]
response['data']['items'] = [p.json(data['keys'], request.user) for p in qs]
elif 'position' in query:
ids = [i.get_id() for i in qs]
data['conditions'] = data['conditions'] + {
'value': data['position'],
'key': query['sort'][0]['key'],
'operator': '^'
}
query = parse_query(data, request.user)
qs = order_query(query['qs'], query['sort'])
if qs.count() > 0:
response['data']['position'] = utils.get_positions(ids, [qs[0].itemId])[0]
elif 'positions' in data:
ids = [i.get_id() for i in qs]
response['data']['positions'] = utils.get_positions(ids, data['positions'])
else:
response['data']['items'] = qs.count()
return render_to_json_response(response)
actions.register(findTitles)