forked from 0x2620/pandora
merge
This commit is contained in:
commit
876e360e76
13 changed files with 68 additions and 90 deletions
3
README
3
README
|
@ -12,7 +12,8 @@ python, bazaar, pip and virtualenv and several other python modules:
|
||||||
apt-get install bzr git subversion mercurial \
|
apt-get install bzr git subversion mercurial \
|
||||||
python-setuptools python-pip python-virtualenv ipython \
|
python-setuptools python-pip python-virtualenv ipython \
|
||||||
python-dev python-imaging python-numpy python-psycopg2 \
|
python-dev python-imaging python-numpy python-psycopg2 \
|
||||||
python-geoip postgresql rabbitmq-server
|
python-geoip python-html5lib python-lxml \
|
||||||
|
postgresql rabbitmq-server
|
||||||
apt-get install oxframe oxtimeline
|
apt-get install oxframe oxtimeline
|
||||||
|
|
||||||
* Pan.do/ra
|
* Pan.do/ra
|
||||||
|
|
|
@ -26,6 +26,7 @@ def parseCondition(condition, user):
|
||||||
'in': 'start',
|
'in': 'start',
|
||||||
'out': 'end',
|
'out': 'end',
|
||||||
'id': 'public_id',
|
'id': 'public_id',
|
||||||
|
'value': 'findvalue',
|
||||||
}.get(k, k)
|
}.get(k, k)
|
||||||
if not k:
|
if not k:
|
||||||
k = 'name'
|
k = 'name'
|
||||||
|
|
|
@ -34,6 +34,7 @@ class Annotation(models.Model):
|
||||||
|
|
||||||
layer = models.CharField(max_length=255, db_index=True)
|
layer = models.CharField(max_length=255, db_index=True)
|
||||||
value = models.TextField()
|
value = models.TextField()
|
||||||
|
findvalue = models.TextField()
|
||||||
sortvalue = models.CharField(max_length=1000, null=True, blank=True, db_index=True)
|
sortvalue = models.CharField(max_length=1000, null=True, blank=True, db_index=True)
|
||||||
|
|
||||||
def editable(self, user):
|
def editable(self, user):
|
||||||
|
@ -44,23 +45,25 @@ class Annotation(models.Model):
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def html(self):
|
|
||||||
if self.layer == 'string':
|
|
||||||
return utils.html_parser(self.value)
|
|
||||||
else:
|
|
||||||
return self.value
|
|
||||||
|
|
||||||
def set_public_id(self):
|
def set_public_id(self):
|
||||||
if self.id:
|
if self.id:
|
||||||
public_id = Annotation.objects.filter(item=self.item, id__lt=self.id).count() + 1
|
public_id = Annotation.objects.filter(item=self.item, id__lt=self.id).count() + 1
|
||||||
self.public_id = "%s/%s" % (self.item.itemId, ox.toAZ(public_id))
|
self.public_id = "%s/%s" % (self.item.itemId, ox.toAZ(public_id))
|
||||||
Annotation.objects.filter(id=self.id).update(public_id=self.public_id)
|
Annotation.objects.filter(id=self.id).update(public_id=self.public_id)
|
||||||
|
|
||||||
|
def get_layer(self):
|
||||||
|
for layer in settings.CONFIG['layers']:
|
||||||
|
if layer['id'] == self.layer:
|
||||||
|
return layer
|
||||||
|
return {}
|
||||||
|
|
||||||
def save(self, *args, **kwargs):
|
def save(self, *args, **kwargs):
|
||||||
set_public_id = not self.id or not self.public_id
|
set_public_id = not self.id or not self.public_id
|
||||||
|
layer = self.get_layer()
|
||||||
if self.value:
|
if self.value:
|
||||||
sortvalue = ox.stripTags(self.value).strip()
|
self.value = utils.cleanup_value(self.value, layer['type'])
|
||||||
sortvalue = sort_string(sortvalue)
|
self.findvalue = ox.stripTags(self.value).strip()
|
||||||
|
sortvalue = sort_string(self.findvalue)
|
||||||
if sortvalue:
|
if sortvalue:
|
||||||
self.sortvalue = sortvalue[:1000]
|
self.sortvalue = sortvalue[:1000]
|
||||||
else:
|
else:
|
||||||
|
@ -69,12 +72,7 @@ class Annotation(models.Model):
|
||||||
self.sortvalue = None
|
self.sortvalue = None
|
||||||
|
|
||||||
#no clip or update clip
|
#no clip or update clip
|
||||||
def get_layer(id):
|
private = layer.get('private', False)
|
||||||
for l in settings.CONFIG['layers']:
|
|
||||||
if l['id'] == id:
|
|
||||||
return l
|
|
||||||
return {}
|
|
||||||
private = get_layer(self.layer).get('private', False)
|
|
||||||
if not private:
|
if not private:
|
||||||
if not self.clip or self.start != self.clip.start or self.end != self.clip.end:
|
if not self.clip or self.start != self.clip.start or self.end != self.clip.end:
|
||||||
self.clip, created = Clip.get_or_create(self.item, self.start, self.end)
|
self.clip, created = Clip.get_or_create(self.item, self.start, self.end)
|
||||||
|
|
|
@ -2,37 +2,17 @@
|
||||||
# ci:si:et:sw=4:sts=4:ts=4
|
# ci:si:et:sw=4:sts=4:ts=4
|
||||||
import re
|
import re
|
||||||
import ox
|
import ox
|
||||||
|
import html5lib
|
||||||
|
|
||||||
|
|
||||||
def html_parser(text, nofollow=True):
|
def cleanup_value(value, layer_type):
|
||||||
text = text.replace('<i>', '__i__').replace('</i>', '__/i__')
|
#FIXME: what about other types? location etc
|
||||||
text = text.replace('<b>', '__b__').replace('</b>', '__/b__')
|
if layer_type == 'text':
|
||||||
#truns links into wiki links, make sure to only take http links
|
value = sanitize_fragment(value)
|
||||||
text = re.sub('<a .*?href="(http.*?)".*?>(.*?)</a>', '[\\1 \\2]', text)
|
|
||||||
text = ox.escape(text)
|
|
||||||
text = text.replace('__i__', '<i>').replace('__/i__', '</i>')
|
|
||||||
text = text.replace('__b__', '<b>').replace('__/b__', '</b>')
|
|
||||||
if nofollow:
|
|
||||||
nofollow_rel = ' rel="nofollow"'
|
|
||||||
else:
|
else:
|
||||||
nofollow_rel = ''
|
value = ox.stripTags(value)
|
||||||
|
return value
|
||||||
|
|
||||||
links = re.compile('(\[(http.*?) (.*?)\])').findall(text)
|
def sanitize_fragment(html):
|
||||||
for t, link, txt in links:
|
return html5lib.parseFragment(html).toxml().decode('utf-8')
|
||||||
link = link.replace('http', '__LINK__').replace('.', '__DOT__')
|
|
||||||
ll = '<a href="%s"%s>%s</a>' % (link, nofollow_rel, txt)
|
|
||||||
text = text.replace(t, ll)
|
|
||||||
links = re.compile('(\[(http.*?)\])').findall(text)
|
|
||||||
for t, link in links:
|
|
||||||
link = link.replace('http', '__LINK__').replace('.', '__DOT__')
|
|
||||||
ll = '<a href="%s"%s>%s</a>' % (link, nofollow_rel, link)
|
|
||||||
text = text.replace(t, ll)
|
|
||||||
|
|
||||||
text = ox.urlize(text, nofollow=nofollow)
|
|
||||||
|
|
||||||
#inpage links
|
|
||||||
text = re.sub('\[(/.+?) (.+?)\]', '<a href="\\1">\\2</a>', text)
|
|
||||||
|
|
||||||
text = text.replace('__LINK__', 'http').replace('__DOT__', '.')
|
|
||||||
text = text.replace("\n", '<br />')
|
|
||||||
return text
|
|
||||||
|
|
|
@ -27,7 +27,7 @@ def parseCondition(condition, user):
|
||||||
'in': 'start',
|
'in': 'start',
|
||||||
'out': 'end',
|
'out': 'end',
|
||||||
'place': 'annotations__places__id',
|
'place': 'annotations__places__id',
|
||||||
'text': 'annotations__value',
|
'text': 'annotations__findvalue',
|
||||||
'user': 'annotations__user__username',
|
'user': 'annotations__user__username',
|
||||||
}.get(k, k)
|
}.get(k, k)
|
||||||
if not k:
|
if not k:
|
||||||
|
@ -40,7 +40,7 @@ def parseCondition(condition, user):
|
||||||
for l in filter(lambda l: not l.get('private', False),
|
for l in filter(lambda l: not l.get('private', False),
|
||||||
settings.CONFIG['layers'])]
|
settings.CONFIG['layers'])]
|
||||||
if k in public_layers:
|
if k in public_layers:
|
||||||
return parseCondition({'key': 'annotations__value',
|
return parseCondition({'key': 'annotations__findvalue',
|
||||||
'value': v,
|
'value': v,
|
||||||
'operator': op}, user) \
|
'operator': op}, user) \
|
||||||
& parseCondition({'key': 'annotations__layer',
|
& parseCondition({'key': 'annotations__layer',
|
||||||
|
|
|
@ -48,7 +48,7 @@ def order_query(qs, sort):
|
||||||
if key.startswith('clip:'):
|
if key.startswith('clip:'):
|
||||||
key = e['key'][len('clip:'):]
|
key = e['key'][len('clip:'):]
|
||||||
key = {
|
key = {
|
||||||
'text': 'annotations__value',
|
'text': 'annotations__sortvalue',
|
||||||
'position': 'start',
|
'position': 'start',
|
||||||
}.get(key, key)
|
}.get(key, key)
|
||||||
elif key not in clip_keys:
|
elif key not in clip_keys:
|
||||||
|
@ -111,7 +111,7 @@ def findClips(request):
|
||||||
Annotation.objects.filter(layer=layer, clip__in=ids))
|
Annotation.objects.filter(layer=layer, clip__in=ids))
|
||||||
elif 'position' in query:
|
elif 'position' in query:
|
||||||
qs = order_query(qs, query['sort'])
|
qs = order_query(qs, query['sort'])
|
||||||
ids = [i.public_id for i in qs]
|
ids = [i['public_id'] for i in qs.values('public_id')]
|
||||||
data['conditions'] = data['conditions'] + {
|
data['conditions'] = data['conditions'] + {
|
||||||
'value': data['position'],
|
'value': data['position'],
|
||||||
'key': query['sort'][0]['key'],
|
'key': query['sort'][0]['key'],
|
||||||
|
@ -123,7 +123,7 @@ def findClips(request):
|
||||||
response['data']['position'] = utils.get_positions(ids, [qs[0].itemId])[0]
|
response['data']['position'] = utils.get_positions(ids, [qs[0].itemId])[0]
|
||||||
elif 'positions' in data:
|
elif 'positions' in data:
|
||||||
qs = order_query(qs, query['sort'])
|
qs = order_query(qs, query['sort'])
|
||||||
ids = [i.public_id for i in qs]
|
ids = [i['public_id'] for i in qs.values('public_id')]
|
||||||
response['data']['positions'] = utils.get_positions(ids, data['positions'])
|
response['data']['positions'] = utils.get_positions(ids, data['positions'])
|
||||||
else:
|
else:
|
||||||
response['data']['items'] = qs.count()
|
response['data']['items'] = qs.count()
|
||||||
|
|
|
@ -39,6 +39,10 @@ def parseCondition(condition, user):
|
||||||
else:
|
else:
|
||||||
exclude = False
|
exclude = False
|
||||||
|
|
||||||
|
facet_keys = models.Item.facet_keys + ['title']
|
||||||
|
for f in settings.CONFIG['filters']:
|
||||||
|
if f['id'] not in facet_keys:
|
||||||
|
facet_keys.append(f['id'])
|
||||||
key_type = settings.CONFIG['keys'].get(k, {'type':'string'}).get('type')
|
key_type = settings.CONFIG['keys'].get(k, {'type':'string'}).get('type')
|
||||||
if isinstance(key_type, list):
|
if isinstance(key_type, list):
|
||||||
key_type = key_type[0]
|
key_type = key_type[0]
|
||||||
|
@ -48,8 +52,8 @@ def parseCondition(condition, user):
|
||||||
'text': 'string',
|
'text': 'string',
|
||||||
'year': 'string',
|
'year': 'string',
|
||||||
'length': 'string',
|
'length': 'string',
|
||||||
'list': 'list',
|
|
||||||
'layer': 'string',
|
'layer': 'string',
|
||||||
|
'list': 'list',
|
||||||
}.get(key_type, key_type)
|
}.get(key_type, key_type)
|
||||||
if k == 'list':
|
if k == 'list':
|
||||||
key_type = ''
|
key_type = ''
|
||||||
|
@ -95,7 +99,7 @@ def parseCondition(condition, user):
|
||||||
value_key = 'find__value'
|
value_key = 'find__value'
|
||||||
else:
|
else:
|
||||||
value_key = k
|
value_key = k
|
||||||
if k in models.Item.facet_keys + ['title']:
|
if k in facet_keys:
|
||||||
in_find = False
|
in_find = False
|
||||||
facet_value = 'facets__value%s' % {
|
facet_value = 'facets__value%s' % {
|
||||||
'==': '__iexact',
|
'==': '__iexact',
|
||||||
|
|
|
@ -545,7 +545,7 @@ class Item(models.Model):
|
||||||
'\n'.join([f.path for f in self.files.all()]))
|
'\n'.join([f.path for f in self.files.all()]))
|
||||||
elif key['type'] == 'layer':
|
elif key['type'] == 'layer':
|
||||||
qs = Annotation.objects.filter(layer=i, item=self).order_by('start')
|
qs = Annotation.objects.filter(layer=i, item=self).order_by('start')
|
||||||
save(i, '\n'.join([l.value for l in qs]))
|
save(i, u'\n'.join([l.findvalue for l in qs]))
|
||||||
elif i != '*' and i not in self.facet_keys:
|
elif i != '*' and i not in self.facet_keys:
|
||||||
value = self.get(i)
|
value = self.get(i)
|
||||||
if isinstance(value, list):
|
if isinstance(value, list):
|
||||||
|
@ -737,6 +737,22 @@ class Item(models.Model):
|
||||||
#update cached values in clips
|
#update cached values in clips
|
||||||
self.clips.all().update(director=s.director, title=s.title)
|
self.clips.all().update(director=s.director, title=s.title)
|
||||||
|
|
||||||
|
def update_layer_facets(self):
|
||||||
|
filters = [f['id'] for f in settings.CONFIG['filters']]
|
||||||
|
for layer in settings.CONFIG['layers']:
|
||||||
|
if layer['id'] in filters:
|
||||||
|
key = layer['id']
|
||||||
|
current_values = [a['value']
|
||||||
|
for a in self.annotations.filter(layer=key).distinct().values('value')]
|
||||||
|
saved_values = [i.value for i in Facet.objects.filter(item=self, key=key)]
|
||||||
|
removed_values = filter(lambda i: i not in current_values, saved_values)
|
||||||
|
if removed_values:
|
||||||
|
Facet.objects.filter(item=self, key=key, value__in=removed_values).delete()
|
||||||
|
for value in current_values:
|
||||||
|
if value not in saved_values:
|
||||||
|
sortvalue = value
|
||||||
|
Facet.objects.get_or_create(item=self, key=key, value=value, sortvalue=sortvalue)
|
||||||
|
|
||||||
def update_facets(self):
|
def update_facets(self):
|
||||||
for key in self.facet_keys + ['title']:
|
for key in self.facet_keys + ['title']:
|
||||||
current_values = self.get(key, [])
|
current_values = self.get(key, [])
|
||||||
|
@ -773,9 +789,11 @@ class Item(models.Model):
|
||||||
if key in self.person_keys + ['name']:
|
if key in self.person_keys + ['name']:
|
||||||
sortvalue = get_name_sort(value)
|
sortvalue = get_name_sort(value)
|
||||||
Facet.objects.get_or_create(item=self, key=key, value=value, sortvalue=sortvalue)
|
Facet.objects.get_or_create(item=self, key=key, value=value, sortvalue=sortvalue)
|
||||||
|
self.update_layer_facets()
|
||||||
|
|
||||||
def path(self, name=''):
|
def path(self, name=''):
|
||||||
h = self.itemId
|
h = self.itemId
|
||||||
|
h = (7-len(h))*'0' + h
|
||||||
return os.path.join('items', h[:2], h[2:4], h[4:6], h[6:], name)
|
return os.path.join('items', h[:2], h[2:4], h[4:6], h[6:], name)
|
||||||
|
|
||||||
'''
|
'''
|
||||||
|
|
|
@ -122,7 +122,7 @@ class List(models.Model):
|
||||||
return response
|
return response
|
||||||
|
|
||||||
def path(self, name=''):
|
def path(self, name=''):
|
||||||
h = "%06d" % self.id
|
h = "%07d" % self.id
|
||||||
return os.path.join('lists', h[:2], h[2:4], h[4:6], h[6:], name)
|
return os.path.join('lists', h[:2], h[2:4], h[4:6], h[6:], name)
|
||||||
|
|
||||||
def update_icon(self):
|
def update_icon(self):
|
||||||
|
|
|
@ -45,16 +45,11 @@
|
||||||
{"id": "collection", "title": "Collection", "type": "string"},
|
{"id": "collection", "title": "Collection", "type": "string"},
|
||||||
{"id": "source", "title": "Source", "type": "string"},
|
{"id": "source", "title": "Source", "type": "string"},
|
||||||
{"id": "director", "title": "Director", "type": "string"},
|
{"id": "director", "title": "Director", "type": "string"},
|
||||||
{"id": "location", "title": "Location", "type": "string"},
|
{"id": "locations", "title": "Location", "type": "string"},
|
||||||
{"id": "year", "title": "Year", "type": "integer"},
|
{"id": "year", "title": "Year", "type": "integer"},
|
||||||
{"id": "language", "title": "Language", "type": "string"},
|
{"id": "language", "title": "Language", "type": "string"},
|
||||||
{"id": "category", "title": "Category", "type": "string"},
|
{"id": "category", "title": "Category", "type": "string"},
|
||||||
{"id": "writer", "title": "Writer", "type": "string"},
|
{"id": "keywords", "title": "Keyword", "type": "string"}
|
||||||
{"id": "producer", "title": "Producer", "type": "string"},
|
|
||||||
{"id": "cinematographer", "title": "Cinematographer", "type": "string"},
|
|
||||||
{"id": "editor", "title": "Editor", "type": "string"},
|
|
||||||
{"id": "actor", "title": "Actor", "type": "string"},
|
|
||||||
{"id": "keyword", "title": "Keyword", "type": "string"}
|
|
||||||
],
|
],
|
||||||
/*
|
/*
|
||||||
An itemKey must have the following properties:
|
An itemKey must have the following properties:
|
||||||
|
@ -174,30 +169,6 @@
|
||||||
"filter": true,
|
"filter": true,
|
||||||
"find": true
|
"find": true
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"id": "genre",
|
|
||||||
"title": "Genre",
|
|
||||||
"type": ["string"],
|
|
||||||
"autocomplete": true,
|
|
||||||
"columnWidth": 120,
|
|
||||||
"filter": true,
|
|
||||||
"find": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "keyword",
|
|
||||||
"title": "Keyword",
|
|
||||||
"type": ["string"],
|
|
||||||
"autocomplete": true,
|
|
||||||
"filter": true,
|
|
||||||
"find": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "numberofkeywords",
|
|
||||||
"title": "Number of Keywords",
|
|
||||||
"type": "integer",
|
|
||||||
"columnWidth": 60,
|
|
||||||
"value": {"key": "keyword", "type": "length"}
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"id": "description",
|
"id": "description",
|
||||||
"title": "Description",
|
"title": "Description",
|
||||||
|
@ -415,7 +386,7 @@
|
||||||
"id": "keywords",
|
"id": "keywords",
|
||||||
"title": "Keywords",
|
"title": "Keywords",
|
||||||
"overlap": true,
|
"overlap": true,
|
||||||
"type": "text"
|
"type": "string"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "descriptions",
|
"id": "descriptions",
|
||||||
|
@ -503,11 +474,11 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"filters": [
|
"filters": [
|
||||||
{"id": "collection", "sort": [{"key": "name", "operator": "-"}]},
|
{"id": "collection", "sort": [{"key": "name", "operator": "+"}]},
|
||||||
{"id": "source", "sort": [{"key": "items", "operator": "-"}]},
|
{"id": "source", "sort": [{"key": "name", "operator": "+"}]},
|
||||||
{"id": "category", "sort": [{"key": "items", "operator": "-"}]},
|
{"id": "category", "sort": [{"key": "items", "operator": "-"}]},
|
||||||
{"id": "keyword", "sort": [{"key": "items", "operator": "-"}]},
|
{"id": "keywords", "sort": [{"key": "items", "operator": "-"}]},
|
||||||
{"id": "location", "sort": [{"key": "items", "operator": "-"}]}
|
{"id": "locations", "sort": [{"key": "items", "operator": "-"}]}
|
||||||
],
|
],
|
||||||
"filtersSize": 176,
|
"filtersSize": 176,
|
||||||
"find": {"conditions": [], "operator": "&"},
|
"find": {"conditions": [], "operator": "&"},
|
||||||
|
|
|
@ -187,6 +187,7 @@ def user_post_save(sender, instance, **kwargs):
|
||||||
profile, new = UserProfile.objects.get_or_create(user=instance)
|
profile, new = UserProfile.objects.get_or_create(user=instance)
|
||||||
if new and instance.is_superuser:
|
if new and instance.is_superuser:
|
||||||
profile.level = len(settings.CONFIG['userLevels']) - 1
|
profile.level = len(settings.CONFIG['userLevels']) - 1
|
||||||
|
profile.newsletter = settings.CONFIG['user']['newsletter']
|
||||||
profile.save()
|
profile.save()
|
||||||
SessionData.objects.filter(user=instance).update(level=profile.level,
|
SessionData.objects.filter(user=instance).update(level=profile.level,
|
||||||
username=instance.username)
|
username=instance.username)
|
||||||
|
|
|
@ -12,3 +12,4 @@ django-celery>2.1.1
|
||||||
-e git://github.com/bit/django-extensions.git#egg=django_extensions
|
-e git://github.com/bit/django-extensions.git#egg=django_extensions
|
||||||
-e git+git://github.com/dcramer/django-devserver#egg=django_devserver
|
-e git+git://github.com/dcramer/django-devserver#egg=django_devserver
|
||||||
gunicorn
|
gunicorn
|
||||||
|
html5lib
|
||||||
|
|
|
@ -30,7 +30,10 @@ pandora.ui.placesDialog = function() {
|
||||||
pandora.api.findClips({
|
pandora.api.findClips({
|
||||||
query: {
|
query: {
|
||||||
conditions: names.map(function(name) {
|
conditions: names.map(function(name) {
|
||||||
return {key: 'subtitles', value: name, operator: '='};
|
//FIXME: this should be more generic
|
||||||
|
return Ox.getObjectById(pandora.site.layers, 'subtitles')
|
||||||
|
? {key: 'subtitles', value: name, operator: '='}
|
||||||
|
: {key: 'locations', value: name, operator: '=='};
|
||||||
}),
|
}),
|
||||||
operator: names.length == 1 ? '&' : '|'
|
operator: names.length == 1 ? '&' : '|'
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue