annotation cleanup

This commit is contained in:
j 2011-12-26 20:00:30 +05:30
parent 93381e4ab8
commit 6524ceea8a
7 changed files with 27 additions and 45 deletions

3
README
View file

@ -12,7 +12,8 @@ python, bazaar, pip and virtualenv and several other python modules:
apt-get install bzr git subversion mercurial \ apt-get install bzr git subversion mercurial \
python-setuptools python-pip python-virtualenv ipython \ python-setuptools python-pip python-virtualenv ipython \
python-dev python-imaging python-numpy python-psycopg2 \ python-dev python-imaging python-numpy python-psycopg2 \
python-geoip postgresql rabbitmq-server python-geoip python-html5lib python-lxml \
postgresql rabbitmq-server
apt-get install oxframe oxtimeline apt-get install oxframe oxtimeline
* Pan.do/ra * Pan.do/ra

View file

@ -34,6 +34,7 @@ class Annotation(models.Model):
layer = models.CharField(max_length=255, db_index=True) layer = models.CharField(max_length=255, db_index=True)
value = models.TextField() value = models.TextField()
findvalue = models.TextField()
sortvalue = models.CharField(max_length=1000, null=True, blank=True, db_index=True) sortvalue = models.CharField(max_length=1000, null=True, blank=True, db_index=True)
def editable(self, user): def editable(self, user):
@ -44,22 +45,25 @@ class Annotation(models.Model):
return True return True
return False return False
def html(self):
if self.layer == 'string':
return utils.html_parser(self.value)
else:
return self.value
def set_public_id(self): def set_public_id(self):
if self.id: if self.id:
public_id = Annotation.objects.filter(item=self.item, id__lt=self.id).count() + 1 public_id = Annotation.objects.filter(item=self.item, id__lt=self.id).count() + 1
self.public_id = "%s/%s" % (self.item.itemId, ox.toAZ(public_id)) self.public_id = "%s/%s" % (self.item.itemId, ox.toAZ(public_id))
Annotation.objects.filter(id=self.id).update(public_id=self.public_id) Annotation.objects.filter(id=self.id).update(public_id=self.public_id)
def get_layer(self):
for layer in settings.CONFIG['layers']:
if layer['id'] == self.layer:
return layer
return {}
def save(self, *args, **kwargs): def save(self, *args, **kwargs):
set_public_id = not self.id or not self.public_id set_public_id = not self.id or not self.public_id
layer = self.get_layer()
if self.value: if self.value:
sortvalue = ox.stripTags(self.value).strip() self.value = utils.cleanup_value(self.value, self.layer['tyoe'])
self.findvalue = ox.stripTags(self.value).strip()
sortvalue = self.findvalue
sortvalue = sort_string(sortvalue) sortvalue = sort_string(sortvalue)
if sortvalue: if sortvalue:
self.sortvalue = sortvalue[:1000] self.sortvalue = sortvalue[:1000]
@ -69,12 +73,7 @@ class Annotation(models.Model):
self.sortvalue = None self.sortvalue = None
#no clip or update clip #no clip or update clip
def get_layer(id): private = layer.get('private', False)
for l in settings.CONFIG['layers']:
if l['id'] == id:
return l
return {}
private = get_layer(self.layer).get('private', False)
if not private: if not private:
if not self.clip or self.start != self.clip.start or self.end != self.clip.end: if not self.clip or self.start != self.clip.start or self.end != self.clip.end:
self.clip, created = Clip.get_or_create(self.item, self.start, self.end) self.clip, created = Clip.get_or_create(self.item, self.start, self.end)

View file

@ -2,37 +2,17 @@
# ci:si:et:sw=4:sts=4:ts=4 # ci:si:et:sw=4:sts=4:ts=4
import re import re
import ox import ox
import html5lib
def html_parser(text, nofollow=True): def cleanup_value(value, layer_type):
text = text.replace('<i>', '__i__').replace('</i>', '__/i__') #FIXME: what about other types? location etc
text = text.replace('<b>', '__b__').replace('</b>', '__/b__') if layer_type == 'text':
#truns links into wiki links, make sure to only take http links value = sanitize_fragment(value)
text = re.sub('<a .*?href="(http.*?)".*?>(.*?)</a>', '[\\1 \\2]', text)
text = ox.escape(text)
text = text.replace('__i__', '<i>').replace('__/i__', '</i>')
text = text.replace('__b__', '<b>').replace('__/b__', '</b>')
if nofollow:
nofollow_rel = ' rel="nofollow"'
else: else:
nofollow_rel = '' value = ox.stripTags(value)
return value
links = re.compile('(\[(http.*?) (.*?)\])').findall(text) def sanitize_fragment(html):
for t, link, txt in links: return html5lib.parseFragment(html).toxml()
link = link.replace('http', '__LINK__').replace('.', '__DOT__')
ll = '<a href="%s"%s>%s</a>' % (link, nofollow_rel, txt)
text = text.replace(t, ll)
links = re.compile('(\[(http.*?)\])').findall(text)
for t, link in links:
link = link.replace('http', '__LINK__').replace('.', '__DOT__')
ll = '<a href="%s"%s>%s</a>' % (link, nofollow_rel, link)
text = text.replace(t, ll)
text = ox.urlize(text, nofollow=nofollow)
#inpage links
text = re.sub('\[(/.+?) (.+?)\]', '<a href="\\1">\\2</a>', text)
text = text.replace('__LINK__', 'http').replace('__DOT__', '.')
text = text.replace("\n", '<br />')
return text

View file

@ -44,7 +44,7 @@ def sort_string(string):
#pad numbered titles #pad numbered titles
string = re.sub('(\d+)', lambda x: '%010d' % int(x.group(0)), string) string = re.sub('(\d+)', lambda x: '%010d' % int(x.group(0)), string)
return unicodedata.normalize('NFKD', string) return unicodedata.normalize('NFKD', string).lower()
def sort_title(title): def sort_title(title):

View file

@ -415,7 +415,7 @@
"id": "keywords", "id": "keywords",
"title": "Keywords", "title": "Keywords",
"overlap": true, "overlap": true,
"type": "text" "type": "string"
}, },
{ {
"id": "descriptions", "id": "descriptions",

View file

@ -187,6 +187,7 @@ def user_post_save(sender, instance, **kwargs):
profile, new = UserProfile.objects.get_or_create(user=instance) profile, new = UserProfile.objects.get_or_create(user=instance)
if new and instance.is_superuser: if new and instance.is_superuser:
profile.level = len(settings.CONFIG['userLevels']) - 1 profile.level = len(settings.CONFIG['userLevels']) - 1
profile.newsletter = settings.CONFIG['user']['newsletter']
profile.save() profile.save()
SessionData.objects.filter(user=instance).update(level=profile.level, SessionData.objects.filter(user=instance).update(level=profile.level,
username=instance.username) username=instance.username)

View file

@ -12,3 +12,4 @@ django-celery>2.1.1
-e git://github.com/bit/django-extensions.git#egg=django_extensions -e git://github.com/bit/django-extensions.git#egg=django_extensions
-e git+git://github.com/dcramer/django-devserver#egg=django_devserver -e git+git://github.com/dcramer/django-devserver#egg=django_devserver
gunicorn gunicorn
html5lib