merge

README

@@ -12,7 +12,8 @@ python, bazaar, pip and virtualenv and several other python modules:
     apt-get install bzr git subversion mercurial \
             python-setuptools python-pip python-virtualenv ipython \
             python-dev python-imaging python-numpy python-psycopg2 \
-            python-geoip postgresql rabbitmq-server
+            python-geoip python-html5lib python-lxml \
+            postgresql rabbitmq-server
     apt-get install oxframe oxtimeline 
 
 * Pan.do/ra

pandora/annotation/managers.py

@@ -26,6 +26,7 @@ def parseCondition(condition, user):
         'in': 'start',
         'out': 'end',
         'id': 'public_id',
+        'value': 'findvalue',
     }.get(k, k)
     if not k:
         k = 'name'

pandora/annotation/models.py

@@ -34,6 +34,7 @@ class Annotation(models.Model):
 
     layer = models.CharField(max_length=255, db_index=True)
     value = models.TextField()
+    findvalue = models.TextField()
     sortvalue = models.CharField(max_length=1000, null=True, blank=True, db_index=True)
 
     def editable(self, user):
@@ -44,23 +45,25 @@ class Annotation(models.Model):
                 return True
         return False
 
-    def html(self):
-        if self.layer == 'string':
-            return utils.html_parser(self.value)
-        else:
-            return self.value
-    
     def set_public_id(self):
         if self.id:
             public_id = Annotation.objects.filter(item=self.item, id__lt=self.id).count() + 1
             self.public_id = "%s/%s" % (self.item.itemId, ox.toAZ(public_id))
             Annotation.objects.filter(id=self.id).update(public_id=self.public_id)
 
+    def get_layer(self):
+        for layer in settings.CONFIG['layers']:
+            if layer['id'] == self.layer:
+                return layer
+        return {}
+
     def save(self, *args, **kwargs):
         set_public_id = not self.id or not self.public_id
+        layer = self.get_layer()
         if self.value:
-            sortvalue = ox.stripTags(self.value).strip()
+            self.value = utils.cleanup_value(self.value, layer['type'])
-            sortvalue = sort_string(sortvalue)
+            self.findvalue = ox.stripTags(self.value).strip()
+            sortvalue = sort_string(self.findvalue)
             if sortvalue:
                 self.sortvalue = sortvalue[:1000]
             else:
@@ -69,12 +72,7 @@ class Annotation(models.Model):
             self.sortvalue = None
 
         #no clip or update clip
-        def get_layer(id):
+        private = layer.get('private', False)
-            for l in settings.CONFIG['layers']:
-                if l['id'] == id:
-                    return l
-            return {}
-        private = get_layer(self.layer).get('private', False)
         if not private:
             if not self.clip or self.start != self.clip.start or self.end != self.clip.end:
                 self.clip, created = Clip.get_or_create(self.item, self.start, self.end)

pandora/annotation/utils.py

@@ -2,37 +2,17 @@
 # ci:si:et:sw=4:sts=4:ts=4
 import re
 import ox
+import html5lib
 
 
-def html_parser(text, nofollow=True):
+def cleanup_value(value, layer_type):
-    text = text.replace('<i>', '__i__').replace('</i>', '__/i__')
+    #FIXME: what about other types? location etc
-    text = text.replace('<b>', '__b__').replace('</b>', '__/b__')
+    if layer_type == 'text':
-    #truns links into wiki links, make sure to only take http links
+        value = sanitize_fragment(value)
-    text = re.sub('<a .*?href="(http.*?)".*?>(.*?)</a>', '[\\1 \\2]', text)
-    text = ox.escape(text)
-    text = text.replace('__i__', '<i>').replace('__/i__', '</i>')
-    text = text.replace('__b__', '<b>').replace('__/b__', '</b>')
-    if nofollow:
-        nofollow_rel = ' rel="nofollow"'
     else:
-        nofollow_rel = ''
+        value = ox.stripTags(value)
+    return value
 
-    links = re.compile('(\[(http.*?) (.*?)\])').findall(text)
+def sanitize_fragment(html):
-    for t, link, txt in links:
+    return html5lib.parseFragment(html).toxml().decode('utf-8')
-        link = link.replace('http', '__LINK__').replace('.', '__DOT__')
-        ll = '<a href="%s"%s>%s</a>' % (link, nofollow_rel, txt)
-        text = text.replace(t, ll)
-    links = re.compile('(\[(http.*?)\])').findall(text)
-    for t, link in links:
-        link = link.replace('http', '__LINK__').replace('.', '__DOT__')
-        ll = '<a href="%s"%s>%s</a>' % (link, nofollow_rel, link)
-        text = text.replace(t, ll)
 
-    text = ox.urlize(text, nofollow=nofollow)
-
-    #inpage links
-    text = re.sub('\[(/.+?) (.+?)\]', '<a href="\\1">\\2</a>', text)
-
-    text = text.replace('__LINK__', 'http').replace('__DOT__', '.')
-    text = text.replace("\n", '<br />')
-    return text

pandora/clip/managers.py

@@ -27,7 +27,7 @@ def parseCondition(condition, user):
         'in': 'start',
         'out': 'end',
         'place': 'annotations__places__id',
-        'text': 'annotations__value',
+        'text': 'annotations__findvalue',
         'user': 'annotations__user__username',
     }.get(k, k)
     if not k:
@@ -40,7 +40,7 @@ def parseCondition(condition, user):
                      for l in filter(lambda l: not l.get('private', False),
                                      settings.CONFIG['layers'])]
     if k in public_layers:
-        return parseCondition({'key': 'annotations__value',
+        return parseCondition({'key': 'annotations__findvalue',
                                'value': v,
                                'operator': op}, user) \
              & parseCondition({'key': 'annotations__layer',

pandora/clip/views.py

@@ -48,7 +48,7 @@ def order_query(qs, sort):
         if key.startswith('clip:'):
             key = e['key'][len('clip:'):]
             key = {
-                'text': 'annotations__value',
+                'text': 'annotations__sortvalue',
                 'position': 'start',
             }.get(key, key)
         elif key not in clip_keys:
@@ -111,7 +111,7 @@ def findClips(request):
                     Annotation.objects.filter(layer=layer, clip__in=ids))
     elif 'position' in query:
         qs = order_query(qs, query['sort'])
-        ids = [i.public_id for i in qs]
+        ids = [i['public_id'] for i in qs.values('public_id')]
         data['conditions'] = data['conditions'] + {
             'value': data['position'],
             'key': query['sort'][0]['key'],
@@ -123,7 +123,7 @@ def findClips(request):
             response['data']['position'] = utils.get_positions(ids, [qs[0].itemId])[0]
     elif 'positions' in data:
         qs = order_query(qs, query['sort'])
-        ids = [i.public_id for i in qs]
+        ids = [i['public_id'] for i in qs.values('public_id')]
         response['data']['positions'] = utils.get_positions(ids, data['positions'])
     else:
         response['data']['items'] = qs.count()

pandora/item/managers.py

@@ -39,6 +39,10 @@ def parseCondition(condition, user):
     else:
         exclude = False
 
+    facet_keys = models.Item.facet_keys + ['title']
+    for f in settings.CONFIG['filters']:
+        if f['id'] not in facet_keys:
+            facet_keys.append(f['id'])
     key_type = settings.CONFIG['keys'].get(k, {'type':'string'}).get('type')
     if isinstance(key_type, list):
         key_type = key_type[0]
@@ -48,8 +52,8 @@ def parseCondition(condition, user):
         'text': 'string',
         'year': 'string',
         'length': 'string',
-        'list': 'list',
         'layer': 'string',
+        'list': 'list',
     }.get(key_type, key_type)
     if k == 'list':
         key_type = ''
@@ -95,7 +99,7 @@ def parseCondition(condition, user):
             value_key = 'find__value'
         else:
             value_key = k
-        if k in models.Item.facet_keys + ['title']:
+        if k in facet_keys:
             in_find = False
             facet_value = 'facets__value%s' % {
                 '==': '__iexact',

pandora/item/models.py

@@ -545,7 +545,7 @@ class Item(models.Model):
                         '\n'.join([f.path for f in self.files.all()]))
                 elif key['type'] == 'layer':
                     qs = Annotation.objects.filter(layer=i, item=self).order_by('start')
-                    save(i, '\n'.join([l.value for l in qs]))
+                    save(i, u'\n'.join([l.findvalue for l in qs]))
                 elif i != '*' and i not in self.facet_keys:
                     value = self.get(i)
                     if isinstance(value, list):
@@ -737,6 +737,22 @@ class Item(models.Model):
         #update cached values in clips
         self.clips.all().update(director=s.director, title=s.title)
 
+    def update_layer_facets(self):
+        filters = [f['id'] for f in settings.CONFIG['filters']]
+        for layer in settings.CONFIG['layers']:
+            if layer['id'] in filters:
+                key = layer['id']
+                current_values = [a['value']
+                    for a in self.annotations.filter(layer=key).distinct().values('value')]
+                saved_values = [i.value for i in Facet.objects.filter(item=self, key=key)]
+                removed_values = filter(lambda i: i not in current_values, saved_values)
+                if removed_values:
+                    Facet.objects.filter(item=self, key=key, value__in=removed_values).delete()
+                for value in current_values:
+                    if value not in saved_values:
+                        sortvalue = value
+                        Facet.objects.get_or_create(item=self, key=key, value=value, sortvalue=sortvalue)
+
     def update_facets(self):
         for key in self.facet_keys + ['title']:
             current_values = self.get(key, [])
@@ -773,9 +789,11 @@ class Item(models.Model):
                     if key in self.person_keys + ['name']:
                         sortvalue = get_name_sort(value)
                     Facet.objects.get_or_create(item=self, key=key, value=value, sortvalue=sortvalue)
+        self.update_layer_facets()
 
     def path(self, name=''):
         h = self.itemId
+        h = (7-len(h))*'0' + h
         return os.path.join('items', h[:2], h[2:4], h[4:6], h[6:], name)
 
     '''

pandora/itemlist/models.py

@@ -122,7 +122,7 @@ class List(models.Model):
         return response
 
     def path(self, name=''):
-        h = "%06d" % self.id
+        h = "%07d" % self.id
         return os.path.join('lists', h[:2], h[2:4], h[4:6], h[6:], name)
 
     def update_icon(self):

pandora/padma.jsonc

@@ -45,16 +45,11 @@
         {"id": "collection", "title": "Collection", "type": "string"},
         {"id": "source", "title": "Source", "type": "string"},
         {"id": "director", "title": "Director", "type": "string"},
-        {"id": "location", "title": "Location", "type": "string"},
+        {"id": "locations", "title": "Location", "type": "string"},
         {"id": "year", "title": "Year", "type": "integer"},
         {"id": "language", "title": "Language", "type": "string"},
         {"id": "category", "title": "Category", "type": "string"},
-        {"id": "writer", "title": "Writer", "type": "string"},
+        {"id": "keywords", "title": "Keyword", "type": "string"}
-        {"id": "producer", "title": "Producer", "type": "string"},
-        {"id": "cinematographer", "title": "Cinematographer", "type": "string"},
-        {"id": "editor", "title": "Editor", "type": "string"},
-        {"id": "actor", "title": "Actor", "type": "string"},
-        {"id": "keyword", "title": "Keyword", "type": "string"}
     ],
     /*
         An itemKey must have the following properties:
@@ -174,30 +169,6 @@
             "filter": true,
             "find": true
         },
-        {
-            "id": "genre", 
-            "title": "Genre",
-            "type": ["string"],
-            "autocomplete": true,
-            "columnWidth": 120,
-            "filter": true,
-            "find": true
-        },
-        {
-            "id": "keyword", 
-            "title": "Keyword",
-            "type": ["string"],
-            "autocomplete": true,
-            "filter": true,
-            "find": true
-        },
-        {
-            "id": "numberofkeywords", 
-            "title": "Number of Keywords", 
-            "type": "integer",
-            "columnWidth": 60,
-            "value": {"key": "keyword", "type": "length"}
-        },
         {
             "id": "description",
             "title": "Description",
@@ -415,7 +386,7 @@
             "id": "keywords",
             "title": "Keywords",
             "overlap": true,
-            "type": "text"
+            "type": "string"
         },
         {
             "id": "descriptions",
@@ -503,11 +474,11 @@
                 }
             },
             "filters": [
-                {"id": "collection", "sort": [{"key": "name", "operator": "-"}]},
+                {"id": "collection", "sort": [{"key": "name", "operator": "+"}]},
-                {"id": "source", "sort": [{"key": "items", "operator": "-"}]},
+                {"id": "source", "sort": [{"key": "name", "operator": "+"}]},
                 {"id": "category", "sort": [{"key": "items", "operator": "-"}]},
-                {"id": "keyword", "sort": [{"key": "items", "operator": "-"}]},
+                {"id": "keywords", "sort": [{"key": "items", "operator": "-"}]},
-                {"id": "location", "sort": [{"key": "items", "operator": "-"}]}
+                {"id": "locations", "sort": [{"key": "items", "operator": "-"}]}
             ],
             "filtersSize": 176,
             "find": {"conditions": [], "operator": "&"},

pandora/user/models.py

@@ -187,6 +187,7 @@ def user_post_save(sender, instance, **kwargs):
     profile, new = UserProfile.objects.get_or_create(user=instance)
     if new and instance.is_superuser:
         profile.level = len(settings.CONFIG['userLevels']) - 1
+        profile.newsletter = settings.CONFIG['user']['newsletter']
         profile.save()
     SessionData.objects.filter(user=instance).update(level=profile.level,
                                                      username=instance.username)

requirements.txt

@@ -12,3 +12,4 @@ django-celery>2.1.1
 -e git://github.com/bit/django-extensions.git#egg=django_extensions
 -e git+git://github.com/dcramer/django-devserver#egg=django_devserver
 gunicorn
+html5lib

static/js/pandora/placesDialog.js

@@ -30,7 +30,10 @@ pandora.ui.placesDialog = function() {
                 pandora.api.findClips({
                     query: {
                         conditions: names.map(function(name) {
-                            return {key: 'subtitles', value: name, operator: '='};
+                            //FIXME: this should be more generic
+                            return Ox.getObjectById(pandora.site.layers, 'subtitles')
+                                ? {key: 'subtitles', value: name, operator: '='}
+                                : {key: 'locations', value: name, operator: '=='};
                         }),
                         operator: names.length == 1 ? '&' : '|'
                     }