store clip values in clip db, cleanup if all annotations are gone, add additionalSort

pandora/0xdb.jsonc

@@ -4,6 +4,11 @@
     You can edit this file.
 */
 {
+    "additionalSort": [
+        {"key": "director", "operator": "+"},
+        {"key": "year", "operator": "-"},
+        {"key": "title", "operator": "+"}
+    ],
     "annotations": {
         "showUsers": true
     },
@@ -47,6 +52,12 @@
         {"id": "lightness", "title": "Lightness", "type": "float"},
         {"id": "volume", "title": "Volume", "type": "float"}
     ],
+    /*
+        clipLayers is the ordered list of public layers that will appear as the
+        text of clips. Excluding a layer from this list means it will not be
+        included in find annotations.
+    */
+    "clipLayers": ["subtitles"],
     // fixme: either this, or filter: true in itemKeys, but not both
     "filters": [
         {"id": "director", "title": "Director", "type": "string"},
@@ -93,6 +104,7 @@
             "id": "title",
             "title": "Title",
             "type": "string",
+            "additionalSort": [{"key": "year", "operator": "-"}, {"key": "director", "operator": "+"}],
             "autocomplete": true,
             "autocompleteSortKey": "votes",
             "columnRequired": true,
@@ -104,6 +116,7 @@
             "id": "director",
             "title": "Director",
             "type": ["string"],
+            "additionalSort": [{"key": "year", "operator": "-"}, {"key": "title", "operator": "-"}],
             "autocomplete": true,
             "columnRequired": true,
             "columnWidth": 180,
@@ -124,6 +137,7 @@
             "id": "year",
             "title": "Year",
             "type": "year",
+            "additionalSort": [{"key": "director", "operator": "+"}, {"key": "title", "operator": "+"}],
             "autocomplete": true,
             "columnWidth": 60,
             "filter": true,

pandora/annotation/models.py

@@ -133,10 +133,11 @@ class Annotation(models.Model):
             self.sortvalue = None
 
         #no clip or update clip
-        private = layer.get('private', False)
+        if self.layer in settings.CONFIG['clipLayers']:
-        if not private:
             if not self.clip or self.start != self.clip.start or self.end != self.clip.end:
                 self.clip, created = Clip.get_or_create(self.item, self.start, self.end)
+        elif self.clip:
+            self.clip = None
 
         super(Annotation, self).save(*args, **kwargs)
         if set_public_id:
@@ -147,6 +148,8 @@ class Annotation(models.Model):
                 'id': self.clip.id,
                 self.layer: False
             }).update(**{self.layer: True})
+            #update clip.findvalue
+            self.clip.save()
 
         if filter(lambda l: l['type'] == 'place' or l.get('hasPlaces'),
                   settings.CONFIG['layers']):

pandora/annotation/views.py

@@ -165,6 +165,8 @@ def removeAnnotation(request):
     a = get_object_or_404_json(models.Annotation, public_id=data['id'])
     if a.editable(request.user):
         a.delete()
+        if a.clip.annotations.count() == 0:
+            a.clip.delete()
     else:
         response = json_response(status=403, text='permission denied')
     return render_to_json_response(response)

pandora/clip/managers.py

@@ -27,8 +27,8 @@ def parseCondition(condition, user):
         'in': 'start',
         'out': 'end',
         'place': 'annotations__places__id',
-        'text': 'annotations__findvalue',
+        'text': 'findvalue',
-        'annotations': 'annotations__findvalue',
+        'annotations': 'findvalue',
         'user': 'annotations__user__username',
     }.get(k, k)
     if not k:
@@ -37,10 +37,7 @@ def parseCondition(condition, user):
     op = condition.get('operator')
     if not op:
         op = ''
-    public_layers = [l['id']
+    if k in settings.CONFIG['clipLayers']:
-                     for l in filter(lambda l: not l.get('private', False),
-                                     settings.CONFIG['layers'])]
-    if k in public_layers:
         return parseCondition({'key': 'annotations__findvalue',
                                'value': v,
                                'operator': op}, user) \
@@ -141,10 +138,7 @@ class ClipManager(Manager):
         return QuerySet(self.model)
 
     def filter_annotations(self, data, user):
-        public_layers = [l['id']
+        keys = settings.CONFIG['clipLayers'] + ['annotations', 'text', '*']
-                         for l in filter(lambda l: not l.get('private', False),
-                         settings.CONFIG['layers'])]
-        keys = public_layers + ['annotations', 'text', '*']
         conditions = data.get('query', {}).get('conditions', [])
         conditions = filter(lambda c: c['key'] in keys, conditions)
         operator = data.get('query', {}).get('operator', '&')
@@ -160,7 +154,7 @@ class ClipManager(Manager):
                 '$': '__iendswith',
             }.get(condition.get('opterator', ''), '__icontains'))
             q = Q(**{key: condition['value']})
-            if condition['key'] in public_layers:
+            if condition['key'] in settings.CONFIG['clipLayers']:
                 q = q & Q(layer=condition['key'])
             return q
         conditions = map(parse, conditions)
@@ -205,6 +199,6 @@ class ClipManager(Manager):
         if conditions:
             qs = qs.filter(conditions)
         if 'keys' in data:
-            for l in filter(lambda k: k in self.model.layers, data['keys']):
+            for l in filter(lambda k: k in settings.CONFIG['clipLayers'], data['keys']):
                 qs = qs.filter(**{l: True})
         return qs

pandora/clip/models.py

@@ -35,8 +35,19 @@ class MetaClip:
             streams = self.item.streams()
             if streams:
                 self.aspect_ratio = streams[0].aspect_ratio
+        sortvalue = ''
+        findvalue = ''
+        for l in settings.CONFIG['clipLayers']:
+            sortvalue += ''.join(filter(lambda s: s,
+                                 [a.sortvalue
+                                  for a in self.annotations.filter(layer=l).order_by('sortvalue')]))
+        if sortvalue:
+            self.sortvalue = sortvalue[:1000]
+        else:
+            self.sortvalue = None
+        self.findvalue = '\n'.join([a.findvalue for a in self.annotations.all()])
         if self.id:
-            for l in self.layers:
+            for l in settings.CONFIG['clipLayers']:
                 setattr(self, l, self.annotations.filter(layer=l).count()>0)
         models.Model.save(self, *args, **kwargs)
 
@@ -60,7 +71,7 @@ class MetaClip:
                     del j[key]
             #needed here to make item find with clips work
             if 'annotations' in keys:
-                annotations = self.annotations.filter(layer__in=self.layers)
+                annotations = self.annotations.filter(layer__in=settings.CONFIG['clipLayers'])
                 if qs:
                     annotations = annotations.filter(qs)
                 j['annotations'] = [a.json(keys=['value', 'id', 'layer'])
@@ -118,13 +129,11 @@ attrs = {
 
     'director': models.CharField(max_length=1000, null=True, db_index=True),
     'title': models.CharField(max_length=1000, db_index=True),
+    'sortvalue': models.CharField(max_length=1000, null=True, db_index=True),
+    'findvalue': models.TextField(),
 }
-public_layers = [l['id']
+for name in settings.CONFIG['clipLayers']:
-                 for l in filter(lambda l: not l.get('private', False),
-                                 settings.CONFIG['layers'])]
-for name in public_layers:
     attrs[name] = models.BooleanField(default=False, db_index=True)
 
 Clip = type('Clip', (MetaClip,models.Model), attrs)
-Clip.layers = public_layers
 

pandora/clip/views.py

@@ -36,20 +36,20 @@ def order_query(qs, sort):
         if operator != '-':
             operator = ''
         clip_keys = ('public_id', 'start', 'end', 'hue', 'saturation', 'lightness', 'volume',
-                     'duration', 'annotations__sortvalue', 'videoRatio',
+                     'duration', 'sortvalue', 'videoRatio',
                      'director', 'title')
         key = {
             'id': 'public_id',
             'in': 'start',
             'out': 'end',
             'position': 'start',
-            'text': 'annotations__sortvalue',
+            'text': 'sortvalue',
             'videoRatio': 'aspect_ratio',
         }.get(e['key'], e['key'])
         if key.startswith('clip:'):
             key = e['key'][len('clip:'):]
             key = {
-                'text': 'annotations__sortvalue',
+                'text': 'sortvalue',
                 'position': 'start',
             }.get(key, key)
         elif key not in clip_keys:
@@ -85,7 +85,8 @@ def findClips(request):
         qs = qs[query['range'][0]:query['range'][1]]
 
         ids = []
-        keys = filter(lambda k: k not in models.Clip.layers + ['annotations'], data['keys'])
+        keys = filter(lambda k: k not in settings.CONFIG['clipLayers'] + ['annotations'],
+                      data['keys'])
         if filter(lambda k: k not in models.Clip.clip_keys, keys):
             qs = qs.select_related('item__sort')
 
@@ -116,9 +117,9 @@ def findClips(request):
         if response['data']['items']:
             if 'annotations' in keys:
                 add_annotations('annotations',
-                    Annotation.objects.filter(layer__in=models.Clip.layers, clip__in=ids),
+                    Annotation.objects.filter(layer__in=settings.CONFIG['clipLayers'],
-                    True)
+                                              clip__in=ids), True)
-            for layer in filter(lambda l: l in keys, models.Clip.layers):
+            for layer in filter(lambda l: l in keys, settings.CONFIG['clipLayers']):
                 add_annotations(layer,
                     Annotation.objects.filter(layer=layer, clip__in=ids))
     elif 'position' in query:

pandora/item/models.py

@@ -40,7 +40,6 @@ import archive.models
 from person.models import get_name_sort
 from title.models import get_title_sort
 
-
 def get_id(info):
     q = Item.objects.all()
     for key in ('title', 'director', 'year'):
@@ -325,9 +324,6 @@ class Item(models.Model):
             if self.poster_frame == -1 and self.sort.duration:
                 self.poster_frame = self.sort.duration/2
                 update_poster = True
-            if not self.get('runtime') and self.sort.duration:
-                self.data['runtime'] = self.sort.duration
-                self.update_sort()
         self.json = self.get_json()
         super(Item, self).save(*args, **kwargs)
         if update_ids:
@@ -1143,7 +1139,7 @@ class Item(models.Model):
         return icon
 
     def load_subtitles(self):
-        if not filter(lambda l: l['id'] == 'subtitles', settings.CONFIG['layers']):
+        if not utils.get_by_id(settings.CONFIG['layers'], 'subtitles'):
             return
         with transaction.commit_on_success():
             layer = 'subtitles'
@@ -1220,7 +1216,8 @@ pre_delete.connect(delete_item, sender=Item)
 
 Item.facet_keys = []
 for key in settings.CONFIG['itemKeys']:
-    if 'autocomplete' in key and not 'autocompleteSortKey' in key:
+    if 'autocomplete' in key and not 'autocompleteSortKey' in key or \
+            key.get('filter'):
         Item.facet_keys.append(key['id'])
 
 Item.person_keys = []

pandora/item/utils.py

@@ -78,3 +78,7 @@ def get_positions(ids, pos):
         except:                         
             pass                                    
     return positions
+
+def get_by_id(objects, id):
+    obj = filter(lambda o: o['id'] == id, objects)
+    return obj and obj[0] or None

pandora/item/views.py

@@ -33,23 +33,15 @@ from clip.models import Clip
 
 from ox.django.api import actions
 
+import utils
+
 
 def _order_query(qs, sort, prefix='sort__'):
     order_by = []
     if len(sort) == 1:
-        if sort[0]['key'] == 'title':
+        key = utils.get_by_id(settings.CONFIG['itemKeys'], sort[0]['key'])
-            sort.append({'operator': '-', 'key': 'year'})
+        for s in key.get('additionalSort', settings.CONFIG.get('additionalSort', [])):
-            sort.append({'operator': '+', 'key': 'director'})
+            sort.append(s)
-        elif sort[0]['key'] == 'director':
-            sort.append({'operator': '-', 'key': 'year'})
-            sort.append({'operator': '-', 'key': 'title'})
-        elif sort[0]['key'] == 'year':
-            sort.append({'operator': '+', 'key': 'director'})
-            sort.append({'operator': '+', 'key': 'title'})
-        elif not sort[0]['key'] in ('value', 'sortvalue'):
-            sort.append({'operator': '+', 'key': 'director'})
-            sort.append({'operator': '-', 'key': 'year'})
-            sort.append({'operator': '+', 'key': 'title'})
     for e in sort:
         operator = e['operator']
         if operator != '-':
@@ -273,14 +265,21 @@ Positions
             Sum('pixels'),
             Sum('size')
         )
-        response['data']['duration'] = r['duration__sum']
+        totals = [i['id'] for i in settings.CONFIG['totals']]
-        response['data']['files'] = files.count()
+        if 'duration' in totals:
-        response['data']['items'] = items.count()
+            response['data']['duration'] = r['duration__sum']
-        response['data']['pixels'] = r['pixels__sum']
+        if 'files' in totals:
-        response['data']['runtime'] = items.aggregate(Sum('sort__runtime'))['sort__runtime__sum']
+            response['data']['files'] = files.count()
-        response['data']['size'] = r['size__sum']
+        if 'items' in totals:
+            response['data']['items'] = items.count()
+        if 'pixels' in totals:
+            response['data']['pixels'] = r['pixels__sum']
+        if 'runtime' in totals:
+            response['data']['runtime'] = items.aggregate(Sum('sort__runtime'))['sort__runtime__sum'] or 0
+        if 'size' in totals:
+            response['data']['size'] = r['size__sum']
         for key in ('runtime', 'duration', 'pixels', 'size'):
-            if response['data'][key] == None:
+            if key in totals and response['data'][key] == None:
                 response['data'][key] = 0 
     return render_to_json_response(response)
 actions.register(find)
@@ -769,8 +768,8 @@ def video(request, id, resolution, format, index=None):
             response = HttpResponse(extract.chop(path, t[0], t[1]), content_type=content_type)
             filename = "Clip of %s - %s-%s - %s %s%s" % (
                 item.get('title'),
-                ox.formatDuration(t[0] * 1000),
+                ox.formatDuration(t[0] * 1000).replace(':', '.')[:-4],
-                ox.formatDuration(t[1] * 1000),
+                ox.formatDuration(t[1] * 1000).replace(':', '.')[:-4],
                 settings.SITENAME,
                 item.itemId,
                 ext

pandora/padma.jsonc

@@ -4,6 +4,9 @@
     You can edit this file.
 */
 {
+    "additionalSort": [
+        {"key": "title", "operator": "+"}
+    ],
     "annotations": {
         "showUsers": true
     },
@@ -47,20 +50,23 @@
         {"id": "lightness", "title": "Lightness", "type": "float"},
         {"id": "volume", "title": "Volume", "type": "float"}
     ],
+    /*
+        clipLayers is the ordered list of public layers that will appear as the
+        text of clips. Excluding a layer from this list means it will not be
+        included in find annotations.
+    */
+    "clipLayers": ["transcripts", "keywords", "places", "events", "descriptions"],
     // fixme: either this, or filter: true in itemKeys, but not both
     "filters": [
         {"id": "source", "title": "Sources", "type": "string"},
         {"id": "project", "title": "Projects", "type": "string"},
         {"id": "topic", "title": "Topics", "type": "string"},
         {"id": "name", "title": "People", "type": "string"},
-        {"id": "keywords", "title": "Keywords", "type": "string"},
         {"id": "language", "title": "Languages", "type": "string"},
+        {"id": "license", "title": "License", "type": "string"},
         {"id": "places", "title": "Places", "type": "string"},
-        //{"id": "year", "title": "Years", "type": "integer"},
+        {"id": "events", "title": "Events", "type": "string"},
-        {"id": "features", "title": "Features", "type": "string"},
+        {"id": "keywords", "title": "Keywords", "type": "string"}
-        {"id": "director", "title": "Directors", "type": "string"},
-        {"id": "cinematographer", "title": "Cinematographers", "type": "string"},
-        {"id": "license", "title": "License", "type": "string"}
     ],
     /*
         An itemKey must have the following properties:
@@ -137,12 +143,6 @@
             "autocomplete": true,
             "find": true
         },
-        {
-            "id": "annotations",
-            "title": "Annotation",
-            "type": "string",
-            "find": true
-        },
         {
             "id": "keywords",
             "title": "Keywords",
@@ -155,7 +155,6 @@
             "autocomplete": true,
             "columnRequired": true,
             "columnWidth": 180,
-            "filter": true,
             "sort": "person"
         },
         {
@@ -164,12 +163,11 @@
             "type": ["string"],
             "autocomplete": true,
             "columnWidth": 180,
-            "filter": true,
             "sort": "person"
         },
         {
-            "id": "features",
+            "id": "featuring",
-            "title": "Features",
+            "title": "Featuring",
             "type": ["string"],
             "autocomplete": true,
             "columnRequired": true,
@@ -177,15 +175,6 @@
             "filter": true,
             "sort": "person"
         },
-        {
-            "id": "year",
-            "title": "Year",
-            "type": "year",
-            "autocomplete": true,
-            "columnWidth": 60,
-            "filter": true
-            //"find": true
-        },
         {
             "id": "language",
             "title": "Language",
@@ -195,13 +184,6 @@
             "filter": true,
             "find": true
         },
-        {
-            "id": "runtime", 
-            "title": "Runtime", 
-            "type": "time",
-            "columnWidth": 60,
-            "format": {"type": "duration", "args": [0, "medium"]}
-        },
         {
             "id": "location",
             "title": "Location",
@@ -211,19 +193,19 @@
             "filter": true,
             "find": true
         },
+        {
+            "id": "date",
+            "title": "Date",
+            "type": "string",
+            "columnWidth": 120
+            //"format": {"type": "date", "args": ["%a, %b %e, %Y"]}
+        },
         {
             "id": "description",
             "title": "Description",
             "type": "text",
             "find": true
         },
-        {
-            "id": "wordsinsummary",
-            "title": "Words in Summary",
-            "type": "integer",
-            "columnWidth": 60,
-            "value": {"key": "description", "type": "words"}
-        },
         {
             "id": "created",
             "title": "Date Created",
@@ -237,6 +219,12 @@
             "type": "string",
             "columnWidth": 90
         },
+        {
+            "id": "annotations",
+            "title": "Annotation",
+            "type": "string",
+            "find": true
+        },
         {
             "id": "places",
             "title": "Places",
@@ -268,7 +256,8 @@
             "id": "resolution",
             "title": "Resolution",
             "type": ["integer"],
-            "columnWidth": 90
+            "columnWidth": 90,
+            "format": {"type": "resolution", "args": ["px"]}
         },
         {
             "id": "aspectratio",
@@ -392,7 +381,6 @@
             "title": "License",
             "type": ["string"],
             "columnWidth": 120,
-            "autocomplete": true,
             "filter": true
         },
         {
@@ -517,7 +505,6 @@
     ],
     "totals": [
         {"id": "items"},
-        {"id": "runtime"},
         {"id": "files", "admin": true},
         {"id": "duration", "admin": true},
         {"id": "size", "admin": true},
@@ -558,7 +545,7 @@
             "itemFind": {"conditions": [], "operator": "&"},
             "itemSort": [{"key": "position", "operator": "+"}],
             "itemView": "info",
-            "listColumns": ["title", "source", "project", "director", "language", "duration"],
+            "listColumns": ["title", "source", "project", "topics", "language", "duration"],
             "listColumnWidth": {},
             "listSelection": [],
             "listSort": [{"key": "title", "operator": "+"}],