add layer flags to clips to avoid slow join with annotation/layer tables

pandora/clip/managers.py

@@ -165,10 +165,6 @@ class ClipManager(Manager):
         if conditions:
             qs = qs.filter(conditions)
         if 'keys' in data:
-            public_layers = [l['id']
+            for l in filter(lambda k: k in self.model.layers, data['keys']):
-                             for l in filter(lambda l: not l.get('private', False),
+                qs = qs.filter(**{l: True})
-                                             settings.CONFIG['layers'])]
-            filter_layers = filter(lambda k: k in public_layers, data['keys'])
-            if filter_layers:
-                qs = qs.filter(annotations__layer__name__in=filter_layers)
         return qs

pandora/clip/models.py

@@ -9,37 +9,7 @@ from archive import extract
 import managers
 
 
-class Clip(models.Model):
+class MetaClip:
-    '''
-    CREATE INDEX clip_clip_title_idx ON clip_clip (title ASC NULLS LAST);
-    CREATE INDEX clip_clip_director_idx ON clip_clip (director ASC NULLS LAST);
-    '''
-    class Meta:
-        unique_together = ("item", "start", "end")
-
-    objects = managers.ClipManager()
-
-    created = models.DateTimeField(auto_now_add=True)
-    modified = models.DateTimeField(auto_now=True)
-    public_id = models.CharField(max_length=128, unique=True)
-    aspect_ratio = models.FloatField(default=0)
-
-    item = models.ForeignKey('item.Item', related_name='clips')
-
-    #seconds
-    start = models.FloatField(default=-1, db_index=True)
-    end = models.FloatField(default=-1)
-    duration = models.FloatField(default=0, db_index=True)
-
-    #get from annotation
-    hue = models.FloatField(default=0, db_index=True)
-    saturation = models.FloatField(default=0, db_index=True)
-    lightness = models.FloatField(default=0, db_index=True)
-    volume = models.FloatField(default=0, null=True, db_index=True)
-
-    director = models.CharField(max_length=1000, db_index=True)
-    title = models.CharField(max_length=1000, db_index=True)
-
     def update_calculated_values(self):
         self.duration = self.end - self.start
         if self.duration > 0:
@@ -60,7 +30,9 @@ class Clip(models.Model):
             streams = self.item.streams()
             if streams:
                 self.aspect_ratio = streams[0].aspect_ratio
-        super(Clip, self).save(*args, **kwargs)
+        for l in self.layers:
+            setattr(self, l, self.annotations.filter(layer__name=l).count()>0)
+        models.Model.save(self, *args, **kwargs)
 
     def json(self, keys=None):
         j = {}
@@ -104,3 +76,40 @@ class Clip(models.Model):
     def __unicode__(self):
         return self.public_id
 
+class Meta:
+    unique_together = ("item", "start", "end")
+
+attrs = {
+    '__module__': 'clip.models',
+    'Meta': Meta,
+    'objects': managers.ClipManager(),
+    'created': models.DateTimeField(auto_now_add=True),
+    'modified': models.DateTimeField(auto_now=True),
+    'public_id': models.CharField(max_length=128, unique=True),
+    'aspect_ratio': models.FloatField(default=0),
+
+    'item': models.ForeignKey('item.Item', related_name='clips'),
+
+    #seconds
+    'start': models.FloatField(default=-1, db_index=True),
+    'end': models.FloatField(default=-1),
+    'duration': models.FloatField(default=0, db_index=True),
+
+    #get from annotation
+    'hue': models.FloatField(default=0, db_index=True),
+    'saturation': models.FloatField(default=0, db_index=True),
+    'lightness': models.FloatField(default=0, db_index=True),
+    'volume': models.FloatField(default=0, null=True, db_index=True),
+
+    'director': models.CharField(max_length=1000, db_index=True),
+    'title': models.CharField(max_length=1000, db_index=True),
+}
+public_layers = [l['id']
+                 for l in filter(lambda l: not l.get('private', False),
+                                 settings.CONFIG['layers'])]
+for name in public_layers:
+    attrs[name] = models.BooleanField(default=False, db_index=True)
+
+Clip = type('Clip', (MetaClip,models.Model), attrs)
+Clip.layers = public_layers
+

pandora/clip/views.py

@@ -82,19 +82,21 @@ def findClips(request):
     if 'keys' in data:
         qs = order_query(qs, query['sort'])
         qs = qs[query['range'][0]:query['range'][1]]
-        qs = qs.select_related('item__sort')
+        #qs = qs.select_related('item__sort')
-        response['data']['items'] = [p.json(keys=data['keys']) for p in qs]
+        ids = []
+        keys = filter(lambda k: k not in models.Clip.layers, data['keys'])
+        def add(p):
+            ids.append(p.id)
+            return p.json(keys=keys)
+        response['data']['items'] = [add(p) for p in qs]
 
         keys = data['keys']
-        public_layers = [l['id']
-                         for l in filter(lambda l: not l.get('private', False),
-                                         settings.CONFIG['layers'])]
 
         def add_annotations(layer, qs):
             for a in qs.values('public_id', 'value', 'clip__public_id'):
                 for i in response['data']['items']:
                     if i['id'] == a['clip__public_id']:
-                        if not i[layer]:
+                        if not layer in i:
                             i[layer] = []
                         i[layer].append({
                             'id': a['public_id'],
@@ -103,10 +105,10 @@ def findClips(request):
         if response['data']['items']:
             if 'annotations' in keys:
                 add_annotations('annotations',
-                    Annotation.objects.filter(layer__name__in=public_layers, clip__in=qs))
+                    Annotation.objects.filter(layer__name__in=models.Clip.layers, clip__in=ids))
-            for layer in filter(lambda l: l in keys, public_layers):
+            for layer in filter(lambda l: l in keys, models.Clip.layers):
                 add_annotations(layer,
-                    Annotation.objects.filter(layer__name=layer, clip__in=qs))
+                    Annotation.objects.filter(layer__name=layer, clip__in=ids))
     elif 'position' in query:
         qs = order_query(qs, query['sort'])
         ids = [i.public_id for i in qs]