streams per file and no longer per item

pandora/api/views.py

@@ -14,6 +14,7 @@ from ox.utils import json
 
 from user.models import get_user_json
 from item.models import ItemSort
+from app.models import site_config
 
 from actions import actions
 
@@ -54,26 +55,21 @@ def init(request):
     '''
     #data = json.loads(request.POST['data'])
     response = json_response({})
-    with open(settings.SITE_CONFIG) as f:
+    config = site_config()
-        config = json.load(f)
+    del config['keys'] #is this needed?
 
-        config['site']['id'] = settings.SITEID
+    #populate max values for percent requests
-        config['site']['name'] = settings.SITENAME
+    for key in filter(lambda k: 'format' in k, config['itemKeys']):
-        config['site']['sectionName'] = settings.SITENAME
+        if key['format']['type'] == 'percent' and key['format']['args'][0] == 'auto':
-        config['site']['url'] = settings.URL
+            name = key['id']
+            if name == 'popularity':
+                name = 'item__accessed__accessed'
+                value = ItemSort.objects.aggregate(Sum(name))['%s__sum'%name]
+            else:
+                value = ItemSort.objects.aggregate(Max(name))['%s__max'%name]
+            key['format']['args'][0] = value
 
-        #populate max values for percent requests
+    response['data']['site'] = config
-        for key in filter(lambda k: 'format' in k, config['itemKeys']):
-            if key['format']['type'] == 'percent' and key['format']['args'][0] == 'auto':
-                name = key['id']
-                if name == 'popularity':
-                    name = 'item__accessed__accessed'
-                    value = ItemSort.objects.aggregate(Sum(name))['%s__sum'%name]
-                else:
-                    value = ItemSort.objects.aggregate(Max(name))['%s__max'%name]
-                key['format']['args'][0] = value
-
-        response['data']['site'] = config
     if request.user.is_authenticated():
         response['data']['user'] = get_user_json(request.user)
     else:

pandora/app/models.py

@@ -28,6 +28,15 @@ class SiteSettings(models.Model):
 def site_config():
     with open(settings.SITE_CONFIG) as f:
         site_config = json.load(f)
+
+        site_config['site']['id'] = settings.SITEID
+        site_config['site']['name'] = settings.SITENAME
+        site_config['site']['sectionName'] = settings.SITENAME
+        site_config['site']['url'] = settings.URL
+
+        site_config['formats'] = settings.VIDEO_FORMATS
+        site_config['resolutions'] = settings.VIDEO_RESOLUTIONS
+
         site_config['keys'] = {}
         for key in site_config['itemKeys']:
             site_config['keys'][key['id']] = key

pandora/archive/models.py

@@ -21,6 +21,9 @@ from item import utils
 from item.models import Item
 from person.models import get_name_sort
 
+import extract
+
+
 
 class File(models.Model):
     created = models.DateTimeField(auto_now_add=True)
@@ -160,10 +163,8 @@ class File(models.Model):
         super(File, self).save(*args, **kwargs)
 
     #upload and data handling
-    video = models.FileField(null=True, blank=True,
-                        upload_to=lambda f, x: f.path(settings.VIDEO_PROFILE))
     data = models.FileField(null=True, blank=True,
-                        upload_to=lambda f, x: f.path('data.bin'))
+                            upload_to=lambda f, x: f.path('data.bin'))
 
     def path(self, name):
         h = self.oshash
@@ -245,15 +246,22 @@ class File(models.Model):
         #FIXME: check that user has instance of this file
         return True
 
-    def save_chunk(self, chunk, chunk_id=-1):
+    def save_chunk(self, chunk, chunk_id=-1, done=False):
         if not self.available:
-            if not self.video:
+            stream, created = Stream.objects.get_or_create(
-                self.video.save(settings.VIDEO_PROFILE, chunk)
+                        file=self,
+                        resolution=settings.VIDEO_RESOLUTIONS[0],
+                        format=settings.VIDEO_FORMATS[0])
+            if created:
+                stream.video.save(stream.name(), chunk)
             else:
-                f = open(self.video.path, 'a')
+                f = open(stream.video.path, 'a')
                 #FIXME: should check that chunk_id/offset is right
                 f.write(chunk.read())
                 f.close()
+            if done:
+                stream.available = True
+                stream.save()
             return True
         return False
 
@@ -365,8 +373,7 @@ class File(models.Model):
 
 def delete_file(sender, **kwargs):
     f = kwargs['instance']
-    if f.video:
+    #FIXME: delete streams here
-        f.video.delete()
     if f.data:
         f.data.delete()
 pre_delete.connect(delete_file, sender=File)
@@ -450,3 +457,71 @@ def delete_frame(sender, **kwargs):
     if f.frame:
         f.frame.delete()
 pre_delete.connect(delete_frame, sender=Frame)
+
+
+class Stream(models.Model):
+
+    class Meta:
+        unique_together = ("file", "resolution", "format")
+
+    file = models.ForeignKey(File, related_name='streams')
+    resolution = models.IntegerField(default=96)
+    format = models.CharField(max_length=255, default='webm')
+
+    video = models.FileField(default=None, blank=True, upload_to=lambda f, x: f.path())
+    source = models.ForeignKey('Stream', related_name='derivatives', default=None, null=True)
+    available = models.BooleanField(default=False)
+    info = fields.DictField(default={})
+    
+    @property
+    def timeline_prefix(self):
+        return os.path.join(settings.MEDIA_ROOT, self.path(), 'timeline')
+
+    def name(self):
+        return u"%sp.%s" % (self.resolution, self.format)
+        
+    def __unicode__(self):
+        return u"%s/%s" % (self.file, self.name())
+
+    def path(self, name=''):
+        return self.file.path(name)
+
+    def extract_derivatives(self):
+        self.make_timeline()
+        for resolution in settings.VIDEO_RESOLUTIONS:
+            for f in settings.VIDEO_FORMATS:
+                derivative, created = Stream.objects.get_or_create(file=self.file,
+                                                  resolution=resolution, format=f)
+            if created:
+                derivative.source = self
+                name = derivative.name()
+                derivative.video.name = os.path.join(os.path.dirname(self.video.name), name)
+                derivative.encode()
+                derivative.save()
+        return True
+
+    def encode(self):
+        if self.source:
+            video = self.source.video.path
+            target = self.video.path
+            info = ox.avinfo(video)
+            if extract.stream(video, target, self.name(), info):
+                self.available = True
+            else:
+                self.available = False
+            self.save()
+
+    def make_timeline(self):
+        if self.available and not self.source:
+            extract.timeline(self.video.path, self.timeline_prefix)
+
+    def save(self, *args, **kwargs):
+        if self.video and not self.info:
+            self.info = ox.avinfo(self.video.path)
+        super(Stream, self).save(*args, **kwargs)
+
+def delete_stream(sender, **kwargs):
+    f = kwargs['instance']
+    if f.video:
+        f.video.delete()
+pre_delete.connect(delete_stream, sender=Stream)

pandora/archive/views.py

@@ -111,7 +111,8 @@ actions.register(update, cache=False)
 
 @login_required_json
 def encodingProfile(request):
-    response = json_response({'profile': settings.VIDEO_PROFILE})
+    profile = "%sp.%s" % (settings.VIDEO_RESOLUTIONS[0], settings.VIDEO_FORMATS[0])
+    response = json_response({'profile': profile})
     return render_to_json_response(response)
 actions.register(encodingProfile)
 
@@ -178,7 +179,7 @@ def firefogg_upload(request):
                     'result': 1,
                     'resultUrl': request.build_absolute_uri('/')
                 }
-                if not f.save_chunk(c, chunk_id):
+                if not f.save_chunk(c, chunk_id, form.cleaned_data['done']):
                     response['result'] = -1
                 elif form.cleaned_data['done']:
                     f.available = True

pandora/item/models.py

@@ -392,8 +392,9 @@ class Item(models.Model):
 
     def get_stream(self):
         stream = {}
-        if self.streams.all().count():
+        videos = self.main_videos()
-            s = self.streams.all()[0]
+        for video in videos:
+            s = video.streams.all()[0]
             if s.video and s.info:
                 stream['duration'] = s.info['duration']
                 if 'video' in s.info and s.info['video']:
@@ -404,8 +405,6 @@ class Item(models.Model):
                     stream['baseUrl'] = '/%s' % self.itemId
                 else:
                     stream['baseUrl'] = os.path.dirname(s.video.url)
-                stream['resolutions'] = sorted(list(set(map(lambda s: int(os.path.splitext(s['profile'])[0][:-1]), self.streams.all().values('profile')))))
-                stream['formats'] = list(set(map(lambda s: os.path.splitext(s['profile'])[1][1:], self.streams.all().values('profile'))))
         return stream
 
     def get_layers(self, user=None):
@@ -451,6 +450,11 @@ class Item(models.Model):
         if not keys or 'poster' in keys:
             i['poster'] = self.get_poster()
 
+        videos = self.main_videos()
+        i['duration'] = sum([v.duration for v in videos])
+        i['durations'] = [v.duration for v in videos]
+        i['apsectRatio'] = i.get('aspectratio')
+
         #only needed by admins
         if keys and 'posters' in keys:
             i['posters'] = self.get_posters()
@@ -561,6 +565,7 @@ class Item(models.Model):
             'size',
             'bitrate',
             'numberoffiles',
+            'parts',
             'published',
             'modified',
             'popularity',
@@ -627,12 +632,13 @@ class Item(models.Model):
         if len(videos) > 0:
             s.duration = sum([v.duration for v in videos])
             s.resolution = videos[0].width * videos[0].height
-            s.aspectratio = int(1000 * utils.parse_decimal(v.display_aspect_ratio))
+            s.aspectratio = float(utils.parse_decimal(v.display_aspect_ratio))
             #FIXME: should be average over all files
             if 'bitrate' in videos[0].info:
                 s.bitrate = videos[0].info['bitrate']
             s.pixels = sum([v.pixels for v in videos])
             s.numberoffiles = self.files.all().count()
+            s.parts = len(videos)
             s.size = sum([v.size for v in videos]) #FIXME: only size of movies?
             s.volume = 0
         else:
@@ -645,6 +651,7 @@ class Item(models.Model):
             s.files = None
             s.size = None
             s.volume = None
+            s.parts = 0
 
         if 'color' in self.data:
             s.hue, s.saturation, s.lightness = self.data['color']
@@ -695,24 +702,34 @@ class Item(models.Model):
     '''
         Video related functions
     '''
-
     def frame(self, position, height=128):
-        stream = self.streams.filter(profile=settings.VIDEO_PROFILE)
+        offset = 0
-        if stream.count()>0:
+        videos = self.main_videos()
-            stream = stream[0]
+        for video in videos:
-        else:
+            if video.duration + offset < position:
-            return None
+                offset += video.duration
-        height = min(height, stream.height())
+            else:
-        path = os.path.join(settings.MEDIA_ROOT, self.path(),
+                position = position - offset
-                            'frames', "%dp"%height, "%s.jpg"%position)
+                stream = video.streams.filter(resolution=settings.VIDEO_RESOLUTIONS[0],
-        if not os.path.exists(path) and stream.video:
+                                              format=settings.VIDEO_FORMATS[0])
-            extract.frame(stream.video.path, path, position, height)
+                if stream.count()>0:
-        if not os.path.exists(path):
+                    stream = stream[0]
-            path = os.path.join(settings.STATIC_ROOT, 'png/frame.broken.png')
+                else:
-        return path
+                    return None
+                height = min(height, stream.resolution)
+                path = os.path.join(settings.MEDIA_ROOT, stream.path(),
+                                    'frames', "%dp"%height, "%s.jpg"%position)
+                if not os.path.exists(path) and stream.video:
+                    extract.frame(stream.video.path, path, position, height)
+                if not os.path.exists(path):
+                    return None
+                return path
 
     @property
     def timeline_prefix(self):
+        videos = self.main_videos()
+        if len(videos) == 1:
+            return os.path.join(settings.MEDIA_ROOT, videos[0].path('timeline'))
         return os.path.join(settings.MEDIA_ROOT, self.path(), 'timeline')
 
     def main_videos(self):
@@ -890,9 +907,7 @@ class Item(models.Model):
         return None
 
     def make_timeline(self):
-        stream = self.streams.filter(profile=settings.VIDEO_PROFILE)
+        print "FIXME, needs to build timeline from parts"
-        if stream.count() > 0 and stream[0].video:
-            extract.timeline(stream[0].video.path, self.timeline_prefix)
 
     def make_poster(self, force=False):
         if not self.poster or force:
@@ -1109,53 +1124,6 @@ class Facet(models.Model):
         super(Facet, self).save(*args, **kwargs)
 
 
-class Stream(models.Model):
-
-    class Meta:
-        unique_together = ("item", "profile")
-
-    item = models.ForeignKey(Item, related_name='streams')
-    profile = models.CharField(max_length=255, default='96p.webm')
-    video = models.FileField(default=None, blank=True, upload_to=lambda f, x: f.path())
-    source = models.ForeignKey('Stream', related_name='derivatives', default=None, null=True)
-    available = models.BooleanField(default=False)
-    info = fields.DictField(default={})
-
-    def __unicode__(self):
-        return u"%s/%s" % (self.item.itemId, self.profile)
-
-    def path(self):
-        return self.item.path(self.profile)
-
-    def height(self):
-        return int(self.profile.split('p')[0])
-
-    def extract_derivatives(self):
-        for profile in settings.VIDEO_DERIVATIVES:
-            derivative, created = Stream.objects.get_or_create(profile=profile, item=self.item)
-            if created:
-                derivative.source = self
-                derivative.video.name = self.video.name.replace(self.profile, profile)
-                derivative.encode()
-                derivative.save()
-        return True
-
-    def encode(self):
-        if self.source:
-            video = self.source.video.path
-            target = self.video.path
-            profile = self.profile
-            info = ox.avinfo(video)
-            if extract.stream(video, target, profile, info):
-                self.available=True
-                self.save()
-
-    def save(self, *args, **kwargs):
-        if self.video and not self.info:
-            self.info = ox.avinfo(self.video.path)
-        super(Stream, self).save(*args, **kwargs)
-
-
 class PosterUrl(models.Model):
 
     class Meta:

pandora/item/tasks.py

@@ -32,8 +32,12 @@ def update_streams(itemId):
         create stream, extract timeline and create derivatives
     '''
     item = models.Item.objects.get(itemId=itemId)
-    if item.files.filter(is_main=True, is_video=True, available=False).count() == 0:
+    videos = item.main_videos()
-        item.update_streams()
+    for video in videos:
+        for f in video.streams.filter(source=None):
+            f.extract_derivatives()
+    #if item.files.filter(is_main=True, is_video=True, available=False).count() == 0:
+    #    item.update_streams()
     return True
 
 def load_subtitles(itemId):

pandora/item/urls.py

@@ -13,7 +13,7 @@ urlpatterns = patterns("item.views",
     (r'^(?P<id>[A-Z0-9].+)/timeline(?P<size>\d+)p\.png$', 'timeline_overview'),
 
     #video
-    (r'^(?P<id>[A-Z0-9].+)/(?P<profile>\d+p)(?P<index>\d*)\.(?P<format>webm|ogv|mp4)$', 'video'),
+    (r'^(?P<id>[A-Z0-9].+)/(?P<resolution>\d+)p(?P<index>\d*)\.(?P<format>webm|ogv|mp4)$', 'video'),
 
     #torrent
     (r'^(?P<id>[A-Z0-9][A-Za-z0-9]+)/torrent/(?P<filename>.*?)$', 'torrent'),

pandora/item/views.py

@@ -652,21 +652,25 @@ def torrent(request, id, filename=None):
                                       os.path.basename(filename.encode('utf-8'))
     return response
 
-def video(request, id, profile, index=None, format=None):
+def video(request, id, resolution, format, index=None):
     item = get_object_or_404(models.Item, itemId=id)
     if index:
         index = int(index) - 1
-        path = item.main_videos()[index].video.path
-        #stream = item.streams.filter(profile=profile)[index]
-        #path = stream.video.path
     else:
-        stream = get_object_or_404(item.streams, profile="%s.%s" % (profile, format))
+        index = 0
-        path = stream.video.path
+    videos = item.main_videos()
+    if index > len(videos):
+        raise Http404
+
+    f = videos[index]
+    path = stream.video.path
+
     #server side cutting
+    #FIXME: this needs to join segments if needed
     t = request.GET.get('t')
     if t:
         t = map(float, t.split(','))
-        ext = os.path.splitext(profile)[1]
+        ext = '.%s' % format
         content_type = mimetypes.guess_type(path)[0]
         if len(t) == 2 and t[1] > t[0] and stream.info['duration']>=t[1]:
             response = HttpResponse(extract.chop(path, t[0], t[1]), content_type=content_type)

pandora/settings.py

@@ -174,24 +174,19 @@ VIDEO_DERIVATIVES = []
 
 TRACKER_URL="http://url2torrent.net:6970/announce"
 
+VIDEO_FORMATS=['webm']
+VIDEO_RESOLUTIONS=[96]
+
 #0xdb
 '''
-VIDEO_PROFILE = '96p.webm'
+VIDEO_FORMATS=['webm', 'mp4']
-VIDEO_DERIVATIVES = [
+VIDEO_RESOLUTIONS=[96]
-    '96p.mp4'
-]
 '''
 
 #Pad.ma
 '''
-VIDEO_PROFILE = '480p.webm'
+VIDEO_FORMATS=['webm', 'mp4']
-VIDEO_DERIVATIVES = [
+VIDEO_RESOLUTIONS=[480, 240, 96]
-    '96p.webm',
-    '240p.webm',
-    '96p.mp4',
-    '240p.mp4',
-    '480p.mp4',
-]
 '''
 
 TRANSMISSON_HOST = 'localhost'

static/js/pandora/ui/item.js

@@ -223,11 +223,11 @@ pandora.ui.item = function() {
             var layers = [],
                 video = result.data.stream,
                 cuts = result.data.cuts || [],
-                format = $.support.video.supportedFormat(video.formats),
+                format = $.support.video.supportedFormat(pandora.site.formats),
                 streams = {};
-            video.height = video.resolutions[0];
+            video.height = pandora.site.resolutions[0];
             video.width = parseInt(video.height * video.aspectRatio / 2) * 2;
-            video.resolutions.forEach(function(resolution) {
+            pandora.site.resolutions.forEach(function(resolution) {
                 streams[resolution] = video.baseUrl + '/' + resolution + 'p.' + format;
             });
             $.each(pandora.site.layers, function(i, layer) {
@@ -289,11 +289,11 @@ pandora.ui.item = function() {
             var layers = [],
                 video = result.data.stream,
                 cuts = result.data.cuts || [],
-                format = $.support.video.supportedFormat(video.formats),
+                format = $.support.video.supportedFormat(pandora.site.formats),
                 streams = {};
-            video.height = video.resolutions[0];
+            video.height = pandora.site.resolutions[0];
             video.width = parseInt(video.height * video.aspectRatio / 2) * 2;
-            video.resolutions.forEach(function(resolution) {
+            pandora.site.resolutions.forEach(function(resolution) {
                 streams[resolution] = video.baseUrl + '/' + resolution + 'p.' + format;
             });
             $.each(pandora.site.layers, function(i, layer) {