From fc26ff01bd7d93f6d47a7ac4d0987cc939790093 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Thu, 18 Aug 2011 14:01:37 +0200 Subject: [PATCH] streams per file and no longer per item --- pandora/api/views.py | 32 +++++------ pandora/app/models.py | 9 +++ pandora/archive/models.py | 93 +++++++++++++++++++++++++++--- pandora/archive/views.py | 5 +- pandora/item/models.py | 106 ++++++++++++----------------------- pandora/item/tasks.py | 8 ++- pandora/item/urls.py | 2 +- pandora/item/views.py | 18 +++--- pandora/settings.py | 19 +++---- static/js/pandora/ui/item.js | 12 ++-- 10 files changed, 178 insertions(+), 126 deletions(-) diff --git a/pandora/api/views.py b/pandora/api/views.py index cb0a9e55..0817d472 100644 --- a/pandora/api/views.py +++ b/pandora/api/views.py @@ -14,6 +14,7 @@ from ox.utils import json from user.models import get_user_json from item.models import ItemSort +from app.models import site_config from actions import actions @@ -54,26 +55,21 @@ def init(request): ''' #data = json.loads(request.POST['data']) response = json_response({}) - with open(settings.SITE_CONFIG) as f: - config = json.load(f) + config = site_config() + del config['keys'] #is this needed? - config['site']['id'] = settings.SITEID - config['site']['name'] = settings.SITENAME - config['site']['sectionName'] = settings.SITENAME - config['site']['url'] = settings.URL + #populate max values for percent requests + for key in filter(lambda k: 'format' in k, config['itemKeys']): + if key['format']['type'] == 'percent' and key['format']['args'][0] == 'auto': + name = key['id'] + if name == 'popularity': + name = 'item__accessed__accessed' + value = ItemSort.objects.aggregate(Sum(name))['%s__sum'%name] + else: + value = ItemSort.objects.aggregate(Max(name))['%s__max'%name] + key['format']['args'][0] = value - #populate max values for percent requests - for key in filter(lambda k: 'format' in k, config['itemKeys']): - if key['format']['type'] == 'percent' and key['format']['args'][0] == 'auto': - name = key['id'] - if name == 'popularity': - name = 'item__accessed__accessed' - value = ItemSort.objects.aggregate(Sum(name))['%s__sum'%name] - else: - value = ItemSort.objects.aggregate(Max(name))['%s__max'%name] - key['format']['args'][0] = value - - response['data']['site'] = config + response['data']['site'] = config if request.user.is_authenticated(): response['data']['user'] = get_user_json(request.user) else: diff --git a/pandora/app/models.py b/pandora/app/models.py index 110010c3..2d480804 100644 --- a/pandora/app/models.py +++ b/pandora/app/models.py @@ -28,6 +28,15 @@ class SiteSettings(models.Model): def site_config(): with open(settings.SITE_CONFIG) as f: site_config = json.load(f) + + site_config['site']['id'] = settings.SITEID + site_config['site']['name'] = settings.SITENAME + site_config['site']['sectionName'] = settings.SITENAME + site_config['site']['url'] = settings.URL + + site_config['formats'] = settings.VIDEO_FORMATS + site_config['resolutions'] = settings.VIDEO_RESOLUTIONS + site_config['keys'] = {} for key in site_config['itemKeys']: site_config['keys'][key['id']] = key diff --git a/pandora/archive/models.py b/pandora/archive/models.py index 92b9860b..6553b1f4 100644 --- a/pandora/archive/models.py +++ b/pandora/archive/models.py @@ -21,6 +21,9 @@ from item import utils from item.models import Item from person.models import get_name_sort +import extract + + class File(models.Model): created = models.DateTimeField(auto_now_add=True) @@ -160,10 +163,8 @@ class File(models.Model): super(File, self).save(*args, **kwargs) #upload and data handling - video = models.FileField(null=True, blank=True, - upload_to=lambda f, x: f.path(settings.VIDEO_PROFILE)) data = models.FileField(null=True, blank=True, - upload_to=lambda f, x: f.path('data.bin')) + upload_to=lambda f, x: f.path('data.bin')) def path(self, name): h = self.oshash @@ -245,15 +246,22 @@ class File(models.Model): #FIXME: check that user has instance of this file return True - def save_chunk(self, chunk, chunk_id=-1): + def save_chunk(self, chunk, chunk_id=-1, done=False): if not self.available: - if not self.video: - self.video.save(settings.VIDEO_PROFILE, chunk) + stream, created = Stream.objects.get_or_create( + file=self, + resolution=settings.VIDEO_RESOLUTIONS[0], + format=settings.VIDEO_FORMATS[0]) + if created: + stream.video.save(stream.name(), chunk) else: - f = open(self.video.path, 'a') + f = open(stream.video.path, 'a') #FIXME: should check that chunk_id/offset is right f.write(chunk.read()) f.close() + if done: + stream.available = True + stream.save() return True return False @@ -365,8 +373,7 @@ class File(models.Model): def delete_file(sender, **kwargs): f = kwargs['instance'] - if f.video: - f.video.delete() + #FIXME: delete streams here if f.data: f.data.delete() pre_delete.connect(delete_file, sender=File) @@ -450,3 +457,71 @@ def delete_frame(sender, **kwargs): if f.frame: f.frame.delete() pre_delete.connect(delete_frame, sender=Frame) + + +class Stream(models.Model): + + class Meta: + unique_together = ("file", "resolution", "format") + + file = models.ForeignKey(File, related_name='streams') + resolution = models.IntegerField(default=96) + format = models.CharField(max_length=255, default='webm') + + video = models.FileField(default=None, blank=True, upload_to=lambda f, x: f.path()) + source = models.ForeignKey('Stream', related_name='derivatives', default=None, null=True) + available = models.BooleanField(default=False) + info = fields.DictField(default={}) + + @property + def timeline_prefix(self): + return os.path.join(settings.MEDIA_ROOT, self.path(), 'timeline') + + def name(self): + return u"%sp.%s" % (self.resolution, self.format) + + def __unicode__(self): + return u"%s/%s" % (self.file, self.name()) + + def path(self, name=''): + return self.file.path(name) + + def extract_derivatives(self): + self.make_timeline() + for resolution in settings.VIDEO_RESOLUTIONS: + for f in settings.VIDEO_FORMATS: + derivative, created = Stream.objects.get_or_create(file=self.file, + resolution=resolution, format=f) + if created: + derivative.source = self + name = derivative.name() + derivative.video.name = os.path.join(os.path.dirname(self.video.name), name) + derivative.encode() + derivative.save() + return True + + def encode(self): + if self.source: + video = self.source.video.path + target = self.video.path + info = ox.avinfo(video) + if extract.stream(video, target, self.name(), info): + self.available = True + else: + self.available = False + self.save() + + def make_timeline(self): + if self.available and not self.source: + extract.timeline(self.video.path, self.timeline_prefix) + + def save(self, *args, **kwargs): + if self.video and not self.info: + self.info = ox.avinfo(self.video.path) + super(Stream, self).save(*args, **kwargs) + +def delete_stream(sender, **kwargs): + f = kwargs['instance'] + if f.video: + f.video.delete() +pre_delete.connect(delete_stream, sender=Stream) diff --git a/pandora/archive/views.py b/pandora/archive/views.py index c5f0c952..367c457c 100644 --- a/pandora/archive/views.py +++ b/pandora/archive/views.py @@ -111,7 +111,8 @@ actions.register(update, cache=False) @login_required_json def encodingProfile(request): - response = json_response({'profile': settings.VIDEO_PROFILE}) + profile = "%sp.%s" % (settings.VIDEO_RESOLUTIONS[0], settings.VIDEO_FORMATS[0]) + response = json_response({'profile': profile}) return render_to_json_response(response) actions.register(encodingProfile) @@ -178,7 +179,7 @@ def firefogg_upload(request): 'result': 1, 'resultUrl': request.build_absolute_uri('/') } - if not f.save_chunk(c, chunk_id): + if not f.save_chunk(c, chunk_id, form.cleaned_data['done']): response['result'] = -1 elif form.cleaned_data['done']: f.available = True diff --git a/pandora/item/models.py b/pandora/item/models.py index 62afbb7f..7baa0f34 100644 --- a/pandora/item/models.py +++ b/pandora/item/models.py @@ -392,8 +392,9 @@ class Item(models.Model): def get_stream(self): stream = {} - if self.streams.all().count(): - s = self.streams.all()[0] + videos = self.main_videos() + for video in videos: + s = video.streams.all()[0] if s.video and s.info: stream['duration'] = s.info['duration'] if 'video' in s.info and s.info['video']: @@ -404,8 +405,6 @@ class Item(models.Model): stream['baseUrl'] = '/%s' % self.itemId else: stream['baseUrl'] = os.path.dirname(s.video.url) - stream['resolutions'] = sorted(list(set(map(lambda s: int(os.path.splitext(s['profile'])[0][:-1]), self.streams.all().values('profile'))))) - stream['formats'] = list(set(map(lambda s: os.path.splitext(s['profile'])[1][1:], self.streams.all().values('profile')))) return stream def get_layers(self, user=None): @@ -451,6 +450,11 @@ class Item(models.Model): if not keys or 'poster' in keys: i['poster'] = self.get_poster() + videos = self.main_videos() + i['duration'] = sum([v.duration for v in videos]) + i['durations'] = [v.duration for v in videos] + i['apsectRatio'] = i.get('aspectratio') + #only needed by admins if keys and 'posters' in keys: i['posters'] = self.get_posters() @@ -561,6 +565,7 @@ class Item(models.Model): 'size', 'bitrate', 'numberoffiles', + 'parts', 'published', 'modified', 'popularity', @@ -627,12 +632,13 @@ class Item(models.Model): if len(videos) > 0: s.duration = sum([v.duration for v in videos]) s.resolution = videos[0].width * videos[0].height - s.aspectratio = int(1000 * utils.parse_decimal(v.display_aspect_ratio)) + s.aspectratio = float(utils.parse_decimal(v.display_aspect_ratio)) #FIXME: should be average over all files if 'bitrate' in videos[0].info: s.bitrate = videos[0].info['bitrate'] s.pixels = sum([v.pixels for v in videos]) s.numberoffiles = self.files.all().count() + s.parts = len(videos) s.size = sum([v.size for v in videos]) #FIXME: only size of movies? s.volume = 0 else: @@ -645,6 +651,7 @@ class Item(models.Model): s.files = None s.size = None s.volume = None + s.parts = 0 if 'color' in self.data: s.hue, s.saturation, s.lightness = self.data['color'] @@ -695,24 +702,34 @@ class Item(models.Model): ''' Video related functions ''' - def frame(self, position, height=128): - stream = self.streams.filter(profile=settings.VIDEO_PROFILE) - if stream.count()>0: - stream = stream[0] - else: - return None - height = min(height, stream.height()) - path = os.path.join(settings.MEDIA_ROOT, self.path(), - 'frames', "%dp"%height, "%s.jpg"%position) - if not os.path.exists(path) and stream.video: - extract.frame(stream.video.path, path, position, height) - if not os.path.exists(path): - path = os.path.join(settings.STATIC_ROOT, 'png/frame.broken.png') - return path + offset = 0 + videos = self.main_videos() + for video in videos: + if video.duration + offset < position: + offset += video.duration + else: + position = position - offset + stream = video.streams.filter(resolution=settings.VIDEO_RESOLUTIONS[0], + format=settings.VIDEO_FORMATS[0]) + if stream.count()>0: + stream = stream[0] + else: + return None + height = min(height, stream.resolution) + path = os.path.join(settings.MEDIA_ROOT, stream.path(), + 'frames', "%dp"%height, "%s.jpg"%position) + if not os.path.exists(path) and stream.video: + extract.frame(stream.video.path, path, position, height) + if not os.path.exists(path): + return None + return path @property def timeline_prefix(self): + videos = self.main_videos() + if len(videos) == 1: + return os.path.join(settings.MEDIA_ROOT, videos[0].path('timeline')) return os.path.join(settings.MEDIA_ROOT, self.path(), 'timeline') def main_videos(self): @@ -890,9 +907,7 @@ class Item(models.Model): return None def make_timeline(self): - stream = self.streams.filter(profile=settings.VIDEO_PROFILE) - if stream.count() > 0 and stream[0].video: - extract.timeline(stream[0].video.path, self.timeline_prefix) + print "FIXME, needs to build timeline from parts" def make_poster(self, force=False): if not self.poster or force: @@ -1109,53 +1124,6 @@ class Facet(models.Model): super(Facet, self).save(*args, **kwargs) -class Stream(models.Model): - - class Meta: - unique_together = ("item", "profile") - - item = models.ForeignKey(Item, related_name='streams') - profile = models.CharField(max_length=255, default='96p.webm') - video = models.FileField(default=None, blank=True, upload_to=lambda f, x: f.path()) - source = models.ForeignKey('Stream', related_name='derivatives', default=None, null=True) - available = models.BooleanField(default=False) - info = fields.DictField(default={}) - - def __unicode__(self): - return u"%s/%s" % (self.item.itemId, self.profile) - - def path(self): - return self.item.path(self.profile) - - def height(self): - return int(self.profile.split('p')[0]) - - def extract_derivatives(self): - for profile in settings.VIDEO_DERIVATIVES: - derivative, created = Stream.objects.get_or_create(profile=profile, item=self.item) - if created: - derivative.source = self - derivative.video.name = self.video.name.replace(self.profile, profile) - derivative.encode() - derivative.save() - return True - - def encode(self): - if self.source: - video = self.source.video.path - target = self.video.path - profile = self.profile - info = ox.avinfo(video) - if extract.stream(video, target, profile, info): - self.available=True - self.save() - - def save(self, *args, **kwargs): - if self.video and not self.info: - self.info = ox.avinfo(self.video.path) - super(Stream, self).save(*args, **kwargs) - - class PosterUrl(models.Model): class Meta: diff --git a/pandora/item/tasks.py b/pandora/item/tasks.py index 79a9a9cb..8ade9700 100644 --- a/pandora/item/tasks.py +++ b/pandora/item/tasks.py @@ -32,8 +32,12 @@ def update_streams(itemId): create stream, extract timeline and create derivatives ''' item = models.Item.objects.get(itemId=itemId) - if item.files.filter(is_main=True, is_video=True, available=False).count() == 0: - item.update_streams() + videos = item.main_videos() + for video in videos: + for f in video.streams.filter(source=None): + f.extract_derivatives() + #if item.files.filter(is_main=True, is_video=True, available=False).count() == 0: + # item.update_streams() return True def load_subtitles(itemId): diff --git a/pandora/item/urls.py b/pandora/item/urls.py index 942a9e4a..1ee7264b 100644 --- a/pandora/item/urls.py +++ b/pandora/item/urls.py @@ -13,7 +13,7 @@ urlpatterns = patterns("item.views", (r'^(?P[A-Z0-9].+)/timeline(?P\d+)p\.png$', 'timeline_overview'), #video - (r'^(?P[A-Z0-9].+)/(?P\d+p)(?P\d*)\.(?Pwebm|ogv|mp4)$', 'video'), + (r'^(?P[A-Z0-9].+)/(?P\d+)p(?P\d*)\.(?Pwebm|ogv|mp4)$', 'video'), #torrent (r'^(?P[A-Z0-9][A-Za-z0-9]+)/torrent/(?P.*?)$', 'torrent'), diff --git a/pandora/item/views.py b/pandora/item/views.py index 95e2c281..92ca38dd 100644 --- a/pandora/item/views.py +++ b/pandora/item/views.py @@ -652,21 +652,25 @@ def torrent(request, id, filename=None): os.path.basename(filename.encode('utf-8')) return response -def video(request, id, profile, index=None, format=None): +def video(request, id, resolution, format, index=None): item = get_object_or_404(models.Item, itemId=id) if index: index = int(index) - 1 - path = item.main_videos()[index].video.path - #stream = item.streams.filter(profile=profile)[index] - #path = stream.video.path else: - stream = get_object_or_404(item.streams, profile="%s.%s" % (profile, format)) - path = stream.video.path + index = 0 + videos = item.main_videos() + if index > len(videos): + raise Http404 + + f = videos[index] + path = stream.video.path + #server side cutting + #FIXME: this needs to join segments if needed t = request.GET.get('t') if t: t = map(float, t.split(',')) - ext = os.path.splitext(profile)[1] + ext = '.%s' % format content_type = mimetypes.guess_type(path)[0] if len(t) == 2 and t[1] > t[0] and stream.info['duration']>=t[1]: response = HttpResponse(extract.chop(path, t[0], t[1]), content_type=content_type) diff --git a/pandora/settings.py b/pandora/settings.py index a9bb57e1..dd74448d 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -174,24 +174,19 @@ VIDEO_DERIVATIVES = [] TRACKER_URL="http://url2torrent.net:6970/announce" +VIDEO_FORMATS=['webm'] +VIDEO_RESOLUTIONS=[96] + #0xdb ''' -VIDEO_PROFILE = '96p.webm' -VIDEO_DERIVATIVES = [ - '96p.mp4' -] +VIDEO_FORMATS=['webm', 'mp4'] +VIDEO_RESOLUTIONS=[96] ''' #Pad.ma ''' -VIDEO_PROFILE = '480p.webm' -VIDEO_DERIVATIVES = [ - '96p.webm', - '240p.webm', - '96p.mp4', - '240p.mp4', - '480p.mp4', -] +VIDEO_FORMATS=['webm', 'mp4'] +VIDEO_RESOLUTIONS=[480, 240, 96] ''' TRANSMISSON_HOST = 'localhost' diff --git a/static/js/pandora/ui/item.js b/static/js/pandora/ui/item.js index 932b2bea..e73597ef 100644 --- a/static/js/pandora/ui/item.js +++ b/static/js/pandora/ui/item.js @@ -223,11 +223,11 @@ pandora.ui.item = function() { var layers = [], video = result.data.stream, cuts = result.data.cuts || [], - format = $.support.video.supportedFormat(video.formats), + format = $.support.video.supportedFormat(pandora.site.formats), streams = {}; - video.height = video.resolutions[0]; + video.height = pandora.site.resolutions[0]; video.width = parseInt(video.height * video.aspectRatio / 2) * 2; - video.resolutions.forEach(function(resolution) { + pandora.site.resolutions.forEach(function(resolution) { streams[resolution] = video.baseUrl + '/' + resolution + 'p.' + format; }); $.each(pandora.site.layers, function(i, layer) { @@ -289,11 +289,11 @@ pandora.ui.item = function() { var layers = [], video = result.data.stream, cuts = result.data.cuts || [], - format = $.support.video.supportedFormat(video.formats), + format = $.support.video.supportedFormat(pandora.site.formats), streams = {}; - video.height = video.resolutions[0]; + video.height = pandora.site.resolutions[0]; video.width = parseInt(video.height * video.aspectRatio / 2) * 2; - video.resolutions.forEach(function(resolution) { + pandora.site.resolutions.forEach(function(resolution) { streams[resolution] = video.baseUrl + '/' + resolution + 'p.' + format; }); $.each(pandora.site.layers, function(i, layer) {