From 052aeb7ba4e3f63828a906831b27bc9807190b30 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Fri, 3 Sep 2010 15:28:44 +0200 Subject: [PATCH] stream, videosupport --- pandora/archive/models.py | 113 ++++---------- pandora/archive/views.py | 44 +++--- pandora/backend/extract.py | 77 --------- pandora/backend/models.py | 146 +++++++++++++++--- pandora/backend/utils.py | 10 +- pandora/settings.py | 2 +- .../static/js/jquery/jquery.videosupport.js | 18 +++ pandora/templates/index.html | 1 + 8 files changed, 203 insertions(+), 208 deletions(-) delete mode 100644 pandora/backend/extract.py create mode 100644 pandora/static/js/jquery/jquery.videosupport.js diff --git a/pandora/archive/models.py b/pandora/archive/models.py index 050ba0eac..d21e313c9 100644 --- a/pandora/archive/models.py +++ b/pandora/archive/models.py @@ -34,6 +34,10 @@ def parse_decimal(string): d = string.split('/') return Decimal(d[0]) / Decimal(d[1]) +def file_path(f, name): + h = f.oshash + return os.path.join('file', h[:2], h[2:4], h[4:6], h[6:], name) + class File(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) @@ -136,11 +140,32 @@ class File(models.Model): r[k] = unicode(self[k]) return r + #upload and data handling + video_available = models.BooleanField(default=False) + video = models.FileField(null=True, blank=True, upload_to=lambda f, x: file_path(f, '%s.webm'%settings.VIDEO_PROFILE)) + data = models.FileField(null=True, blank=True, upload_to=lambda f, x: file_path(f, 'data.raw')) + def contents(self): - if self.contents_set.count() > 0: - return self.contents_set.all()[0].data + if self.data: + return self.data.read() return None + def editable(self, user): + #FIXME: check that user has instance of this file + return True + + def save_chunk(self, chunk, chunk_id=-1): + if not self.video_available: + if not self.video: + self.video.save('%s.webm'%settings.VIDEO_PROFILE, chunk) + else: + f = open(self.video.path, 'a') + #FIXME: should check that chunk_id/offset is right + f.write(chunk.read()) + f.close() + return True + return False + class Volume(models.Model): class Meta: unique_together = ("user", "name") @@ -198,88 +223,4 @@ class Frame(models.Model): def __unicode__(self): return u'%s at %s' % (self.file, self.position) -def stream_path(f): - h = f.file.oshash - return os.path.join('stream', h[:2], h[2:4], h[4:6], h[6:], f.profile) - -class Stream(models.Model): - class Meta: - unique_together = ("file", "profile") - - file = models.ForeignKey(File, related_name='streams') - profile = models.CharField(max_length=255, default='96p.webm') - video = models.FileField(default=None, blank=True, upload_to=lambda f, x: stream_path(f)) - source = models.ForeignKey('Stream', related_name='derivatives', default=None, null=True) - available = models.BooleanField(default=False) - - def __unicode__(self): - return self.video - - def extract_derivates(self): - if settings.VIDEO_H264: - profile = self.profile.replace('.webm', '.mp4') - if Stream.objects.filter(profile=profile, source=self).count() == 0: - derivate = Stream(file=self.file, source=self, profile=profile) - derivate.video.name = self.video.name.replace(self.profile, profile) - derivate.encode() - - for p in settings.VIDEO_DERIVATIVES: - profile = p + '.webm' - target = self.video.path.replace(self.profile, profile) - if Stream.objects.filter(profile=profile, source=self).count() == 0: - derivate = Stream(file=self.file, source=self, profile=profile) - derivate.video.name = self.video.name.replace(self.profile, profile) - derivate.encode() - - if settings.VIDEO_H264: - profile = p + '.mp4' - if Stream.objects.filter(profile=profile, source=self).count() == 0: - derivate = Stream(file=self.file, source=self, profile=profile) - derivate.video.name = self.video.name.replace(self.profile, profile) - derivate.encode() - return True - - def encode(self): - if self.source: - video = self.source.video.path - target = self.video.path - profile = self.profile - info = self.file.info - if extract.stream(video, target, profile, info): - self.available=True - self.save() - - def editable(self, user): - #FIXME: possibly needs user setting for stream - return True - - def save_chunk(self, chunk, chunk_id=-1): - if not self.available: - if not self.video: - self.video.save(self.profile, chunk) - else: - f = open(self.video.path, 'a') - #FIXME: should check that chunk_id/offset is right - f.write(chunk.read()) - f.close() - return True - return False - - def save(self, *args, **kwargs): - if self.available and not self.file.available: - self.file.available = True - self.file.save() - super(Stream, self).save(*args, **kwargs) - -class FileContents(models.Model): - created = models.DateTimeField(auto_now_add=True) - modified = models.DateTimeField(auto_now=True) - file = models.ForeignKey(File, related_name="contents_set") - data = models.TextField(default=u'') - - def save(self, *args, **kwargs): - if self.data and not self.file.available: - self.file.available = True - self.file.save() - super(FileContents, self).save(*args, **kwargs) diff --git a/pandora/archive/views.py b/pandora/archive/views.py index c063db578..da4c7985a 100644 --- a/pandora/archive/views.py +++ b/pandora/archive/views.py @@ -183,10 +183,8 @@ def api_upload(request): else: response = json_response(status=403, text='permissino denied') if 'file' in request.FILES: - if f.contents.count() == 0: - contents = models.FileContents(file=f) - contents.data = request.FILES['file'].read() - contents.save() + if not f.data: + f.data.save('data.raw', request.FILES['file']) response = json_response({}) else: response = json_response(status=403, text='permissino denied') @@ -203,44 +201,42 @@ def firefogg_upload(request): oshash = request.GET['oshash'] #handle video upload if request.method == 'POST': - #init upload #post next chunk if 'chunk' in request.FILES and oshash: - stream = get_object_or_404(models.Stream, file__oshash=oshash, profile=profile) - + f = get_object_or_404(models.File, oshash=oshash) form = VideoChunkForm(request.POST, request.FILES) - if form.is_valid() and stream.editable(request.user): + if form.is_valid() and profile == settings.VIDEO_PROFILE and f.editable(request.user): c = form.cleaned_data['chunk'] chunk_id = form.cleaned_data['chunkId'] response = { 'result': 1, 'resultUrl': request.build_absolute_uri('/') } - if not stream.save_chunk(c, chunk_id): + if not f.save_chunk(c, chunk_id): response['result'] = -1 elif form.cleaned_data['done']: #FIXME: send message to encode deamon to create derivates instead - stream.available = True - stream.save() + f.available = True + f.save() response['result'] = 1 response['done'] = 1 return render_to_json_response(response) - #FIXME: check for valid profile - elif oshash: + #init upload + elif oshash and profile == settings.VIDEO_PROFILE: #404 if oshash is not know, files must be registered via update api first f = get_object_or_404(models.File, oshash=oshash) - stream, created = models.Stream.objects.get_or_create(file=f, profile=profile) - if stream.video: #FIXME: check permission here instead of just starting over - stream.video.delete() - stream.available = False - stream.save() - response = { - #is it possible to no hardcode url here? - 'uploadUrl': request.build_absolute_uri('/api/upload/?oshash=%s&profile=%s' % (f.oshash, profile)), - 'result': 1 - } - return render_to_json_response(response) + if f.editable(request.user): + if f.video: + f.video.delete() + f.video_available = False + f.save() + response = { + #is it possible to no hardcode url here? + 'uploadUrl': request.build_absolute_uri('/api/upload/?oshash=%s&profile=%s' % (f.oshash, profile)), + 'result': 1 + } + return render_to_json_response(response) response = json_response(status=400, text='this request requires POST') return render_to_json_response(response) diff --git a/pandora/backend/extract.py b/pandora/backend/extract.py deleted file mode 100644 index 58f6fd920..000000000 --- a/pandora/backend/extract.py +++ /dev/null @@ -1,77 +0,0 @@ -# -*- coding: utf-8 -*- -# vi:si:et:sw=4:sts=4:ts=4 -# GPL 2010 -from __future__ import division -import re -import os -from os.path import abspath, join, dirname, exists -import shutil -import time -import warnings -import subprocess - -import ox -import Image -import simplejson as json - - -img_extension='jpg' - -def run_command(cmd, timeout=10): - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - while timeout > 0: - time.sleep(0.2) - timeout -= 0.2 - if p.poll() != None: - return p.returncode - if p.poll() == None: - os.kill(p.pid, 9) - killedpid, stat = os.waitpid(p.pid, os.WNOHANG) - return p.returncode - -def frame(videoFile, position, baseFolder, width=128, redo=False): - ''' - params: - videoFile - position as float in seconds - baseFolder to write frames to - width of frame - redo boolean to extract file even if it exists - ''' - def frame_path(size): - return os.path.join(baseFolder, "%s.%s.%s" % (ox.ms2time(position*1000), size, img_extension)) - - #not using input file, to slow to extract frame right now - base_size = 320 - frame = frame_path(base_size) - - if exists(videoFile): - if redo or not exists(frame): - if not exists(baseFolder): - os.makedirs(baseFolder) - cmd = ['oggThumb', '-t', str(position), '-n', frame, '-s', '%dx0'%base_size, videoFile] - run_command(cmd) - if width != base_size: - frame_base = frame - frame = frame_path(width) - if not exists(frame): - resize_image(frame_base, frame, width) - return frame - -def resize_image(image_source, image_output, width): - if exists(image_source): - source = Image.open(image_source) - source_width = source.size[0] - source_height = source.size[1] - - height = int(width / (float(source_width) / source_height)) - height = height - height % 2 - - if width < source_width: - resize_method = Image.ANTIALIAS - else: - resize_method = Image.BICUBIC - output = source.resize((width, height), resize_method) - output.save(image_output) - - diff --git a/pandora/backend/models.py b/pandora/backend/models.py index 1a06dce1e..4a1ef52d7 100644 --- a/pandora/backend/models.py +++ b/pandora/backend/models.py @@ -4,8 +4,10 @@ from __future__ import division, with_statement from datetime import datetime import os.path +import math import random import re +import subprocess from django.db import models from django.db.models import Q @@ -23,7 +25,7 @@ from firefogg import Firefogg import managers import load import utils -import extract +from archive import extract def getMovie(info): @@ -204,14 +206,28 @@ class Movie(models.Model): movie[pub_key] = value() else: movie[pub_key] = value - movie['poster'] = self.get_poster() - + if not fields: + movie['poster'] = self.get_poster() + movie['stream'] = self.get_stream() if fields: for f in fields: if f.endswith('.length') and f[:-7] in ('cast', 'genre', 'trivia'): movie[f] = getattr(self.sort, f[:-7]) return movie + def get_stream(self): + stream = {} + if self.streams.all().count(): + s = self.streams.all()[0] + if s.video and s.info: + stream['duration'] = s.info['duration'] + if 'video' in s.info and s.info['video']: + stream['aspectRatio'] = s.info['video'][0]['width'] / s.info['video'][0]['height'] + + stream['baseUrl'] = os.path.dirname(s.video.url) + stream['profiles'] = list(set(map(lambda s: int(os.path.splitext(s['profile'])[0][:-1]), self.streams.all().values('profile')))) + return stream + def fields(self): fields = {} for f in self._meta.fields: @@ -227,13 +243,6 @@ class Movie(models.Model): self.get('series title', ''), self.get('episode title', ''), self.get('season', ''), self.get('episode', '')) - def streams(self): - streams = [] - for f in self.files.filter(is_main=True, available=True): - for s in f.streams.all(): - streams.append(s.video.url) - return streams - def frame(self, position, width=128): #FIXME: compute offset and so on f = self.files.all()[0] @@ -292,15 +301,7 @@ class Movie(models.Model): #title title = canonicalTitle(self.get('title')) - title = re.sub(u'[\'!¿¡,\.;\-"\:\*\[\]]', '', title) - title = title.replace(u'Æ', 'Ae') - #pad numbered titles - numbers = re.compile('^(\d+)').findall(title) - if numbers: - padded = '%010d' % int(numbers[0]) - title = re.sub('^(\d+)', padded, title) - - s.title = title.strip() + s.title = utils.sort_title(title) s.country = ','.join(self.get('countries', [])) s.year = self.get('year', '') @@ -373,6 +374,55 @@ class Movie(models.Model): else: Facet.objects.filter(movie=self, key='year').delete() + def updatePoster(self): + n = self.files.count() * 3 + frame = int(math.floor(n/2)) + part = 1 + for f in self.files.filter(is_main=True, video_available=True): + for frame in f.frames.all(): + path = os.path.abspath(os.path.join(settings.MEDIA_ROOT, poster_path(self))) + path = path.replace('.jpg', '%s.%s.jpg'%(part, frame.pos)) + cmd = ['oxposter', + '-t', self.get('title'), + '-d', self.get('director'), + '-f', frame.frame.path, + '-p', path + ] + if len(self.movieId) == 7: + cmd += ['-i', self.movieId] + else: + cmd += ['-o', self.movieId] + print cmd + subprocess.Popen(cmd) + part += 1 + + def updateStreams(self): + files = {} + for f in self.files.filter(is_main=True, video_available=True): + files[utils.sort_title(f.name)] = f.video.path + + if files: + stream, created = Stream.objects.get_or_create(movie=self, profile='%s.webm' % settings.VIDEO_PROFILE) + stream.video.name = stream_path(stream) + cmd = [] + + for f in sorted(files): + if not cmd: + cmd.append(files[f]) + else: + cmd.append('+') + cmd.append(files[f]) + cmd = [ 'mkvmerge', '-o', stream.video.path ] + cmd + subprocess.Popen(cmd) + stream.save() + + extract.timeline(stream.video.path, os.path.join(stream.video.path[:-len(stream.profile)], 'timeline')) + stream.extract_derivatives() + + #something with poster + self.available = True + self.save() + class MovieFind(models.Model): """ used to search movies, all search values are in here @@ -638,3 +688,61 @@ class Collection(models.Model): def editable(self, user): return self.users.filter(id=user.id).count() > 0 + +def stream_path(f): + h = f.movie.movieId + return os.path.join('stream', h[:2], h[2:4], h[4:6], h[6:], f.profile) + +class Stream(models.Model): + class Meta: + unique_together = ("movie", "profile") + + movie = models.ForeignKey(Movie, related_name='streams') + profile = models.CharField(max_length=255, default='96p.webm') + video = models.FileField(default=None, blank=True, upload_to=lambda f, x: stream_path(f)) + source = models.ForeignKey('Stream', related_name='derivatives', default=None, null=True) + available = models.BooleanField(default=False) + info = fields.DictField(default={}) + + #def __unicode__(self): + # return self.video + + def extract_derivatives(self): + if settings.VIDEO_H264: + profile = self.profile.replace('.webm', '.mp4') + if Stream.objects.filter(profile=profile, source=self).count() == 0: + derivative = Stream(movie=self.movie, source=self, profile=profile) + derivative.video.name = self.video.name.replace(self.profile, profile) + derivative.encode() + + for p in settings.VIDEO_DERIVATIVES: + profile = p + '.webm' + target = self.video.path.replace(self.profile, profile) + if Stream.objects.filter(profile=profile, source=self).count() == 0: + derivative = Stream(movie=movie.file, source=self, profile=profile) + derivative.video.name = self.video.name.replace(self.profile, profile) + derivative.encode() + + if settings.VIDEO_H264: + profile = p + '.mp4' + if Stream.objects.filter(profile=profile, source=self).count() == 0: + derivative = Stream(movie=self.movie, source=self, profile=profile) + derivative.video.name = self.video.name.replace(self.profile, profile) + derivative.encode() + return True + + def encode(self): + if self.source: + video = self.source.video.path + target = self.video.path + profile = self.profile + info = ox.avinfo(video) + if extract.stream(video, target, profile, info): + self.available=True + self.save() + + def save(self, *args, **kwargs): + if self.video and not self.info: + self.info = ox.avinfo(self.video.path) + super(Stream, self).save(*args, **kwargs) + diff --git a/pandora/backend/utils.py b/pandora/backend/utils.py index 5a55af3d9..a289deeff 100644 --- a/pandora/backend/utils.py +++ b/pandora/backend/utils.py @@ -10,7 +10,7 @@ import hashlib import ox import ox.iso -from ox.normalize import normalizeName, normalizeTitle +from ox.normalize import normalizeName, normalizeTitle, canonicalTitle def plural_key(term): @@ -149,3 +149,11 @@ def parse_path(path): season=r['season'], episode=r['episode']) return r +def sort_title(title): + #title + title = re.sub(u'[\'!¿¡,\.;\-"\:\*\[\]]', '', title) + title = title.replace(u'Æ', 'Ae') + #pad numbered titles + title = re.sub('(\d+)', lambda x: '%010d' % int(x.group(0)), title) + return title.strip() + diff --git a/pandora/settings.py b/pandora/settings.py index c8b5af73a..abaa3a206 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -69,7 +69,7 @@ TESTS_ROOT = join(PROJECT_ROOT, 'tests') # URL that handles the media served from MEDIA_ROOT. Make sure to use a # trailing slash if there is a path component (optional in other cases). # Examples: "http://media.lawrence.com", "http://example.com/media/" -MEDIA_URL = '' +MEDIA_URL = '/media/' STATIC_URL = '/static/' diff --git a/pandora/static/js/jquery/jquery.videosupport.js b/pandora/static/js/jquery/jquery.videosupport.js new file mode 100644 index 000000000..ae999c3f8 --- /dev/null +++ b/pandora/static/js/jquery/jquery.videosupport.js @@ -0,0 +1,18 @@ +jQuery.support.video = function() { + jQuery.browser.chrome = /chrome/.test(navigator.userAgent.toLowerCase()); + var video = {}; + var v = document.createElement('video'); + if (v) { + video.support = true; + video.webm = !!(v.canPlayType && v.canPlayType('video/webm; codecs="vp8, vorbis"').replace(/no/, '')); + //Disable WebM on Safari/Perian, seeking does not work + if(video.webm && jQuery.browser.safari && !jQuery.browser.chrome) + video.webm = false; + video.h264 = !!(v.canPlayType && v.canPlayType('video/mp4; codecs="avc1.42E01E, mp4a.40.2"').replace(/no/, '')); + video.ogg = !!(v.canPlayType && v.canPlayType('video/ogg; codecs="theora, vorbis"').replace(/no/, '')); + } else { + video.support = false; + } + return video; +}(); + diff --git a/pandora/templates/index.html b/pandora/templates/index.html index 423e763a4..db3e98540 100644 --- a/pandora/templates/index.html +++ b/pandora/templates/index.html @@ -6,6 +6,7 @@ +