streams per file and no longer per item

This commit is contained in:
j 2011-08-18 14:01:37 +02:00
parent f228a4a5e5
commit fc26ff01bd
10 changed files with 178 additions and 126 deletions

View file

@ -14,6 +14,7 @@ from ox.utils import json
from user.models import get_user_json from user.models import get_user_json
from item.models import ItemSort from item.models import ItemSort
from app.models import site_config
from actions import actions from actions import actions
@ -54,26 +55,21 @@ def init(request):
''' '''
#data = json.loads(request.POST['data']) #data = json.loads(request.POST['data'])
response = json_response({}) response = json_response({})
with open(settings.SITE_CONFIG) as f: config = site_config()
config = json.load(f) del config['keys'] #is this needed?
config['site']['id'] = settings.SITEID #populate max values for percent requests
config['site']['name'] = settings.SITENAME for key in filter(lambda k: 'format' in k, config['itemKeys']):
config['site']['sectionName'] = settings.SITENAME if key['format']['type'] == 'percent' and key['format']['args'][0] == 'auto':
config['site']['url'] = settings.URL name = key['id']
if name == 'popularity':
name = 'item__accessed__accessed'
value = ItemSort.objects.aggregate(Sum(name))['%s__sum'%name]
else:
value = ItemSort.objects.aggregate(Max(name))['%s__max'%name]
key['format']['args'][0] = value
#populate max values for percent requests response['data']['site'] = config
for key in filter(lambda k: 'format' in k, config['itemKeys']):
if key['format']['type'] == 'percent' and key['format']['args'][0] == 'auto':
name = key['id']
if name == 'popularity':
name = 'item__accessed__accessed'
value = ItemSort.objects.aggregate(Sum(name))['%s__sum'%name]
else:
value = ItemSort.objects.aggregate(Max(name))['%s__max'%name]
key['format']['args'][0] = value
response['data']['site'] = config
if request.user.is_authenticated(): if request.user.is_authenticated():
response['data']['user'] = get_user_json(request.user) response['data']['user'] = get_user_json(request.user)
else: else:

View file

@ -28,6 +28,15 @@ class SiteSettings(models.Model):
def site_config(): def site_config():
with open(settings.SITE_CONFIG) as f: with open(settings.SITE_CONFIG) as f:
site_config = json.load(f) site_config = json.load(f)
site_config['site']['id'] = settings.SITEID
site_config['site']['name'] = settings.SITENAME
site_config['site']['sectionName'] = settings.SITENAME
site_config['site']['url'] = settings.URL
site_config['formats'] = settings.VIDEO_FORMATS
site_config['resolutions'] = settings.VIDEO_RESOLUTIONS
site_config['keys'] = {} site_config['keys'] = {}
for key in site_config['itemKeys']: for key in site_config['itemKeys']:
site_config['keys'][key['id']] = key site_config['keys'][key['id']] = key

View file

@ -21,6 +21,9 @@ from item import utils
from item.models import Item from item.models import Item
from person.models import get_name_sort from person.models import get_name_sort
import extract
class File(models.Model): class File(models.Model):
created = models.DateTimeField(auto_now_add=True) created = models.DateTimeField(auto_now_add=True)
@ -160,10 +163,8 @@ class File(models.Model):
super(File, self).save(*args, **kwargs) super(File, self).save(*args, **kwargs)
#upload and data handling #upload and data handling
video = models.FileField(null=True, blank=True,
upload_to=lambda f, x: f.path(settings.VIDEO_PROFILE))
data = models.FileField(null=True, blank=True, data = models.FileField(null=True, blank=True,
upload_to=lambda f, x: f.path('data.bin')) upload_to=lambda f, x: f.path('data.bin'))
def path(self, name): def path(self, name):
h = self.oshash h = self.oshash
@ -245,15 +246,22 @@ class File(models.Model):
#FIXME: check that user has instance of this file #FIXME: check that user has instance of this file
return True return True
def save_chunk(self, chunk, chunk_id=-1): def save_chunk(self, chunk, chunk_id=-1, done=False):
if not self.available: if not self.available:
if not self.video: stream, created = Stream.objects.get_or_create(
self.video.save(settings.VIDEO_PROFILE, chunk) file=self,
resolution=settings.VIDEO_RESOLUTIONS[0],
format=settings.VIDEO_FORMATS[0])
if created:
stream.video.save(stream.name(), chunk)
else: else:
f = open(self.video.path, 'a') f = open(stream.video.path, 'a')
#FIXME: should check that chunk_id/offset is right #FIXME: should check that chunk_id/offset is right
f.write(chunk.read()) f.write(chunk.read())
f.close() f.close()
if done:
stream.available = True
stream.save()
return True return True
return False return False
@ -365,8 +373,7 @@ class File(models.Model):
def delete_file(sender, **kwargs): def delete_file(sender, **kwargs):
f = kwargs['instance'] f = kwargs['instance']
if f.video: #FIXME: delete streams here
f.video.delete()
if f.data: if f.data:
f.data.delete() f.data.delete()
pre_delete.connect(delete_file, sender=File) pre_delete.connect(delete_file, sender=File)
@ -450,3 +457,71 @@ def delete_frame(sender, **kwargs):
if f.frame: if f.frame:
f.frame.delete() f.frame.delete()
pre_delete.connect(delete_frame, sender=Frame) pre_delete.connect(delete_frame, sender=Frame)
class Stream(models.Model):
class Meta:
unique_together = ("file", "resolution", "format")
file = models.ForeignKey(File, related_name='streams')
resolution = models.IntegerField(default=96)
format = models.CharField(max_length=255, default='webm')
video = models.FileField(default=None, blank=True, upload_to=lambda f, x: f.path())
source = models.ForeignKey('Stream', related_name='derivatives', default=None, null=True)
available = models.BooleanField(default=False)
info = fields.DictField(default={})
@property
def timeline_prefix(self):
return os.path.join(settings.MEDIA_ROOT, self.path(), 'timeline')
def name(self):
return u"%sp.%s" % (self.resolution, self.format)
def __unicode__(self):
return u"%s/%s" % (self.file, self.name())
def path(self, name=''):
return self.file.path(name)
def extract_derivatives(self):
self.make_timeline()
for resolution in settings.VIDEO_RESOLUTIONS:
for f in settings.VIDEO_FORMATS:
derivative, created = Stream.objects.get_or_create(file=self.file,
resolution=resolution, format=f)
if created:
derivative.source = self
name = derivative.name()
derivative.video.name = os.path.join(os.path.dirname(self.video.name), name)
derivative.encode()
derivative.save()
return True
def encode(self):
if self.source:
video = self.source.video.path
target = self.video.path
info = ox.avinfo(video)
if extract.stream(video, target, self.name(), info):
self.available = True
else:
self.available = False
self.save()
def make_timeline(self):
if self.available and not self.source:
extract.timeline(self.video.path, self.timeline_prefix)
def save(self, *args, **kwargs):
if self.video and not self.info:
self.info = ox.avinfo(self.video.path)
super(Stream, self).save(*args, **kwargs)
def delete_stream(sender, **kwargs):
f = kwargs['instance']
if f.video:
f.video.delete()
pre_delete.connect(delete_stream, sender=Stream)

View file

@ -111,7 +111,8 @@ actions.register(update, cache=False)
@login_required_json @login_required_json
def encodingProfile(request): def encodingProfile(request):
response = json_response({'profile': settings.VIDEO_PROFILE}) profile = "%sp.%s" % (settings.VIDEO_RESOLUTIONS[0], settings.VIDEO_FORMATS[0])
response = json_response({'profile': profile})
return render_to_json_response(response) return render_to_json_response(response)
actions.register(encodingProfile) actions.register(encodingProfile)
@ -178,7 +179,7 @@ def firefogg_upload(request):
'result': 1, 'result': 1,
'resultUrl': request.build_absolute_uri('/') 'resultUrl': request.build_absolute_uri('/')
} }
if not f.save_chunk(c, chunk_id): if not f.save_chunk(c, chunk_id, form.cleaned_data['done']):
response['result'] = -1 response['result'] = -1
elif form.cleaned_data['done']: elif form.cleaned_data['done']:
f.available = True f.available = True

View file

@ -392,8 +392,9 @@ class Item(models.Model):
def get_stream(self): def get_stream(self):
stream = {} stream = {}
if self.streams.all().count(): videos = self.main_videos()
s = self.streams.all()[0] for video in videos:
s = video.streams.all()[0]
if s.video and s.info: if s.video and s.info:
stream['duration'] = s.info['duration'] stream['duration'] = s.info['duration']
if 'video' in s.info and s.info['video']: if 'video' in s.info and s.info['video']:
@ -404,8 +405,6 @@ class Item(models.Model):
stream['baseUrl'] = '/%s' % self.itemId stream['baseUrl'] = '/%s' % self.itemId
else: else:
stream['baseUrl'] = os.path.dirname(s.video.url) stream['baseUrl'] = os.path.dirname(s.video.url)
stream['resolutions'] = sorted(list(set(map(lambda s: int(os.path.splitext(s['profile'])[0][:-1]), self.streams.all().values('profile')))))
stream['formats'] = list(set(map(lambda s: os.path.splitext(s['profile'])[1][1:], self.streams.all().values('profile'))))
return stream return stream
def get_layers(self, user=None): def get_layers(self, user=None):
@ -451,6 +450,11 @@ class Item(models.Model):
if not keys or 'poster' in keys: if not keys or 'poster' in keys:
i['poster'] = self.get_poster() i['poster'] = self.get_poster()
videos = self.main_videos()
i['duration'] = sum([v.duration for v in videos])
i['durations'] = [v.duration for v in videos]
i['apsectRatio'] = i.get('aspectratio')
#only needed by admins #only needed by admins
if keys and 'posters' in keys: if keys and 'posters' in keys:
i['posters'] = self.get_posters() i['posters'] = self.get_posters()
@ -561,6 +565,7 @@ class Item(models.Model):
'size', 'size',
'bitrate', 'bitrate',
'numberoffiles', 'numberoffiles',
'parts',
'published', 'published',
'modified', 'modified',
'popularity', 'popularity',
@ -627,12 +632,13 @@ class Item(models.Model):
if len(videos) > 0: if len(videos) > 0:
s.duration = sum([v.duration for v in videos]) s.duration = sum([v.duration for v in videos])
s.resolution = videos[0].width * videos[0].height s.resolution = videos[0].width * videos[0].height
s.aspectratio = int(1000 * utils.parse_decimal(v.display_aspect_ratio)) s.aspectratio = float(utils.parse_decimal(v.display_aspect_ratio))
#FIXME: should be average over all files #FIXME: should be average over all files
if 'bitrate' in videos[0].info: if 'bitrate' in videos[0].info:
s.bitrate = videos[0].info['bitrate'] s.bitrate = videos[0].info['bitrate']
s.pixels = sum([v.pixels for v in videos]) s.pixels = sum([v.pixels for v in videos])
s.numberoffiles = self.files.all().count() s.numberoffiles = self.files.all().count()
s.parts = len(videos)
s.size = sum([v.size for v in videos]) #FIXME: only size of movies? s.size = sum([v.size for v in videos]) #FIXME: only size of movies?
s.volume = 0 s.volume = 0
else: else:
@ -645,6 +651,7 @@ class Item(models.Model):
s.files = None s.files = None
s.size = None s.size = None
s.volume = None s.volume = None
s.parts = 0
if 'color' in self.data: if 'color' in self.data:
s.hue, s.saturation, s.lightness = self.data['color'] s.hue, s.saturation, s.lightness = self.data['color']
@ -695,24 +702,34 @@ class Item(models.Model):
''' '''
Video related functions Video related functions
''' '''
def frame(self, position, height=128): def frame(self, position, height=128):
stream = self.streams.filter(profile=settings.VIDEO_PROFILE) offset = 0
if stream.count()>0: videos = self.main_videos()
stream = stream[0] for video in videos:
else: if video.duration + offset < position:
return None offset += video.duration
height = min(height, stream.height()) else:
path = os.path.join(settings.MEDIA_ROOT, self.path(), position = position - offset
'frames', "%dp"%height, "%s.jpg"%position) stream = video.streams.filter(resolution=settings.VIDEO_RESOLUTIONS[0],
if not os.path.exists(path) and stream.video: format=settings.VIDEO_FORMATS[0])
extract.frame(stream.video.path, path, position, height) if stream.count()>0:
if not os.path.exists(path): stream = stream[0]
path = os.path.join(settings.STATIC_ROOT, 'png/frame.broken.png') else:
return path return None
height = min(height, stream.resolution)
path = os.path.join(settings.MEDIA_ROOT, stream.path(),
'frames', "%dp"%height, "%s.jpg"%position)
if not os.path.exists(path) and stream.video:
extract.frame(stream.video.path, path, position, height)
if not os.path.exists(path):
return None
return path
@property @property
def timeline_prefix(self): def timeline_prefix(self):
videos = self.main_videos()
if len(videos) == 1:
return os.path.join(settings.MEDIA_ROOT, videos[0].path('timeline'))
return os.path.join(settings.MEDIA_ROOT, self.path(), 'timeline') return os.path.join(settings.MEDIA_ROOT, self.path(), 'timeline')
def main_videos(self): def main_videos(self):
@ -890,9 +907,7 @@ class Item(models.Model):
return None return None
def make_timeline(self): def make_timeline(self):
stream = self.streams.filter(profile=settings.VIDEO_PROFILE) print "FIXME, needs to build timeline from parts"
if stream.count() > 0 and stream[0].video:
extract.timeline(stream[0].video.path, self.timeline_prefix)
def make_poster(self, force=False): def make_poster(self, force=False):
if not self.poster or force: if not self.poster or force:
@ -1109,53 +1124,6 @@ class Facet(models.Model):
super(Facet, self).save(*args, **kwargs) super(Facet, self).save(*args, **kwargs)
class Stream(models.Model):
class Meta:
unique_together = ("item", "profile")
item = models.ForeignKey(Item, related_name='streams')
profile = models.CharField(max_length=255, default='96p.webm')
video = models.FileField(default=None, blank=True, upload_to=lambda f, x: f.path())
source = models.ForeignKey('Stream', related_name='derivatives', default=None, null=True)
available = models.BooleanField(default=False)
info = fields.DictField(default={})
def __unicode__(self):
return u"%s/%s" % (self.item.itemId, self.profile)
def path(self):
return self.item.path(self.profile)
def height(self):
return int(self.profile.split('p')[0])
def extract_derivatives(self):
for profile in settings.VIDEO_DERIVATIVES:
derivative, created = Stream.objects.get_or_create(profile=profile, item=self.item)
if created:
derivative.source = self
derivative.video.name = self.video.name.replace(self.profile, profile)
derivative.encode()
derivative.save()
return True
def encode(self):
if self.source:
video = self.source.video.path
target = self.video.path
profile = self.profile
info = ox.avinfo(video)
if extract.stream(video, target, profile, info):
self.available=True
self.save()
def save(self, *args, **kwargs):
if self.video and not self.info:
self.info = ox.avinfo(self.video.path)
super(Stream, self).save(*args, **kwargs)
class PosterUrl(models.Model): class PosterUrl(models.Model):
class Meta: class Meta:

View file

@ -32,8 +32,12 @@ def update_streams(itemId):
create stream, extract timeline and create derivatives create stream, extract timeline and create derivatives
''' '''
item = models.Item.objects.get(itemId=itemId) item = models.Item.objects.get(itemId=itemId)
if item.files.filter(is_main=True, is_video=True, available=False).count() == 0: videos = item.main_videos()
item.update_streams() for video in videos:
for f in video.streams.filter(source=None):
f.extract_derivatives()
#if item.files.filter(is_main=True, is_video=True, available=False).count() == 0:
# item.update_streams()
return True return True
def load_subtitles(itemId): def load_subtitles(itemId):

View file

@ -13,7 +13,7 @@ urlpatterns = patterns("item.views",
(r'^(?P<id>[A-Z0-9].+)/timeline(?P<size>\d+)p\.png$', 'timeline_overview'), (r'^(?P<id>[A-Z0-9].+)/timeline(?P<size>\d+)p\.png$', 'timeline_overview'),
#video #video
(r'^(?P<id>[A-Z0-9].+)/(?P<profile>\d+p)(?P<index>\d*)\.(?P<format>webm|ogv|mp4)$', 'video'), (r'^(?P<id>[A-Z0-9].+)/(?P<resolution>\d+)p(?P<index>\d*)\.(?P<format>webm|ogv|mp4)$', 'video'),
#torrent #torrent
(r'^(?P<id>[A-Z0-9][A-Za-z0-9]+)/torrent/(?P<filename>.*?)$', 'torrent'), (r'^(?P<id>[A-Z0-9][A-Za-z0-9]+)/torrent/(?P<filename>.*?)$', 'torrent'),

View file

@ -652,21 +652,25 @@ def torrent(request, id, filename=None):
os.path.basename(filename.encode('utf-8')) os.path.basename(filename.encode('utf-8'))
return response return response
def video(request, id, profile, index=None, format=None): def video(request, id, resolution, format, index=None):
item = get_object_or_404(models.Item, itemId=id) item = get_object_or_404(models.Item, itemId=id)
if index: if index:
index = int(index) - 1 index = int(index) - 1
path = item.main_videos()[index].video.path
#stream = item.streams.filter(profile=profile)[index]
#path = stream.video.path
else: else:
stream = get_object_or_404(item.streams, profile="%s.%s" % (profile, format)) index = 0
path = stream.video.path videos = item.main_videos()
if index > len(videos):
raise Http404
f = videos[index]
path = stream.video.path
#server side cutting #server side cutting
#FIXME: this needs to join segments if needed
t = request.GET.get('t') t = request.GET.get('t')
if t: if t:
t = map(float, t.split(',')) t = map(float, t.split(','))
ext = os.path.splitext(profile)[1] ext = '.%s' % format
content_type = mimetypes.guess_type(path)[0] content_type = mimetypes.guess_type(path)[0]
if len(t) == 2 and t[1] > t[0] and stream.info['duration']>=t[1]: if len(t) == 2 and t[1] > t[0] and stream.info['duration']>=t[1]:
response = HttpResponse(extract.chop(path, t[0], t[1]), content_type=content_type) response = HttpResponse(extract.chop(path, t[0], t[1]), content_type=content_type)

View file

@ -174,24 +174,19 @@ VIDEO_DERIVATIVES = []
TRACKER_URL="http://url2torrent.net:6970/announce" TRACKER_URL="http://url2torrent.net:6970/announce"
VIDEO_FORMATS=['webm']
VIDEO_RESOLUTIONS=[96]
#0xdb #0xdb
''' '''
VIDEO_PROFILE = '96p.webm' VIDEO_FORMATS=['webm', 'mp4']
VIDEO_DERIVATIVES = [ VIDEO_RESOLUTIONS=[96]
'96p.mp4'
]
''' '''
#Pad.ma #Pad.ma
''' '''
VIDEO_PROFILE = '480p.webm' VIDEO_FORMATS=['webm', 'mp4']
VIDEO_DERIVATIVES = [ VIDEO_RESOLUTIONS=[480, 240, 96]
'96p.webm',
'240p.webm',
'96p.mp4',
'240p.mp4',
'480p.mp4',
]
''' '''
TRANSMISSON_HOST = 'localhost' TRANSMISSON_HOST = 'localhost'

View file

@ -223,11 +223,11 @@ pandora.ui.item = function() {
var layers = [], var layers = [],
video = result.data.stream, video = result.data.stream,
cuts = result.data.cuts || [], cuts = result.data.cuts || [],
format = $.support.video.supportedFormat(video.formats), format = $.support.video.supportedFormat(pandora.site.formats),
streams = {}; streams = {};
video.height = video.resolutions[0]; video.height = pandora.site.resolutions[0];
video.width = parseInt(video.height * video.aspectRatio / 2) * 2; video.width = parseInt(video.height * video.aspectRatio / 2) * 2;
video.resolutions.forEach(function(resolution) { pandora.site.resolutions.forEach(function(resolution) {
streams[resolution] = video.baseUrl + '/' + resolution + 'p.' + format; streams[resolution] = video.baseUrl + '/' + resolution + 'p.' + format;
}); });
$.each(pandora.site.layers, function(i, layer) { $.each(pandora.site.layers, function(i, layer) {
@ -289,11 +289,11 @@ pandora.ui.item = function() {
var layers = [], var layers = [],
video = result.data.stream, video = result.data.stream,
cuts = result.data.cuts || [], cuts = result.data.cuts || [],
format = $.support.video.supportedFormat(video.formats), format = $.support.video.supportedFormat(pandora.site.formats),
streams = {}; streams = {};
video.height = video.resolutions[0]; video.height = pandora.site.resolutions[0];
video.width = parseInt(video.height * video.aspectRatio / 2) * 2; video.width = parseInt(video.height * video.aspectRatio / 2) * 2;
video.resolutions.forEach(function(resolution) { pandora.site.resolutions.forEach(function(resolution) {
streams[resolution] = video.baseUrl + '/' + resolution + 'p.' + format; streams[resolution] = video.baseUrl + '/' + resolution + 'p.' + format;
}); });
$.each(pandora.site.layers, function(i, layer) { $.each(pandora.site.layers, function(i, layer) {