0x2620/pandora
2
1
Fork 1
Code Issues 1.3k Pull requests Releases Wiki Activity

streams per file and no longer per item

Browse source
This commit is contained in:
j 2011-08-18 14:01:37 +02:00
parent f228a4a5e5
commit fc26ff01bd
10 changed files with 178 additions and 126 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
pandora/api/views.py
View file

@ -14,6 +14,7 @@ from ox.utils import json
from user.models import get_user_json
from item.models import ItemSort
from app.models import site_config
from actions import actions
@ -54,26 +55,21 @@ def init(request):
'''
#data = json.loads(request.POST['data'])
response = json_response({})
with open(settings.SITE_CONFIG) as f:
config = json.load(f)
config = site_config()
del config['keys'] #is this needed?
config['site']['id'] = settings.SITEID
config['site']['name'] = settings.SITENAME
config['site']['sectionName'] = settings.SITENAME
config['site']['url'] = settings.URL
#populate max values for percent requests
for key in filter(lambda k: 'format' in k, config['itemKeys']):
if key['format']['type'] == 'percent' and key['format']['args'][0] == 'auto':
name = key['id']
if name == 'popularity':
name = 'item__accessed__accessed'
value = ItemSort.objects.aggregate(Sum(name))['%s__sum'%name]
else:
value = ItemSort.objects.aggregate(Max(name))['%s__max'%name]
key['format']['args'][0] = value
#populate max values for percent requests
for key in filter(lambda k: 'format' in k, config['itemKeys']):
if key['format']['type'] == 'percent' and key['format']['args'][0] == 'auto':
name = key['id']
if name == 'popularity':
name = 'item__accessed__accessed'
value = ItemSort.objects.aggregate(Sum(name))['%s__sum'%name]
else:
value = ItemSort.objects.aggregate(Max(name))['%s__max'%name]
key['format']['args'][0] = value
response['data']['site'] = config
response['data']['site'] = config
if request.user.is_authenticated():
response['data']['user'] = get_user_json(request.user)
else:

9
pandora/app/models.py
View file

@ -28,6 +28,15 @@ class SiteSettings(models.Model):
def site_config():
with open(settings.SITE_CONFIG) as f:
site_config = json.load(f)
site_config['site']['id'] = settings.SITEID
site_config['site']['name'] = settings.SITENAME
site_config['site']['sectionName'] = settings.SITENAME
site_config['site']['url'] = settings.URL
site_config['formats'] = settings.VIDEO_FORMATS
site_config['resolutions'] = settings.VIDEO_RESOLUTIONS
site_config['keys'] = {}
for key in site_config['itemKeys']:
site_config['keys'][key['id']] = key

93
pandora/archive/models.py
View file

@ -21,6 +21,9 @@ from item import utils
from item.models import Item
from person.models import get_name_sort
import extract
class File(models.Model):
created = models.DateTimeField(auto_now_add=True)
@ -160,10 +163,8 @@ class File(models.Model):
super(File, self).save(*args, **kwargs)
#upload and data handling
video = models.FileField(null=True, blank=True,
upload_to=lambda f, x: f.path(settings.VIDEO_PROFILE))
data = models.FileField(null=True, blank=True,
upload_to=lambda f, x: f.path('data.bin'))
upload_to=lambda f, x: f.path('data.bin'))
def path(self, name):
h = self.oshash
@ -245,15 +246,22 @@ class File(models.Model):
#FIXME: check that user has instance of this file
return True
def save_chunk(self, chunk, chunk_id=-1):
def save_chunk(self, chunk, chunk_id=-1, done=False):
if not self.available:
if not self.video:
self.video.save(settings.VIDEO_PROFILE, chunk)
stream, created = Stream.objects.get_or_create(
file=self,
resolution=settings.VIDEO_RESOLUTIONS[0],
format=settings.VIDEO_FORMATS[0])
if created:
stream.video.save(stream.name(), chunk)
else:
f = open(self.video.path, 'a')
f = open(stream.video.path, 'a')
#FIXME: should check that chunk_id/offset is right
f.write(chunk.read())
f.close()
if done:
stream.available = True
stream.save()
return True
return False
@ -365,8 +373,7 @@ class File(models.Model):
def delete_file(sender, **kwargs):
f = kwargs['instance']
if f.video:
f.video.delete()
#FIXME: delete streams here
if f.data:
f.data.delete()
pre_delete.connect(delete_file, sender=File)
@ -450,3 +457,71 @@ def delete_frame(sender, **kwargs):
if f.frame:
f.frame.delete()
pre_delete.connect(delete_frame, sender=Frame)
class Stream(models.Model):
class Meta:
unique_together = ("file", "resolution", "format")
file = models.ForeignKey(File, related_name='streams')
resolution = models.IntegerField(default=96)
format = models.CharField(max_length=255, default='webm')
video = models.FileField(default=None, blank=True, upload_to=lambda f, x: f.path())
source = models.ForeignKey('Stream', related_name='derivatives', default=None, null=True)
available = models.BooleanField(default=False)
info = fields.DictField(default={})
@property
def timeline_prefix(self):
return os.path.join(settings.MEDIA_ROOT, self.path(), 'timeline')
def name(self):
return u"%sp.%s" % (self.resolution, self.format)
def __unicode__(self):
return u"%s/%s" % (self.file, self.name())
def path(self, name=''):
return self.file.path(name)
def extract_derivatives(self):
self.make_timeline()
for resolution in settings.VIDEO_RESOLUTIONS:
for f in settings.VIDEO_FORMATS:
derivative, created = Stream.objects.get_or_create(file=self.file,
resolution=resolution, format=f)
if created:
derivative.source = self
name = derivative.name()
derivative.video.name = os.path.join(os.path.dirname(self.video.name), name)
derivative.encode()
derivative.save()
return True
def encode(self):
if self.source:
video = self.source.video.path
target = self.video.path
info = ox.avinfo(video)
if extract.stream(video, target, self.name(), info):
self.available = True
else:
self.available = False
self.save()
def make_timeline(self):
if self.available and not self.source:
extract.timeline(self.video.path, self.timeline_prefix)
def save(self, *args, **kwargs):
if self.video and not self.info:
self.info = ox.avinfo(self.video.path)
super(Stream, self).save(*args, **kwargs)
def delete_stream(sender, **kwargs):
f = kwargs['instance']
if f.video:
f.video.delete()
pre_delete.connect(delete_stream, sender=Stream)

5
pandora/archive/views.py
View file

@ -111,7 +111,8 @@ actions.register(update, cache=False)
@login_required_json
def encodingProfile(request):
response = json_response({'profile': settings.VIDEO_PROFILE})
profile = "%sp.%s" % (settings.VIDEO_RESOLUTIONS[0], settings.VIDEO_FORMATS[0])
response = json_response({'profile': profile})
return render_to_json_response(response)
actions.register(encodingProfile)
@ -178,7 +179,7 @@ def firefogg_upload(request):
'result': 1,
'resultUrl': request.build_absolute_uri('/')
}
if not f.save_chunk(c, chunk_id):
if not f.save_chunk(c, chunk_id, form.cleaned_data['done']):
response['result'] = -1
elif form.cleaned_data['done']:
f.available = True

106
pandora/item/models.py
View file

@ -392,8 +392,9 @@ class Item(models.Model):
def get_stream(self):
stream = {}
if self.streams.all().count():
s = self.streams.all()[0]
videos = self.main_videos()
for video in videos:
s = video.streams.all()[0]
if s.video and s.info:
stream['duration'] = s.info['duration']
if 'video' in s.info and s.info['video']:
@ -404,8 +405,6 @@ class Item(models.Model):
stream['baseUrl'] = '/%s' % self.itemId
else:
stream['baseUrl'] = os.path.dirname(s.video.url)
stream['resolutions'] = sorted(list(set(map(lambda s: int(os.path.splitext(s['profile'])[0][:-1]), self.streams.all().values('profile')))))
stream['formats'] = list(set(map(lambda s: os.path.splitext(s['profile'])[1][1:], self.streams.all().values('profile'))))
return stream
def get_layers(self, user=None):
@ -451,6 +450,11 @@ class Item(models.Model):
if not keys or 'poster' in keys:
i['poster'] = self.get_poster()
videos = self.main_videos()
i['duration'] = sum([v.duration for v in videos])
i['durations'] = [v.duration for v in videos]
i['apsectRatio'] = i.get('aspectratio')
#only needed by admins
if keys and 'posters' in keys:
i['posters'] = self.get_posters()
@ -561,6 +565,7 @@ class Item(models.Model):
'size',
'bitrate',
'numberoffiles',
'parts',
'published',
'modified',
'popularity',
@ -627,12 +632,13 @@ class Item(models.Model):
if len(videos) > 0:
s.duration = sum([v.duration for v in videos])
s.resolution = videos[0].width * videos[0].height
s.aspectratio = int(1000 * utils.parse_decimal(v.display_aspect_ratio))
s.aspectratio = float(utils.parse_decimal(v.display_aspect_ratio))
#FIXME: should be average over all files
if 'bitrate' in videos[0].info:
s.bitrate = videos[0].info['bitrate']
s.pixels = sum([v.pixels for v in videos])
s.numberoffiles = self.files.all().count()
s.parts = len(videos)
s.size = sum([v.size for v in videos]) #FIXME: only size of movies?
s.volume = 0
else:
@ -645,6 +651,7 @@ class Item(models.Model):
s.files = None
s.size = None
s.volume = None
s.parts = 0
if 'color' in self.data:
s.hue, s.saturation, s.lightness = self.data['color']
@ -695,24 +702,34 @@ class Item(models.Model):
'''
Video related functions
'''
def frame(self, position, height=128):
stream = self.streams.filter(profile=settings.VIDEO_PROFILE)
if stream.count()>0:
stream = stream[0]
else:
return None
height = min(height, stream.height())
path = os.path.join(settings.MEDIA_ROOT, self.path(),
'frames', "%dp"%height, "%s.jpg"%position)
if not os.path.exists(path) and stream.video:
extract.frame(stream.video.path, path, position, height)
if not os.path.exists(path):
path = os.path.join(settings.STATIC_ROOT, 'png/frame.broken.png')
return path
offset = 0
videos = self.main_videos()
for video in videos:
if video.duration + offset < position:
offset += video.duration
else:
position = position - offset
stream = video.streams.filter(resolution=settings.VIDEO_RESOLUTIONS[0],
format=settings.VIDEO_FORMATS[0])
if stream.count()>0:
stream = stream[0]
else:
return None
height = min(height, stream.resolution)
path = os.path.join(settings.MEDIA_ROOT, stream.path(),
'frames', "%dp"%height, "%s.jpg"%position)
if not os.path.exists(path) and stream.video:
extract.frame(stream.video.path, path, position, height)
if not os.path.exists(path):
return None
return path
@property
def timeline_prefix(self):
videos = self.main_videos()
if len(videos) == 1:
return os.path.join(settings.MEDIA_ROOT, videos[0].path('timeline'))
return os.path.join(settings.MEDIA_ROOT, self.path(), 'timeline')
def main_videos(self):
@ -890,9 +907,7 @@ class Item(models.Model):
return None
def make_timeline(self):
stream = self.streams.filter(profile=settings.VIDEO_PROFILE)
if stream.count() > 0 and stream[0].video:
extract.timeline(stream[0].video.path, self.timeline_prefix)
print "FIXME, needs to build timeline from parts"
def make_poster(self, force=False):
if not self.poster or force:
@ -1109,53 +1124,6 @@ class Facet(models.Model):
super(Facet, self).save(*args, **kwargs)
class Stream(models.Model):
class Meta:
unique_together = ("item", "profile")
item = models.ForeignKey(Item, related_name='streams')
profile = models.CharField(max_length=255, default='96p.webm')
video = models.FileField(default=None, blank=True, upload_to=lambda f, x: f.path())
source = models.ForeignKey('Stream', related_name='derivatives', default=None, null=True)
available = models.BooleanField(default=False)
info = fields.DictField(default={})
def __unicode__(self):
return u"%s/%s" % (self.item.itemId, self.profile)
def path(self):
return self.item.path(self.profile)
def height(self):
return int(self.profile.split('p')[0])
def extract_derivatives(self):
for profile in settings.VIDEO_DERIVATIVES:
derivative, created = Stream.objects.get_or_create(profile=profile, item=self.item)
if created:
derivative.source = self
derivative.video.name = self.video.name.replace(self.profile, profile)
derivative.encode()
derivative.save()
return True
def encode(self):
if self.source:
video = self.source.video.path
target = self.video.path
profile = self.profile
info = ox.avinfo(video)
if extract.stream(video, target, profile, info):
self.available=True
self.save()
def save(self, *args, **kwargs):
if self.video and not self.info:
self.info = ox.avinfo(self.video.path)
super(Stream, self).save(*args, **kwargs)
class PosterUrl(models.Model):
class Meta:

8
pandora/item/tasks.py
View file

@ -32,8 +32,12 @@ def update_streams(itemId):
create stream, extract timeline and create derivatives
'''
item = models.Item.objects.get(itemId=itemId)
if item.files.filter(is_main=True, is_video=True, available=False).count() == 0:
item.update_streams()
videos = item.main_videos()
for video in videos:
for f in video.streams.filter(source=None):
f.extract_derivatives()
#if item.files.filter(is_main=True, is_video=True, available=False).count() == 0:
# item.update_streams()
return True
def load_subtitles(itemId):

2
pandora/item/urls.py
View file

@ -13,7 +13,7 @@ urlpatterns = patterns("item.views",
(r'^(?P<id>[A-Z0-9].+)/timeline(?P<size>\d+)p\.png$', 'timeline_overview'),
#video
(r'^(?P<id>[A-Z0-9].+)/(?P<profile>\d+p)(?P<index>\d*)\.(?P<format>webm|ogv|mp4)$', 'video'),
(r'^(?P<id>[A-Z0-9].+)/(?P<resolution>\d+)p(?P<index>\d*)\.(?P<format>webm|ogv|mp4)$', 'video'),
#torrent
(r'^(?P<id>[A-Z0-9][A-Za-z0-9]+)/torrent/(?P<filename>.*?)$', 'torrent'),

18
pandora/item/views.py
View file

@ -652,21 +652,25 @@ def torrent(request, id, filename=None):
os.path.basename(filename.encode('utf-8'))
return response
def video(request, id, profile, index=None, format=None):
def video(request, id, resolution, format, index=None):
item = get_object_or_404(models.Item, itemId=id)
if index:
index = int(index) - 1
path = item.main_videos()[index].video.path
#stream = item.streams.filter(profile=profile)[index]
#path = stream.video.path
else:
stream = get_object_or_404(item.streams, profile="%s.%s" % (profile, format))
path = stream.video.path
index = 0
videos = item.main_videos()
if index > len(videos):
raise Http404
f = videos[index]
path = stream.video.path
#server side cutting
#FIXME: this needs to join segments if needed
t = request.GET.get('t')
if t:
t = map(float, t.split(','))
ext = os.path.splitext(profile)[1]
ext = '.%s' % format
content_type = mimetypes.guess_type(path)[0]
if len(t) == 2 and t[1] > t[0] and stream.info['duration']>=t[1]:
response = HttpResponse(extract.chop(path, t[0], t[1]), content_type=content_type)

19
pandora/settings.py
View file

@ -174,24 +174,19 @@ VIDEO_DERIVATIVES = []
TRACKER_URL="http://url2torrent.net:6970/announce"
VIDEO_FORMATS=['webm']
VIDEO_RESOLUTIONS=[96]
#0xdb
'''
VIDEO_PROFILE = '96p.webm'
VIDEO_DERIVATIVES = [
'96p.mp4'
]
VIDEO_FORMATS=['webm', 'mp4']
VIDEO_RESOLUTIONS=[96]
'''
#Pad.ma
'''
VIDEO_PROFILE = '480p.webm'
VIDEO_DERIVATIVES = [
'96p.webm',
'240p.webm',
'96p.mp4',
'240p.mp4',
'480p.mp4',
]
VIDEO_FORMATS=['webm', 'mp4']
VIDEO_RESOLUTIONS=[480, 240, 96]
'''
TRANSMISSON_HOST = 'localhost'

12
static/js/pandora/ui/item.js
View file

@ -223,11 +223,11 @@ pandora.ui.item = function() {
var layers = [],
video = result.data.stream,
cuts = result.data.cuts || [],
format = $.support.video.supportedFormat(video.formats),
format = $.support.video.supportedFormat(pandora.site.formats),
streams = {};
video.height = video.resolutions[0];
video.height = pandora.site.resolutions[0];
video.width = parseInt(video.height * video.aspectRatio / 2) * 2;
video.resolutions.forEach(function(resolution) {
pandora.site.resolutions.forEach(function(resolution) {
streams[resolution] = video.baseUrl + '/' + resolution + 'p.' + format;
});
$.each(pandora.site.layers, function(i, layer) {
@ -289,11 +289,11 @@ pandora.ui.item = function() {
var layers = [],
video = result.data.stream,
cuts = result.data.cuts || [],
format = $.support.video.supportedFormat(video.formats),
format = $.support.video.supportedFormat(pandora.site.formats),
streams = {};
video.height = video.resolutions[0];
video.height = pandora.site.resolutions[0];
video.width = parseInt(video.height * video.aspectRatio / 2) * 2;
video.resolutions.forEach(function(resolution) {
pandora.site.resolutions.forEach(function(resolution) {
streams[resolution] = video.baseUrl + '/' + resolution + 'p.' + format;
});
$.each(pandora.site.layers, function(i, layer) {