0x2620/pandora
2
1
Fork 1
Code Issues 1.3k Pull requests Releases Wiki Activity

add support for direct video uploads with server side transcoding, ticket #1351

Browse source
This commit is contained in:
j 2013-03-26 11:54:03 +00:00
parent d0c59e05c2
commit 9153c80844
6 changed files with 290 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

171
pandora/archive/migrations/0006_encoding.py Normal file
View file

@ -0,0 +1,171 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'File.encoding'
db.add_column('archive_file', 'encoding',
self.gf('django.db.models.fields.BooleanField')(default=False),
keep_default=False)
def backwards(self, orm):
# Deleting field 'File.encoding'
db.delete_column('archive_file', 'encoding')
models = {
'archive.file': {
'Meta': {'object_name': 'File'},
'audio_codec': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'bits_per_pixel': ('django.db.models.fields.FloatField', [], {'default': '-1'}),
'channels': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'data': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
'display_aspect_ratio': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
'encoding': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'extension': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True'}),
'framerate': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'height': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'info': ('ox.django.fields.DictField', [], {'default': '{}'}),
'is_audio': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_subtitle': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_video': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'item': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'files'", 'null': 'True', 'to': "orm['item.Item']"}),
'language': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '8', 'null': 'True'}),
'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'oshash': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '16'}),
'part': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True'}),
'part_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True'}),
'path': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '2048'}),
'pixel_format': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'pixels': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}),
'samplerate': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'selected': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'size': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}),
'sort_path': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '2048'}),
'type': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}),
'uploading': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'version': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True'}),
'video_codec': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'wanted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'width': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
'archive.frame': {
'Meta': {'unique_together': "(('file', 'position'),)", 'object_name': 'Frame'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'file': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'frames'", 'to': "orm['archive.File']"}),
'frame': ('django.db.models.fields.files.ImageField', [], {'default': 'None', 'max_length': '100', 'null': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'position': ('django.db.models.fields.FloatField', [], {})
},
'archive.instance': {
'Meta': {'unique_together': "(('path', 'volume'),)", 'object_name': 'Instance'},
'atime': ('django.db.models.fields.IntegerField', [], {'default': '1364297821'}),
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'ctime': ('django.db.models.fields.IntegerField', [], {'default': '1364297821'}),
'file': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'instances'", 'to': "orm['archive.File']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'ignore': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'mtime': ('django.db.models.fields.IntegerField', [], {'default': '1364297821'}),
'path': ('django.db.models.fields.CharField', [], {'max_length': '2048'}),
'volume': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'files'", 'to': "orm['archive.Volume']"})
},
'archive.stream': {
'Meta': {'unique_together': "(('file', 'resolution', 'format'),)", 'object_name': 'Stream'},
'aspect_ratio': ('django.db.models.fields.FloatField', [], {'default': '0'}),
'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'color': ('ox.django.fields.TupleField', [], {'default': '[]'}),
'cuts': ('ox.django.fields.TupleField', [], {'default': '[]'}),
'duration': ('django.db.models.fields.FloatField', [], {'default': '0'}),
'file': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'streams'", 'to': "orm['archive.File']"}),
'format': ('django.db.models.fields.CharField', [], {'default': "'webm'", 'max_length': '255'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'info': ('ox.django.fields.DictField', [], {'default': '{}'}),
'oshash': ('django.db.models.fields.CharField', [], {'max_length': '16', 'null': 'True', 'db_index': 'True'}),
'resolution': ('django.db.models.fields.IntegerField', [], {'default': '96'}),
'source': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'derivatives'", 'null': 'True', 'to': "orm['archive.Stream']"}),
'video': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '100', 'blank': 'True'}),
'volume': ('django.db.models.fields.FloatField', [], {'default': '0'})
},
'archive.volume': {
'Meta': {'unique_together': "(('user', 'name'),)", 'object_name': 'Volume'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '1024'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'volumes'", 'to': "orm['auth.User']"})
},
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'item.item': {
'Meta': {'object_name': 'Item'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'data': ('ox.django.fields.DictField', [], {'default': '{}'}),
'external_data': ('ox.django.fields.DictField', [], {'default': '{}'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'items'", 'blank': 'True', 'to': "orm['auth.Group']"}),
'icon': ('django.db.models.fields.files.ImageField', [], {'default': 'None', 'max_length': '100', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'itemId': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'blank': 'True'}),
'json': ('ox.django.fields.DictField', [], {'default': '{}'}),
'level': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}),
'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'oxdbId': ('django.db.models.fields.CharField', [], {'max_length': '42', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'poster': ('django.db.models.fields.files.ImageField', [], {'default': 'None', 'max_length': '100', 'blank': 'True'}),
'poster_frame': ('django.db.models.fields.FloatField', [], {'default': '-1'}),
'poster_height': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'poster_source': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'poster_width': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'rendered': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
'stream_aspect': ('django.db.models.fields.FloatField', [], {'default': '1.3333333333333333'}),
'stream_info': ('ox.django.fields.DictField', [], {'default': '{}'}),
'torrent': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '1000', 'blank': 'True'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'items'", 'null': 'True', 'to': "orm['auth.User']"})
}
}
complete_apps = ['archive']

30
pandora/archive/models.py
View file

@ -76,6 +76,7 @@ class File(models.Model):
available = models.BooleanField(default = False) available = models.BooleanField(default = False)
selected = models.BooleanField(default = False) selected = models.BooleanField(default = False)
uploading = models.BooleanField(default = False) uploading = models.BooleanField(default = False)
encoding = models.BooleanField(default = False)
wanted = models.BooleanField(default = False) wanted = models.BooleanField(default = False)
is_audio = models.BooleanField(default=False) is_audio = models.BooleanField(default=False)
@ -248,6 +249,25 @@ class File(models.Model):
(not self.item or self.item.user == user) (not self.item or self.item.user == user)
def save_chunk(self, chunk, chunk_id=-1, done=False): def save_chunk(self, chunk, chunk_id=-1, done=False):
if not self.available:
if not self.data:
name = 'data.%s' % self.info.get('extension', 'avi')
self.data.name = self.get_path(name)
ox.makedirs(os.path.dirname(self.data.path))
with open(self.data.path, 'w') as f:
f.write(chunk.read())
self.save()
else:
with open(self.data.path, 'a') as f:
f.write(chunk.read())
if done:
self.info.update(ox.avinfo(self.data.path))
self.parse_info()
self.save()
return True
return False
def save_chunk_stream(self, chunk, chunk_id=-1, done=False):
if not self.available: if not self.available:
config = settings.CONFIG['video'] config = settings.CONFIG['video']
stream, created = Stream.objects.get_or_create( stream, created = Stream.objects.get_or_create(
@ -282,6 +302,7 @@ class File(models.Model):
'audioCodec': self.audio_codec, 'audioCodec': self.audio_codec,
'available': self.available, 'available': self.available,
'duration': duration, 'duration': duration,
'encoding': self.encoding,
'framerate': self.framerate, 'framerate': self.framerate,
'id': self.oshash, 'id': self.oshash,
'instances': [i.json() for i in self.instances.all()], 'instances': [i.json() for i in self.instances.all()],
@ -472,6 +493,15 @@ class Stream(models.Model):
else: else:
self.available = False self.available = False
self.save() self.save()
elif self.file.data:
video = self.file.data.path
target = self.video.path
info = ox.avinfo(video)
if extract.stream(video, target, self.name(), info):
self.available = True
else:
self.available = False
self.save()
def make_timeline(self): def make_timeline(self):
if self.available and not self.source: if self.available and not self.source:

32
pandora/archive/tasks.py
View file

@ -1,10 +1,12 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4 # vi:si:et:sw=4:sts=4:ts=4
import os
from glob import glob from glob import glob
from celery.task import task from celery.task import task
from django.conf import settings from django.conf import settings
import ox
from item.models import Item from item.models import Item
import models import models
@ -86,12 +88,17 @@ def update_files(user, volume, files):
@task(queue="encoding") @task(queue="encoding")
def process_stream(fileId): def process_stream(fileId):
'''
process uploaded stream
'''
file = models.File.objects.get(id=fileId) file = models.File.objects.get(id=fileId)
streams = file.streams.filter(source=None) streams = file.streams.filter(source=None)
if streams.count() > 0: if streams.count() > 0:
stream = streams[0] stream = streams[0]
models.File.objects.filter(id=fileId).update(encoding=True)
stream.make_timeline() stream.make_timeline()
stream.extract_derivatives() stream.extract_derivatives()
models.File.objects.filter(id=fileId).update(encoding=False)
file.item.update_selected() file.item.update_selected()
if not file.item.rendered: if not file.item.rendered:
file.item.update_timeline() file.item.update_timeline()
@ -99,6 +106,31 @@ def process_stream(fileId):
file.item.save() file.item.save()
return True return True
@task(queue="encoding")
def extract_stream(fileId):
'''
extract stream from direct upload
'''
file = models.File.objects.get(id=fileId)
if file.data:
config = settings.CONFIG['video']
stream, created = models.Stream.objects.get_or_create(
file=file, resolution=max(config['resolutions']),
format=config['formats'][0])
if created:
models.File.objects.filter(id=fileId).update(encoding=True)
stream.video.name = stream.path(stream.name())
stream.encode()
if stream.available:
stream.make_timeline()
stream.extract_derivatives()
file.item.update_selected()
if not file.item.rendered:
file.item.update_timeline()
if file.item.rendered:
file.item.save()
models.File.objects.filter(id=fileId).update(encoding=False)
@task(queue="encoding") @task(queue="encoding")
def extract_derivatives(fileId, rebuild=False): def extract_derivatives(fileId, rebuild=False):
file = models.File.objects.get(id=fileId) file = models.File.objects.get(id=fileId)

54
pandora/archive/views.py
View file

@ -160,7 +160,7 @@ def upload(request):
actions.register(upload, cache=False) actions.register(upload, cache=False)
class VideoChunkForm(forms.Form): class ChunkForm(forms.Form):
chunk = forms.FileField() chunk = forms.FileField()
chunkId = forms.IntegerField(required=False) chunkId = forms.IntegerField(required=False)
done = forms.IntegerField(required=False) done = forms.IntegerField(required=False)
@ -209,7 +209,8 @@ def addMedia(request):
else: else:
extension = 'webm' extension = 'webm'
f.selected = True f.selected = True
f.info = data['info'] if 'info' in data and data['info']:
f.info = data['info']
f.info['extension'] = extension f.info['extension'] = extension
f.parse_info() f.parse_info()
f.save() f.save()
@ -230,7 +231,7 @@ def firefogg_upload(request):
#post next chunk #post next chunk
if 'chunk' in request.FILES and oshash: if 'chunk' in request.FILES and oshash:
f = get_object_or_404(models.File, oshash=oshash) f = get_object_or_404(models.File, oshash=oshash)
form = VideoChunkForm(request.POST, request.FILES) form = ChunkForm(request.POST, request.FILES)
if form.is_valid() and profile == video_profile and f.editable(request.user): if form.is_valid() and profile == video_profile and f.editable(request.user):
c = form.cleaned_data['chunk'] c = form.cleaned_data['chunk']
chunk_id = form.cleaned_data['chunkId'] chunk_id = form.cleaned_data['chunkId']
@ -238,7 +239,7 @@ def firefogg_upload(request):
'result': 1, 'result': 1,
'resultUrl': request.build_absolute_uri('/%s'%f.item.itemId) 'resultUrl': request.build_absolute_uri('/%s'%f.item.itemId)
} }
if not f.save_chunk(c, chunk_id, form.cleaned_data['done']): if not f.save_chunk_stream(c, chunk_id, form.cleaned_data['done']):
response['result'] = -1 response['result'] = -1
elif form.cleaned_data['done']: elif form.cleaned_data['done']:
f.uploading = False f.uploading = False
@ -274,6 +275,51 @@ def firefogg_upload(request):
response = json_response(status=400, text='this request requires POST') response = json_response(status=400, text='this request requires POST')
return render_to_json_response(response) return render_to_json_response(response)
@login_required_json
def direct_upload(request):
if 'id' in request.GET:
file = models.File.objects.get(oshash=request.GET['id'])
else:
oshash = request.POST['id']
response = json_response(status=400, text='this request requires POST')
if 'chunk' in request.FILES:
form = ChunkForm(request.POST, request.FILES)
if form.is_valid() and file.editable(request.user):
c = form.cleaned_data['chunk']
chunk_id = form.cleaned_data['chunkId']
response = {
'result': 1,
'resultUrl': request.build_absolute_uri(file.item.get_absolute_url())
}
if not file.save_chunk(c, chunk_id, form.cleaned_data['done']):
response['result'] = -1
if form.cleaned_data['done']:
file.uploading = False
file.save()
#try/execpt so it does not fail if rabitmq is down
try:
t = tasks.extract_stream.delay(file.id)
response['resultUrl'] = t.task_id
except:
pass
response['done'] = 1
return render_to_json_response(response)
#init upload
else:
file, created = models.File.objects.get_or_create(oshash=oshash)
if file.editable(request.user):
file.uploading = True
file.save()
upload_url = request.build_absolute_uri('/api/upload/direct/?id=%s' % file.oshash)
return render_to_json_response({
'uploadUrl': upload_url,
'url': request.build_absolute_uri(file.item.get_absolute_url()),
'result': 1
})
else:
response = json_response(status=403, text='permission denied')
return render_to_json_response(response)
@login_required_json @login_required_json
def taskStatus(request): def taskStatus(request):

1
pandora/urls.py
View file

@ -24,6 +24,7 @@ urlpatterns = patterns('',
(r'^admin/', include(admin.site.urls)), (r'^admin/', include(admin.site.urls)),
(r'^api/upload/text/?$', 'text.views.upload'), (r'^api/upload/text/?$', 'text.views.upload'),
(r'^api/upload/file/?$', 'file.views.upload'), (r'^api/upload/file/?$', 'file.views.upload'),
(r'^api/upload/direct/?$', 'archive.views.direct_upload'),
(r'^api/upload/?$', 'archive.views.firefogg_upload'), (r'^api/upload/?$', 'archive.views.firefogg_upload'),
(r'^url=(?P<url>.*)$', 'app.views.redirect_url'), (r'^url=(?P<url>.*)$', 'app.views.redirect_url'),
(r'^file/(?P<oshash>.*)$', 'archive.views.lookup_file'), (r'^file/(?P<oshash>.*)$', 'archive.views.lookup_file'),

9
static/js/pandora/filesView.js
View file

@ -81,8 +81,11 @@ pandora.ui.filesView = function(options, self) {
format: function(value, data) { format: function(value, data) {
return $('<img>') return $('<img>')
.attr({ .attr({
src: data.wanted ? Ox.UI.getImageURL('symbolUpload') : src: data.encoding
Ox.UI.getImageURL('symbolCheck') ? Ox.UI.getImageURL('symbolSync')
: data.wanted
? Ox.UI.getImageURL('symbolUpload')
: Ox.UI.getImageURL('symbolCheck')
}) })
.css({ .css({
width: '10px', width: '10px',
@ -221,7 +224,7 @@ pandora.ui.filesView = function(options, self) {
query: self.filesQuery query: self.filesQuery
}), callback); }), callback);
}, },
keys: ['wanted', 'instances'], keys: ['encoding', 'instances', 'wanted'],
scrollbarVisible: true, scrollbarVisible: true,
sort: [{key: 'path', operator: '+'}], sort: [{key: 'path', operator: '+'}],
unique: 'id' unique: 'id'