add support for direct video uploads with server side transcoding, ticket #1351

This commit is contained in:
j 2013-03-26 11:54:03 +00:00
parent d0c59e05c2
commit 9153c80844
6 changed files with 290 additions and 7 deletions

View file

@ -0,0 +1,171 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'File.encoding'
db.add_column('archive_file', 'encoding',
self.gf('django.db.models.fields.BooleanField')(default=False),
keep_default=False)
def backwards(self, orm):
# Deleting field 'File.encoding'
db.delete_column('archive_file', 'encoding')
models = {
'archive.file': {
'Meta': {'object_name': 'File'},
'audio_codec': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'bits_per_pixel': ('django.db.models.fields.FloatField', [], {'default': '-1'}),
'channels': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'data': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
'display_aspect_ratio': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
'encoding': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'extension': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True'}),
'framerate': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'height': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'info': ('ox.django.fields.DictField', [], {'default': '{}'}),
'is_audio': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_subtitle': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_video': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'item': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'files'", 'null': 'True', 'to': "orm['item.Item']"}),
'language': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '8', 'null': 'True'}),
'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'oshash': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '16'}),
'part': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True'}),
'part_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True'}),
'path': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '2048'}),
'pixel_format': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'pixels': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}),
'samplerate': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'selected': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'size': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}),
'sort_path': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '2048'}),
'type': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}),
'uploading': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'version': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True'}),
'video_codec': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'wanted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'width': ('django.db.models.fields.IntegerField', [], {'default': '0'})
},
'archive.frame': {
'Meta': {'unique_together': "(('file', 'position'),)", 'object_name': 'Frame'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'file': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'frames'", 'to': "orm['archive.File']"}),
'frame': ('django.db.models.fields.files.ImageField', [], {'default': 'None', 'max_length': '100', 'null': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'position': ('django.db.models.fields.FloatField', [], {})
},
'archive.instance': {
'Meta': {'unique_together': "(('path', 'volume'),)", 'object_name': 'Instance'},
'atime': ('django.db.models.fields.IntegerField', [], {'default': '1364297821'}),
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'ctime': ('django.db.models.fields.IntegerField', [], {'default': '1364297821'}),
'file': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'instances'", 'to': "orm['archive.File']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'ignore': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'mtime': ('django.db.models.fields.IntegerField', [], {'default': '1364297821'}),
'path': ('django.db.models.fields.CharField', [], {'max_length': '2048'}),
'volume': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'files'", 'to': "orm['archive.Volume']"})
},
'archive.stream': {
'Meta': {'unique_together': "(('file', 'resolution', 'format'),)", 'object_name': 'Stream'},
'aspect_ratio': ('django.db.models.fields.FloatField', [], {'default': '0'}),
'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'color': ('ox.django.fields.TupleField', [], {'default': '[]'}),
'cuts': ('ox.django.fields.TupleField', [], {'default': '[]'}),
'duration': ('django.db.models.fields.FloatField', [], {'default': '0'}),
'file': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'streams'", 'to': "orm['archive.File']"}),
'format': ('django.db.models.fields.CharField', [], {'default': "'webm'", 'max_length': '255'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'info': ('ox.django.fields.DictField', [], {'default': '{}'}),
'oshash': ('django.db.models.fields.CharField', [], {'max_length': '16', 'null': 'True', 'db_index': 'True'}),
'resolution': ('django.db.models.fields.IntegerField', [], {'default': '96'}),
'source': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'derivatives'", 'null': 'True', 'to': "orm['archive.Stream']"}),
'video': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '100', 'blank': 'True'}),
'volume': ('django.db.models.fields.FloatField', [], {'default': '0'})
},
'archive.volume': {
'Meta': {'unique_together': "(('user', 'name'),)", 'object_name': 'Volume'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '1024'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'volumes'", 'to': "orm['auth.User']"})
},
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'item.item': {
'Meta': {'object_name': 'Item'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'data': ('ox.django.fields.DictField', [], {'default': '{}'}),
'external_data': ('ox.django.fields.DictField', [], {'default': '{}'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'items'", 'blank': 'True', 'to': "orm['auth.Group']"}),
'icon': ('django.db.models.fields.files.ImageField', [], {'default': 'None', 'max_length': '100', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'itemId': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'blank': 'True'}),
'json': ('ox.django.fields.DictField', [], {'default': '{}'}),
'level': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}),
'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'oxdbId': ('django.db.models.fields.CharField', [], {'max_length': '42', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'poster': ('django.db.models.fields.files.ImageField', [], {'default': 'None', 'max_length': '100', 'blank': 'True'}),
'poster_frame': ('django.db.models.fields.FloatField', [], {'default': '-1'}),
'poster_height': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'poster_source': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'poster_width': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'rendered': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
'stream_aspect': ('django.db.models.fields.FloatField', [], {'default': '1.3333333333333333'}),
'stream_info': ('ox.django.fields.DictField', [], {'default': '{}'}),
'torrent': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '1000', 'blank': 'True'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'items'", 'null': 'True', 'to': "orm['auth.User']"})
}
}
complete_apps = ['archive']

View file

@ -76,6 +76,7 @@ class File(models.Model):
available = models.BooleanField(default = False) available = models.BooleanField(default = False)
selected = models.BooleanField(default = False) selected = models.BooleanField(default = False)
uploading = models.BooleanField(default = False) uploading = models.BooleanField(default = False)
encoding = models.BooleanField(default = False)
wanted = models.BooleanField(default = False) wanted = models.BooleanField(default = False)
is_audio = models.BooleanField(default=False) is_audio = models.BooleanField(default=False)
@ -248,6 +249,25 @@ class File(models.Model):
(not self.item or self.item.user == user) (not self.item or self.item.user == user)
def save_chunk(self, chunk, chunk_id=-1, done=False): def save_chunk(self, chunk, chunk_id=-1, done=False):
if not self.available:
if not self.data:
name = 'data.%s' % self.info.get('extension', 'avi')
self.data.name = self.get_path(name)
ox.makedirs(os.path.dirname(self.data.path))
with open(self.data.path, 'w') as f:
f.write(chunk.read())
self.save()
else:
with open(self.data.path, 'a') as f:
f.write(chunk.read())
if done:
self.info.update(ox.avinfo(self.data.path))
self.parse_info()
self.save()
return True
return False
def save_chunk_stream(self, chunk, chunk_id=-1, done=False):
if not self.available: if not self.available:
config = settings.CONFIG['video'] config = settings.CONFIG['video']
stream, created = Stream.objects.get_or_create( stream, created = Stream.objects.get_or_create(
@ -282,6 +302,7 @@ class File(models.Model):
'audioCodec': self.audio_codec, 'audioCodec': self.audio_codec,
'available': self.available, 'available': self.available,
'duration': duration, 'duration': duration,
'encoding': self.encoding,
'framerate': self.framerate, 'framerate': self.framerate,
'id': self.oshash, 'id': self.oshash,
'instances': [i.json() for i in self.instances.all()], 'instances': [i.json() for i in self.instances.all()],
@ -472,6 +493,15 @@ class Stream(models.Model):
else: else:
self.available = False self.available = False
self.save() self.save()
elif self.file.data:
video = self.file.data.path
target = self.video.path
info = ox.avinfo(video)
if extract.stream(video, target, self.name(), info):
self.available = True
else:
self.available = False
self.save()
def make_timeline(self): def make_timeline(self):
if self.available and not self.source: if self.available and not self.source:

View file

@ -1,10 +1,12 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4 # vi:si:et:sw=4:sts=4:ts=4
import os
from glob import glob from glob import glob
from celery.task import task from celery.task import task
from django.conf import settings from django.conf import settings
import ox
from item.models import Item from item.models import Item
import models import models
@ -86,12 +88,17 @@ def update_files(user, volume, files):
@task(queue="encoding") @task(queue="encoding")
def process_stream(fileId): def process_stream(fileId):
'''
process uploaded stream
'''
file = models.File.objects.get(id=fileId) file = models.File.objects.get(id=fileId)
streams = file.streams.filter(source=None) streams = file.streams.filter(source=None)
if streams.count() > 0: if streams.count() > 0:
stream = streams[0] stream = streams[0]
models.File.objects.filter(id=fileId).update(encoding=True)
stream.make_timeline() stream.make_timeline()
stream.extract_derivatives() stream.extract_derivatives()
models.File.objects.filter(id=fileId).update(encoding=False)
file.item.update_selected() file.item.update_selected()
if not file.item.rendered: if not file.item.rendered:
file.item.update_timeline() file.item.update_timeline()
@ -99,6 +106,31 @@ def process_stream(fileId):
file.item.save() file.item.save()
return True return True
@task(queue="encoding")
def extract_stream(fileId):
'''
extract stream from direct upload
'''
file = models.File.objects.get(id=fileId)
if file.data:
config = settings.CONFIG['video']
stream, created = models.Stream.objects.get_or_create(
file=file, resolution=max(config['resolutions']),
format=config['formats'][0])
if created:
models.File.objects.filter(id=fileId).update(encoding=True)
stream.video.name = stream.path(stream.name())
stream.encode()
if stream.available:
stream.make_timeline()
stream.extract_derivatives()
file.item.update_selected()
if not file.item.rendered:
file.item.update_timeline()
if file.item.rendered:
file.item.save()
models.File.objects.filter(id=fileId).update(encoding=False)
@task(queue="encoding") @task(queue="encoding")
def extract_derivatives(fileId, rebuild=False): def extract_derivatives(fileId, rebuild=False):
file = models.File.objects.get(id=fileId) file = models.File.objects.get(id=fileId)

View file

@ -160,7 +160,7 @@ def upload(request):
actions.register(upload, cache=False) actions.register(upload, cache=False)
class VideoChunkForm(forms.Form): class ChunkForm(forms.Form):
chunk = forms.FileField() chunk = forms.FileField()
chunkId = forms.IntegerField(required=False) chunkId = forms.IntegerField(required=False)
done = forms.IntegerField(required=False) done = forms.IntegerField(required=False)
@ -209,7 +209,8 @@ def addMedia(request):
else: else:
extension = 'webm' extension = 'webm'
f.selected = True f.selected = True
f.info = data['info'] if 'info' in data and data['info']:
f.info = data['info']
f.info['extension'] = extension f.info['extension'] = extension
f.parse_info() f.parse_info()
f.save() f.save()
@ -230,7 +231,7 @@ def firefogg_upload(request):
#post next chunk #post next chunk
if 'chunk' in request.FILES and oshash: if 'chunk' in request.FILES and oshash:
f = get_object_or_404(models.File, oshash=oshash) f = get_object_or_404(models.File, oshash=oshash)
form = VideoChunkForm(request.POST, request.FILES) form = ChunkForm(request.POST, request.FILES)
if form.is_valid() and profile == video_profile and f.editable(request.user): if form.is_valid() and profile == video_profile and f.editable(request.user):
c = form.cleaned_data['chunk'] c = form.cleaned_data['chunk']
chunk_id = form.cleaned_data['chunkId'] chunk_id = form.cleaned_data['chunkId']
@ -238,7 +239,7 @@ def firefogg_upload(request):
'result': 1, 'result': 1,
'resultUrl': request.build_absolute_uri('/%s'%f.item.itemId) 'resultUrl': request.build_absolute_uri('/%s'%f.item.itemId)
} }
if not f.save_chunk(c, chunk_id, form.cleaned_data['done']): if not f.save_chunk_stream(c, chunk_id, form.cleaned_data['done']):
response['result'] = -1 response['result'] = -1
elif form.cleaned_data['done']: elif form.cleaned_data['done']:
f.uploading = False f.uploading = False
@ -274,6 +275,51 @@ def firefogg_upload(request):
response = json_response(status=400, text='this request requires POST') response = json_response(status=400, text='this request requires POST')
return render_to_json_response(response) return render_to_json_response(response)
@login_required_json
def direct_upload(request):
if 'id' in request.GET:
file = models.File.objects.get(oshash=request.GET['id'])
else:
oshash = request.POST['id']
response = json_response(status=400, text='this request requires POST')
if 'chunk' in request.FILES:
form = ChunkForm(request.POST, request.FILES)
if form.is_valid() and file.editable(request.user):
c = form.cleaned_data['chunk']
chunk_id = form.cleaned_data['chunkId']
response = {
'result': 1,
'resultUrl': request.build_absolute_uri(file.item.get_absolute_url())
}
if not file.save_chunk(c, chunk_id, form.cleaned_data['done']):
response['result'] = -1
if form.cleaned_data['done']:
file.uploading = False
file.save()
#try/execpt so it does not fail if rabitmq is down
try:
t = tasks.extract_stream.delay(file.id)
response['resultUrl'] = t.task_id
except:
pass
response['done'] = 1
return render_to_json_response(response)
#init upload
else:
file, created = models.File.objects.get_or_create(oshash=oshash)
if file.editable(request.user):
file.uploading = True
file.save()
upload_url = request.build_absolute_uri('/api/upload/direct/?id=%s' % file.oshash)
return render_to_json_response({
'uploadUrl': upload_url,
'url': request.build_absolute_uri(file.item.get_absolute_url()),
'result': 1
})
else:
response = json_response(status=403, text='permission denied')
return render_to_json_response(response)
@login_required_json @login_required_json
def taskStatus(request): def taskStatus(request):

View file

@ -24,6 +24,7 @@ urlpatterns = patterns('',
(r'^admin/', include(admin.site.urls)), (r'^admin/', include(admin.site.urls)),
(r'^api/upload/text/?$', 'text.views.upload'), (r'^api/upload/text/?$', 'text.views.upload'),
(r'^api/upload/file/?$', 'file.views.upload'), (r'^api/upload/file/?$', 'file.views.upload'),
(r'^api/upload/direct/?$', 'archive.views.direct_upload'),
(r'^api/upload/?$', 'archive.views.firefogg_upload'), (r'^api/upload/?$', 'archive.views.firefogg_upload'),
(r'^url=(?P<url>.*)$', 'app.views.redirect_url'), (r'^url=(?P<url>.*)$', 'app.views.redirect_url'),
(r'^file/(?P<oshash>.*)$', 'archive.views.lookup_file'), (r'^file/(?P<oshash>.*)$', 'archive.views.lookup_file'),

View file

@ -81,8 +81,11 @@ pandora.ui.filesView = function(options, self) {
format: function(value, data) { format: function(value, data) {
return $('<img>') return $('<img>')
.attr({ .attr({
src: data.wanted ? Ox.UI.getImageURL('symbolUpload') : src: data.encoding
Ox.UI.getImageURL('symbolCheck') ? Ox.UI.getImageURL('symbolSync')
: data.wanted
? Ox.UI.getImageURL('symbolUpload')
: Ox.UI.getImageURL('symbolCheck')
}) })
.css({ .css({
width: '10px', width: '10px',
@ -221,7 +224,7 @@ pandora.ui.filesView = function(options, self) {
query: self.filesQuery query: self.filesQuery
}), callback); }), callback);
}, },
keys: ['wanted', 'instances'], keys: ['encoding', 'instances', 'wanted'],
scrollbarVisible: true, scrollbarVisible: true,
sort: [{key: 'path', operator: '+'}], sort: [{key: 'path', operator: '+'}],
unique: 'id' unique: 'id'