From 9153c80844acb2aa91632c46bcc1fc814c2efb00 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Tue, 26 Mar 2013 11:54:03 +0000 Subject: [PATCH] add support for direct video uploads with server side transcoding, ticket #1351 --- pandora/archive/migrations/0006_encoding.py | 171 ++++++++++++++++++++ pandora/archive/models.py | 30 ++++ pandora/archive/tasks.py | 32 ++++ pandora/archive/views.py | 54 ++++++- pandora/urls.py | 1 + static/js/pandora/filesView.js | 9 +- 6 files changed, 290 insertions(+), 7 deletions(-) create mode 100644 pandora/archive/migrations/0006_encoding.py diff --git a/pandora/archive/migrations/0006_encoding.py b/pandora/archive/migrations/0006_encoding.py new file mode 100644 index 00000000..2159ddc3 --- /dev/null +++ b/pandora/archive/migrations/0006_encoding.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'File.encoding' + db.add_column('archive_file', 'encoding', + self.gf('django.db.models.fields.BooleanField')(default=False), + keep_default=False) + + + def backwards(self, orm): + # Deleting field 'File.encoding' + db.delete_column('archive_file', 'encoding') + + + models = { + 'archive.file': { + 'Meta': {'object_name': 'File'}, + 'audio_codec': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'bits_per_pixel': ('django.db.models.fields.FloatField', [], {'default': '-1'}), + 'channels': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'data': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'display_aspect_ratio': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'encoding': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'extension': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True'}), + 'framerate': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'height': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'info': ('ox.django.fields.DictField', [], {'default': '{}'}), + 'is_audio': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_subtitle': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_video': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'item': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'files'", 'null': 'True', 'to': "orm['item.Item']"}), + 'language': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '8', 'null': 'True'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'oshash': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '16'}), + 'part': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True'}), + 'part_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True'}), + 'path': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '2048'}), + 'pixel_format': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pixels': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'samplerate': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'selected': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'size': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'sort_path': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '2048'}), + 'type': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}), + 'uploading': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'version': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True'}), + 'video_codec': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'wanted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'width': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + 'archive.frame': { + 'Meta': {'unique_together': "(('file', 'position'),)", 'object_name': 'Frame'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'file': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'frames'", 'to': "orm['archive.File']"}), + 'frame': ('django.db.models.fields.files.ImageField', [], {'default': 'None', 'max_length': '100', 'null': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'position': ('django.db.models.fields.FloatField', [], {}) + }, + 'archive.instance': { + 'Meta': {'unique_together': "(('path', 'volume'),)", 'object_name': 'Instance'}, + 'atime': ('django.db.models.fields.IntegerField', [], {'default': '1364297821'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'ctime': ('django.db.models.fields.IntegerField', [], {'default': '1364297821'}), + 'file': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'instances'", 'to': "orm['archive.File']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ignore': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'mtime': ('django.db.models.fields.IntegerField', [], {'default': '1364297821'}), + 'path': ('django.db.models.fields.CharField', [], {'max_length': '2048'}), + 'volume': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'files'", 'to': "orm['archive.Volume']"}) + }, + 'archive.stream': { + 'Meta': {'unique_together': "(('file', 'resolution', 'format'),)", 'object_name': 'Stream'}, + 'aspect_ratio': ('django.db.models.fields.FloatField', [], {'default': '0'}), + 'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'color': ('ox.django.fields.TupleField', [], {'default': '[]'}), + 'cuts': ('ox.django.fields.TupleField', [], {'default': '[]'}), + 'duration': ('django.db.models.fields.FloatField', [], {'default': '0'}), + 'file': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'streams'", 'to': "orm['archive.File']"}), + 'format': ('django.db.models.fields.CharField', [], {'default': "'webm'", 'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'info': ('ox.django.fields.DictField', [], {'default': '{}'}), + 'oshash': ('django.db.models.fields.CharField', [], {'max_length': '16', 'null': 'True', 'db_index': 'True'}), + 'resolution': ('django.db.models.fields.IntegerField', [], {'default': '96'}), + 'source': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'derivatives'", 'null': 'True', 'to': "orm['archive.Stream']"}), + 'video': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '100', 'blank': 'True'}), + 'volume': ('django.db.models.fields.FloatField', [], {'default': '0'}) + }, + 'archive.volume': { + 'Meta': {'unique_together': "(('user', 'name'),)", 'object_name': 'Volume'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '1024'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'volumes'", 'to': "orm['auth.User']"}) + }, + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'item.item': { + 'Meta': {'object_name': 'Item'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'data': ('ox.django.fields.DictField', [], {'default': '{}'}), + 'external_data': ('ox.django.fields.DictField', [], {'default': '{}'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'items'", 'blank': 'True', 'to': "orm['auth.Group']"}), + 'icon': ('django.db.models.fields.files.ImageField', [], {'default': 'None', 'max_length': '100', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'itemId': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'blank': 'True'}), + 'json': ('ox.django.fields.DictField', [], {'default': '{}'}), + 'level': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'oxdbId': ('django.db.models.fields.CharField', [], {'max_length': '42', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'poster': ('django.db.models.fields.files.ImageField', [], {'default': 'None', 'max_length': '100', 'blank': 'True'}), + 'poster_frame': ('django.db.models.fields.FloatField', [], {'default': '-1'}), + 'poster_height': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'poster_source': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'poster_width': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'rendered': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}), + 'stream_aspect': ('django.db.models.fields.FloatField', [], {'default': '1.3333333333333333'}), + 'stream_info': ('ox.django.fields.DictField', [], {'default': '{}'}), + 'torrent': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '1000', 'blank': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'items'", 'null': 'True', 'to': "orm['auth.User']"}) + } + } + + complete_apps = ['archive'] \ No newline at end of file diff --git a/pandora/archive/models.py b/pandora/archive/models.py index d1127d49..395c57ba 100644 --- a/pandora/archive/models.py +++ b/pandora/archive/models.py @@ -76,6 +76,7 @@ class File(models.Model): available = models.BooleanField(default = False) selected = models.BooleanField(default = False) uploading = models.BooleanField(default = False) + encoding = models.BooleanField(default = False) wanted = models.BooleanField(default = False) is_audio = models.BooleanField(default=False) @@ -248,6 +249,25 @@ class File(models.Model): (not self.item or self.item.user == user) def save_chunk(self, chunk, chunk_id=-1, done=False): + if not self.available: + if not self.data: + name = 'data.%s' % self.info.get('extension', 'avi') + self.data.name = self.get_path(name) + ox.makedirs(os.path.dirname(self.data.path)) + with open(self.data.path, 'w') as f: + f.write(chunk.read()) + self.save() + else: + with open(self.data.path, 'a') as f: + f.write(chunk.read()) + if done: + self.info.update(ox.avinfo(self.data.path)) + self.parse_info() + self.save() + return True + return False + + def save_chunk_stream(self, chunk, chunk_id=-1, done=False): if not self.available: config = settings.CONFIG['video'] stream, created = Stream.objects.get_or_create( @@ -282,6 +302,7 @@ class File(models.Model): 'audioCodec': self.audio_codec, 'available': self.available, 'duration': duration, + 'encoding': self.encoding, 'framerate': self.framerate, 'id': self.oshash, 'instances': [i.json() for i in self.instances.all()], @@ -472,6 +493,15 @@ class Stream(models.Model): else: self.available = False self.save() + elif self.file.data: + video = self.file.data.path + target = self.video.path + info = ox.avinfo(video) + if extract.stream(video, target, self.name(), info): + self.available = True + else: + self.available = False + self.save() def make_timeline(self): if self.available and not self.source: diff --git a/pandora/archive/tasks.py b/pandora/archive/tasks.py index 4247e6cf..a9656dd1 100644 --- a/pandora/archive/tasks.py +++ b/pandora/archive/tasks.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 +import os from glob import glob from celery.task import task from django.conf import settings +import ox from item.models import Item import models @@ -86,12 +88,17 @@ def update_files(user, volume, files): @task(queue="encoding") def process_stream(fileId): + ''' + process uploaded stream + ''' file = models.File.objects.get(id=fileId) streams = file.streams.filter(source=None) if streams.count() > 0: stream = streams[0] + models.File.objects.filter(id=fileId).update(encoding=True) stream.make_timeline() stream.extract_derivatives() + models.File.objects.filter(id=fileId).update(encoding=False) file.item.update_selected() if not file.item.rendered: file.item.update_timeline() @@ -99,6 +106,31 @@ def process_stream(fileId): file.item.save() return True +@task(queue="encoding") +def extract_stream(fileId): + ''' + extract stream from direct upload + ''' + file = models.File.objects.get(id=fileId) + if file.data: + config = settings.CONFIG['video'] + stream, created = models.Stream.objects.get_or_create( + file=file, resolution=max(config['resolutions']), + format=config['formats'][0]) + if created: + models.File.objects.filter(id=fileId).update(encoding=True) + stream.video.name = stream.path(stream.name()) + stream.encode() + if stream.available: + stream.make_timeline() + stream.extract_derivatives() + file.item.update_selected() + if not file.item.rendered: + file.item.update_timeline() + if file.item.rendered: + file.item.save() + models.File.objects.filter(id=fileId).update(encoding=False) + @task(queue="encoding") def extract_derivatives(fileId, rebuild=False): file = models.File.objects.get(id=fileId) diff --git a/pandora/archive/views.py b/pandora/archive/views.py index 1a13d901..6a48b4ef 100644 --- a/pandora/archive/views.py +++ b/pandora/archive/views.py @@ -160,7 +160,7 @@ def upload(request): actions.register(upload, cache=False) -class VideoChunkForm(forms.Form): +class ChunkForm(forms.Form): chunk = forms.FileField() chunkId = forms.IntegerField(required=False) done = forms.IntegerField(required=False) @@ -209,7 +209,8 @@ def addMedia(request): else: extension = 'webm' f.selected = True - f.info = data['info'] + if 'info' in data and data['info']: + f.info = data['info'] f.info['extension'] = extension f.parse_info() f.save() @@ -230,7 +231,7 @@ def firefogg_upload(request): #post next chunk if 'chunk' in request.FILES and oshash: f = get_object_or_404(models.File, oshash=oshash) - form = VideoChunkForm(request.POST, request.FILES) + form = ChunkForm(request.POST, request.FILES) if form.is_valid() and profile == video_profile and f.editable(request.user): c = form.cleaned_data['chunk'] chunk_id = form.cleaned_data['chunkId'] @@ -238,7 +239,7 @@ def firefogg_upload(request): 'result': 1, 'resultUrl': request.build_absolute_uri('/%s'%f.item.itemId) } - if not f.save_chunk(c, chunk_id, form.cleaned_data['done']): + if not f.save_chunk_stream(c, chunk_id, form.cleaned_data['done']): response['result'] = -1 elif form.cleaned_data['done']: f.uploading = False @@ -274,6 +275,51 @@ def firefogg_upload(request): response = json_response(status=400, text='this request requires POST') return render_to_json_response(response) +@login_required_json +def direct_upload(request): + if 'id' in request.GET: + file = models.File.objects.get(oshash=request.GET['id']) + else: + oshash = request.POST['id'] + response = json_response(status=400, text='this request requires POST') + if 'chunk' in request.FILES: + form = ChunkForm(request.POST, request.FILES) + if form.is_valid() and file.editable(request.user): + c = form.cleaned_data['chunk'] + chunk_id = form.cleaned_data['chunkId'] + response = { + 'result': 1, + 'resultUrl': request.build_absolute_uri(file.item.get_absolute_url()) + } + if not file.save_chunk(c, chunk_id, form.cleaned_data['done']): + response['result'] = -1 + if form.cleaned_data['done']: + file.uploading = False + file.save() + #try/execpt so it does not fail if rabitmq is down + try: + t = tasks.extract_stream.delay(file.id) + response['resultUrl'] = t.task_id + except: + pass + response['done'] = 1 + return render_to_json_response(response) + #init upload + else: + file, created = models.File.objects.get_or_create(oshash=oshash) + if file.editable(request.user): + file.uploading = True + file.save() + upload_url = request.build_absolute_uri('/api/upload/direct/?id=%s' % file.oshash) + return render_to_json_response({ + 'uploadUrl': upload_url, + 'url': request.build_absolute_uri(file.item.get_absolute_url()), + 'result': 1 + }) + else: + response = json_response(status=403, text='permission denied') + return render_to_json_response(response) + @login_required_json def taskStatus(request): diff --git a/pandora/urls.py b/pandora/urls.py index 6dfd98e4..cef1d8ac 100644 --- a/pandora/urls.py +++ b/pandora/urls.py @@ -24,6 +24,7 @@ urlpatterns = patterns('', (r'^admin/', include(admin.site.urls)), (r'^api/upload/text/?$', 'text.views.upload'), (r'^api/upload/file/?$', 'file.views.upload'), + (r'^api/upload/direct/?$', 'archive.views.direct_upload'), (r'^api/upload/?$', 'archive.views.firefogg_upload'), (r'^url=(?P.*)$', 'app.views.redirect_url'), (r'^file/(?P.*)$', 'archive.views.lookup_file'), diff --git a/static/js/pandora/filesView.js b/static/js/pandora/filesView.js index a0c2bce5..86a8590f 100644 --- a/static/js/pandora/filesView.js +++ b/static/js/pandora/filesView.js @@ -81,8 +81,11 @@ pandora.ui.filesView = function(options, self) { format: function(value, data) { return $('') .attr({ - src: data.wanted ? Ox.UI.getImageURL('symbolUpload') : - Ox.UI.getImageURL('symbolCheck') + src: data.encoding + ? Ox.UI.getImageURL('symbolSync') + : data.wanted + ? Ox.UI.getImageURL('symbolUpload') + : Ox.UI.getImageURL('symbolCheck') }) .css({ width: '10px', @@ -221,7 +224,7 @@ pandora.ui.filesView = function(options, self) { query: self.filesQuery }), callback); }, - keys: ['wanted', 'instances'], + keys: ['encoding', 'instances', 'wanted'], scrollbarVisible: true, sort: [{key: 'path', operator: '+'}], unique: 'id'