add support for direct video uploads with server side transcoding, ticket #1351

pandora/archive/migrations/0006_encoding.py

@@ -0,0 +1,171 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'File.encoding'
+        db.add_column('archive_file', 'encoding',
+                      self.gf('django.db.models.fields.BooleanField')(default=False),
+                      keep_default=False)
+
+
+    def backwards(self, orm):
+        # Deleting field 'File.encoding'
+        db.delete_column('archive_file', 'encoding')
+
+
+    models = {
+        'archive.file': {
+            'Meta': {'object_name': 'File'},
+            'audio_codec': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'bits_per_pixel': ('django.db.models.fields.FloatField', [], {'default': '-1'}),
+            'channels': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'data': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'display_aspect_ratio': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
+            'encoding': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'extension': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True'}),
+            'framerate': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'height': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'info': ('ox.django.fields.DictField', [], {'default': '{}'}),
+            'is_audio': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_subtitle': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_video': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'item': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'files'", 'null': 'True', 'to': "orm['item.Item']"}),
+            'language': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '8', 'null': 'True'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'oshash': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '16'}),
+            'part': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True'}),
+            'part_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True'}),
+            'path': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '2048'}),
+            'pixel_format': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'pixels': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}),
+            'samplerate': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'selected': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'size': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}),
+            'sort_path': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '2048'}),
+            'type': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255'}),
+            'uploading': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'version': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True'}),
+            'video_codec': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'wanted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'width': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        'archive.frame': {
+            'Meta': {'unique_together': "(('file', 'position'),)", 'object_name': 'Frame'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'file': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'frames'", 'to': "orm['archive.File']"}),
+            'frame': ('django.db.models.fields.files.ImageField', [], {'default': 'None', 'max_length': '100', 'null': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'position': ('django.db.models.fields.FloatField', [], {})
+        },
+        'archive.instance': {
+            'Meta': {'unique_together': "(('path', 'volume'),)", 'object_name': 'Instance'},
+            'atime': ('django.db.models.fields.IntegerField', [], {'default': '1364297821'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'ctime': ('django.db.models.fields.IntegerField', [], {'default': '1364297821'}),
+            'file': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'instances'", 'to': "orm['archive.File']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'ignore': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'mtime': ('django.db.models.fields.IntegerField', [], {'default': '1364297821'}),
+            'path': ('django.db.models.fields.CharField', [], {'max_length': '2048'}),
+            'volume': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'files'", 'to': "orm['archive.Volume']"})
+        },
+        'archive.stream': {
+            'Meta': {'unique_together': "(('file', 'resolution', 'format'),)", 'object_name': 'Stream'},
+            'aspect_ratio': ('django.db.models.fields.FloatField', [], {'default': '0'}),
+            'available': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'color': ('ox.django.fields.TupleField', [], {'default': '[]'}),
+            'cuts': ('ox.django.fields.TupleField', [], {'default': '[]'}),
+            'duration': ('django.db.models.fields.FloatField', [], {'default': '0'}),
+            'file': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'streams'", 'to': "orm['archive.File']"}),
+            'format': ('django.db.models.fields.CharField', [], {'default': "'webm'", 'max_length': '255'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'info': ('ox.django.fields.DictField', [], {'default': '{}'}),
+            'oshash': ('django.db.models.fields.CharField', [], {'max_length': '16', 'null': 'True', 'db_index': 'True'}),
+            'resolution': ('django.db.models.fields.IntegerField', [], {'default': '96'}),
+            'source': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'derivatives'", 'null': 'True', 'to': "orm['archive.Stream']"}),
+            'video': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '100', 'blank': 'True'}),
+            'volume': ('django.db.models.fields.FloatField', [], {'default': '0'})
+        },
+        'archive.volume': {
+            'Meta': {'unique_together': "(('user', 'name'),)", 'object_name': 'Volume'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '1024'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'volumes'", 'to': "orm['auth.User']"})
+        },
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'item.item': {
+            'Meta': {'object_name': 'Item'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'data': ('ox.django.fields.DictField', [], {'default': '{}'}),
+            'external_data': ('ox.django.fields.DictField', [], {'default': '{}'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'items'", 'blank': 'True', 'to': "orm['auth.Group']"}),
+            'icon': ('django.db.models.fields.files.ImageField', [], {'default': 'None', 'max_length': '100', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'itemId': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128', 'blank': 'True'}),
+            'json': ('ox.django.fields.DictField', [], {'default': '{}'}),
+            'level': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'oxdbId': ('django.db.models.fields.CharField', [], {'max_length': '42', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'poster': ('django.db.models.fields.files.ImageField', [], {'default': 'None', 'max_length': '100', 'blank': 'True'}),
+            'poster_frame': ('django.db.models.fields.FloatField', [], {'default': '-1'}),
+            'poster_height': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'poster_source': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'poster_width': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'rendered': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'stream_aspect': ('django.db.models.fields.FloatField', [], {'default': '1.3333333333333333'}),
+            'stream_info': ('ox.django.fields.DictField', [], {'default': '{}'}),
+            'torrent': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '1000', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'items'", 'null': 'True', 'to': "orm['auth.User']"})
+        }
+    }
+
+    complete_apps = ['archive']

pandora/archive/models.py

@@ -76,6 +76,7 @@ class File(models.Model):
     available = models.BooleanField(default = False)
     selected = models.BooleanField(default = False)
     uploading = models.BooleanField(default = False)
+    encoding = models.BooleanField(default = False)
     wanted = models.BooleanField(default = False)
 
     is_audio = models.BooleanField(default=False)
@@ -248,6 +249,25 @@ class File(models.Model):
             (not self.item or self.item.user == user)
 
     def save_chunk(self, chunk, chunk_id=-1, done=False):
+        if not self.available:
+            if not self.data:
+                name = 'data.%s' % self.info.get('extension', 'avi')
+                self.data.name = self.get_path(name)
+                ox.makedirs(os.path.dirname(self.data.path))
+                with open(self.data.path, 'w') as f:
+                    f.write(chunk.read())
+                self.save()
+            else:
+                with open(self.data.path, 'a') as f:
+                    f.write(chunk.read())
+            if done:
+                self.info.update(ox.avinfo(self.data.path))
+                self.parse_info()
+                self.save()
+            return True
+        return False
+
+    def save_chunk_stream(self, chunk, chunk_id=-1, done=False):
         if not self.available:
             config = settings.CONFIG['video']
             stream, created = Stream.objects.get_or_create(
@@ -282,6 +302,7 @@ class File(models.Model):
             'audioCodec': self.audio_codec,
             'available': self.available,
             'duration': duration,
+            'encoding': self.encoding,
             'framerate': self.framerate,
             'id': self.oshash,
             'instances': [i.json() for i in self.instances.all()],
@@ -472,6 +493,15 @@ class Stream(models.Model):
             else:
                 self.available = False
             self.save()
+        elif self.file.data:
+            video = self.file.data.path
+            target = self.video.path
+            info = ox.avinfo(video)
+            if extract.stream(video, target, self.name(), info):
+                self.available = True
+            else:
+                self.available = False
+            self.save()
 
     def make_timeline(self):
         if self.available and not self.source:

pandora/archive/tasks.py

@@ -1,10 +1,12 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
+import os
 from glob import glob
 
 from celery.task import task
 
 from django.conf import settings
+import ox
 
 from item.models import Item
 import models
@@ -86,12 +88,17 @@ def update_files(user, volume, files):
 
 @task(queue="encoding")
 def process_stream(fileId):
+    '''
+        process uploaded stream
+    '''
     file = models.File.objects.get(id=fileId)
     streams = file.streams.filter(source=None)
     if streams.count() > 0:
         stream = streams[0]
+        models.File.objects.filter(id=fileId).update(encoding=True)
         stream.make_timeline()
         stream.extract_derivatives()
+        models.File.objects.filter(id=fileId).update(encoding=False)
     file.item.update_selected()
     if not file.item.rendered:
         file.item.update_timeline()
@@ -99,6 +106,31 @@ def process_stream(fileId):
         file.item.save()
     return True
 
+@task(queue="encoding")
+def extract_stream(fileId):
+    '''
+        extract stream from direct upload
+    '''
+    file = models.File.objects.get(id=fileId)
+    if file.data:
+        config = settings.CONFIG['video']
+        stream, created = models.Stream.objects.get_or_create(
+            file=file, resolution=max(config['resolutions']),
+            format=config['formats'][0])
+        if created:
+            models.File.objects.filter(id=fileId).update(encoding=True)
+            stream.video.name = stream.path(stream.name())
+            stream.encode()
+            if stream.available:
+                stream.make_timeline()
+                stream.extract_derivatives()
+                file.item.update_selected()
+                if not file.item.rendered:
+                    file.item.update_timeline()
+                if file.item.rendered:
+                    file.item.save()
+            models.File.objects.filter(id=fileId).update(encoding=False)
+
 @task(queue="encoding")
 def extract_derivatives(fileId, rebuild=False):
     file = models.File.objects.get(id=fileId)

pandora/archive/views.py

@@ -160,7 +160,7 @@ def upload(request):
 actions.register(upload, cache=False)
 
 
-class VideoChunkForm(forms.Form):
+class ChunkForm(forms.Form):
     chunk = forms.FileField()
     chunkId = forms.IntegerField(required=False)
     done = forms.IntegerField(required=False)
@@ -209,7 +209,8 @@ def addMedia(request):
         else:
             extension = 'webm'
         f.selected = True
-        f.info = data['info']
+        if 'info' in data and data['info']:
+            f.info = data['info']
         f.info['extension'] = extension
         f.parse_info()
         f.save()
@@ -230,7 +231,7 @@ def firefogg_upload(request):
         #post next chunk
         if 'chunk' in request.FILES and oshash:
             f = get_object_or_404(models.File, oshash=oshash)
-            form = VideoChunkForm(request.POST, request.FILES)
+            form = ChunkForm(request.POST, request.FILES)
             if form.is_valid() and profile == video_profile and f.editable(request.user):
                 c = form.cleaned_data['chunk']
                 chunk_id = form.cleaned_data['chunkId']
@@ -238,7 +239,7 @@ def firefogg_upload(request):
                     'result': 1,
                     'resultUrl': request.build_absolute_uri('/%s'%f.item.itemId)
                 }
-                if not f.save_chunk(c, chunk_id, form.cleaned_data['done']):
+                if not f.save_chunk_stream(c, chunk_id, form.cleaned_data['done']):
                     response['result'] = -1
                 elif form.cleaned_data['done']:
                     f.uploading = False
@@ -274,6 +275,51 @@ def firefogg_upload(request):
     response = json_response(status=400, text='this request requires POST')
     return render_to_json_response(response)
 
+@login_required_json
+def direct_upload(request):
+    if 'id' in request.GET:
+        file = models.File.objects.get(oshash=request.GET['id'])
+    else:
+        oshash = request.POST['id']
+    response = json_response(status=400, text='this request requires POST')
+    if 'chunk' in request.FILES:
+        form = ChunkForm(request.POST, request.FILES)
+        if form.is_valid() and file.editable(request.user):
+            c = form.cleaned_data['chunk']
+            chunk_id = form.cleaned_data['chunkId']
+            response = {
+                'result': 1,
+                'resultUrl': request.build_absolute_uri(file.item.get_absolute_url())
+            }
+            if not file.save_chunk(c, chunk_id, form.cleaned_data['done']):
+                response['result'] = -1
+            if form.cleaned_data['done']:
+                file.uploading = False
+                file.save()
+                #try/execpt so it does not fail if rabitmq is down
+                try:
+                    t = tasks.extract_stream.delay(file.id)
+                    response['resultUrl'] = t.task_id
+                except:
+                    pass
+                response['done'] = 1
+            return render_to_json_response(response)
+    #init upload
+    else:
+        file, created = models.File.objects.get_or_create(oshash=oshash)
+        if file.editable(request.user):
+            file.uploading = True
+            file.save()
+            upload_url = request.build_absolute_uri('/api/upload/direct/?id=%s' % file.oshash)
+            return render_to_json_response({
+                'uploadUrl': upload_url,
+                'url': request.build_absolute_uri(file.item.get_absolute_url()),
+                'result': 1
+            })
+        else:
+            response = json_response(status=403, text='permission denied')
+    return render_to_json_response(response)
+
 
 @login_required_json
 def taskStatus(request):

pandora/urls.py

@@ -24,6 +24,7 @@ urlpatterns = patterns('',
     (r'^admin/', include(admin.site.urls)),
     (r'^api/upload/text/?$', 'text.views.upload'),
     (r'^api/upload/file/?$', 'file.views.upload'),
+    (r'^api/upload/direct/?$', 'archive.views.direct_upload'),
     (r'^api/upload/?$', 'archive.views.firefogg_upload'),
     (r'^url=(?P<url>.*)$', 'app.views.redirect_url'),
     (r'^file/(?P<oshash>.*)$', 'archive.views.lookup_file'),

static/js/pandora/filesView.js

@@ -81,8 +81,11 @@ pandora.ui.filesView = function(options, self) {
                     format: function(value, data) {
                         return $('<img>')
                             .attr({
-                                src: data.wanted ? Ox.UI.getImageURL('symbolUpload') :
+                                src: data.encoding
-                                                   Ox.UI.getImageURL('symbolCheck')
+                                    ? Ox.UI.getImageURL('symbolSync')
+                                    : data.wanted
+                                    ? Ox.UI.getImageURL('symbolUpload')
+                                    : Ox.UI.getImageURL('symbolCheck')
                             })
                             .css({
                                 width: '10px',
@@ -221,7 +224,7 @@ pandora.ui.filesView = function(options, self) {
                     query: self.filesQuery
                 }), callback);
             },
-            keys: ['wanted', 'instances'],
+            keys: ['encoding', 'instances', 'wanted'],
             scrollbarVisible: true,
             sort: [{key: 'path', operator: '+'}],
             unique: 'id'