refactor chunk upload: return offset, continue at server offset

pandora/archive/chunk.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+from __future__ import division, with_statement
+
+import os
+
+import ox
+from django import forms
+
+class ChunkForm(forms.Form):
+    chunk = forms.FileField()
+    chunkId = forms.IntegerField(required=False)
+    offset = forms.IntegerField(required=False)
+    done = forms.IntegerField(required=False)
+
+def save_chunk(self, file, chunk, offset, name, done_cb=None):
+    print 'file', file
+    print 'chunk', chunk
+    if not file:
+        file.name = name
+        ox.makedirs(os.path.dirname(file.path))
+        with open(file.path, 'w') as f:
+            f.write(chunk.read())
+        self.save()
+    else:
+        path = file.path
+        size = file.size
+        if offset == None:
+            offset = size
+        elif offset > size:
+            return False, size
+        with open(path, 'r+') as f:
+            f.seek(offset)
+            f.write(chunk.read())
+    return done_cb() if done_cb else True, file.size
+
+def process_chunk(request, _save_chunk):
+    response = {
+        'result': 1,
+    }
+    form = ChunkForm(request.POST, request.FILES)
+    if form.is_valid():
+        d = form.cleaned_data
+        ok, _offset = _save_chunk(d['chunk'], d['offset'], d['done'])
+        response['offset'] = _offset
+        response['result'] = 1 if ok else -1
+        if d['done']:
+            response['done'] = 1
+    else:
+        response['result'] = -1
+    return response

pandora/archive/models.py

@@ -18,6 +18,7 @@ from item import utils
 import item.models
 from person.models import get_name_sort
 
+from chunk import save_chunk
 import extract
 
 class File(models.Model):
@@ -299,32 +300,32 @@ class File(models.Model):
 
     def save_chunk(self, chunk, offset=None, done=False):
         if not self.available:
-            if not self.data:
+            name = 'data.%s' % self.info.get('extension', 'avi')
-                name = 'data.%s' % self.info.get('extension', 'avi')
+            name = self.get_path(name)
-                self.data.name = self.get_path(name)
+
-                ox.makedirs(os.path.dirname(self.data.path))
+            def done_cb():
-                with open(self.data.path, 'w') as f:
+                if done:
-                    f.write(chunk.read())
+                    self.info.update(ox.avinfo(self.data.path))
-                self.save()
+                    self.parse_info()
-            else:
+                    # reject invalid uploads
-                if offset == None:
+                    if self.info.get('oshash') != self.oshash:
-                    offset = self.data.size
+                        self.data.delete()
-                elif offset > self.data.size:
+                        self.save()
-                    return False
+                        return False, 0
-                with open(self.data.path, 'r+') as f:
-                    f.seek(offset)
-                    f.write(chunk.read())
-            if done:
-                self.info.update(ox.avinfo(self.data.path))
-                self.parse_info()
-                # reject invalid uploads
-                if self.info.get('oshash') != self.oshash:
-                    self.data.delete()
                     self.save()
-                    return False
+                return True, self.data.size
-                self.save()
+            return save_chunk(self, self.data, chunk, offset, name, done_cb)
-            return True
+        else:
-        return False
+            return False, 0
+
+    def save_chunk_stream(self, chunk, offset, resolution, format, done):
+        if not self.available:
+            config = settings.CONFIG['video']
+            stream, created = Stream.objects.get_or_create(
+                        file=self, resolution=resolution, format=format)
+            name = stream.path(stream.name())
+            return save_chunk(stream, stream.media, chunk, offset, name)
+        return False, 0
 
     def stream_resolution(self):
         config = settings.CONFIG['video']
@@ -334,32 +335,6 @@ class File(models.Model):
                 return resolution
         return resolution
 
-    def save_chunk_stream(self, chunk, offset, resolution, format, done):
-        if not self.available:
-            config = settings.CONFIG['video']
-            stream, created = Stream.objects.get_or_create(
-                        file=self, resolution=resolution, format=format)
-            if created:
-                stream.media.name = stream.path(stream.name())
-                ox.makedirs(os.path.dirname(stream.media.path))
-                with open(stream.media.path, 'w') as f:
-                    f.write(chunk.read())
-                stream.save()
-            else:
-                if offset == -1:
-                    offset = stream.media.size
-                elif offset > stream.media.size:
-                    return False
-                with open(stream.media.path, 'r+') as f:
-                    f.seek(offset)
-                    f.write(chunk.read())
-            if done:
-                stream.available = True
-                stream.info = {}
-                stream.save()
-            return True
-        return False
-
     def json(self, keys=None, user=None):
         resolution = (self.width, self.height)
         if resolution == (0, 0) or self.type != 'video':

pandora/archive/views.py

@@ -23,6 +23,7 @@ from ox.django.api import actions
 
 import models
 import tasks
+from chunk import process_chunk
 
 
 @login_required_json
@@ -159,12 +160,6 @@ def upload(request):
 actions.register(upload, cache=False)
 
 
-class ChunkForm(forms.Form):
-    chunk = forms.FileField()
-    chunkId = forms.IntegerField(required=False)
-    offset = forms.IntegerField(required=False)
-    done = forms.IntegerField(required=False)
-
 @login_required_json
 def addMedia(request):
     '''
@@ -238,18 +233,12 @@ def firefogg_upload(request):
         #post next chunk
         if 'chunk' in request.FILES and oshash:
             f = get_object_or_404(models.File, oshash=oshash)
-            form = ChunkForm(request.POST, request.FILES)
+            if f.editable(request.user):
-            if form.is_valid() and f.editable(request.user):
+                def save_chunk(chunk, offset, done):
-                c = form.cleaned_data['chunk']
+                    return f.save_chunk_stream(chunk, offset, resolution, format, done)
-                offset = form.cleaned_data['offset']
+                response = process_chunk(request, save_chunk)
-                response = {
+                response['resultUrl'] = request.build_absolute_uri('/%s'%f.item.itemId)
-                    'result': 1,
+                if response.get('done'):
-                    'resultUrl': request.build_absolute_uri('/%s'%f.item.itemId)
-                }
-                if not f.save_chunk_stream(c, offset, resolution, format,
-                        form.cleaned_data['done']):
-                    response['result'] = -1
-                elif form.cleaned_data['done']:
                     f.uploading = False
                     if response['result'] == 1:
                         f.queued = True
@@ -264,8 +253,6 @@ def firefogg_upload(request):
                         response['resultUrl'] = t.task_id
                     except:
                         pass
-                    response['result'] = 1
-                    response['done'] = 1
                 return render_to_json_response(response)
         #init upload
         elif oshash:
@@ -298,17 +285,10 @@ def direct_upload(request):
         oshash = request.POST['id']
     response = json_response(status=400, text='this request requires POST')
     if 'chunk' in request.FILES:
-        form = ChunkForm(request.POST, request.FILES)
+        if file.editable(request.user):
-        if form.is_valid() and file.editable(request.user):
+            response = process_chunk(request, file.save_chunk)
-            c = form.cleaned_data['chunk']
+            response['resultUrl'] = request.build_absolute_uri(file.item.get_absolute_url())
-            offset = form.cleaned_data['offset']
+            if response.get('done'):
-            response = {
-                'result': 1,
-                'resultUrl': request.build_absolute_uri(file.item.get_absolute_url())
-            }
-            if not file.save_chunk(c, offset, form.cleaned_data['done']):
-                response['result'] = -1
-            if form.cleaned_data['done']:
                 file.uploading = False
                 if response['result'] == 1:
                     file.queued = True
@@ -323,7 +303,6 @@ def direct_upload(request):
                     response['resultUrl'] = t.task_id
                 except:
                     pass
-                response['done'] = 1
             return render_to_json_response(response)
     #init upload
     else:

pandora/document/models.py

@@ -17,6 +17,7 @@ import ox
 
 from item.models import Item
 from archive.extract import resize_image
+from archive.chunk import save_chunk
 
 import managers
 import utils
@@ -180,29 +181,20 @@ class Document(models.Model):
 
     def save_chunk(self, chunk, offset=None, done=False):
         if self.uploading:
-            if not self.file:
+            name = 'data.%s' % self.extension
-                name = 'data.%s' % self.extension
+            name = self.path(name)
-                self.file.name = self.path(name)
+
-                ox.makedirs(os.path.dirname(self.file.path))
+            def done_cb():
-                with open(self.file.path, 'w') as f:
+                if done:
-                    f.write(chunk.read())
+                    self.uploading = False
-                self.save()
+                    self.get_info()
-            else:
+                    self.get_ratio()
-                if offset == None:
+                    self.oshash = ox.oshash(self.file.path)
-                    offset = self.file.size
+                    self.save()
-                elif offset > self.file.size:
+                return True, self.file.size
-                    return False
+
-                with open(self.file.path, 'r+') as f:
+            return save_chunk(self, self.file, chunk, offset, name, done_cb)
-                    f.seek(offset)
+        return False, 0
-                    f.write(chunk.read())
-            if done:
-                self.uploading = False
-                self.get_info()
-                self.get_ratio()
-                self.oshash = ox.oshash(self.file.path)
-                self.save()
-            return True
-        return False
 
     def thumbnail(self, size=None, page=None):
         src = self.file.path

pandora/document/views.py

@@ -13,6 +13,8 @@ from django.db.models import Sum
 from item import utils
 from item.models import Item
 from itemlist.models import List
+from archive.chunk import process_chunk
+
 import models
 
 def get_document_or_404_json(id):
@@ -281,11 +283,6 @@ def thumbnail(request, id, size=256, page=None):
     document = models.Document.get(id)
     return HttpFileResponse(document.thumbnail(size, page=page))
 
-class ChunkForm(forms.Form):
-    chunk = forms.FileField()
-    offset = forms.IntegerField(required=False)
-    done = forms.IntegerField(required=False)
-
 @login_required_json
 def upload(request):
     if 'id' in request.GET:
@@ -297,19 +294,9 @@ def upload(request):
         extension = extension[-1].lower()
     response = json_response(status=400, text='this request requires POST')
     if 'chunk' in request.FILES:
-        form = ChunkForm(request.POST, request.FILES)
+        if file.editable(request.user):
-        if form.is_valid() and file.editable(request.user):
+            response = process_chunk(request, file.save_chunk)
-            c = form.cleaned_data['chunk']
+            response['resultUrl'] = request.build_absolute_uri(file.get_absolute_url())
-            offset = form.cleaned_data['offset']
-            response = {
-                'result': 1,
-                'id': file.get_id(),
-                'resultUrl': request.build_absolute_uri(file.get_absolute_url())
-            }
-            if not file.save_chunk(c, offset, form.cleaned_data['done']):
-                response['result'] = -1
-            if form.cleaned_data['done']:
-                response['done'] = 1
             return render_to_json_response(response)
     #init upload
     else:

pandora/text/models.py

@@ -16,6 +16,7 @@ import ox
 from ox.django.fields import DictField, TupleField
 
 from archive import extract
+from archive.chunk import save_chunk
 
 import managers
 
@@ -278,25 +279,16 @@ class Text(models.Model):
 
     def save_chunk(self, chunk, offset=None, done=False):
         if self.uploading:
-            if not self.file:
+            name = self.path('data.pdf')
-                self.file.name = self.path('data.pdf')
+
-                ox.makedirs(os.path.dirname(self.file.path))
+            def done_cb():
-                with open(self.file.path, 'w') as f:
+                if done:
-                    f.write(chunk.read())
+                    self.uploading = False
-                self.save()
+                    self.save()
-            else:
+                return True, self.file.size
-                if offset == None:
+
-                    offset = self.file.size
+            return save_chunk(self, self.file, chunk, offset, name, done_cb)
-                elif offset > self.file.size:
+        return False, 0
-                    return False
-                with open(self.file.path, 'r+') as f:
-                    f.seek(offset)
-                    f.write(chunk.read())
-            if done:
-                self.uploading = False
-                self.save()
-            return True
-        return False
 
 def delete_file(sender, **kwargs):
     t = kwargs['instance']

pandora/text/views.py

@@ -17,6 +17,7 @@ from django.shortcuts import render_to_response
 from django.template import RequestContext
 
 from item import utils
+from archive.chunk import process_chunk
 import models
 
 def get_text_or_404_json(id):
@@ -377,11 +378,6 @@ def icon(request, id, size=16):
         icon = os.path.join(settings.STATIC_ROOT, 'jpg/list256.jpg')
     return HttpFileResponse(icon, content_type='image/jpeg')
 
-class ChunkForm(forms.Form):
-    chunk = forms.FileField()
-    offset = forms.IntegerField(required=False)
-    done = forms.IntegerField(required=False)
-
 def pdf_viewer(request, id):
     text = get_text_or_404_json(id)
     if text.type == 'pdf' and text.file and not text.uploading:
@@ -409,18 +405,10 @@ def upload(request):
     if text.editable(request.user):
         #post next chunk
         if 'chunk' in request.FILES:
-            form = ChunkForm(request.POST, request.FILES)
+            if text.editable(request.user):
-            if form.is_valid() and text.editable(request.user):
+                response = process_chunk(request, text.save_chunk)
-                c = form.cleaned_data['chunk']
+                response['resultUrl'] = request.build_absolute_uri(text.get_absolute_url())
-                offset = form.cleaned_data['offset']
+                return render_to_json_response(response)
-                response = {
-                    'result': 1,
-                    'resultUrl': request.build_absolute_uri(text.get_absolute_url())
-                }
-                if not text.save_chunk(c, offset, form.cleaned_data['done']):
-                    response['result'] = -1
-                if form.cleaned_data['done']:
-                    response['done'] = 1
                 return render_to_json_response(response)
         #init upload
         else:

static/js/chunkupload.js

@@ -162,7 +162,15 @@ pandora.chunkupload = function(options) {
                     nextChunkId = chunkId + 1;
                     that.triggerEvent('paused', {next: nextChunkId});
                 } else {
-                    uploadChunk(chunkId + 1);
+                    if (Ox.isUndefined(response.offset) ||
+                        response.offset ==  (chunkId +1) * chunkSize) {
+                        uploadChunk(chunkId + 1);
+                    } else {
+                        // continue at chunk closest to offset from server
+                        console.log('server offset', response.offset,
+                            'next chunk', Math.floor(response.offset / chuknSize));
+                        uploadChunk(Math.floor(response.offset / chuknSize));
+                    }
                 }
             } else {
                 // failed to upload, try again in 5 second