upload chunks with offset. fail if uploaded file does not match oshash

pandora/archive/models.py

@@ -297,7 +297,7 @@ class File(models.Model):
             self.instances.filter(volume__user=user).count() > 0 or \
             (not self.item or self.item.user == user)
 
-    def save_chunk(self, chunk, chunk_id=-1, done=False):
+    def save_chunk(self, chunk, offset=None, done=False):
         if not self.available:
             if not self.data:
                 name = 'data.%s' % self.info.get('extension', 'avi')
@@ -307,11 +307,21 @@ class File(models.Model):
                     f.write(chunk.read())
                 self.save()
             else:
-                with open(self.data.path, 'a') as f:
+                if offset == None:
+                    offset = self.data.size
+                elif offset > self.data.size:
+                    return False
+                with open(self.data.path, 'r+') as f:
+                    f.seek(offset)
                     f.write(chunk.read())
             if done:
                 self.info.update(ox.avinfo(self.data.path))
                 self.parse_info()
+                # reject invalid uploads
+                if self.info.get('oshash') != self.oshash:
+                    self.data.delete()
+                    self.save()
+                    return False
                 self.save()
             return True
         return False
@@ -324,7 +334,7 @@ class File(models.Model):
                 return resolution
         return resolution
 
-    def save_chunk_stream(self, chunk, chunk_id, resolution, format, done):
+    def save_chunk_stream(self, chunk, offset, resolution, format, done):
         if not self.available:
             config = settings.CONFIG['video']
             stream, created = Stream.objects.get_or_create(
@@ -336,8 +346,12 @@ class File(models.Model):
                     f.write(chunk.read())
                 stream.save()
             else:
-                with open(stream.media.path, 'a') as f:
+                if offset == -1:
-                    #FIXME: should check that chunk_id/offset is right
+                    offset = stream.media.size
+                elif offset > stream.media.size:
+                    return False
+                with open(stream.media.path, 'r+') as f:
+                    f.seek(offset)
                     f.write(chunk.read())
             if done:
                 stream.available = True
@@ -351,7 +365,7 @@ class File(models.Model):
         if resolution == (0, 0) or self.type != 'video':
             resolution = None
         duration = self.duration
-        if self.type != 'video':
+        if self.type not in ('audio', 'video'):
             duration = None
         state = ''
         error = ''

pandora/archive/views.py

@@ -162,6 +162,7 @@ actions.register(upload, cache=False)
 class ChunkForm(forms.Form):
     chunk = forms.FileField()
     chunkId = forms.IntegerField(required=False)
+    offset = forms.IntegerField(required=False)
     done = forms.IntegerField(required=False)
 
 @login_required_json
@@ -240,17 +241,22 @@ def firefogg_upload(request):
             form = ChunkForm(request.POST, request.FILES)
             if form.is_valid() and f.editable(request.user):
                 c = form.cleaned_data['chunk']
-                chunk_id = form.cleaned_data['chunkId']
+                offset = form.cleaned_data['offset']
                 response = {
                     'result': 1,
                     'resultUrl': request.build_absolute_uri('/%s'%f.item.itemId)
                 }
-                if not f.save_chunk_stream(c, chunk_id, resolution, format,
+                if not f.save_chunk_stream(c, offset, resolution, format,
                         form.cleaned_data['done']):
                     response['result'] = -1
                 elif form.cleaned_data['done']:
                     f.uploading = False
+                    if response['result'] == 1:
                         f.queued = True
+                        f.wanted = False
+                    else:
+                        f.queued = False
+                        f.wanted = True
                     f.save()
                     #FIXME: this fails badly if rabbitmq goes down
                     try:
@@ -295,16 +301,21 @@ def direct_upload(request):
         form = ChunkForm(request.POST, request.FILES)
         if form.is_valid() and file.editable(request.user):
             c = form.cleaned_data['chunk']
-            chunk_id = form.cleaned_data['chunkId']
+            offset = form.cleaned_data['offset']
             response = {
                 'result': 1,
                 'resultUrl': request.build_absolute_uri(file.item.get_absolute_url())
             }
-            if not file.save_chunk(c, chunk_id, form.cleaned_data['done']):
+            if not file.save_chunk(c, offset, form.cleaned_data['done']):
                 response['result'] = -1
             if form.cleaned_data['done']:
                 file.uploading = False
+                if response['result'] == 1:
                     file.queued = True
+                    file.wanted = False
+                else:
+                    file.queued = False
+                    file.wanted = True
                 file.save()
                 #try/execpt so it does not fail if rabitmq is down
                 try:

pandora/document/models.py

@@ -178,7 +178,7 @@ class Document(models.Model):
         h = (7-len(h))*'0' + h
         return os.path.join('documents', h[:2], h[2:4], h[4:6], h[6:], name)
 
-    def save_chunk(self, chunk, chunk_id=-1, done=False):
+    def save_chunk(self, chunk, offset=None, done=False):
         if self.uploading:
             if not self.file:
                 name = 'data.%s' % self.extension
@@ -188,7 +188,12 @@ class Document(models.Model):
                     f.write(chunk.read())
                 self.save()
             else:
-                with open(self.file.path, 'a') as f:
+                if offset == None:
+                    offset = self.file.size
+                elif offset > self.file.size:
+                    return False
+                with open(self.file.path, 'r+') as f:
+                    f.seek(offset)
                     f.write(chunk.read())
             if done:
                 self.uploading = False

pandora/document/views.py

@@ -283,7 +283,7 @@ def thumbnail(request, id, size=256, page=None):
 
 class ChunkForm(forms.Form):
     chunk = forms.FileField()
-    chunkId = forms.IntegerField(required=False)
+    offset = forms.IntegerField(required=False)
     done = forms.IntegerField(required=False)
 
 @login_required_json
@@ -300,13 +300,13 @@ def upload(request):
         form = ChunkForm(request.POST, request.FILES)
         if form.is_valid() and file.editable(request.user):
             c = form.cleaned_data['chunk']
-            chunk_id = form.cleaned_data['chunkId']
+            offset = form.cleaned_data['offset']
             response = {
                 'result': 1,
                 'id': file.get_id(),
                 'resultUrl': request.build_absolute_uri(file.get_absolute_url())
             }
-            if not file.save_chunk(c, chunk_id, form.cleaned_data['done']):
+            if not file.save_chunk(c, offset, form.cleaned_data['done']):
                 response['result'] = -1
             if form.cleaned_data['done']:
                 response['done'] = 1

pandora/text/models.py

@@ -276,7 +276,7 @@ class Text(models.Model):
                 path = source
         return path
 
-    def save_chunk(self, chunk, chunk_id=-1, done=False):
+    def save_chunk(self, chunk, offset=None, done=False):
         if self.uploading:
             if not self.file:
                 self.file.name = self.path('data.pdf')
@@ -285,7 +285,12 @@ class Text(models.Model):
                     f.write(chunk.read())
                 self.save()
             else:
-                with open(self.file.path, 'a') as f:
+                if offset == None:
+                    offset = self.file.size
+                elif offset > self.file.size:
+                    return False
+                with open(self.file.path, 'r+') as f:
+                    f.seek(offset)
                     f.write(chunk.read())
             if done:
                 self.uploading = False

pandora/text/views.py

@@ -379,7 +379,7 @@ def icon(request, id, size=16):
 
 class ChunkForm(forms.Form):
     chunk = forms.FileField()
-    chunkId = forms.IntegerField(required=False)
+    offset = forms.IntegerField(required=False)
     done = forms.IntegerField(required=False)
 
 def pdf_viewer(request, id):
@@ -412,12 +412,12 @@ def upload(request):
             form = ChunkForm(request.POST, request.FILES)
             if form.is_valid() and text.editable(request.user):
                 c = form.cleaned_data['chunk']
-                chunk_id = form.cleaned_data['chunkId']
+                offset = form.cleaned_data['offset']
                 response = {
                     'result': 1,
                     'resultUrl': request.build_absolute_uri(text.get_absolute_url())
                 }
-                if not text.save_chunk(c, chunk_id, form.cleaned_data['done']):
+                if not text.save_chunk(c, offset, form.cleaned_data['done']):
                     response['result'] = -1
                 if form.cleaned_data['done']:
                     response['done'] = 1

static/js/chunkupload.js

@@ -26,7 +26,8 @@
   });
 */
 pandora.chunkupload = function(options) {
-    var chunkSize = options.size || 1024 * 1024,
+    var aborted = false,
+        chunkSize = options.size || 1024 * 1024,
         chunkURL,
         file = options.file,
         maxRetry = -1,
@@ -120,6 +121,10 @@ pandora.chunkupload = function(options) {
             chunk,
             chunkOffset = chunkId * chunkSize;
 
+        if (aborted) {
+            return;
+        }
+
         if (file.mozSlice) {
             chunk = file.mozSlice(chunkOffset, chunkOffset+chunkSize, file.type);
         } else if (file.webkitSlice) {
@@ -145,7 +150,8 @@ pandora.chunkupload = function(options) {
             if (response.done == 1) {
                 //upload finished
                 that.resultUrl = response.resultUrl;
-                that.progress = 1;
+                // set progress to -1 if overall upload failed
+                that.progress = response.result;
                 that.status = 'done';
                 done();
             } else if (response.result == 1) {
@@ -210,7 +216,7 @@ pandora.chunkupload = function(options) {
         Object.keys(options.data).forEach(function(key) {
             formData.append(key, options.data[key]);
         });
-        formData.append('chunkId', chunkId);
+        formData.append('offset', chunkOffset);
         if (bytesAvailable <= chunkOffset + chunkSize) {
             formData.append('done', 1);
         }
@@ -220,9 +226,13 @@ pandora.chunkupload = function(options) {
     }
 
     that.abort = function() {
+        aborted = true;
         if (request) {
             request.abort();
             request = null;
+        } else {
+            that.progress = -1;
+            done();
         }
         return that;
     };

static/js/uploadVideoDialog.js

@@ -64,6 +64,7 @@ pandora.ui.uploadVideoDialog = function(data) {
                 }).bindEvent({
                     click: function(data) {
                         if (data.files.length) {
+                            cancelled = false;
                             $actionButton.hide();
                             $closeButton.options('title', Ox._('Cancel'));
                             upload(data.files[0]);