From 7cd152b1d5eccd2e82debed70a2ecd679378fb62 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Fri, 11 Apr 2014 16:56:22 +0000 Subject: [PATCH] refactor chunk upload: return offset, continue at server offset --- pandora/archive/chunk.py | 51 +++++++++++++++++++++++++ pandora/archive/models.py | 77 +++++++++++++------------------------- pandora/archive/views.py | 43 ++++++--------------- pandora/document/models.py | 38 ++++++++----------- pandora/document/views.py | 23 +++--------- pandora/text/models.py | 30 ++++++--------- pandora/text/views.py | 22 +++-------- static/js/chunkupload.js | 10 ++++- 8 files changed, 133 insertions(+), 161 deletions(-) create mode 100644 pandora/archive/chunk.py diff --git a/pandora/archive/chunk.py b/pandora/archive/chunk.py new file mode 100644 index 00000000..97a5bb2a --- /dev/null +++ b/pandora/archive/chunk.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from __future__ import division, with_statement + +import os + +import ox +from django import forms + +class ChunkForm(forms.Form): + chunk = forms.FileField() + chunkId = forms.IntegerField(required=False) + offset = forms.IntegerField(required=False) + done = forms.IntegerField(required=False) + +def save_chunk(self, file, chunk, offset, name, done_cb=None): + print 'file', file + print 'chunk', chunk + if not file: + file.name = name + ox.makedirs(os.path.dirname(file.path)) + with open(file.path, 'w') as f: + f.write(chunk.read()) + self.save() + else: + path = file.path + size = file.size + if offset == None: + offset = size + elif offset > size: + return False, size + with open(path, 'r+') as f: + f.seek(offset) + f.write(chunk.read()) + return done_cb() if done_cb else True, file.size + +def process_chunk(request, _save_chunk): + response = { + 'result': 1, + } + form = ChunkForm(request.POST, request.FILES) + if form.is_valid(): + d = form.cleaned_data + ok, _offset = _save_chunk(d['chunk'], d['offset'], d['done']) + response['offset'] = _offset + response['result'] = 1 if ok else -1 + if d['done']: + response['done'] = 1 + else: + response['result'] = -1 + return response diff --git a/pandora/archive/models.py b/pandora/archive/models.py index 4bbd6847..b5dbe4ed 100644 --- a/pandora/archive/models.py +++ b/pandora/archive/models.py @@ -18,6 +18,7 @@ from item import utils import item.models from person.models import get_name_sort +from chunk import save_chunk import extract class File(models.Model): @@ -299,32 +300,32 @@ class File(models.Model): def save_chunk(self, chunk, offset=None, done=False): if not self.available: - if not self.data: - name = 'data.%s' % self.info.get('extension', 'avi') - self.data.name = self.get_path(name) - ox.makedirs(os.path.dirname(self.data.path)) - with open(self.data.path, 'w') as f: - f.write(chunk.read()) - self.save() - else: - if offset == None: - offset = self.data.size - elif offset > self.data.size: - return False - with open(self.data.path, 'r+') as f: - f.seek(offset) - f.write(chunk.read()) - if done: - self.info.update(ox.avinfo(self.data.path)) - self.parse_info() - # reject invalid uploads - if self.info.get('oshash') != self.oshash: - self.data.delete() + name = 'data.%s' % self.info.get('extension', 'avi') + name = self.get_path(name) + + def done_cb(): + if done: + self.info.update(ox.avinfo(self.data.path)) + self.parse_info() + # reject invalid uploads + if self.info.get('oshash') != self.oshash: + self.data.delete() + self.save() + return False, 0 self.save() - return False - self.save() - return True - return False + return True, self.data.size + return save_chunk(self, self.data, chunk, offset, name, done_cb) + else: + return False, 0 + + def save_chunk_stream(self, chunk, offset, resolution, format, done): + if not self.available: + config = settings.CONFIG['video'] + stream, created = Stream.objects.get_or_create( + file=self, resolution=resolution, format=format) + name = stream.path(stream.name()) + return save_chunk(stream, stream.media, chunk, offset, name) + return False, 0 def stream_resolution(self): config = settings.CONFIG['video'] @@ -334,32 +335,6 @@ class File(models.Model): return resolution return resolution - def save_chunk_stream(self, chunk, offset, resolution, format, done): - if not self.available: - config = settings.CONFIG['video'] - stream, created = Stream.objects.get_or_create( - file=self, resolution=resolution, format=format) - if created: - stream.media.name = stream.path(stream.name()) - ox.makedirs(os.path.dirname(stream.media.path)) - with open(stream.media.path, 'w') as f: - f.write(chunk.read()) - stream.save() - else: - if offset == -1: - offset = stream.media.size - elif offset > stream.media.size: - return False - with open(stream.media.path, 'r+') as f: - f.seek(offset) - f.write(chunk.read()) - if done: - stream.available = True - stream.info = {} - stream.save() - return True - return False - def json(self, keys=None, user=None): resolution = (self.width, self.height) if resolution == (0, 0) or self.type != 'video': diff --git a/pandora/archive/views.py b/pandora/archive/views.py index 97ae2e29..6756a225 100644 --- a/pandora/archive/views.py +++ b/pandora/archive/views.py @@ -23,6 +23,7 @@ from ox.django.api import actions import models import tasks +from chunk import process_chunk @login_required_json @@ -159,12 +160,6 @@ def upload(request): actions.register(upload, cache=False) -class ChunkForm(forms.Form): - chunk = forms.FileField() - chunkId = forms.IntegerField(required=False) - offset = forms.IntegerField(required=False) - done = forms.IntegerField(required=False) - @login_required_json def addMedia(request): ''' @@ -238,18 +233,12 @@ def firefogg_upload(request): #post next chunk if 'chunk' in request.FILES and oshash: f = get_object_or_404(models.File, oshash=oshash) - form = ChunkForm(request.POST, request.FILES) - if form.is_valid() and f.editable(request.user): - c = form.cleaned_data['chunk'] - offset = form.cleaned_data['offset'] - response = { - 'result': 1, - 'resultUrl': request.build_absolute_uri('/%s'%f.item.itemId) - } - if not f.save_chunk_stream(c, offset, resolution, format, - form.cleaned_data['done']): - response['result'] = -1 - elif form.cleaned_data['done']: + if f.editable(request.user): + def save_chunk(chunk, offset, done): + return f.save_chunk_stream(chunk, offset, resolution, format, done) + response = process_chunk(request, save_chunk) + response['resultUrl'] = request.build_absolute_uri('/%s'%f.item.itemId) + if response.get('done'): f.uploading = False if response['result'] == 1: f.queued = True @@ -264,8 +253,6 @@ def firefogg_upload(request): response['resultUrl'] = t.task_id except: pass - response['result'] = 1 - response['done'] = 1 return render_to_json_response(response) #init upload elif oshash: @@ -298,17 +285,10 @@ def direct_upload(request): oshash = request.POST['id'] response = json_response(status=400, text='this request requires POST') if 'chunk' in request.FILES: - form = ChunkForm(request.POST, request.FILES) - if form.is_valid() and file.editable(request.user): - c = form.cleaned_data['chunk'] - offset = form.cleaned_data['offset'] - response = { - 'result': 1, - 'resultUrl': request.build_absolute_uri(file.item.get_absolute_url()) - } - if not file.save_chunk(c, offset, form.cleaned_data['done']): - response['result'] = -1 - if form.cleaned_data['done']: + if file.editable(request.user): + response = process_chunk(request, file.save_chunk) + response['resultUrl'] = request.build_absolute_uri(file.item.get_absolute_url()) + if response.get('done'): file.uploading = False if response['result'] == 1: file.queued = True @@ -323,7 +303,6 @@ def direct_upload(request): response['resultUrl'] = t.task_id except: pass - response['done'] = 1 return render_to_json_response(response) #init upload else: diff --git a/pandora/document/models.py b/pandora/document/models.py index 781cfbf2..cb2d3557 100644 --- a/pandora/document/models.py +++ b/pandora/document/models.py @@ -17,6 +17,7 @@ import ox from item.models import Item from archive.extract import resize_image +from archive.chunk import save_chunk import managers import utils @@ -180,29 +181,20 @@ class Document(models.Model): def save_chunk(self, chunk, offset=None, done=False): if self.uploading: - if not self.file: - name = 'data.%s' % self.extension - self.file.name = self.path(name) - ox.makedirs(os.path.dirname(self.file.path)) - with open(self.file.path, 'w') as f: - f.write(chunk.read()) - self.save() - else: - if offset == None: - offset = self.file.size - elif offset > self.file.size: - return False - with open(self.file.path, 'r+') as f: - f.seek(offset) - f.write(chunk.read()) - if done: - self.uploading = False - self.get_info() - self.get_ratio() - self.oshash = ox.oshash(self.file.path) - self.save() - return True - return False + name = 'data.%s' % self.extension + name = self.path(name) + + def done_cb(): + if done: + self.uploading = False + self.get_info() + self.get_ratio() + self.oshash = ox.oshash(self.file.path) + self.save() + return True, self.file.size + + return save_chunk(self, self.file, chunk, offset, name, done_cb) + return False, 0 def thumbnail(self, size=None, page=None): src = self.file.path diff --git a/pandora/document/views.py b/pandora/document/views.py index 5e6da818..505e4f79 100644 --- a/pandora/document/views.py +++ b/pandora/document/views.py @@ -13,6 +13,8 @@ from django.db.models import Sum from item import utils from item.models import Item from itemlist.models import List +from archive.chunk import process_chunk + import models def get_document_or_404_json(id): @@ -281,11 +283,6 @@ def thumbnail(request, id, size=256, page=None): document = models.Document.get(id) return HttpFileResponse(document.thumbnail(size, page=page)) -class ChunkForm(forms.Form): - chunk = forms.FileField() - offset = forms.IntegerField(required=False) - done = forms.IntegerField(required=False) - @login_required_json def upload(request): if 'id' in request.GET: @@ -297,19 +294,9 @@ def upload(request): extension = extension[-1].lower() response = json_response(status=400, text='this request requires POST') if 'chunk' in request.FILES: - form = ChunkForm(request.POST, request.FILES) - if form.is_valid() and file.editable(request.user): - c = form.cleaned_data['chunk'] - offset = form.cleaned_data['offset'] - response = { - 'result': 1, - 'id': file.get_id(), - 'resultUrl': request.build_absolute_uri(file.get_absolute_url()) - } - if not file.save_chunk(c, offset, form.cleaned_data['done']): - response['result'] = -1 - if form.cleaned_data['done']: - response['done'] = 1 + if file.editable(request.user): + response = process_chunk(request, file.save_chunk) + response['resultUrl'] = request.build_absolute_uri(file.get_absolute_url()) return render_to_json_response(response) #init upload else: diff --git a/pandora/text/models.py b/pandora/text/models.py index cef4a1a8..e3278b15 100644 --- a/pandora/text/models.py +++ b/pandora/text/models.py @@ -16,6 +16,7 @@ import ox from ox.django.fields import DictField, TupleField from archive import extract +from archive.chunk import save_chunk import managers @@ -278,25 +279,16 @@ class Text(models.Model): def save_chunk(self, chunk, offset=None, done=False): if self.uploading: - if not self.file: - self.file.name = self.path('data.pdf') - ox.makedirs(os.path.dirname(self.file.path)) - with open(self.file.path, 'w') as f: - f.write(chunk.read()) - self.save() - else: - if offset == None: - offset = self.file.size - elif offset > self.file.size: - return False - with open(self.file.path, 'r+') as f: - f.seek(offset) - f.write(chunk.read()) - if done: - self.uploading = False - self.save() - return True - return False + name = self.path('data.pdf') + + def done_cb(): + if done: + self.uploading = False + self.save() + return True, self.file.size + + return save_chunk(self, self.file, chunk, offset, name, done_cb) + return False, 0 def delete_file(sender, **kwargs): t = kwargs['instance'] diff --git a/pandora/text/views.py b/pandora/text/views.py index f642e848..832450db 100644 --- a/pandora/text/views.py +++ b/pandora/text/views.py @@ -17,6 +17,7 @@ from django.shortcuts import render_to_response from django.template import RequestContext from item import utils +from archive.chunk import process_chunk import models def get_text_or_404_json(id): @@ -377,11 +378,6 @@ def icon(request, id, size=16): icon = os.path.join(settings.STATIC_ROOT, 'jpg/list256.jpg') return HttpFileResponse(icon, content_type='image/jpeg') -class ChunkForm(forms.Form): - chunk = forms.FileField() - offset = forms.IntegerField(required=False) - done = forms.IntegerField(required=False) - def pdf_viewer(request, id): text = get_text_or_404_json(id) if text.type == 'pdf' and text.file and not text.uploading: @@ -409,18 +405,10 @@ def upload(request): if text.editable(request.user): #post next chunk if 'chunk' in request.FILES: - form = ChunkForm(request.POST, request.FILES) - if form.is_valid() and text.editable(request.user): - c = form.cleaned_data['chunk'] - offset = form.cleaned_data['offset'] - response = { - 'result': 1, - 'resultUrl': request.build_absolute_uri(text.get_absolute_url()) - } - if not text.save_chunk(c, offset, form.cleaned_data['done']): - response['result'] = -1 - if form.cleaned_data['done']: - response['done'] = 1 + if text.editable(request.user): + response = process_chunk(request, text.save_chunk) + response['resultUrl'] = request.build_absolute_uri(text.get_absolute_url()) + return render_to_json_response(response) return render_to_json_response(response) #init upload else: diff --git a/static/js/chunkupload.js b/static/js/chunkupload.js index 902649a6..70ecb2eb 100644 --- a/static/js/chunkupload.js +++ b/static/js/chunkupload.js @@ -162,7 +162,15 @@ pandora.chunkupload = function(options) { nextChunkId = chunkId + 1; that.triggerEvent('paused', {next: nextChunkId}); } else { - uploadChunk(chunkId + 1); + if (Ox.isUndefined(response.offset) || + response.offset == (chunkId +1) * chunkSize) { + uploadChunk(chunkId + 1); + } else { + // continue at chunk closest to offset from server + console.log('server offset', response.offset, + 'next chunk', Math.floor(response.offset / chuknSize)); + uploadChunk(Math.floor(response.offset / chuknSize)); + } } } else { // failed to upload, try again in 5 second