refactor chunk upload: return offset, continue at server offset

This commit is contained in:
j 2014-04-11 16:56:22 +00:00
parent 22e3a9eedd
commit 7cd152b1d5
8 changed files with 133 additions and 161 deletions

51
pandora/archive/chunk.py Normal file
View file

@ -0,0 +1,51 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, with_statement
import os
import ox
from django import forms
class ChunkForm(forms.Form):
chunk = forms.FileField()
chunkId = forms.IntegerField(required=False)
offset = forms.IntegerField(required=False)
done = forms.IntegerField(required=False)
def save_chunk(self, file, chunk, offset, name, done_cb=None):
print 'file', file
print 'chunk', chunk
if not file:
file.name = name
ox.makedirs(os.path.dirname(file.path))
with open(file.path, 'w') as f:
f.write(chunk.read())
self.save()
else:
path = file.path
size = file.size
if offset == None:
offset = size
elif offset > size:
return False, size
with open(path, 'r+') as f:
f.seek(offset)
f.write(chunk.read())
return done_cb() if done_cb else True, file.size
def process_chunk(request, _save_chunk):
response = {
'result': 1,
}
form = ChunkForm(request.POST, request.FILES)
if form.is_valid():
d = form.cleaned_data
ok, _offset = _save_chunk(d['chunk'], d['offset'], d['done'])
response['offset'] = _offset
response['result'] = 1 if ok else -1
if d['done']:
response['done'] = 1
else:
response['result'] = -1
return response

View file

@ -18,6 +18,7 @@ from item import utils
import item.models import item.models
from person.models import get_name_sort from person.models import get_name_sort
from chunk import save_chunk
import extract import extract
class File(models.Model): class File(models.Model):
@ -299,21 +300,10 @@ class File(models.Model):
def save_chunk(self, chunk, offset=None, done=False): def save_chunk(self, chunk, offset=None, done=False):
if not self.available: if not self.available:
if not self.data:
name = 'data.%s' % self.info.get('extension', 'avi') name = 'data.%s' % self.info.get('extension', 'avi')
self.data.name = self.get_path(name) name = self.get_path(name)
ox.makedirs(os.path.dirname(self.data.path))
with open(self.data.path, 'w') as f: def done_cb():
f.write(chunk.read())
self.save()
else:
if offset == None:
offset = self.data.size
elif offset > self.data.size:
return False
with open(self.data.path, 'r+') as f:
f.seek(offset)
f.write(chunk.read())
if done: if done:
self.info.update(ox.avinfo(self.data.path)) self.info.update(ox.avinfo(self.data.path))
self.parse_info() self.parse_info()
@ -321,10 +311,21 @@ class File(models.Model):
if self.info.get('oshash') != self.oshash: if self.info.get('oshash') != self.oshash:
self.data.delete() self.data.delete()
self.save() self.save()
return False return False, 0
self.save() self.save()
return True return True, self.data.size
return False return save_chunk(self, self.data, chunk, offset, name, done_cb)
else:
return False, 0
def save_chunk_stream(self, chunk, offset, resolution, format, done):
if not self.available:
config = settings.CONFIG['video']
stream, created = Stream.objects.get_or_create(
file=self, resolution=resolution, format=format)
name = stream.path(stream.name())
return save_chunk(stream, stream.media, chunk, offset, name)
return False, 0
def stream_resolution(self): def stream_resolution(self):
config = settings.CONFIG['video'] config = settings.CONFIG['video']
@ -334,32 +335,6 @@ class File(models.Model):
return resolution return resolution
return resolution return resolution
def save_chunk_stream(self, chunk, offset, resolution, format, done):
if not self.available:
config = settings.CONFIG['video']
stream, created = Stream.objects.get_or_create(
file=self, resolution=resolution, format=format)
if created:
stream.media.name = stream.path(stream.name())
ox.makedirs(os.path.dirname(stream.media.path))
with open(stream.media.path, 'w') as f:
f.write(chunk.read())
stream.save()
else:
if offset == -1:
offset = stream.media.size
elif offset > stream.media.size:
return False
with open(stream.media.path, 'r+') as f:
f.seek(offset)
f.write(chunk.read())
if done:
stream.available = True
stream.info = {}
stream.save()
return True
return False
def json(self, keys=None, user=None): def json(self, keys=None, user=None):
resolution = (self.width, self.height) resolution = (self.width, self.height)
if resolution == (0, 0) or self.type != 'video': if resolution == (0, 0) or self.type != 'video':

View file

@ -23,6 +23,7 @@ from ox.django.api import actions
import models import models
import tasks import tasks
from chunk import process_chunk
@login_required_json @login_required_json
@ -159,12 +160,6 @@ def upload(request):
actions.register(upload, cache=False) actions.register(upload, cache=False)
class ChunkForm(forms.Form):
chunk = forms.FileField()
chunkId = forms.IntegerField(required=False)
offset = forms.IntegerField(required=False)
done = forms.IntegerField(required=False)
@login_required_json @login_required_json
def addMedia(request): def addMedia(request):
''' '''
@ -238,18 +233,12 @@ def firefogg_upload(request):
#post next chunk #post next chunk
if 'chunk' in request.FILES and oshash: if 'chunk' in request.FILES and oshash:
f = get_object_or_404(models.File, oshash=oshash) f = get_object_or_404(models.File, oshash=oshash)
form = ChunkForm(request.POST, request.FILES) if f.editable(request.user):
if form.is_valid() and f.editable(request.user): def save_chunk(chunk, offset, done):
c = form.cleaned_data['chunk'] return f.save_chunk_stream(chunk, offset, resolution, format, done)
offset = form.cleaned_data['offset'] response = process_chunk(request, save_chunk)
response = { response['resultUrl'] = request.build_absolute_uri('/%s'%f.item.itemId)
'result': 1, if response.get('done'):
'resultUrl': request.build_absolute_uri('/%s'%f.item.itemId)
}
if not f.save_chunk_stream(c, offset, resolution, format,
form.cleaned_data['done']):
response['result'] = -1
elif form.cleaned_data['done']:
f.uploading = False f.uploading = False
if response['result'] == 1: if response['result'] == 1:
f.queued = True f.queued = True
@ -264,8 +253,6 @@ def firefogg_upload(request):
response['resultUrl'] = t.task_id response['resultUrl'] = t.task_id
except: except:
pass pass
response['result'] = 1
response['done'] = 1
return render_to_json_response(response) return render_to_json_response(response)
#init upload #init upload
elif oshash: elif oshash:
@ -298,17 +285,10 @@ def direct_upload(request):
oshash = request.POST['id'] oshash = request.POST['id']
response = json_response(status=400, text='this request requires POST') response = json_response(status=400, text='this request requires POST')
if 'chunk' in request.FILES: if 'chunk' in request.FILES:
form = ChunkForm(request.POST, request.FILES) if file.editable(request.user):
if form.is_valid() and file.editable(request.user): response = process_chunk(request, file.save_chunk)
c = form.cleaned_data['chunk'] response['resultUrl'] = request.build_absolute_uri(file.item.get_absolute_url())
offset = form.cleaned_data['offset'] if response.get('done'):
response = {
'result': 1,
'resultUrl': request.build_absolute_uri(file.item.get_absolute_url())
}
if not file.save_chunk(c, offset, form.cleaned_data['done']):
response['result'] = -1
if form.cleaned_data['done']:
file.uploading = False file.uploading = False
if response['result'] == 1: if response['result'] == 1:
file.queued = True file.queued = True
@ -323,7 +303,6 @@ def direct_upload(request):
response['resultUrl'] = t.task_id response['resultUrl'] = t.task_id
except: except:
pass pass
response['done'] = 1
return render_to_json_response(response) return render_to_json_response(response)
#init upload #init upload
else: else:

View file

@ -17,6 +17,7 @@ import ox
from item.models import Item from item.models import Item
from archive.extract import resize_image from archive.extract import resize_image
from archive.chunk import save_chunk
import managers import managers
import utils import utils
@ -180,29 +181,20 @@ class Document(models.Model):
def save_chunk(self, chunk, offset=None, done=False): def save_chunk(self, chunk, offset=None, done=False):
if self.uploading: if self.uploading:
if not self.file:
name = 'data.%s' % self.extension name = 'data.%s' % self.extension
self.file.name = self.path(name) name = self.path(name)
ox.makedirs(os.path.dirname(self.file.path))
with open(self.file.path, 'w') as f: def done_cb():
f.write(chunk.read())
self.save()
else:
if offset == None:
offset = self.file.size
elif offset > self.file.size:
return False
with open(self.file.path, 'r+') as f:
f.seek(offset)
f.write(chunk.read())
if done: if done:
self.uploading = False self.uploading = False
self.get_info() self.get_info()
self.get_ratio() self.get_ratio()
self.oshash = ox.oshash(self.file.path) self.oshash = ox.oshash(self.file.path)
self.save() self.save()
return True return True, self.file.size
return False
return save_chunk(self, self.file, chunk, offset, name, done_cb)
return False, 0
def thumbnail(self, size=None, page=None): def thumbnail(self, size=None, page=None):
src = self.file.path src = self.file.path

View file

@ -13,6 +13,8 @@ from django.db.models import Sum
from item import utils from item import utils
from item.models import Item from item.models import Item
from itemlist.models import List from itemlist.models import List
from archive.chunk import process_chunk
import models import models
def get_document_or_404_json(id): def get_document_or_404_json(id):
@ -281,11 +283,6 @@ def thumbnail(request, id, size=256, page=None):
document = models.Document.get(id) document = models.Document.get(id)
return HttpFileResponse(document.thumbnail(size, page=page)) return HttpFileResponse(document.thumbnail(size, page=page))
class ChunkForm(forms.Form):
chunk = forms.FileField()
offset = forms.IntegerField(required=False)
done = forms.IntegerField(required=False)
@login_required_json @login_required_json
def upload(request): def upload(request):
if 'id' in request.GET: if 'id' in request.GET:
@ -297,19 +294,9 @@ def upload(request):
extension = extension[-1].lower() extension = extension[-1].lower()
response = json_response(status=400, text='this request requires POST') response = json_response(status=400, text='this request requires POST')
if 'chunk' in request.FILES: if 'chunk' in request.FILES:
form = ChunkForm(request.POST, request.FILES) if file.editable(request.user):
if form.is_valid() and file.editable(request.user): response = process_chunk(request, file.save_chunk)
c = form.cleaned_data['chunk'] response['resultUrl'] = request.build_absolute_uri(file.get_absolute_url())
offset = form.cleaned_data['offset']
response = {
'result': 1,
'id': file.get_id(),
'resultUrl': request.build_absolute_uri(file.get_absolute_url())
}
if not file.save_chunk(c, offset, form.cleaned_data['done']):
response['result'] = -1
if form.cleaned_data['done']:
response['done'] = 1
return render_to_json_response(response) return render_to_json_response(response)
#init upload #init upload
else: else:

View file

@ -16,6 +16,7 @@ import ox
from ox.django.fields import DictField, TupleField from ox.django.fields import DictField, TupleField
from archive import extract from archive import extract
from archive.chunk import save_chunk
import managers import managers
@ -278,25 +279,16 @@ class Text(models.Model):
def save_chunk(self, chunk, offset=None, done=False): def save_chunk(self, chunk, offset=None, done=False):
if self.uploading: if self.uploading:
if not self.file: name = self.path('data.pdf')
self.file.name = self.path('data.pdf')
ox.makedirs(os.path.dirname(self.file.path)) def done_cb():
with open(self.file.path, 'w') as f:
f.write(chunk.read())
self.save()
else:
if offset == None:
offset = self.file.size
elif offset > self.file.size:
return False
with open(self.file.path, 'r+') as f:
f.seek(offset)
f.write(chunk.read())
if done: if done:
self.uploading = False self.uploading = False
self.save() self.save()
return True return True, self.file.size
return False
return save_chunk(self, self.file, chunk, offset, name, done_cb)
return False, 0
def delete_file(sender, **kwargs): def delete_file(sender, **kwargs):
t = kwargs['instance'] t = kwargs['instance']

View file

@ -17,6 +17,7 @@ from django.shortcuts import render_to_response
from django.template import RequestContext from django.template import RequestContext
from item import utils from item import utils
from archive.chunk import process_chunk
import models import models
def get_text_or_404_json(id): def get_text_or_404_json(id):
@ -377,11 +378,6 @@ def icon(request, id, size=16):
icon = os.path.join(settings.STATIC_ROOT, 'jpg/list256.jpg') icon = os.path.join(settings.STATIC_ROOT, 'jpg/list256.jpg')
return HttpFileResponse(icon, content_type='image/jpeg') return HttpFileResponse(icon, content_type='image/jpeg')
class ChunkForm(forms.Form):
chunk = forms.FileField()
offset = forms.IntegerField(required=False)
done = forms.IntegerField(required=False)
def pdf_viewer(request, id): def pdf_viewer(request, id):
text = get_text_or_404_json(id) text = get_text_or_404_json(id)
if text.type == 'pdf' and text.file and not text.uploading: if text.type == 'pdf' and text.file and not text.uploading:
@ -409,18 +405,10 @@ def upload(request):
if text.editable(request.user): if text.editable(request.user):
#post next chunk #post next chunk
if 'chunk' in request.FILES: if 'chunk' in request.FILES:
form = ChunkForm(request.POST, request.FILES) if text.editable(request.user):
if form.is_valid() and text.editable(request.user): response = process_chunk(request, text.save_chunk)
c = form.cleaned_data['chunk'] response['resultUrl'] = request.build_absolute_uri(text.get_absolute_url())
offset = form.cleaned_data['offset'] return render_to_json_response(response)
response = {
'result': 1,
'resultUrl': request.build_absolute_uri(text.get_absolute_url())
}
if not text.save_chunk(c, offset, form.cleaned_data['done']):
response['result'] = -1
if form.cleaned_data['done']:
response['done'] = 1
return render_to_json_response(response) return render_to_json_response(response)
#init upload #init upload
else: else:

View file

@ -162,7 +162,15 @@ pandora.chunkupload = function(options) {
nextChunkId = chunkId + 1; nextChunkId = chunkId + 1;
that.triggerEvent('paused', {next: nextChunkId}); that.triggerEvent('paused', {next: nextChunkId});
} else { } else {
if (Ox.isUndefined(response.offset) ||
response.offset == (chunkId +1) * chunkSize) {
uploadChunk(chunkId + 1); uploadChunk(chunkId + 1);
} else {
// continue at chunk closest to offset from server
console.log('server offset', response.offset,
'next chunk', Math.floor(response.offset / chuknSize));
uploadChunk(Math.floor(response.offset / chuknSize));
}
} }
} else { } else {
// failed to upload, try again in 5 second // failed to upload, try again in 5 second