0x2620
/
pandora
2
1
Fork 1
Code Issues 1.3k Pull Requests Releases Wiki Activity

refactor chunk upload: return offset, continue at server offset

This commit is contained in:
j 2014-04-11 16:56:22 +00:00
parent 22e3a9eedd
commit 7cd152b1d5
8 changed files with 133 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
pandora/archive/chunk.py Normal file
View File

@ -0,0 +1,51 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, with_statement
import os
import ox
from django import forms
class ChunkForm(forms.Form):
chunk = forms.FileField()
chunkId = forms.IntegerField(required=False)
offset = forms.IntegerField(required=False)
done = forms.IntegerField(required=False)
def save_chunk(self, file, chunk, offset, name, done_cb=None):
print 'file', file
print 'chunk', chunk
if not file:
file.name = name
ox.makedirs(os.path.dirname(file.path))
with open(file.path, 'w') as f:
f.write(chunk.read())
self.save()
else:
path = file.path
size = file.size
if offset == None:
offset = size
elif offset > size:
return False, size
with open(path, 'r+') as f:
f.seek(offset)
f.write(chunk.read())
return done_cb() if done_cb else True, file.size
def process_chunk(request, _save_chunk):
response = {
'result': 1,
}
form = ChunkForm(request.POST, request.FILES)
if form.is_valid():
d = form.cleaned_data
ok, _offset = _save_chunk(d['chunk'], d['offset'], d['done'])
response['offset'] = _offset
response['result'] = 1 if ok else -1
if d['done']:
response['done'] = 1
else:
response['result'] = -1
return response

77
pandora/archive/models.py
View File

@ -18,6 +18,7 @@ from item import utils
import item.models
from person.models import get_name_sort
from chunk import save_chunk
import extract
class File(models.Model):
@ -299,32 +300,32 @@ class File(models.Model):
def save_chunk(self, chunk, offset=None, done=False):
if not self.available:
if not self.data:
name = 'data.%s' % self.info.get('extension', 'avi')
self.data.name = self.get_path(name)
ox.makedirs(os.path.dirname(self.data.path))
with open(self.data.path, 'w') as f:
f.write(chunk.read())
self.save()
else:
if offset == None:
offset = self.data.size
elif offset > self.data.size:
return False
with open(self.data.path, 'r+') as f:
f.seek(offset)
f.write(chunk.read())
if done:
self.info.update(ox.avinfo(self.data.path))
self.parse_info()
# reject invalid uploads
if self.info.get('oshash') != self.oshash:
self.data.delete()
name = 'data.%s' % self.info.get('extension', 'avi')
name = self.get_path(name)
def done_cb():
if done:
self.info.update(ox.avinfo(self.data.path))
self.parse_info()
# reject invalid uploads
if self.info.get('oshash') != self.oshash:
self.data.delete()
self.save()
return False, 0
self.save()
return False
self.save()
return True
return False
return True, self.data.size
return save_chunk(self, self.data, chunk, offset, name, done_cb)
else:
return False, 0
def save_chunk_stream(self, chunk, offset, resolution, format, done):
if not self.available:
config = settings.CONFIG['video']
stream, created = Stream.objects.get_or_create(
file=self, resolution=resolution, format=format)
name = stream.path(stream.name())
return save_chunk(stream, stream.media, chunk, offset, name)
return False, 0
def stream_resolution(self):
config = settings.CONFIG['video']
@ -334,32 +335,6 @@ class File(models.Model):
return resolution
return resolution
def save_chunk_stream(self, chunk, offset, resolution, format, done):
if not self.available:
config = settings.CONFIG['video']
stream, created = Stream.objects.get_or_create(
file=self, resolution=resolution, format=format)
if created:
stream.media.name = stream.path(stream.name())
ox.makedirs(os.path.dirname(stream.media.path))
with open(stream.media.path, 'w') as f:
f.write(chunk.read())
stream.save()
else:
if offset == -1:
offset = stream.media.size
elif offset > stream.media.size:
return False
with open(stream.media.path, 'r+') as f:
f.seek(offset)
f.write(chunk.read())
if done:
stream.available = True
stream.info = {}
stream.save()
return True
return False
def json(self, keys=None, user=None):
resolution = (self.width, self.height)
if resolution == (0, 0) or self.type != 'video':

43
pandora/archive/views.py
View File

@ -23,6 +23,7 @@ from ox.django.api import actions
import models
import tasks
from chunk import process_chunk
@login_required_json
@ -159,12 +160,6 @@ def upload(request):
actions.register(upload, cache=False)
class ChunkForm(forms.Form):
chunk = forms.FileField()
chunkId = forms.IntegerField(required=False)
offset = forms.IntegerField(required=False)
done = forms.IntegerField(required=False)
@login_required_json
def addMedia(request):
'''
@ -238,18 +233,12 @@ def firefogg_upload(request):
#post next chunk
if 'chunk' in request.FILES and oshash:
f = get_object_or_404(models.File, oshash=oshash)
form = ChunkForm(request.POST, request.FILES)
if form.is_valid() and f.editable(request.user):
c = form.cleaned_data['chunk']
offset = form.cleaned_data['offset']
response = {
'result': 1,
'resultUrl': request.build_absolute_uri('/%s'%f.item.itemId)
}
if not f.save_chunk_stream(c, offset, resolution, format,
form.cleaned_data['done']):
response['result'] = -1
elif form.cleaned_data['done']:
if f.editable(request.user):
def save_chunk(chunk, offset, done):
return f.save_chunk_stream(chunk, offset, resolution, format, done)
response = process_chunk(request, save_chunk)
response['resultUrl'] = request.build_absolute_uri('/%s'%f.item.itemId)
if response.get('done'):
f.uploading = False
if response['result'] == 1:
f.queued = True
@ -264,8 +253,6 @@ def firefogg_upload(request):
response['resultUrl'] = t.task_id
except:
pass
response['result'] = 1
response['done'] = 1
return render_to_json_response(response)
#init upload
elif oshash:
@ -298,17 +285,10 @@ def direct_upload(request):
oshash = request.POST['id']
response = json_response(status=400, text='this request requires POST')
if 'chunk' in request.FILES:
form = ChunkForm(request.POST, request.FILES)
if form.is_valid() and file.editable(request.user):
c = form.cleaned_data['chunk']
offset = form.cleaned_data['offset']
response = {
'result': 1,
'resultUrl': request.build_absolute_uri(file.item.get_absolute_url())
}
if not file.save_chunk(c, offset, form.cleaned_data['done']):
response['result'] = -1
if form.cleaned_data['done']:
if file.editable(request.user):
response = process_chunk(request, file.save_chunk)
response['resultUrl'] = request.build_absolute_uri(file.item.get_absolute_url())
if response.get('done'):
file.uploading = False
if response['result'] == 1:
file.queued = True
@ -323,7 +303,6 @@ def direct_upload(request):
response['resultUrl'] = t.task_id
except:
pass
response['done'] = 1
return render_to_json_response(response)
#init upload
else:

38
pandora/document/models.py
View File

@ -17,6 +17,7 @@ import ox
from item.models import Item
from archive.extract import resize_image
from archive.chunk import save_chunk
import managers
import utils
@ -180,29 +181,20 @@ class Document(models.Model):
def save_chunk(self, chunk, offset=None, done=False):
if self.uploading:
if not self.file:
name = 'data.%s' % self.extension
self.file.name = self.path(name)
ox.makedirs(os.path.dirname(self.file.path))
with open(self.file.path, 'w') as f:
f.write(chunk.read())
self.save()
else:
if offset == None:
offset = self.file.size
elif offset > self.file.size:
return False
with open(self.file.path, 'r+') as f:
f.seek(offset)
f.write(chunk.read())
if done:
self.uploading = False
self.get_info()
self.get_ratio()
self.oshash = ox.oshash(self.file.path)
self.save()
return True
return False
name = 'data.%s' % self.extension
name = self.path(name)
def done_cb():
if done:
self.uploading = False
self.get_info()
self.get_ratio()
self.oshash = ox.oshash(self.file.path)
self.save()
return True, self.file.size
return save_chunk(self, self.file, chunk, offset, name, done_cb)
return False, 0
def thumbnail(self, size=None, page=None):
src = self.file.path

23
pandora/document/views.py
View File

@ -13,6 +13,8 @@ from django.db.models import Sum
from item import utils
from item.models import Item
from itemlist.models import List
from archive.chunk import process_chunk
import models
def get_document_or_404_json(id):
@ -281,11 +283,6 @@ def thumbnail(request, id, size=256, page=None):
document = models.Document.get(id)
return HttpFileResponse(document.thumbnail(size, page=page))
class ChunkForm(forms.Form):
chunk = forms.FileField()
offset = forms.IntegerField(required=False)
done = forms.IntegerField(required=False)
@login_required_json
def upload(request):
if 'id' in request.GET:
@ -297,19 +294,9 @@ def upload(request):
extension = extension[-1].lower()
response = json_response(status=400, text='this request requires POST')
if 'chunk' in request.FILES:
form = ChunkForm(request.POST, request.FILES)
if form.is_valid() and file.editable(request.user):
c = form.cleaned_data['chunk']
offset = form.cleaned_data['offset']
response = {
'result': 1,
'id': file.get_id(),
'resultUrl': request.build_absolute_uri(file.get_absolute_url())
}
if not file.save_chunk(c, offset, form.cleaned_data['done']):
response['result'] = -1
if form.cleaned_data['done']:
response['done'] = 1
if file.editable(request.user):
response = process_chunk(request, file.save_chunk)
response['resultUrl'] = request.build_absolute_uri(file.get_absolute_url())
return render_to_json_response(response)
#init upload
else:

30
pandora/text/models.py
View File

@ -16,6 +16,7 @@ import ox
from ox.django.fields import DictField, TupleField
from archive import extract
from archive.chunk import save_chunk
import managers
@ -278,25 +279,16 @@ class Text(models.Model):
def save_chunk(self, chunk, offset=None, done=False):
if self.uploading:
if not self.file:
self.file.name = self.path('data.pdf')
ox.makedirs(os.path.dirname(self.file.path))
with open(self.file.path, 'w') as f:
f.write(chunk.read())
self.save()
else:
if offset == None:
offset = self.file.size
elif offset > self.file.size:
return False
with open(self.file.path, 'r+') as f:
f.seek(offset)
f.write(chunk.read())
if done:
self.uploading = False
self.save()
return True
return False
name = self.path('data.pdf')
def done_cb():
if done:
self.uploading = False
self.save()
return True, self.file.size
return save_chunk(self, self.file, chunk, offset, name, done_cb)
return False, 0
def delete_file(sender, **kwargs):
t = kwargs['instance']

22
pandora/text/views.py
View File

@ -17,6 +17,7 @@ from django.shortcuts import render_to_response
from django.template import RequestContext
from item import utils
from archive.chunk import process_chunk
import models
def get_text_or_404_json(id):
@ -377,11 +378,6 @@ def icon(request, id, size=16):
icon = os.path.join(settings.STATIC_ROOT, 'jpg/list256.jpg')
return HttpFileResponse(icon, content_type='image/jpeg')
class ChunkForm(forms.Form):
chunk = forms.FileField()
offset = forms.IntegerField(required=False)
done = forms.IntegerField(required=False)
def pdf_viewer(request, id):
text = get_text_or_404_json(id)
if text.type == 'pdf' and text.file and not text.uploading:
@ -409,18 +405,10 @@ def upload(request):
if text.editable(request.user):
#post next chunk
if 'chunk' in request.FILES:
form = ChunkForm(request.POST, request.FILES)
if form.is_valid() and text.editable(request.user):
c = form.cleaned_data['chunk']
offset = form.cleaned_data['offset']
response = {
'result': 1,
'resultUrl': request.build_absolute_uri(text.get_absolute_url())
}
if not text.save_chunk(c, offset, form.cleaned_data['done']):
response['result'] = -1
if form.cleaned_data['done']:
response['done'] = 1
if text.editable(request.user):
response = process_chunk(request, text.save_chunk)
response['resultUrl'] = request.build_absolute_uri(text.get_absolute_url())
return render_to_json_response(response)
return render_to_json_response(response)
#init upload
else:

10
static/js/chunkupload.js
View File

@ -162,7 +162,15 @@ pandora.chunkupload = function(options) {
nextChunkId = chunkId + 1;
that.triggerEvent('paused', {next: nextChunkId});
} else {
uploadChunk(chunkId + 1);
if (Ox.isUndefined(response.offset) ||
response.offset == (chunkId +1) * chunkSize) {
uploadChunk(chunkId + 1);
} else {
// continue at chunk closest to offset from server
console.log('server offset', response.offset,
'next chunk', Math.floor(response.offset / chuknSize));
uploadChunk(Math.floor(response.offset / chuknSize));
}
}
} else {
// failed to upload, try again in 5 second