0x2620/pandora
2
1
Fork 1
Code Issues 1.3k Pull requests Releases Wiki Activity

upload chunks with offset. fail if uploaded file does not match oshash

Browse source
This commit is contained in:
j 2014-04-11 12:08:14 +00:00
parent 4dca348b6e
commit 22e3a9eedd
8 changed files with 71 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
pandora/archive/models.py
View file

@ -297,7 +297,7 @@ class File(models.Model):
self.instances.filter(volume__user=user).count() > 0 or \
(not self.item or self.item.user == user)
def save_chunk(self, chunk, chunk_id=-1, done=False):
def save_chunk(self, chunk, offset=None, done=False):
if not self.available:
if not self.data:
name = 'data.%s' % self.info.get('extension', 'avi')
@ -307,11 +307,21 @@ class File(models.Model):
f.write(chunk.read())
self.save()
else:
with open(self.data.path, 'a') as f:
if offset == None:
offset = self.data.size
elif offset > self.data.size:
return False
with open(self.data.path, 'r+') as f:
f.seek(offset)
f.write(chunk.read())
if done:
self.info.update(ox.avinfo(self.data.path))
self.parse_info()
# reject invalid uploads
if self.info.get('oshash') != self.oshash:
self.data.delete()
self.save()
return False
self.save()
return True
return False
@ -324,7 +334,7 @@ class File(models.Model):
return resolution
return resolution
def save_chunk_stream(self, chunk, chunk_id, resolution, format, done):
def save_chunk_stream(self, chunk, offset, resolution, format, done):
if not self.available:
config = settings.CONFIG['video']
stream, created = Stream.objects.get_or_create(
@ -336,8 +346,12 @@ class File(models.Model):
f.write(chunk.read())
stream.save()
else:
with open(stream.media.path, 'a') as f:
#FIXME: should check that chunk_id/offset is right
if offset == -1:
offset = stream.media.size
elif offset > stream.media.size:
return False
with open(stream.media.path, 'r+') as f:
f.seek(offset)
f.write(chunk.read())
if done:
stream.available = True
@ -351,7 +365,7 @@ class File(models.Model):
if resolution == (0, 0) or self.type != 'video':
resolution = None
duration = self.duration
if self.type != 'video':
if self.type not in ('audio', 'video'):
duration = None
state = ''
error = ''

23
pandora/archive/views.py
View file

@ -162,6 +162,7 @@ actions.register(upload, cache=False)
class ChunkForm(forms.Form):
chunk = forms.FileField()
chunkId = forms.IntegerField(required=False)
offset = forms.IntegerField(required=False)
done = forms.IntegerField(required=False)
@login_required_json
@ -240,17 +241,22 @@ def firefogg_upload(request):
form = ChunkForm(request.POST, request.FILES)
if form.is_valid() and f.editable(request.user):
c = form.cleaned_data['chunk']
chunk_id = form.cleaned_data['chunkId']
offset = form.cleaned_data['offset']
response = {
'result': 1,
'resultUrl': request.build_absolute_uri('/%s'%f.item.itemId)
}
if not f.save_chunk_stream(c, chunk_id, resolution, format,
if not f.save_chunk_stream(c, offset, resolution, format,
form.cleaned_data['done']):
response['result'] = -1
elif form.cleaned_data['done']:
f.uploading = False
f.queued = True
if response['result'] == 1:
f.queued = True
f.wanted = False
else:
f.queued = False
f.wanted = True
f.save()
#FIXME: this fails badly if rabbitmq goes down
try:
@ -295,16 +301,21 @@ def direct_upload(request):
form = ChunkForm(request.POST, request.FILES)
if form.is_valid() and file.editable(request.user):
c = form.cleaned_data['chunk']
chunk_id = form.cleaned_data['chunkId']
offset = form.cleaned_data['offset']
response = {
'result': 1,
'resultUrl': request.build_absolute_uri(file.item.get_absolute_url())
}
if not file.save_chunk(c, chunk_id, form.cleaned_data['done']):
if not file.save_chunk(c, offset, form.cleaned_data['done']):
response['result'] = -1
if form.cleaned_data['done']:
file.uploading = False
file.queued = True
if response['result'] == 1:
file.queued = True
file.wanted = False
else:
file.queued = False
file.wanted = True
file.save()
#try/execpt so it does not fail if rabitmq is down
try:

9
pandora/document/models.py
View file

@ -178,7 +178,7 @@ class Document(models.Model):
h = (7-len(h))*'0' + h
return os.path.join('documents', h[:2], h[2:4], h[4:6], h[6:], name)
def save_chunk(self, chunk, chunk_id=-1, done=False):
def save_chunk(self, chunk, offset=None, done=False):
if self.uploading:
if not self.file:
name = 'data.%s' % self.extension
@ -188,7 +188,12 @@ class Document(models.Model):
f.write(chunk.read())
self.save()
else:
with open(self.file.path, 'a') as f:
if offset == None:
offset = self.file.size
elif offset > self.file.size:
return False
with open(self.file.path, 'r+') as f:
f.seek(offset)
f.write(chunk.read())
if done:
self.uploading = False

6
pandora/document/views.py
View file

@ -283,7 +283,7 @@ def thumbnail(request, id, size=256, page=None):
class ChunkForm(forms.Form):
chunk = forms.FileField()
chunkId = forms.IntegerField(required=False)
offset = forms.IntegerField(required=False)
done = forms.IntegerField(required=False)
@login_required_json
@ -300,13 +300,13 @@ def upload(request):
form = ChunkForm(request.POST, request.FILES)
if form.is_valid() and file.editable(request.user):
c = form.cleaned_data['chunk']
chunk_id = form.cleaned_data['chunkId']
offset = form.cleaned_data['offset']
response = {
'result': 1,
'id': file.get_id(),
'resultUrl': request.build_absolute_uri(file.get_absolute_url())
}
if not file.save_chunk(c, chunk_id, form.cleaned_data['done']):
if not file.save_chunk(c, offset, form.cleaned_data['done']):
response['result'] = -1
if form.cleaned_data['done']:
response['done'] = 1

9
pandora/text/models.py
View file

@ -276,7 +276,7 @@ class Text(models.Model):
path = source
return path
def save_chunk(self, chunk, chunk_id=-1, done=False):
def save_chunk(self, chunk, offset=None, done=False):
if self.uploading:
if not self.file:
self.file.name = self.path('data.pdf')
@ -285,7 +285,12 @@ class Text(models.Model):
f.write(chunk.read())
self.save()
else:
with open(self.file.path, 'a') as f:
if offset == None:
offset = self.file.size
elif offset > self.file.size:
return False
with open(self.file.path, 'r+') as f:
f.seek(offset)
f.write(chunk.read())
if done:
self.uploading = False

6
pandora/text/views.py
View file

@ -379,7 +379,7 @@ def icon(request, id, size=16):
class ChunkForm(forms.Form):
chunk = forms.FileField()
chunkId = forms.IntegerField(required=False)
offset = forms.IntegerField(required=False)
done = forms.IntegerField(required=False)
def pdf_viewer(request, id):
@ -412,12 +412,12 @@ def upload(request):
form = ChunkForm(request.POST, request.FILES)
if form.is_valid() and text.editable(request.user):
c = form.cleaned_data['chunk']
chunk_id = form.cleaned_data['chunkId']
offset = form.cleaned_data['offset']
response = {
'result': 1,
'resultUrl': request.build_absolute_uri(text.get_absolute_url())
}
if not text.save_chunk(c, chunk_id, form.cleaned_data['done']):
if not text.save_chunk(c, offset, form.cleaned_data['done']):
response['result'] = -1
if form.cleaned_data['done']:
response['done'] = 1

16
static/js/chunkupload.js
View file

@ -26,7 +26,8 @@
});
*/
pandora.chunkupload = function(options) {
var chunkSize = options.size || 1024 * 1024,
var aborted = false,
chunkSize = options.size || 1024 * 1024,
chunkURL,
file = options.file,
maxRetry = -1,
@ -120,6 +121,10 @@ pandora.chunkupload = function(options) {
chunk,
chunkOffset = chunkId * chunkSize;
if (aborted) {
return;
}
if (file.mozSlice) {
chunk = file.mozSlice(chunkOffset, chunkOffset+chunkSize, file.type);
} else if (file.webkitSlice) {
@ -145,7 +150,8 @@ pandora.chunkupload = function(options) {
if (response.done == 1) {
//upload finished
that.resultUrl = response.resultUrl;
that.progress = 1;
// set progress to -1 if overall upload failed
that.progress = response.result;
that.status = 'done';
done();
} else if (response.result == 1) {
@ -210,7 +216,7 @@ pandora.chunkupload = function(options) {
Object.keys(options.data).forEach(function(key) {
formData.append(key, options.data[key]);
});
formData.append('chunkId', chunkId);
formData.append('offset', chunkOffset);
if (bytesAvailable <= chunkOffset + chunkSize) {
formData.append('done', 1);
}
@ -220,9 +226,13 @@ pandora.chunkupload = function(options) {
}
that.abort = function() {
aborted = true;
if (request) {
request.abort();
request = null;
} else {
that.progress = -1;
done();
}
return that;
};

1
static/js/uploadVideoDialog.js
View file

@ -64,6 +64,7 @@ pandora.ui.uploadVideoDialog = function(data) {
}).bindEvent({
click: function(data) {
if (data.files.length) {
cancelled = false;
$actionButton.hide();
$closeButton.options('title', Ox._('Cancel'));
upload(data.files[0]);