From 8eb70f55128db80bc367c5710f0268954180946e Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Tue, 19 Feb 2013 16:39:50 +0000 Subject: [PATCH] add pdf upload support --- ...eld_text_uploading__add_field_text_file.py | 95 ++++++++ pandora/text/models.py | 24 ++ pandora/text/views.py | 47 ++++ pandora/urls.py | 2 + static/js/pandora/chunkupload.js | 205 ++++++++++++++++++ static/js/pandora/text.js | 33 ++- 6 files changed, 399 insertions(+), 7 deletions(-) create mode 100644 pandora/text/migrations/0004_auto__add_field_text_uploading__add_field_text_file.py create mode 100644 static/js/pandora/chunkupload.js diff --git a/pandora/text/migrations/0004_auto__add_field_text_uploading__add_field_text_file.py b/pandora/text/migrations/0004_auto__add_field_text_uploading__add_field_text_file.py new file mode 100644 index 000000000..733dcf08d --- /dev/null +++ b/pandora/text/migrations/0004_auto__add_field_text_uploading__add_field_text_file.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'Text.uploading' + db.add_column('text_text', 'uploading', + self.gf('django.db.models.fields.BooleanField')(default=False), + keep_default=False) + + # Adding field 'Text.file' + db.add_column('text_text', 'file', + self.gf('django.db.models.fields.files.FileField')(default=None, max_length=100, null=True, blank=True), + keep_default=False) + + + def backwards(self, orm): + # Deleting field 'Text.uploading' + db.delete_column('text_text', 'uploading') + + # Deleting field 'Text.file' + db.delete_column('text_text', 'file') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'text.position': { + 'Meta': {'unique_together': "(('user', 'text', 'section'),)", 'object_name': 'Position'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'position': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'section': ('django.db.models.fields.CharField', [], {'max_length': "'255'"}), + 'text': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'position'", 'to': "orm['text.Text']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'text_position'", 'to': "orm['auth.User']"}) + }, + 'text.text': { + 'Meta': {'unique_together': "(('user', 'name'),)", 'object_name': 'Text'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'file': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'icon': ('django.db.models.fields.files.ImageField', [], {'default': 'None', 'max_length': '100', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'links': ('ox.django.fields.DictField', [], {'default': '{}'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'poster_frames': ('ox.django.fields.TupleField', [], {'default': '[]'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'private'", 'max_length': '20'}), + 'subscribed_users': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'subscribed_texts'", 'symmetrical': 'False', 'to': "orm['auth.User']"}), + 'text': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'type': ('django.db.models.fields.CharField', [], {'default': "'html'", 'max_length': '255'}), + 'uploading': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'texts'", 'to': "orm['auth.User']"}) + } + } + + complete_apps = ['text'] \ No newline at end of file diff --git a/pandora/text/models.py b/pandora/text/models.py index 00e23d702..a0248f3de 100644 --- a/pandora/text/models.py +++ b/pandora/text/models.py @@ -41,12 +41,17 @@ class Text(models.Model): subscribed_users = models.ManyToManyField(User, related_name='subscribed_texts') objects = managers.TextManager() + uploading = models.BooleanField(default = False) + file = models.FileField(default=None, blank=True,null=True, upload_to=lambda f, x: f.path(x)) def save(self, *args, **kwargs): super(Text, self).save(*args, **kwargs) def __unicode__(self): return self.get_id() + + def get_absolute_url(self): + return '/texts/%s' % self.get_id() def get_id(self): return u'%s:%s' % (self.user.username, self.name) @@ -82,6 +87,8 @@ class Text(models.Model): response[key] = self.subscribed_users.filter(id=user.id).exists() elif hasattr(self, _map.get(key, key)): response[key] = getattr(self, _map.get(key,key)) + if self.type == 'pdf': + response['available'] = True if self.file and not self.uploading else False return response def path(self, name=''): @@ -144,6 +151,23 @@ class Text(models.Model): path = source return path + def save_chunk(self, chunk, chunk_id=-1, done=False): + if self.uploading: + if not self.file: + self.file.name = self.path('data.pdf') + ox.makedirs(os.path.dirname(self.file.path)) + with open(self.file.path, 'w') as f: + f.write(chunk.read()) + self.save() + else: + with open(self.file.path, 'a') as f: + f.write(chunk.read()) + if done: + self.uploading = False + self.save() + return True + return False + class Position(models.Model): class Meta: diff --git a/pandora/text/views.py b/pandora/text/views.py index 27f6bb50a..33e1b1050 100644 --- a/pandora/text/views.py +++ b/pandora/text/views.py @@ -10,6 +10,7 @@ from ox.django.api import actions from ox.django.decorators import login_required_json from ox.django.http import HttpFileResponse from ox.django.shortcuts import render_to_json_response, get_object_or_404_json, json_response +from django import forms from django.db.models import Count, Q, Sum, Max from django.conf import settings @@ -451,3 +452,49 @@ def icon(request, id, size=16): else: icon = os.path.join(settings.STATIC_ROOT, 'jpg/list256.jpg') return HttpFileResponse(icon, content_type='image/jpeg') + +class ChunkForm(forms.Form): + chunk = forms.FileField() + chunkId = forms.IntegerField(required=False) + done = forms.IntegerField(required=False) + +def pdf(request, id): + text = get_text_or_404_json(id) + if text.type == 'pdf' and text.file and not text.uploading: + return HttpFileResponse(text.file.path, content_type='application/pdf') + response = json_response(status=404, text='file not found') + +@login_required_json +def upload(request): + text = get_text_or_404_json(request.POST['id']) + if text.editable(request.user): + #post next chunk + if 'chunk' in request.FILES: + form = ChunkForm(request.POST, request.FILES) + if form.is_valid() and text.editable(request.user): + c = form.cleaned_data['chunk'] + chunk_id = form.cleaned_data['chunkId'] + response = { + 'result': 1, + 'resultUrl': request.build_absolute_uri(text.get_absolute_url()) + } + if not text.save_chunk(c, chunk_id, form.cleaned_data['done']): + response['result'] = -1 + if form.cleaned_data['done']: + response['done'] = 1 + return render_to_json_response(response) + #init upload + else: + text.uploading = True + if text.file: + text.file.delete() + text.save() + return render_to_json_response({ + 'uploadUrl': request.build_absolute_uri('/api/upload/text'), + 'url': request.build_absolute_uri(text.get_absolute_url()), + 'result': 1 + }) + else: + response = json_response(status=404, text='permission denied') + response = json_response(status=400, text='this request requires POST') + return render_to_json_response(response) diff --git a/pandora/urls.py b/pandora/urls.py index c1abf5e35..7763c8db2 100644 --- a/pandora/urls.py +++ b/pandora/urls.py @@ -22,6 +22,7 @@ urlpatterns = patterns('', # Uncomment the admin/doc line below to enable admin documentation: # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), (r'^admin/', include(admin.site.urls)), + (r'^api/upload/text/?$', 'text.views.upload'), (r'^api/upload/?$', 'archive.views.firefogg_upload'), (r'^url=(?P.*)$', 'app.views.redirect_url'), (r'^file/(?P.*)$', 'archive.views.lookup_file'), @@ -29,6 +30,7 @@ urlpatterns = patterns('', (r'^resetUI$', 'user.views.reset_ui'), (r'^list/(?P.*?)/icon(?P\d*).jpg$', 'itemlist.views.icon'), (r'^text/(?P.*?)/icon(?P\d*).jpg$', 'text.views.icon'), + (r'^text/(?P.*?)/text.pdf$', 'text.views.pdf'), (r'^robots.txt$', serve_static_file, {'location': os.path.join(settings.STATIC_ROOT, 'robots.txt'), 'content_type': 'text/plain'}), (r'^favicon.ico$', serve_static_file, {'location': os.path.join(settings.STATIC_ROOT, 'png/icon.16.png'), 'content_type': 'image/x-icon'}), (r'^opensearch.xml$', 'app.views.opensearch_xml'), diff --git a/static/js/pandora/chunkupload.js b/static/js/pandora/chunkupload.js new file mode 100644 index 000000000..d7f5791d0 --- /dev/null +++ b/static/js/pandora/chunkupload.js @@ -0,0 +1,205 @@ +// vi:si:et:sw=4:sts=4:ts=4 +// GPL2+/MIT 2012 +'use strict'; +/* + Usage: + pandora.chunkupload({ + file: file, + url: '/add', + data: {'name': file.name} + }).bindEvent({ + progress: function(data) { + console.log(data.progress); + }, + done: function(result) { + if(result.progress == 1) { + var response = JSON.parse(result.responseText); + if(response.resultUrl) { + document.location.href = response.resultUrl; + } else { + alert(response.status; + } + } else { + alert('!!!'); + } + } + }); +*/ +pandora.chunkupload = function(options) { + var chunkSize = options.size || 1024*1024, + chunkUrl, + file = options.file, + maxRetry = -1, + retries = 0, + request, + that = Ox.Element(); + + initUpload(); + + function done() { + that.triggerEvent('done', { + status: that.status, + progress: that.progress, + responseText: that.responseText + }); + } + + function initUpload() { + //request upload slot from server + that.status = 'requesting chunk upload'; + that.progress = 0; + request = new XMLHttpRequest(); + request.addEventListener('load', function (evt) { + var response = {}; + that.responseText = evt.target.responseText; + try { + response = JSON.parse(evt.target.responseText); + } catch(e) { + response = {}; + that.status = 'failed to parse response'; + that.progress = -1; + done(); + } + if (response.maxRetry) { + maxRetry = response.maxRetry; + } + chunkUrl = response.uploadUrl; + if (document.location.protocol == 'https:') { + chunkUrl = chunkUrl.replace(/http:\/\//, 'https://'); + } + if (chunkUrl) { + that.status = 'uploading'; + that.progress = 0.0; + //start upload + uploadChunk(0); + } else { + that.status = 'upload failed, no upload url provided'; + that.progress = -1; + done(); + } + }, false); + request.addEventListener('error', function (evt) { + that.status = 'uplaod failed'; + that.progress = -1; + that.responseText = evt.target.responseText; + done(); + }, false); + request.addEventListener('abort', function (evt) { + that.status = 'aborted'; + that.progress = -1; + done(); + }, false); + var formData = new FormData(); + + Object.keys(options.data).forEach(function(key) { + formData.append(key, options.data[key]); + }); + request.open('POST', options.url); + request.send(formData); + } + + function progress(p) { + that.progress = p; + that.triggerEvent('progress', { + progress: that.progress, + status: that.status + }); + } + + function uploadChunk(chunkId) { + var bytesAvailable = file.size, + chunk, + chunkOffset = chunkId * chunkSize; + + if(file.mozSlice) { + chunk = file.mozSlice(chunkOffset, chunkOffset+chunkSize, file.type); + } else if(file.webkitSlice) { + chunk = file.webkitSlice(chunkOffset, chunkOffset+chunkSize, file.type); + } else if(file.slice) { + chunk = file.slice(chunkOffset, chunkOffset+chunkSize, file.type); + } else { + that.status = 'Sorry, your browser is currently not supported.'; + done() + } + + progress(parseFloat(chunkOffset)/bytesAvailable); + + request = new XMLHttpRequest(); + request.addEventListener('load', function (evt) { + var response; + that.responseText = evt.target.responseText; + try { + response = JSON.parse(evt.target.responseText); + } catch(e) { + response = {}; + } + if (response.done == 1) { + //upload finished + that.resultUrl = response.resultUrl; + that.progress = 1; + that.status = 'done'; + done(); + } else if (response.result == 1) { + //reset retry counter + retries = 0; + //start uploading next chunk + uploadChunk(chunkId + 1); + } else { + //failed to upload, try again in 5 second + retries++; + if (maxRetry > 0 && retries > maxRetry) { + that.status = 'uplaod failed'; + that.progress = -1; + done(); + } else { + setTimeout(function() { + uploadChunk(chunkId); + }, 5000); + } + } + }, false); + request.addEventListener('error', function (evt) { + //failed to upload, try again in 3 second + retries++; + if (maxRetry > 0 && retries > maxRetry) { + that.status = 'uplaod failed'; + that.progress = -1; + done(); + } else { + setTimeout(function() { + uploadChunk(chunkId); + }, 3000); + } + }, false); + request.upload.addEventListener('progress', function (evt) { + if (evt.lengthComputable) { + progress(parseFloat(chunkOffset + evt.loaded) / bytesAvailable); + } + }, false); + request.addEventListener('abort', function (evt) { + that.status = 'aborted'; + that.progress = -1; + done(); + }, false); + + var formData = new FormData(); + Object.keys(options.data).forEach(function(key) { + formData.append(key, options.data[key]); + }); + formData.append('chunkId', chunkId); + if (bytesAvailable <= chunkOffset + chunkSize) { + formData.append('done', 1); + } + formData.append('chunk', chunk); + request.open('POST', chunkUrl, true); + request.send(formData); + } + + that.abort = function() { + if (request) { + request.abort(); + request = null; + } + }; + return that; +}; diff --git a/static/js/pandora/text.js b/static/js/pandora/text.js index b98666e6e..3ee93a5ff 100644 --- a/static/js/pandora/text.js +++ b/static/js/pandora/text.js @@ -25,13 +25,32 @@ pandora.ui.text = function() { pandora.UI.set({text: ''}); } else if (result.data.type == 'pdf') { $text && $text.remove(); - $text = Ox.Editable({ - clickLink: pandora.clickLink, - editable: false, - type: 'textarea', - value: 'REPLACE ME WITH PDF VIEWER' - }) - .appendTo(that); + $text = Ox.Element().appendTo(that); + if(result.data.available) { + $text.append( + $('