diff --git a/README b/README index c37ef78b..6b9cf462 100644 --- a/README +++ b/README @@ -30,7 +30,7 @@ To run pan.do/ra you need to install and setup: python-gst0.10 gstreamer0.10-plugins-good gstreamer0.10-plugins-bad \ postgresql postgresql-contrib rabbitmq-server \ ffmpeg2theora libav-tools libavcodec-extra-53 \ - python-ox oxframe + python-ox oxframe imagemagick * Prepare Environment diff --git a/pandora/file/__init__.py b/pandora/file/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pandora/file/managers.py b/pandora/file/managers.py new file mode 100644 index 00000000..cadc9976 --- /dev/null +++ b/pandora/file/managers.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from django.db.models import Q, Manager + + +def parseCondition(condition, user): + ''' + ''' + k = condition.get('key', 'name') + k = { + 'user': 'user__username', + }.get(k, k) + if not k: + k = 'name' + v = condition['value'] + op = condition.get('operator') + if not op: + op = '=' + if op.startswith('!'): + op = op[1:] + exclude = True + else: + exclude = False + if k == 'id': + try: + public_id = v.split(':') + username = public_id[0] + name = ":".join(public_id[1:]) + extension = name.split('.') + name = '.'.join(extension[:-1]) + extension = extension[-1].lower() + q = Q(user__username=username, name=name, extension=extension) + except: + q = Q(id__in=[]) + return q + if isinstance(v, bool): #featured and public flag + key = k + else: + key = "%s%s" % (k, { + '==': '__iexact', + '^': '__istartswith', + '$': '__iendswith', + }.get(op, '__icontains')) + key = str(key) + if exclude: + q = ~Q(**{key: v}) + else: + q = Q(**{key: v}) + return q + +def parseConditions(conditions, operator, user): + ''' + conditions: [ + { + value: "war" + } + { + key: "year", + value: "1970-1980, + operator: "!=" + }, + { + key: "country", + value: "f", + operator: "^" + } + ], + operator: "&" + ''' + conn = [] + for condition in conditions: + if 'conditions' in condition: + q = parseConditions(condition['conditions'], + condition.get('operator', '&'), user) + if q: + conn.append(q) + pass + else: + conn.append(parseCondition(condition, user)) + if conn: + q = conn[0] + for c in conn[1:]: + if operator == '|': + q = q | c + else: + q = q & c + return q + return None + + +class FileManager(Manager): + + def get_query_set(self): + return super(FileManager, self).get_query_set() + + def find(self, data, user): + ''' + query: { + conditions: [ + { + value: "war" + } + { + key: "year", + value: "1970-1980, + operator: "!=" + }, + { + key: "country", + value: "f", + operator: "^" + } + ], + operator: "&" + } + ''' + + #join query with operator + qs = self.get_query_set() + conditions = parseConditions(data['query'].get('conditions', []), + data['query'].get('operator', '&'), + user) + if conditions: + qs = qs.filter(conditions) + + return qs + diff --git a/pandora/file/migrations/0001_initial.py b/pandora/file/migrations/0001_initial.py new file mode 100644 index 00000000..6c4b997b --- /dev/null +++ b/pandora/file/migrations/0001_initial.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'File' + db.create_table('file_file', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), + ('modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)), + ('user', self.gf('django.db.models.fields.related.ForeignKey')(related_name='files', to=orm['auth.User'])), + ('name', self.gf('django.db.models.fields.CharField')(max_length=255)), + ('extension', self.gf('django.db.models.fields.CharField')(max_length=255)), + ('size', self.gf('django.db.models.fields.IntegerField')(default=0)), + ('matches', self.gf('django.db.models.fields.IntegerField')(default=0)), + ('ratio', self.gf('django.db.models.fields.FloatField')(default=1)), + ('description', self.gf('django.db.models.fields.TextField')(default='')), + ('oshash', self.gf('django.db.models.fields.CharField')(max_length=16, unique=True, null=True)), + ('file', self.gf('django.db.models.fields.files.FileField')(default=None, max_length=100, null=True, blank=True)), + ('uploading', self.gf('django.db.models.fields.BooleanField')(default=False)), + ('name_sort', self.gf('django.db.models.fields.CharField')(max_length=255)), + ('description_sort', self.gf('django.db.models.fields.CharField')(max_length=512)), + )) + db.send_create_signal('file', ['File']) + + # Adding unique constraint on 'File', fields ['user', 'name', 'extension'] + db.create_unique('file_file', ['user_id', 'name', 'extension']) + + + def backwards(self, orm): + # Removing unique constraint on 'File', fields ['user', 'name', 'extension'] + db.delete_unique('file_file', ['user_id', 'name', 'extension']) + + # Deleting model 'File' + db.delete_table('file_file') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'file.file': { + 'Meta': {'unique_together': "(('user', 'name', 'extension'),)", 'object_name': 'File'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'description_sort': ('django.db.models.fields.CharField', [], {'max_length': '512'}), + 'extension': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'file': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'matches': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'name_sort': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'oshash': ('django.db.models.fields.CharField', [], {'max_length': '16', 'unique': 'True', 'null': 'True'}), + 'ratio': ('django.db.models.fields.FloatField', [], {'default': '1'}), + 'size': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'uploading': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'files'", 'to': "orm['auth.User']"}) + } + } + + complete_apps = ['file'] \ No newline at end of file diff --git a/pandora/file/migrations/__init__.py b/pandora/file/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pandora/file/models.py b/pandora/file/models.py new file mode 100644 index 00000000..ba94d418 --- /dev/null +++ b/pandora/file/models.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from __future__ import division, with_statement +import os +import re +import subprocess +from urllib import quote + +from django.db import models +from django.contrib.auth.models import User +from django.db.models.signals import pre_delete + +import Image +import ox + +import managers + + +class File(models.Model): + + class Meta: + unique_together = ("user", "name", "extension") + + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + + user = models.ForeignKey(User, related_name='files') + name = models.CharField(max_length=255) + extension = models.CharField(max_length=255) + size = models.IntegerField(default=0) + matches = models.IntegerField(default=0) + ratio = models.FloatField(default=1) + description = models.TextField(default="") + oshash = models.CharField(max_length=16, unique=True, null=True) + + file = models.FileField(default=None, blank=True,null=True, upload_to=lambda f, x: f.path(x)) + + objects = managers.FileManager() + uploading = models.BooleanField(default = False) + + name_sort = models.CharField(max_length=255) + description_sort = models.CharField(max_length=512) + + def save(self, *args, **kwargs): + if not self.uploading: + if self.file: + self.size = self.file.size + if self.extension == 'pdf' and not os.path.exists(self.thumbnail()): + self.make_thumbnail() + + self.name_sort = ox.sort_string(self.name or u'')[:255] + self.description_sort = ox.sort_string(self.description or u'')[:512] + + super(File, self).save(*args, **kwargs) + + def __unicode__(self): + return self.get_id() + + @classmethod + def get(cls, id): + username, name, extension = cls.parse_id(id) + return cls.objects.get(user__username=username, name=name, extension=extension) + + @classmethod + def parse_id(cls, id): + public_id = id.split(':') + username = public_id[0] + name = ":".join(public_id[1:]) + extension = name.split('.') + name = '.'.join(extension[:-1]) + extension = extension[-1].lower() + return username, name, extension + + def get_absolute_url(self): + return '/files/%s' % quote(self.get_id()) + + def get_id(self): + return u'%s:%s.%s' % (self.user.username, self.name, self.extension) + + def editable(self, user): + if not user or user.is_anonymous(): + return False + if self.user == user or \ + user.is_staff or \ + user.get_profile().capability('canEditFiles') == True: + return True + return False + + def edit(self, data, user): + for key in data: + if key == 'name': + data['name'] = re.sub(' \[\d+\]$', '', data['name']).strip() + if not data['name']: + data['name'] = "Untitled" + name = data['name'] + num = 1 + while File.objects.filter(name=name, user=self.user, extension=self.extension).exclude(id=self.id).count()>0: + num += 1 + name = data['name'] + ' [%d]' % num + self.name = name + elif key == 'description': + self.description = ox.sanitize_html(data['description']) + + def json(self, keys=None, user=None): + if not keys: + keys=[ + 'description', + 'editable', + 'id', + 'name', + 'extension', + 'oshash', + 'size', + 'ratio', + 'user' + ] + response = {} + _map = { + } + for key in keys: + if key == 'id': + response[key] = self.get_id() + elif key == 'editable': + response[key] = self.editable(user) + elif key == 'user': + response[key] = self.user.username + elif hasattr(self, _map.get(key, key)): + response[key] = getattr(self, _map.get(key,key)) + return response + + def path(self, name=''): + h = "%07d" % self.id + return os.path.join('uploads', h[:2], h[2:4], h[4:6], h[6:], name) + + def save_chunk(self, chunk, chunk_id=-1, done=False): + if self.uploading: + if not self.file: + name = 'data.%s' % self.extension + self.file.name = self.path(name) + ox.makedirs(os.path.dirname(self.file.path)) + with open(self.file.path, 'w') as f: + f.write(chunk.read()) + self.save() + else: + with open(self.file.path, 'a') as f: + f.write(chunk.read()) + if done: + self.uploading = False + self.get_ratio() + self.oshash = ox.oshash(self.file.path) + self.save() + return True + return False + + def thumbnail(self): + return '%s.jpg' % self.file.path + + def make_thumbnail(self, force=False): + thumb = self.thumbnail() + if not os.path.exists(thumb) or force: + cmd = ['convert', '%s[0]' % self.file.path, + '-background', 'white', '-flatten', '-resize', '256x256', thumb] + p = subprocess.Popen(cmd) + p.wait() + + def get_ratio(self): + if self.extension == 'pdf': + self.make_thumbnail() + image = self.thumbnail() + else: + image = self.file.path + try: + size = Image.open(image).size + except: + size = [1,1] + self.ratio = size[0] / size[1] + + + +def delete_file(sender, **kwargs): + t = kwargs['instance'] + if t.file: + if t.extension == 'pdf': + thumb = t.thumbnail() + if os.path.exists(thumb): + os.unlink(thumb) + t.file.delete() +pre_delete.connect(delete_file, sender=File) + diff --git a/pandora/file/views.py b/pandora/file/views.py new file mode 100644 index 00000000..d11c8678 --- /dev/null +++ b/pandora/file/views.py @@ -0,0 +1,203 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from __future__ import division + +from ox.utils import json +from ox.django.api import actions +from ox.django.decorators import login_required_json +from ox.django.http import HttpFileResponse +from ox.django.shortcuts import render_to_json_response, get_object_or_404_json, json_response +from django import forms + +from item import utils +import models + +def get_file_or_404_json(id): + username, name, extension = models.File.parse_id(id) + return get_object_or_404_json(models.File, user__username=username, name=name, extension=extension) + +@login_required_json +def editFile(request): + ''' + takes { + id: string + name: string + description: string + } + returns { + id: + ... + } + ''' + response = json_response() + data = json.loads(request.POST['data']) + if data['id']: + file = models.File.get(data['id']) + if file.editable(request.user): + file.edit(data, request.user) + file.save() + response['data'] = file.json(user=request.user) + else: + response = json_response(status=403, file='permission denied') + else: + response = json_response(status=500, file='invalid request') + return render_to_json_response(response) +actions.register(editFile, cache=False) + + +def _order_query(qs, sort): + order_by = [] + for e in sort: + operator = e['operator'] + if operator != '-': + operator = '' + key = { + 'name': 'name_sort', + 'description': 'description_sort', + }.get(e['key'], e['key']) + order = '%s%s' % (operator, key) + order_by.append(order) + if order_by: + qs = qs.order_by(*order_by) + qs = qs.distinct() + return qs + +def parse_query(data, user): + query = {} + query['range'] = [0, 100] + query['sort'] = [{'key':'user', 'operator':'+'}, {'key':'name', 'operator':'+'}] + for key in ('keys', 'group', 'file', 'range', 'position', 'positions', 'sort'): + if key in data: + query[key] = data[key] + query['qs'] = models.File.objects.find(data, user).exclude(name='') + return query + + +def findFiles(request): + ''' + takes { + query: { + conditions: [ + { + key: 'user', + value: 'something', + operator: '=' + } + ] + operator: "," + }, + sort: [{key: 'name', operator: '+'}], + range: [0, 100] + keys: [] + } + + possible query keys: + name, user, extension, size + + possible keys: + name, user, extension, size + + } + returns { + items: [object] + } + ''' + data = json.loads(request.POST['data']) + query = parse_query(data, request.user) + + #order + qs = _order_query(query['qs'], query['sort']) + response = json_response() + if 'keys' in data: + qs = qs[query['range'][0]:query['range'][1]] + + response['data']['items'] = [l.json(data['keys'], request.user) for l in qs] + elif 'position' in data: + #FIXME: actually implement position requests + response['data']['position'] = 0 + elif 'positions' in data: + ids = [i.get_id() for i in qs] + response['data']['positions'] = utils.get_positions(ids, query['positions']) + else: + response['data']['items'] = qs.count() + return render_to_json_response(response) +actions.register(findFiles) + +@login_required_json +def removeFile(request): + ''' + takes { + id: string, + } + returns { + } + ''' + data = json.loads(request.POST['data']) + response = json_response() + file = models.File.get(data['id']) + if file.editable(request.user): + file.delete() + else: + response = json_response(status=403, file='not allowed') + return render_to_json_response(response) +actions.register(removeFile, cache=False) + + +def file(request, id): + file = models.File.get(id) + return HttpFileResponse(file.file.path) + +def thumbnail(request, id): + file = models.File.get(id) + return HttpFileResponse(file.thumbnail()) + +class ChunkForm(forms.Form): + chunk = forms.FileField() + chunkId = forms.IntegerField(required=False) + done = forms.IntegerField(required=False) + +@login_required_json +def upload(request): + if 'id' in request.GET: + file = models.File.get(request.GET['id']) + else: + extension = request.POST['filename'].split('.') + name = '.'.join(extension[:-1]) + extension = extension[-1].lower() + response = json_response(status=400, text='this request requires POST') + if 'chunk' in request.FILES: + form = ChunkForm(request.POST, request.FILES) + if form.is_valid() and file.editable(request.user): + c = form.cleaned_data['chunk'] + chunk_id = form.cleaned_data['chunkId'] + response = { + 'result': 1, + 'resultUrl': request.build_absolute_uri(file.get_absolute_url()) + } + if not file.save_chunk(c, chunk_id, form.cleaned_data['done']): + response['result'] = -1 + if form.cleaned_data['done']: + response['done'] = 1 + return render_to_json_response(response) + #init upload + else: + created = False + num = 1 + _name = name + while not created: + file, created = models.File.objects.get_or_create( + user=request.user, name=name, extension=extension) + if not created: + num += 1 + name = _name + ' [%d]' % num + file.name = name + file.extension = extension + file.uploading = True + file.save() + upload_url = request.build_absolute_uri('/api/upload/file?id=%s' % file.get_id()) + return render_to_json_response({ + 'uploadUrl': upload_url, + 'url': request.build_absolute_uri(file.get_absolute_url()), + 'result': 1 + }) + return render_to_json_response(response) diff --git a/pandora/settings.py b/pandora/settings.py index deb96d4a..737e1b5f 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -130,6 +130,7 @@ INSTALLED_APPS = ( 'user', 'urlalias', 'tv', + 'file', ) # Log errors into db diff --git a/pandora/urls.py b/pandora/urls.py index e2ca5e3b..6dfd98e4 100644 --- a/pandora/urls.py +++ b/pandora/urls.py @@ -23,11 +23,14 @@ urlpatterns = patterns('', # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), (r'^admin/', include(admin.site.urls)), (r'^api/upload/text/?$', 'text.views.upload'), + (r'^api/upload/file/?$', 'file.views.upload'), (r'^api/upload/?$', 'archive.views.firefogg_upload'), (r'^url=(?P.*)$', 'app.views.redirect_url'), (r'^file/(?P.*)$', 'archive.views.lookup_file'), (r'^api/?$', include(ox.django.api.urls)), (r'^resetUI$', 'user.views.reset_ui'), + (r'^files/(?P.*?.pdf).jpg$', 'file.views.thumbnail'), + (r'^files/(?P.*?.)$', 'file.views.file'), (r'^list/(?P.*?)/icon(?P\d*).jpg$', 'itemlist.views.icon'), (r'^text/(?P.*?)/icon(?P\d*).jpg$', 'text.views.icon'), (r'^texts/(?P.*?)/text.pdf$', 'text.views.pdf'), diff --git a/vm/build.sh b/vm/build.sh index f2733d1f..4ae3e210 100755 --- a/vm/build.sh +++ b/vm/build.sh @@ -46,6 +46,7 @@ sudo vmbuilder vbox ubuntu --suite=precise \ --addpkg libavcodec-extra-53 \ --addpkg libav-tools \ --addpkg ffmpeg2theora \ + --addpkg imagemagick \ --execscript=$base/install.sh \ --firstboot=$base/firstboot.sh