add files backend

README

@@ -30,7 +30,7 @@ To run pan.do/ra you need to install and setup:
             python-gst0.10 gstreamer0.10-plugins-good gstreamer0.10-plugins-bad \
             postgresql postgresql-contrib rabbitmq-server \
             ffmpeg2theora libav-tools libavcodec-extra-53 \
-            python-ox oxframe
+            python-ox oxframe imagemagick
 
 
 * Prepare Environment

pandora/file/managers.py

@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+from django.db.models import Q, Manager
+
+
+def parseCondition(condition, user):
+    '''
+    '''
+    k = condition.get('key', 'name')
+    k = {
+        'user': 'user__username',
+    }.get(k, k)
+    if not k:
+        k = 'name'
+    v = condition['value']
+    op = condition.get('operator')
+    if not op:
+        op = '='
+    if op.startswith('!'):
+        op = op[1:]
+        exclude = True
+    else:
+        exclude = False
+    if k == 'id':
+        try:
+            public_id = v.split(':')
+            username = public_id[0]
+            name = ":".join(public_id[1:])
+            extension = name.split('.')
+            name = '.'.join(extension[:-1])
+            extension = extension[-1].lower()
+            q = Q(user__username=username, name=name, extension=extension)
+        except:
+            q = Q(id__in=[])
+        return q
+    if isinstance(v, bool): #featured and public flag
+        key = k
+    else:
+        key = "%s%s" % (k, {
+            '==': '__iexact',
+            '^': '__istartswith',
+            '$': '__iendswith',
+        }.get(op, '__icontains'))
+    key = str(key)
+    if exclude:
+        q = ~Q(**{key: v})
+    else:
+        q = Q(**{key: v})
+    return q
+
+def parseConditions(conditions, operator, user):
+    '''
+    conditions: [
+        {
+            value: "war"
+        }
+        {
+            key: "year",
+            value: "1970-1980,
+            operator: "!="
+        },
+        {
+            key: "country",
+            value: "f",
+            operator: "^"
+        }
+    ],
+    operator: "&"
+    '''
+    conn = []
+    for condition in conditions:
+        if 'conditions' in condition:
+            q = parseConditions(condition['conditions'],
+                             condition.get('operator', '&'), user)
+            if q:
+                conn.append(q)
+            pass
+        else:
+            conn.append(parseCondition(condition, user))
+    if conn:
+        q = conn[0]
+        for c in conn[1:]:
+            if operator == '|':
+                q = q | c
+            else:
+                q = q & c
+        return q
+    return None
+
+
+class FileManager(Manager):
+
+    def get_query_set(self):
+        return super(FileManager, self).get_query_set()
+
+    def find(self, data, user):
+        '''
+            query: {
+                conditions: [
+                    {
+                        value: "war"
+                    }
+                    {
+                        key: "year",
+                        value: "1970-1980,
+                        operator: "!="
+                    },
+                    {
+                        key: "country",
+                        value: "f",
+                        operator: "^"
+                    }
+                ],
+                operator: "&"
+            }
+        '''
+
+        #join query with operator
+        qs = self.get_query_set()
+        conditions = parseConditions(data['query'].get('conditions', []),
+                                     data['query'].get('operator', '&'),
+                                     user)
+        if conditions:
+            qs = qs.filter(conditions)
+
+        return qs
+

pandora/file/migrations/0001_initial.py

@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'File'
+        db.create_table('file_file', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+            ('modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)),
+            ('user', self.gf('django.db.models.fields.related.ForeignKey')(related_name='files', to=orm['auth.User'])),
+            ('name', self.gf('django.db.models.fields.CharField')(max_length=255)),
+            ('extension', self.gf('django.db.models.fields.CharField')(max_length=255)),
+            ('size', self.gf('django.db.models.fields.IntegerField')(default=0)),
+            ('matches', self.gf('django.db.models.fields.IntegerField')(default=0)),
+            ('ratio', self.gf('django.db.models.fields.FloatField')(default=1)),
+            ('description', self.gf('django.db.models.fields.TextField')(default='')),
+            ('oshash', self.gf('django.db.models.fields.CharField')(max_length=16, unique=True, null=True)),
+            ('file', self.gf('django.db.models.fields.files.FileField')(default=None, max_length=100, null=True, blank=True)),
+            ('uploading', self.gf('django.db.models.fields.BooleanField')(default=False)),
+            ('name_sort', self.gf('django.db.models.fields.CharField')(max_length=255)),
+            ('description_sort', self.gf('django.db.models.fields.CharField')(max_length=512)),
+        ))
+        db.send_create_signal('file', ['File'])
+
+        # Adding unique constraint on 'File', fields ['user', 'name', 'extension']
+        db.create_unique('file_file', ['user_id', 'name', 'extension'])
+
+
+    def backwards(self, orm):
+        # Removing unique constraint on 'File', fields ['user', 'name', 'extension']
+        db.delete_unique('file_file', ['user_id', 'name', 'extension'])
+
+        # Deleting model 'File'
+        db.delete_table('file_file')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'file.file': {
+            'Meta': {'unique_together': "(('user', 'name', 'extension'),)", 'object_name': 'File'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'default': "''"}),
+            'description_sort': ('django.db.models.fields.CharField', [], {'max_length': '512'}),
+            'extension': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'file': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'matches': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'name_sort': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'oshash': ('django.db.models.fields.CharField', [], {'max_length': '16', 'unique': 'True', 'null': 'True'}),
+            'ratio': ('django.db.models.fields.FloatField', [], {'default': '1'}),
+            'size': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'uploading': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'files'", 'to': "orm['auth.User']"})
+        }
+    }
+
+    complete_apps = ['file']

pandora/file/models.py

@@ -0,0 +1,189 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+from __future__ import division, with_statement
+import os
+import re
+import subprocess
+from urllib import quote
+
+from django.db import models
+from django.contrib.auth.models import User
+from django.db.models.signals import pre_delete
+
+import Image
+import ox
+
+import managers
+
+
+class File(models.Model):
+
+    class Meta:
+        unique_together = ("user", "name", "extension")
+
+    created = models.DateTimeField(auto_now_add=True)
+    modified = models.DateTimeField(auto_now=True)
+
+    user = models.ForeignKey(User, related_name='files')
+    name = models.CharField(max_length=255)
+    extension = models.CharField(max_length=255)
+    size = models.IntegerField(default=0)
+    matches = models.IntegerField(default=0)
+    ratio = models.FloatField(default=1)
+    description = models.TextField(default="")
+    oshash = models.CharField(max_length=16, unique=True, null=True)
+
+    file = models.FileField(default=None, blank=True,null=True, upload_to=lambda f, x: f.path(x))
+
+    objects = managers.FileManager()
+    uploading = models.BooleanField(default = False)
+
+    name_sort = models.CharField(max_length=255)
+    description_sort = models.CharField(max_length=512)
+
+    def save(self, *args, **kwargs):
+        if not self.uploading:
+            if self.file:
+                self.size = self.file.size
+                if self.extension == 'pdf' and not os.path.exists(self.thumbnail()):
+                    self.make_thumbnail()
+
+        self.name_sort = ox.sort_string(self.name or u'')[:255]
+        self.description_sort = ox.sort_string(self.description or u'')[:512]
+
+        super(File, self).save(*args, **kwargs)
+
+    def __unicode__(self):
+        return self.get_id()
+
+    @classmethod
+    def get(cls, id):
+        username, name, extension = cls.parse_id(id)
+        return cls.objects.get(user__username=username, name=name, extension=extension)
+
+    @classmethod
+    def parse_id(cls, id):
+        public_id = id.split(':')
+        username = public_id[0]
+        name = ":".join(public_id[1:])
+        extension = name.split('.')
+        name = '.'.join(extension[:-1])
+        extension = extension[-1].lower()
+        return username, name, extension
+
+    def get_absolute_url(self):
+        return '/files/%s' % quote(self.get_id())
+
+    def get_id(self):
+        return u'%s:%s.%s' % (self.user.username, self.name, self.extension)
+
+    def editable(self, user):
+        if not user or user.is_anonymous():
+            return False
+        if self.user == user or \
+           user.is_staff or \
+           user.get_profile().capability('canEditFiles') == True:
+            return True
+        return False
+
+    def edit(self, data, user):
+        for key in data:
+            if key == 'name':
+                data['name'] = re.sub(' \[\d+\]$', '', data['name']).strip()
+                if not data['name']:
+                    data['name'] = "Untitled"
+                name = data['name']
+                num = 1
+                while File.objects.filter(name=name, user=self.user, extension=self.extension).exclude(id=self.id).count()>0:
+                    num += 1
+                    name = data['name'] + ' [%d]' % num
+                self.name = name
+            elif key == 'description':
+                self.description = ox.sanitize_html(data['description'])
+
+    def json(self, keys=None, user=None):
+        if not keys:
+             keys=[
+                'description',
+                'editable',
+                'id',
+                'name',
+                'extension',
+                'oshash',
+                'size',
+                'ratio',
+                'user'
+            ]
+        response = {}
+        _map = {
+        }
+        for key in keys:
+            if key == 'id':
+                response[key] = self.get_id()
+            elif key == 'editable':
+                response[key] = self.editable(user)
+            elif key == 'user':
+                response[key] = self.user.username
+            elif hasattr(self, _map.get(key, key)):
+                response[key] = getattr(self, _map.get(key,key))
+        return response
+
+    def path(self, name=''):
+        h = "%07d" % self.id
+        return os.path.join('uploads', h[:2], h[2:4], h[4:6], h[6:], name)
+
+    def save_chunk(self, chunk, chunk_id=-1, done=False):
+        if self.uploading:
+            if not self.file:
+                name = 'data.%s' % self.extension
+                self.file.name = self.path(name)
+                ox.makedirs(os.path.dirname(self.file.path))
+                with open(self.file.path, 'w') as f:
+                    f.write(chunk.read())
+                self.save()
+            else:
+                with open(self.file.path, 'a') as f:
+                    f.write(chunk.read())
+            if done:
+                self.uploading = False
+                self.get_ratio()
+                self.oshash = ox.oshash(self.file.path)
+                self.save()
+            return True
+        return False
+
+    def thumbnail(self):
+        return '%s.jpg' % self.file.path
+
+    def make_thumbnail(self, force=False):
+        thumb = self.thumbnail()
+        if not os.path.exists(thumb) or force:
+            cmd = ['convert', '%s[0]' % self.file.path,
+                '-background', 'white', '-flatten', '-resize', '256x256', thumb]
+            p = subprocess.Popen(cmd)
+            p.wait()
+
+    def get_ratio(self):
+        if self.extension == 'pdf':
+            self.make_thumbnail()
+            image = self.thumbnail()
+        else:
+            image = self.file.path
+        try:
+            size = Image.open(image).size
+        except:
+            size = [1,1]
+        self.ratio = size[0] / size[1]
+
+
+
+def delete_file(sender, **kwargs):
+    t = kwargs['instance']
+    if t.file:
+        if t.extension == 'pdf':
+            thumb = t.thumbnail()
+            if os.path.exists(thumb):
+                os.unlink(thumb)
+        t.file.delete()
+pre_delete.connect(delete_file, sender=File)
+

pandora/file/views.py

@@ -0,0 +1,203 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+from __future__ import division
+
+from ox.utils import json
+from ox.django.api import actions
+from ox.django.decorators import login_required_json
+from ox.django.http import HttpFileResponse
+from ox.django.shortcuts import render_to_json_response, get_object_or_404_json, json_response
+from django import forms
+
+from item import utils
+import models
+
+def get_file_or_404_json(id):
+    username, name, extension = models.File.parse_id(id)
+    return get_object_or_404_json(models.File, user__username=username, name=name, extension=extension)
+
+@login_required_json
+def editFile(request):
+    '''
+        takes {
+            id: string
+            name: string
+            description: string
+        }
+        returns {
+            id:
+            ...
+        }
+    '''
+    response = json_response()
+    data = json.loads(request.POST['data'])
+    if data['id']:
+        file = models.File.get(data['id'])
+        if file.editable(request.user):
+            file.edit(data, request.user)
+            file.save()
+            response['data'] = file.json(user=request.user)
+        else:
+            response = json_response(status=403, file='permission denied')
+    else:
+        response = json_response(status=500, file='invalid request')
+    return render_to_json_response(response)
+actions.register(editFile, cache=False)
+
+
+def _order_query(qs, sort):
+    order_by = []
+    for e in sort:
+        operator = e['operator']
+        if operator != '-':
+            operator = ''
+        key = {
+            'name': 'name_sort',
+            'description': 'description_sort',
+        }.get(e['key'], e['key'])
+        order = '%s%s' % (operator, key)
+        order_by.append(order)
+    if order_by:
+        qs = qs.order_by(*order_by)
+    qs = qs.distinct()
+    return qs
+
+def parse_query(data, user):
+    query = {}
+    query['range'] = [0, 100]
+    query['sort'] = [{'key':'user', 'operator':'+'}, {'key':'name', 'operator':'+'}]
+    for key in ('keys', 'group', 'file', 'range', 'position', 'positions', 'sort'):
+        if key in data:
+            query[key] = data[key]
+    query['qs'] = models.File.objects.find(data, user).exclude(name='')
+    return query
+
+
+def findFiles(request):
+    '''
+        takes {
+            query: {
+                conditions: [
+                    {
+                        key: 'user',
+                        value: 'something',
+                        operator: '='
+                    }
+                ]
+                operator: ","
+            },
+            sort: [{key: 'name', operator: '+'}],
+            range: [0, 100]
+            keys: []
+        }
+
+        possible query keys:
+            name, user, extension, size
+
+        possible keys:
+            name, user, extension, size
+
+        }
+        returns {
+            items: [object]
+        }
+    '''
+    data = json.loads(request.POST['data'])
+    query = parse_query(data, request.user)
+
+    #order
+    qs = _order_query(query['qs'], query['sort'])
+    response = json_response()
+    if 'keys' in data:
+        qs = qs[query['range'][0]:query['range'][1]]
+
+        response['data']['items'] = [l.json(data['keys'], request.user) for l in qs]
+    elif 'position' in data:
+        #FIXME: actually implement position requests
+        response['data']['position'] = 0
+    elif 'positions' in data:
+        ids = [i.get_id() for i in qs]
+        response['data']['positions'] = utils.get_positions(ids, query['positions'])
+    else:
+        response['data']['items'] = qs.count()
+    return render_to_json_response(response)
+actions.register(findFiles)
+
+@login_required_json
+def removeFile(request):
+    '''
+        takes {
+            id: string,
+        }
+        returns {
+        }
+    '''
+    data = json.loads(request.POST['data'])
+    response = json_response()
+    file = models.File.get(data['id'])
+    if file.editable(request.user):
+        file.delete()
+    else:
+        response = json_response(status=403, file='not allowed')
+    return render_to_json_response(response)
+actions.register(removeFile, cache=False)
+
+
+def file(request, id):
+    file = models.File.get(id)
+    return HttpFileResponse(file.file.path)
+
+def thumbnail(request, id):
+    file = models.File.get(id)
+    return HttpFileResponse(file.thumbnail())
+
+class ChunkForm(forms.Form):
+    chunk = forms.FileField()
+    chunkId = forms.IntegerField(required=False)
+    done = forms.IntegerField(required=False)
+
+@login_required_json
+def upload(request):
+    if 'id' in request.GET:
+        file = models.File.get(request.GET['id'])
+    else:
+        extension = request.POST['filename'].split('.')
+        name = '.'.join(extension[:-1])
+        extension = extension[-1].lower()
+    response = json_response(status=400, text='this request requires POST')
+    if 'chunk' in request.FILES:
+        form = ChunkForm(request.POST, request.FILES)
+        if form.is_valid() and file.editable(request.user):
+            c = form.cleaned_data['chunk']
+            chunk_id = form.cleaned_data['chunkId']
+            response = {
+                'result': 1,
+                'resultUrl': request.build_absolute_uri(file.get_absolute_url())
+            }
+            if not file.save_chunk(c, chunk_id, form.cleaned_data['done']):
+                response['result'] = -1
+            if form.cleaned_data['done']:
+                response['done'] = 1
+            return render_to_json_response(response)
+    #init upload
+    else:
+        created = False
+        num = 1
+        _name = name
+        while not created:
+            file, created = models.File.objects.get_or_create(
+                user=request.user, name=name, extension=extension)
+            if not created:
+                num += 1
+                name = _name + ' [%d]' % num
+        file.name = name
+        file.extension = extension
+        file.uploading = True
+        file.save()
+        upload_url = request.build_absolute_uri('/api/upload/file?id=%s' % file.get_id())
+        return render_to_json_response({
+            'uploadUrl': upload_url,
+            'url': request.build_absolute_uri(file.get_absolute_url()),
+            'result': 1
+        })
+    return render_to_json_response(response)

pandora/settings.py

@@ -130,6 +130,7 @@ INSTALLED_APPS = (
     'user',
     'urlalias',
     'tv',
+    'file',
 )
 
 # Log errors into db

pandora/urls.py

@@ -23,11 +23,14 @@ urlpatterns = patterns('',
     # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
     (r'^admin/', include(admin.site.urls)),
     (r'^api/upload/text/?$', 'text.views.upload'),
+    (r'^api/upload/file/?$', 'file.views.upload'),
     (r'^api/upload/?$', 'archive.views.firefogg_upload'),
     (r'^url=(?P<url>.*)$', 'app.views.redirect_url'),
     (r'^file/(?P<oshash>.*)$', 'archive.views.lookup_file'),
     (r'^api/?$', include(ox.django.api.urls)),
     (r'^resetUI$', 'user.views.reset_ui'),
+    (r'^files/(?P<id>.*?.pdf).jpg$', 'file.views.thumbnail'),
+    (r'^files/(?P<id>.*?.)$', 'file.views.file'),
     (r'^list/(?P<id>.*?)/icon(?P<size>\d*).jpg$', 'itemlist.views.icon'),
     (r'^text/(?P<id>.*?)/icon(?P<size>\d*).jpg$', 'text.views.icon'),
     (r'^texts/(?P<id>.*?)/text.pdf$', 'text.views.pdf'),

vm/build.sh

@@ -46,6 +46,7 @@ sudo  vmbuilder vbox ubuntu --suite=precise \
     --addpkg libavcodec-extra-53 \
     --addpkg libav-tools \
     --addpkg ffmpeg2theora \
+    --addpkg imagemagick \
     --execscript=$base/install.sh \
     --firstboot=$base/firstboot.sh