add files backend

This commit is contained in:
j 2013-03-24 12:28:57 +00:00
parent 488b5431de
commit c4fd2fb48c
10 changed files with 625 additions and 1 deletions

2
README
View File

@ -30,7 +30,7 @@ To run pan.do/ra you need to install and setup:
python-gst0.10 gstreamer0.10-plugins-good gstreamer0.10-plugins-bad \
postgresql postgresql-contrib rabbitmq-server \
ffmpeg2theora libav-tools libavcodec-extra-53 \
python-ox oxframe
python-ox oxframe imagemagick
* Prepare Environment

0
pandora/file/__init__.py Normal file
View File

127
pandora/file/managers.py Normal file
View File

@ -0,0 +1,127 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from django.db.models import Q, Manager
def parseCondition(condition, user):
'''
'''
k = condition.get('key', 'name')
k = {
'user': 'user__username',
}.get(k, k)
if not k:
k = 'name'
v = condition['value']
op = condition.get('operator')
if not op:
op = '='
if op.startswith('!'):
op = op[1:]
exclude = True
else:
exclude = False
if k == 'id':
try:
public_id = v.split(':')
username = public_id[0]
name = ":".join(public_id[1:])
extension = name.split('.')
name = '.'.join(extension[:-1])
extension = extension[-1].lower()
q = Q(user__username=username, name=name, extension=extension)
except:
q = Q(id__in=[])
return q
if isinstance(v, bool): #featured and public flag
key = k
else:
key = "%s%s" % (k, {
'==': '__iexact',
'^': '__istartswith',
'$': '__iendswith',
}.get(op, '__icontains'))
key = str(key)
if exclude:
q = ~Q(**{key: v})
else:
q = Q(**{key: v})
return q
def parseConditions(conditions, operator, user):
'''
conditions: [
{
value: "war"
}
{
key: "year",
value: "1970-1980,
operator: "!="
},
{
key: "country",
value: "f",
operator: "^"
}
],
operator: "&"
'''
conn = []
for condition in conditions:
if 'conditions' in condition:
q = parseConditions(condition['conditions'],
condition.get('operator', '&'), user)
if q:
conn.append(q)
pass
else:
conn.append(parseCondition(condition, user))
if conn:
q = conn[0]
for c in conn[1:]:
if operator == '|':
q = q | c
else:
q = q & c
return q
return None
class FileManager(Manager):
def get_query_set(self):
return super(FileManager, self).get_query_set()
def find(self, data, user):
'''
query: {
conditions: [
{
value: "war"
}
{
key: "year",
value: "1970-1980,
operator: "!="
},
{
key: "country",
value: "f",
operator: "^"
}
],
operator: "&"
}
'''
#join query with operator
qs = self.get_query_set()
conditions = parseConditions(data['query'].get('conditions', []),
data['query'].get('operator', '&'),
user)
if conditions:
qs = qs.filter(conditions)
return qs

View File

@ -0,0 +1,100 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'File'
db.create_table('file_file', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
('modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)),
('user', self.gf('django.db.models.fields.related.ForeignKey')(related_name='files', to=orm['auth.User'])),
('name', self.gf('django.db.models.fields.CharField')(max_length=255)),
('extension', self.gf('django.db.models.fields.CharField')(max_length=255)),
('size', self.gf('django.db.models.fields.IntegerField')(default=0)),
('matches', self.gf('django.db.models.fields.IntegerField')(default=0)),
('ratio', self.gf('django.db.models.fields.FloatField')(default=1)),
('description', self.gf('django.db.models.fields.TextField')(default='')),
('oshash', self.gf('django.db.models.fields.CharField')(max_length=16, unique=True, null=True)),
('file', self.gf('django.db.models.fields.files.FileField')(default=None, max_length=100, null=True, blank=True)),
('uploading', self.gf('django.db.models.fields.BooleanField')(default=False)),
('name_sort', self.gf('django.db.models.fields.CharField')(max_length=255)),
('description_sort', self.gf('django.db.models.fields.CharField')(max_length=512)),
))
db.send_create_signal('file', ['File'])
# Adding unique constraint on 'File', fields ['user', 'name', 'extension']
db.create_unique('file_file', ['user_id', 'name', 'extension'])
def backwards(self, orm):
# Removing unique constraint on 'File', fields ['user', 'name', 'extension']
db.delete_unique('file_file', ['user_id', 'name', 'extension'])
# Deleting model 'File'
db.delete_table('file_file')
models = {
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'file.file': {
'Meta': {'unique_together': "(('user', 'name', 'extension'),)", 'object_name': 'File'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'description': ('django.db.models.fields.TextField', [], {'default': "''"}),
'description_sort': ('django.db.models.fields.CharField', [], {'max_length': '512'}),
'extension': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'file': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '100', 'null': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'matches': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'name_sort': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'oshash': ('django.db.models.fields.CharField', [], {'max_length': '16', 'unique': 'True', 'null': 'True'}),
'ratio': ('django.db.models.fields.FloatField', [], {'default': '1'}),
'size': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'uploading': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'files'", 'to': "orm['auth.User']"})
}
}
complete_apps = ['file']

View File

189
pandora/file/models.py Normal file
View File

@ -0,0 +1,189 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, with_statement
import os
import re
import subprocess
from urllib import quote
from django.db import models
from django.contrib.auth.models import User
from django.db.models.signals import pre_delete
import Image
import ox
import managers
class File(models.Model):
class Meta:
unique_together = ("user", "name", "extension")
created = models.DateTimeField(auto_now_add=True)
modified = models.DateTimeField(auto_now=True)
user = models.ForeignKey(User, related_name='files')
name = models.CharField(max_length=255)
extension = models.CharField(max_length=255)
size = models.IntegerField(default=0)
matches = models.IntegerField(default=0)
ratio = models.FloatField(default=1)
description = models.TextField(default="")
oshash = models.CharField(max_length=16, unique=True, null=True)
file = models.FileField(default=None, blank=True,null=True, upload_to=lambda f, x: f.path(x))
objects = managers.FileManager()
uploading = models.BooleanField(default = False)
name_sort = models.CharField(max_length=255)
description_sort = models.CharField(max_length=512)
def save(self, *args, **kwargs):
if not self.uploading:
if self.file:
self.size = self.file.size
if self.extension == 'pdf' and not os.path.exists(self.thumbnail()):
self.make_thumbnail()
self.name_sort = ox.sort_string(self.name or u'')[:255]
self.description_sort = ox.sort_string(self.description or u'')[:512]
super(File, self).save(*args, **kwargs)
def __unicode__(self):
return self.get_id()
@classmethod
def get(cls, id):
username, name, extension = cls.parse_id(id)
return cls.objects.get(user__username=username, name=name, extension=extension)
@classmethod
def parse_id(cls, id):
public_id = id.split(':')
username = public_id[0]
name = ":".join(public_id[1:])
extension = name.split('.')
name = '.'.join(extension[:-1])
extension = extension[-1].lower()
return username, name, extension
def get_absolute_url(self):
return '/files/%s' % quote(self.get_id())
def get_id(self):
return u'%s:%s.%s' % (self.user.username, self.name, self.extension)
def editable(self, user):
if not user or user.is_anonymous():
return False
if self.user == user or \
user.is_staff or \
user.get_profile().capability('canEditFiles') == True:
return True
return False
def edit(self, data, user):
for key in data:
if key == 'name':
data['name'] = re.sub(' \[\d+\]$', '', data['name']).strip()
if not data['name']:
data['name'] = "Untitled"
name = data['name']
num = 1
while File.objects.filter(name=name, user=self.user, extension=self.extension).exclude(id=self.id).count()>0:
num += 1
name = data['name'] + ' [%d]' % num
self.name = name
elif key == 'description':
self.description = ox.sanitize_html(data['description'])
def json(self, keys=None, user=None):
if not keys:
keys=[
'description',
'editable',
'id',
'name',
'extension',
'oshash',
'size',
'ratio',
'user'
]
response = {}
_map = {
}
for key in keys:
if key == 'id':
response[key] = self.get_id()
elif key == 'editable':
response[key] = self.editable(user)
elif key == 'user':
response[key] = self.user.username
elif hasattr(self, _map.get(key, key)):
response[key] = getattr(self, _map.get(key,key))
return response
def path(self, name=''):
h = "%07d" % self.id
return os.path.join('uploads', h[:2], h[2:4], h[4:6], h[6:], name)
def save_chunk(self, chunk, chunk_id=-1, done=False):
if self.uploading:
if not self.file:
name = 'data.%s' % self.extension
self.file.name = self.path(name)
ox.makedirs(os.path.dirname(self.file.path))
with open(self.file.path, 'w') as f:
f.write(chunk.read())
self.save()
else:
with open(self.file.path, 'a') as f:
f.write(chunk.read())
if done:
self.uploading = False
self.get_ratio()
self.oshash = ox.oshash(self.file.path)
self.save()
return True
return False
def thumbnail(self):
return '%s.jpg' % self.file.path
def make_thumbnail(self, force=False):
thumb = self.thumbnail()
if not os.path.exists(thumb) or force:
cmd = ['convert', '%s[0]' % self.file.path,
'-background', 'white', '-flatten', '-resize', '256x256', thumb]
p = subprocess.Popen(cmd)
p.wait()
def get_ratio(self):
if self.extension == 'pdf':
self.make_thumbnail()
image = self.thumbnail()
else:
image = self.file.path
try:
size = Image.open(image).size
except:
size = [1,1]
self.ratio = size[0] / size[1]
def delete_file(sender, **kwargs):
t = kwargs['instance']
if t.file:
if t.extension == 'pdf':
thumb = t.thumbnail()
if os.path.exists(thumb):
os.unlink(thumb)
t.file.delete()
pre_delete.connect(delete_file, sender=File)

203
pandora/file/views.py Normal file
View File

@ -0,0 +1,203 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division
from ox.utils import json
from ox.django.api import actions
from ox.django.decorators import login_required_json
from ox.django.http import HttpFileResponse
from ox.django.shortcuts import render_to_json_response, get_object_or_404_json, json_response
from django import forms
from item import utils
import models
def get_file_or_404_json(id):
username, name, extension = models.File.parse_id(id)
return get_object_or_404_json(models.File, user__username=username, name=name, extension=extension)
@login_required_json
def editFile(request):
'''
takes {
id: string
name: string
description: string
}
returns {
id:
...
}
'''
response = json_response()
data = json.loads(request.POST['data'])
if data['id']:
file = models.File.get(data['id'])
if file.editable(request.user):
file.edit(data, request.user)
file.save()
response['data'] = file.json(user=request.user)
else:
response = json_response(status=403, file='permission denied')
else:
response = json_response(status=500, file='invalid request')
return render_to_json_response(response)
actions.register(editFile, cache=False)
def _order_query(qs, sort):
order_by = []
for e in sort:
operator = e['operator']
if operator != '-':
operator = ''
key = {
'name': 'name_sort',
'description': 'description_sort',
}.get(e['key'], e['key'])
order = '%s%s' % (operator, key)
order_by.append(order)
if order_by:
qs = qs.order_by(*order_by)
qs = qs.distinct()
return qs
def parse_query(data, user):
query = {}
query['range'] = [0, 100]
query['sort'] = [{'key':'user', 'operator':'+'}, {'key':'name', 'operator':'+'}]
for key in ('keys', 'group', 'file', 'range', 'position', 'positions', 'sort'):
if key in data:
query[key] = data[key]
query['qs'] = models.File.objects.find(data, user).exclude(name='')
return query
def findFiles(request):
'''
takes {
query: {
conditions: [
{
key: 'user',
value: 'something',
operator: '='
}
]
operator: ","
},
sort: [{key: 'name', operator: '+'}],
range: [0, 100]
keys: []
}
possible query keys:
name, user, extension, size
possible keys:
name, user, extension, size
}
returns {
items: [object]
}
'''
data = json.loads(request.POST['data'])
query = parse_query(data, request.user)
#order
qs = _order_query(query['qs'], query['sort'])
response = json_response()
if 'keys' in data:
qs = qs[query['range'][0]:query['range'][1]]
response['data']['items'] = [l.json(data['keys'], request.user) for l in qs]
elif 'position' in data:
#FIXME: actually implement position requests
response['data']['position'] = 0
elif 'positions' in data:
ids = [i.get_id() for i in qs]
response['data']['positions'] = utils.get_positions(ids, query['positions'])
else:
response['data']['items'] = qs.count()
return render_to_json_response(response)
actions.register(findFiles)
@login_required_json
def removeFile(request):
'''
takes {
id: string,
}
returns {
}
'''
data = json.loads(request.POST['data'])
response = json_response()
file = models.File.get(data['id'])
if file.editable(request.user):
file.delete()
else:
response = json_response(status=403, file='not allowed')
return render_to_json_response(response)
actions.register(removeFile, cache=False)
def file(request, id):
file = models.File.get(id)
return HttpFileResponse(file.file.path)
def thumbnail(request, id):
file = models.File.get(id)
return HttpFileResponse(file.thumbnail())
class ChunkForm(forms.Form):
chunk = forms.FileField()
chunkId = forms.IntegerField(required=False)
done = forms.IntegerField(required=False)
@login_required_json
def upload(request):
if 'id' in request.GET:
file = models.File.get(request.GET['id'])
else:
extension = request.POST['filename'].split('.')
name = '.'.join(extension[:-1])
extension = extension[-1].lower()
response = json_response(status=400, text='this request requires POST')
if 'chunk' in request.FILES:
form = ChunkForm(request.POST, request.FILES)
if form.is_valid() and file.editable(request.user):
c = form.cleaned_data['chunk']
chunk_id = form.cleaned_data['chunkId']
response = {
'result': 1,
'resultUrl': request.build_absolute_uri(file.get_absolute_url())
}
if not file.save_chunk(c, chunk_id, form.cleaned_data['done']):
response['result'] = -1
if form.cleaned_data['done']:
response['done'] = 1
return render_to_json_response(response)
#init upload
else:
created = False
num = 1
_name = name
while not created:
file, created = models.File.objects.get_or_create(
user=request.user, name=name, extension=extension)
if not created:
num += 1
name = _name + ' [%d]' % num
file.name = name
file.extension = extension
file.uploading = True
file.save()
upload_url = request.build_absolute_uri('/api/upload/file?id=%s' % file.get_id())
return render_to_json_response({
'uploadUrl': upload_url,
'url': request.build_absolute_uri(file.get_absolute_url()),
'result': 1
})
return render_to_json_response(response)

View File

@ -130,6 +130,7 @@ INSTALLED_APPS = (
'user',
'urlalias',
'tv',
'file',
)
# Log errors into db

View File

@ -23,11 +23,14 @@ urlpatterns = patterns('',
# url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
(r'^admin/', include(admin.site.urls)),
(r'^api/upload/text/?$', 'text.views.upload'),
(r'^api/upload/file/?$', 'file.views.upload'),
(r'^api/upload/?$', 'archive.views.firefogg_upload'),
(r'^url=(?P<url>.*)$', 'app.views.redirect_url'),
(r'^file/(?P<oshash>.*)$', 'archive.views.lookup_file'),
(r'^api/?$', include(ox.django.api.urls)),
(r'^resetUI$', 'user.views.reset_ui'),
(r'^files/(?P<id>.*?.pdf).jpg$', 'file.views.thumbnail'),
(r'^files/(?P<id>.*?.)$', 'file.views.file'),
(r'^list/(?P<id>.*?)/icon(?P<size>\d*).jpg$', 'itemlist.views.icon'),
(r'^text/(?P<id>.*?)/icon(?P<size>\d*).jpg$', 'text.views.icon'),
(r'^texts/(?P<id>.*?)/text.pdf$', 'text.views.pdf'),

View File

@ -46,6 +46,7 @@ sudo vmbuilder vbox ubuntu --suite=precise \
--addpkg libavcodec-extra-53 \
--addpkg libav-tools \
--addpkg ffmpeg2theora \
--addpkg imagemagick \
--execscript=$base/install.sh \
--firstboot=$base/firstboot.sh