migrate to celery tasks, add Access-Control-Allow-Origin * for now, update links, query stats

This commit is contained in:
j 2010-06-26 16:32:08 +02:00
parent 53683ebf0e
commit d24296df71
5 changed files with 59 additions and 150 deletions

View file

@ -1,33 +0,0 @@
from carrot.connection import DjangoBrokerConnection
from carrot.messaging import Consumer, Publisher
from django.conf import settings
import load
import models
def send_bg_message(msg):
conn = DjangoBrokerConnection()
publisher = Publisher(connection=conn, exchange="pandora-bg",
routing_key="pandora-bg")
publisher.send(msg)
publisher.close()
def run():
conn = DjangoBrokerConnection()
consumer = Consumer(connection=conn, queue="pandora-bg",
exchange="pandora-bg",
routing_key="pandora-bg")
def handle_background_tasks_callback(data, message):
print("Got bg message")
print data
if 'loadIMDb' in data:
imdbId = data['loadIMDb']
load.loadIMDb(imdbId)
elif 'findMovie' in data:
f = models.File.objects.get(pk=data['findMovie'])
f.findMovie()
message.ack()
consumer.register_callback(handle_background_tasks_callback)
consumer.wait() # Go into the consumer loop.

View file

@ -1,44 +0,0 @@
from carrot.connection import DjangoBrokerConnection
from carrot.messaging import Consumer, Publisher
from django.conf import settings
import load
import models
def send_encoder_message(msg):
conn = DjangoBrokerConnection()
publisher = Publisher(connection=conn, exchange="pandora-encoder",
routing_key="pandora-encoder")
publisher.send(msg)
publisher.close()
def run():
conn = DjangoBrokerConnection()
consumer = Consumer(connection=conn, queue="pandora-encoder",
exchange="pandora-encoder",
routing_key="pandora-encoder")
def handle_background_tasks_callback(data, message):
print("Got encoder message")
print data
if 'extract' in data:
'''
update file stuff
create derivates and other related stuff for a file
'''
fileId = data['fileId']
f = models.File.objects.get(pk=fileId)
f.extract()
elif 'updateMovie' in data:
'''
update movie
create proxy stream and other related files extracted from movieFiles
'''
movieId = data['movieId']
m = models.Movie.objects.get(pk=fileId)
m.extract()
message.ack()
consumer.register_callback(handle_background_tasks_callback)
consumer.wait() # Go into the consumer loop.

View file

@ -1,22 +0,0 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import os
from os.path import join, dirname, basename, splitext, exists
from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
from ... import encoder
class Command(BaseCommand):
"""
listen to rabbitmq and execute encoding tasks.
"""
help = 'listen to rabbitmq and execute encoding tasks.'
args = ''
def handle(self, **options):
encoder.run()

41
pandora/backend/tasks.py Normal file
View file

@ -0,0 +1,41 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from datetime import timedelta
from celery.decorators import task, periodic_task
import load
import models
@periodic_task(run_every=timedelta(days=1))
def cronjob(**kwargs):
print "do some cleanup stuff once a day"
@task(ignore_resulsts=True)
def loadIMDb(imdbId):
load.loadIMDb(imdbId)
@task(ignore_resulsts=True)
def findMovie(fileId):
f = models.File.objects.get(pk=fileId)
f.findMovie()
@task(ignore_resulsts=True, exchange="encoding")
def extractData(fileId):
'''
update file stuff
create derivates and other related stuff for a file
'''
f = models.File.objects.get(pk=fileId)
f.extract()
@task(ignore_resulsts=True, exchange="encoding")
def updateMovie(movidId):
'''
update movie
create proxy stream and other related files extracted from movieFiles
'''
m = models.Movie.objects.get(pk=movieId)
m.extract()

View file

@ -54,7 +54,7 @@ def api(request):
else: else:
response = render_to_json_response(json_response(status=400, response = render_to_json_response(json_response(status=400,
text='Unknown function %s' % function)) text='Unknown function %s' % function))
#response['Access-Control-Allow-Origin'] = '*' response['Access-Control-Allow-Origin'] = '*'
return response return response
def api_hello(request): def api_hello(request):
@ -111,47 +111,13 @@ def _parse_query(data, user):
query = {} query = {}
query['range'] = [0, 100] query['range'] = [0, 100]
query['sort'] = (['title', 'asc']) query['sort'] = (['title', 'asc'])
_dicts = ['p', ] for key in ('sort', 'keys', 'group', 'list', 'range'):
_ints = ['n', ]
for key in ('sort', 'keys', 'group', 'list', 'range', 'n'):
if key in data: if key in data:
query[key] = data[key] query[key] = data[key]
print query print query
query['qs'] = models.Movie.objects.find(data, user) query['qs'] = models.Movie.objects.find(data, user)
#group by only allows sorting by name or number of itmes #group by only allows sorting by name or number of itmes
return query return query
'''
def _parse_query(data, user):
query = {}
query['i'] = 0
query['o'] = 100
query['s'] = 'title:asc'
def parse_dict(s):
d = s.split(",")
return [i.strip() for i in d]
_dicts = ['p', ]
_ints = ['n', ]
for key in ('s', 'p', 'g', 'l', 'n'):
if key in data:
if key in _ints:
query[key] = int(data[key])
elif key in _dicts:
query[key] = parse_dict(data[key])
else:
query[key] = data[key]
print query
query['q'] = models.Movie.objects.find(data, user)
if 'r' in data:
r = data['r'].split(':')
if len(r) == 1: r.append(0)
if r[0] == '': r[0] = 0
if r[1] == '': r[0] = -1
query['i'] = int(r[0])
query['o'] = int(r[1])
#group by only allows sorting by name or number of itmes
return query
'''
def api_find(request): def api_find(request):
''' '''
@ -159,7 +125,7 @@ def api_find(request):
{'query': query, 'sort': string, 'range': array} {'query': query, 'sort': string, 'range': array}
query: query object, more on query syntax at query: query object, more on query syntax at
http://wiki.0xdb.org/wiki/QuerySyntax https://wiki.0x2620.org/wiki/0xdb2QuerySyntax
sort: string or arrays of keys; to sort key in descending order prefix with - sort: string or arrays of keys; to sort key in descending order prefix with -
default: ['director', '-year'] default: ['director', '-year']
range: result range, array [from, to] range: result range, array [from, to]
@ -184,25 +150,25 @@ def api_find(request):
'data': {items=int, files=int, pixels=int, size=int, duration=int}} 'data': {items=int, files=int, pixels=int, size=int, duration=int}}
''' '''
data = json.loads(request.POST['data']) data = json.loads(request.POST['data'])
if settings.JSON_DEBUG:
print json.dumps(data, indent=2)
query = _parse_query(data, request.user) query = _parse_query(data, request.user)
response = json_response({}) response = json_response({})
if 'keys' in query: if 'keys' in query:
response['data']['items'] = [] response['data']['items'] = []
qs = _order_query(query['qs'], query['sort']) qs = _order_query(query['qs'], query['sort'])
if 'n' in query: _p = query['keys']
response = {'items': qs.count()} def only_p(m):
else: r = {}
_p = query['keys'] if m:
def only_p(m): m = json.loads(m)
r = {} for p in _p:
if m: r[p] = m[p]
m = json.loads(m) return r
for p in _p: qs = qs[query['range'][0]:query['range'][1]]
r[p] = m[p]
return r
qs = qs[query['range'][0]:query['range'][1]]
response['data']['items'] = [only_p(m['json']) for m in qs.values('json')] response['data']['items'] = [only_p(m['json']) for m in qs.values('json')]
elif 'group' in query: elif 'group' in query:
if query['s'].split(':')[0] not in ('name', 'items'): if query['s'].split(':')[0] not in ('name', 'items'):
@ -242,7 +208,8 @@ def api_find(request):
else: else:
#FIXME: also filter lists here #FIXME: also filter lists here
movies = models.Movie.objects.filter(available=True) movies = models.Movie.objects.filter(available=True)
files = models.File.objects.all() movies = query['qs']
files = models.File.objects.all().filter(movie__in=movies)
response['data']['items'] = movies.count() response['data']['items'] = movies.count()
response['data']['files'] = files.count() response['data']['files'] = files.count()
r = files.aggregate(Count('size'), Count('pixels'), Count('duration')) r = files.aggregate(Count('size'), Count('pixels'), Count('duration'))