migrations, add celery tasks

This commit is contained in:
j 2010-08-08 14:39:37 +02:00
parent 7b74c5f2ec
commit f6c44a68a7
12 changed files with 251 additions and 63 deletions

View file

@ -9,3 +9,4 @@ bin
lib lib
src src
include include
local_urls.py

8
README
View file

@ -15,3 +15,11 @@ also it uses system python moduels, so make sure you have
ipython, dbmodule(psycopg2, python-mysqldb,..) ipython, dbmodule(psycopg2, python-mysqldb,..)
installed via apt-get/pip/easy_install in your system path. installed via apt-get/pip/easy_install in your system path.
Install rabbitmq and carrot:
sudo apt-get install rabbitmq-server
sudo rabbitmqctl add_user oxdata ox
sudo rabbitmqctl add_vhost /oxdata
sudo rabbitmqctl set_permissions -p /oxdata oxdata ".*" ".*" ".*"

View file

@ -0,0 +1,47 @@
# encoding: utf-8
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Deleting model 'karagarga'
db.delete_table('lookup_karagarga')
def backwards(self, orm):
# Adding model 'karagarga'
db.create_table('lookup_karagarga', (
('movie_id', self.gf('django.db.models.fields.related.ForeignKey')(default=None, related_name='karagarga_ids', to=orm['lookup.MovieId'])),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('karagarga_id', self.gf('django.db.models.fields.IntegerField')(unique=True)),
))
db.send_create_signal('lookup', ['karagarga'])
models = {
'lookup.movieid': {
'Meta': {'object_name': 'MovieId'},
'amg_id': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'criterion_id': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'director': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
'episode': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
'episode_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'imdb_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '7', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'impawards_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'oxdb_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '42', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'season': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
'series_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
'title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
'wikipedia_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'year': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '4', 'blank': 'True'})
}
}
complete_apps = ['lookup']

19
oxdata/lookup/tasks.py Normal file
View file

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from datetime import timedelta
from celery.decorators import task, periodic_task
import models
import cache
@periodic_task(run_every=timedelta(days=1))
def cronjob(**kwargs):
print "do some cleanup stuff once a day"
cache.getIds()
@task(ignore_resulsts=True, queue='default')
def updateImdb(id):
movie_id = models.MovieId.objects.get(pk=id)
movie_id.updateFromImdb()

View file

@ -9,22 +9,31 @@ from oxdjango.shortcuts import render_to_json_response
import models import models
def get_movie_id(request):
movie_id = None
if 'movieId' in request.GET:
movieId = request.GET['movieId']
if len(movieId) == 7:
movie_id = models.MovieId.objects.get(imdb_id=movieId)
else:
movie_id = models.MovieId.objects.get(oxdb_id=movieId)
if 'imdb' in request.GET:
imdbId = request.GET['imdb']
movie_id = models.MovieId.objects.get(imdb_id=imdbId)
elif 'oxdb' in request.GET:
oxdbId = request.GET['oxdb']
movie_id = models.MovieId.objects.get(oxdb_id=oxdbId)
elif 'criterion' in request.GET:
criterion_id = request.GET['criterion']
movie_id = models.MovieId.objects.get(criterion_id=criterion_id)
if movie_id:
movie_id.updateFromImdb()
return movie_id
def ids(request): def ids(request):
json = {} json = {}
movie = None movie = get_movie_id(request)
if 'imdb' in request.GET:
imdb_id = request.GET['imdb']
movie = models.getMovieIdByImdbId(imdb_id)
elif 'oxdb' in request.GET:
try:
movie = models.MovieId.objects.get(oxdb_id=request.GET['oxdb'])
except models.MovieId.DoesNotExist:
pass
if 'criterion' in request.GET:
criterion_id = request.GET['criterion']
movie = models.MovieId.objects.get(criterion_id=criterion_id)
if movie: if movie:
movie.updateFromImdb()
json = movie.json() json = movie.json()
return render_to_json_response(json) return render_to_json_response(json)

View file

@ -0,0 +1,92 @@
# encoding: utf-8
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'PosterCache.status'
db.add_column('poster_postercache', 'status', self.gf('django.db.models.fields.CharField')(default='200', max_length=1024), keep_default=False)
# Deleting field 'poster.upload'
db.delete_column('poster_poster', 'upload')
# Deleting field 'poster.oxdb'
db.delete_column('poster_poster', 'oxdb')
# Renaming column for 'Poster.poster' to match new field type.
db.rename_column('poster_poster', 'poster_id', 'poster')
# Changing field 'Poster.poster'
db.add_column('poster_poster', 'poster', self.gf('django.db.models.fields.files.ImageField')(max_length=255))
# Removing index on 'poster', fields ['poster']
db.delete_index('poster_poster', ['poster_id'])
def backwards(self, orm):
# Deleting field 'PosterCache.status'
db.delete_column('poster_postercache', 'status')
# Adding field 'poster.upload'
db.add_column('poster_poster', 'upload', self.gf('django.db.models.fields.files.ImageField')(default=None, max_length=255), keep_default=False)
# Adding field 'poster.oxdb'
db.add_column('poster_poster', 'oxdb', self.gf('django.db.models.fields.files.ImageField')(default=None, max_length=255), keep_default=False)
# Renaming column for 'Poster.poster' to match new field type.
db.rename_column('poster_poster', 'poster', 'poster_id')
# Changing field 'Poster.poster'
db.alter_column('poster_poster', 'poster_id', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['poster.PosterCache'], blank=True))
# Adding index on 'poster', fields ['poster']
db.create_index('poster_poster', ['poster_id'])
models = {
'lookup.movieid': {
'Meta': {'object_name': 'MovieId'},
'amg_id': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'criterion_id': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'director': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
'episode': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
'episode_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'imdb_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '7', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'impawards_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'oxdb_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '42', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'season': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
'series_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
'title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
'wikipedia_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'year': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '4', 'blank': 'True'})
},
'poster.poster': {
'Meta': {'object_name': 'Poster'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'movie_id': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'poster'", 'to': "orm['lookup.MovieId']"}),
'poster': ('django.db.models.fields.files.ImageField', [], {'max_length': '255'})
},
'poster.postercache': {
'Meta': {'object_name': 'PosterCache'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'failed': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'image': ('django.db.models.fields.files.ImageField', [], {'max_length': '255'}),
'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'movie_id': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'postercache'", 'to': "orm['lookup.MovieId']"}),
'site': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'site_id': ('django.db.models.fields.CharField', [], {'max_length': '42'}),
'status': ('django.db.models.fields.CharField', [], {'default': "'200'", 'max_length': '1024'}),
'url': ('django.db.models.fields.CharField', [], {'max_length': '1024'})
}
}
complete_apps = ['poster']

View file

@ -3,6 +3,7 @@
import os.path import os.path
import hashlib import hashlib
from django.conf import settings
from django.db import models from django.db import models
from django.db.models import Q from django.db.models import Q
from django.contrib.auth.models import User from django.contrib.auth.models import User
@ -15,19 +16,20 @@ import ox.web.impawards
from oxdata.lookup.models import MovieId from oxdata.lookup.models import MovieId
def getPosters(movie_id): def getPosters(movie_id, url_prefix='', limit=lambda x, y: x<y):
getPosterUrls(movie_id) getPosterUrls(movie_id)
posters = {} posters = {}
if url_prefix.endswith('/'): url_prefix = url_prefix[:-1]
for p in PosterCache.objects.all().filter(movie_id=movie_id, failed=False).order_by('id'): for p in PosterCache.objects.all().filter(movie_id=movie_id, failed=False).order_by('id'):
if p.site not in posters: if p.site not in posters:
posters[p.site] = [] posters[p.site] = []
poster = p.get() poster = p.get()
if poster: if poster:
pjson = {} pjson = {}
pjson['url'] = poster._get_url() pjson['url'] = url_prefix + poster._get_url()
pjson['width'] = poster.width pjson['width'] = poster.width
pjson['height'] = poster.height pjson['height'] = poster.height
if poster.width < poster.height: if limit(poster.width, poster.height):
posters[p.site].append(pjson) posters[p.site].append(pjson)
for p in posters.keys(): for p in posters.keys():
if not posters[p]: if not posters[p]:
@ -37,32 +39,13 @@ def getPosters(movie_id):
posters['local'] = [p.poster.url] posters['local'] = [p.poster.url]
return posters return posters
def getMovieStills(movie_id):
getPosterUrls(movie_id)
stills = {}
for p in PosterCache.objects.all().filter(movie_id=movie_id, failed=False).order_by('id'):
if p.site not in stills:
stills[p.site] = []
still = p.get()
if still:
pjson = {}
pjson['url'] = still._get_url()
pjson['width'] = still.width
pjson['height'] = still.height
if still.width > still.height:
stills[p.site].append(pjson)
for p in stills.keys():
if not stills[p]:
del stills[p]
return stills
def poster_path(url, filename): def poster_path(url, filename):
url_hash = hashlib.sha1(url).hexdigest() h = hashlib.sha1(url).hexdigest()
ext = 'jpg' ext = 'jpg'
if filename.endswith('.png'): if filename.endswith('.png'):
ext = 'png' ext = 'png'
name = "%s.%s" % (url_hash, ext) name = "%s.%s" % (h, ext)
return os.path.join('posters', url_hash[:2], url_hash[2:4], url_hash[4:6], name) return os.path.join('posters', h[:2], h[2:4], h[4:6], name)
class PosterCache(models.Model): class PosterCache(models.Model):
created = models.DateTimeField(auto_now_add=True) created = models.DateTimeField(auto_now_add=True)
@ -112,26 +95,36 @@ def getPosterUrls(m):
p.save() p.save()
if m.imdb_id: if m.imdb_id:
if settings.DEBUG:
print 'imdb'
poster = ox.web.imdb.getMoviePoster(m.imdb_id) poster = ox.web.imdb.getMoviePoster(m.imdb_id)
if poster: if poster:
addPoster(poster, 'imdb.com', m.imdb_id) addPoster(poster, 'imdb.com', m.imdb_id)
for poster in ox.web.movieposterdb.getData(m.imdb_id)['posters']:
addPoster(poster, 'movieposterdb.com', m.imdb_id) #site is sometimes down
#for poster in ox.web.movieposterdb.getData(m.imdb_id)['posters']:
# addPoster(poster, 'movieposterdb.com', m.imdb_id)
if m.criterion_id: if m.criterion_id:
if settings.DEBUG:
print 'criterion', m.criterion_id print 'criterion', m.criterion_id
for poster in ox.web.criterion.getData(m.criterion_id)['posters']: for poster in ox.web.criterion.getData(m.criterion_id)['posters']:
addPoster(poster, 'criterion.com', m.criterion_id) addPoster(poster, 'criterion.com', m.criterion_id)
if m.wikipedia_id: if m.wikipedia_id:
if settings.DEBUG:
print 'wikipedia'
poster = ox.web.wikipedia.getPosterUrl(m.wikipedia_id) poster = ox.web.wikipedia.getPosterUrl(m.wikipedia_id)
if poster: if poster:
if PosterCache.objects.all().filter(url=poster).count() == 0: if PosterCache.objects.all().filter(url=poster).count() == 0:
addPoster(poster, 'wikipedia.org', m.wikipedia_id) addPoster(poster, 'wikipedia.org', m.wikipedia_id)
if m.impawards_id: if m.impawards_id:
if settings.DEBUG:
print 'impawards'
data = ox.web.impawards.getData(m.impawards_id) data = ox.web.impawards.getData(m.impawards_id)
if data and 'imdbId' in data: if data and 'imdbId' in data:
for poster in data['posters']: for poster in data['posters']:
addPoster(poster, 'impawards.com', m.imdb_id) addPoster(poster, 'impawards.com', m.imdb_id)
print 'done'
#fixme: get 0xdb still, possibly imdb still as fallback? #fixme: get 0xdb still, possibly imdb still as fallback?

26
oxdata/poster/tasks.py Normal file
View file

@ -0,0 +1,26 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from datetime import timedelta
from celery.decorators import task, periodic_task
import models
from lookup.models import getMovieIdByImdbId
'''
@periodic_task(run_every=timedelta(days=1))
def cronjob(**kwargs):
print "do some cleanup stuff once a day"
'''
@task(ignore_resulsts=True, queue='default')
def getMovieposteredb(imdb_id):
m = getMovieIdByImdbId(imdb_id)
def addPoster(url, site, site_id):
if PosterCache.objects.all().filter(url=url).count() == 0:
p = PosterCache(url=url, site=site, site_id=site_id, movie_id=m)
p.save()
for poster in ox.web.movieposterdb.getData(imdb_id)['posters']:
addPoster(poster, 'movieposterdb.com', imdb_id)

View file

@ -1,7 +1,6 @@
from django.conf.urls.defaults import * from django.conf.urls.defaults import *
urlpatterns = patterns('oxdata.poster.views', urlpatterns = patterns('oxdata.poster.views',
(r'^$', 'poster_json'), (r'^$', 'poster'),
(r'^.html$', 'poster_html'),
) )

View file

@ -10,30 +10,17 @@ from django.template import RequestContext
from oxdjango.shortcuts import render_to_json_response from oxdjango.shortcuts import render_to_json_response
from oxdata.lookup.models import MovieId from oxdata.lookup.models import MovieId
import models from oxdata.lookup.views import get_movie_id
def get_movie_id(request): import models
if 'movieId' in request.GET:
movieId = request.GET['movieId']
if len(movieId) == 7:
movie_id = MovieId.objects.get(imdb_id=imdbId)
else:
movie_id = MovieId.objects.get(oxdb_id=oxdbId)
if 'imdb' in request.GET:
imdbId = request.GET['imdb']
movie_id = MovieId.objects.get(imdb_id=imdbId)
elif 'oxdb' in request.GET:
oxdbId = request.GET['oxdb']
movie_id = MovieId.objects.get(oxdb_id=oxdbId)
return movie_id
def poster(request): def poster(request):
movie_id = get_movie_id(request) movie_id = get_movie_id(request)
json = models.getPosters(movie_id) json = models.getPosters(movie_id, request.build_absolute_uri('/'))
return render_to_json_response(json) return render_to_json_response(json)
def still(request): def still(request):
movie_id = get_movie_id(request) movie_id = get_movie_id(request)
json = models.getMovieStills(movie_id) json = models.getPosters(movie_id, request.build_absolute_uri('/'), limit=lambda x, y: x>y)
return render_to_json_response(json) return render_to_json_response(json)

View file

@ -86,6 +86,7 @@ INSTALLED_APPS = (
'django.contrib.admin', 'django.contrib.admin',
'django.contrib.humanize', 'django.contrib.humanize',
'south', 'south',
'djcelery',
'oxdata.lookup', 'oxdata.lookup',
'oxdata.poster', 'oxdata.poster',
@ -94,6 +95,13 @@ INSTALLED_APPS = (
LOGIN_REDIRECT_URL='/' LOGIN_REDIRECT_URL='/'
CELERY_RESULT_BACKEND = "database"
BROKER_HOST = "127.0.0.1"
BROKER_PORT = 5672
BROKER_USER = "oxdata"
BROKER_PASSWORD = "ox"
BROKER_VHOST = "/oxdata"
#overwrite default settings with local settings #overwrite default settings with local settings
try: try:
from local_settings import * from local_settings import *

View file

@ -1,6 +1,5 @@
-e svn+http://code.djangoproject.com/svn/django/branches/releases/1.2.X/#egg=django -e svn+http://code.djangoproject.com/svn/django/branches/releases/1.2.X/#egg=django
South==0.6.1 South
-e bzr+http://code.0x2620.org/python-oxdjango/#egg=python-oxdjango -e bzr+http://code.0x2620.org/python-oxdjango/#egg=python-oxdjango
-e bzr+http://code.0x2620.org/python-ox/#egg=python-ox -e bzr+http://code.0x2620.org/python-ox/#egg=python-ox
django-celery