0x2620/oxdata
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

migrations, add celery tasks

Browse source
This commit is contained in:
j 2010-08-08 14:39:37 +02:00
parent 7b74c5f2ec
commit f6c44a68a7
12 changed files with 251 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.bzrignore
View file

@ -9,3 +9,4 @@ bin
lib
src
include
local_urls.py

8
README
View file

@ -15,3 +15,11 @@ also it uses system python moduels, so make sure you have
ipython, dbmodule(psycopg2, python-mysqldb,..)
installed via apt-get/pip/easy_install in your system path.
Install rabbitmq and carrot:
sudo apt-get install rabbitmq-server
sudo rabbitmqctl add_user oxdata ox
sudo rabbitmqctl add_vhost /oxdata
sudo rabbitmqctl set_permissions -p /oxdata oxdata ".*" ".*" ".*"

47
oxdata/lookup/migrations/0002_lookdown.py Normal file
View file

@ -0,0 +1,47 @@
# encoding: utf-8
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Deleting model 'karagarga'
db.delete_table('lookup_karagarga')
def backwards(self, orm):
# Adding model 'karagarga'
db.create_table('lookup_karagarga', (
('movie_id', self.gf('django.db.models.fields.related.ForeignKey')(default=None, related_name='karagarga_ids', to=orm['lookup.MovieId'])),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('karagarga_id', self.gf('django.db.models.fields.IntegerField')(unique=True)),
))
db.send_create_signal('lookup', ['karagarga'])
models = {
'lookup.movieid': {
'Meta': {'object_name': 'MovieId'},
'amg_id': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'criterion_id': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'director': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
'episode': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
'episode_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'imdb_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '7', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'impawards_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'oxdb_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '42', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'season': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
'series_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
'title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
'wikipedia_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'year': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '4', 'blank': 'True'})
}
}
complete_apps = ['lookup']

19
oxdata/lookup/tasks.py Normal file
View file

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from datetime import timedelta
from celery.decorators import task, periodic_task
import models
import cache
@periodic_task(run_every=timedelta(days=1))
def cronjob(**kwargs):
print "do some cleanup stuff once a day"
cache.getIds()
@task(ignore_resulsts=True, queue='default')
def updateImdb(id):
movie_id = models.MovieId.objects.get(pk=id)
movie_id.updateFromImdb()

35
oxdata/lookup/views.py
View file

@ -9,22 +9,31 @@ from oxdjango.shortcuts import render_to_json_response
import models
def get_movie_id(request):
movie_id = None
if 'movieId' in request.GET:
movieId = request.GET['movieId']
if len(movieId) == 7:
movie_id = models.MovieId.objects.get(imdb_id=movieId)
else:
movie_id = models.MovieId.objects.get(oxdb_id=movieId)
if 'imdb' in request.GET:
imdbId = request.GET['imdb']
movie_id = models.MovieId.objects.get(imdb_id=imdbId)
elif 'oxdb' in request.GET:
oxdbId = request.GET['oxdb']
movie_id = models.MovieId.objects.get(oxdb_id=oxdbId)
elif 'criterion' in request.GET:
criterion_id = request.GET['criterion']
movie_id = models.MovieId.objects.get(criterion_id=criterion_id)
if movie_id:
movie_id.updateFromImdb()
return movie_id
def ids(request):
json = {}
movie = None
if 'imdb' in request.GET:
imdb_id = request.GET['imdb']
movie = models.getMovieIdByImdbId(imdb_id)
elif 'oxdb' in request.GET:
try:
movie = models.MovieId.objects.get(oxdb_id=request.GET['oxdb'])
except models.MovieId.DoesNotExist:
pass
if 'criterion' in request.GET:
criterion_id = request.GET['criterion']
movie = models.MovieId.objects.get(criterion_id=criterion_id)
movie = get_movie_id(request)
if movie:
movie.updateFromImdb()
json = movie.json()
return render_to_json_response(json)

92
oxdata/poster/migrations/0002_reposter.py Normal file
View file

@ -0,0 +1,92 @@
# encoding: utf-8
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'PosterCache.status'
db.add_column('poster_postercache', 'status', self.gf('django.db.models.fields.CharField')(default='200', max_length=1024), keep_default=False)
# Deleting field 'poster.upload'
db.delete_column('poster_poster', 'upload')
# Deleting field 'poster.oxdb'
db.delete_column('poster_poster', 'oxdb')
# Renaming column for 'Poster.poster' to match new field type.
db.rename_column('poster_poster', 'poster_id', 'poster')
# Changing field 'Poster.poster'
db.add_column('poster_poster', 'poster', self.gf('django.db.models.fields.files.ImageField')(max_length=255))
# Removing index on 'poster', fields ['poster']
db.delete_index('poster_poster', ['poster_id'])
def backwards(self, orm):
# Deleting field 'PosterCache.status'
db.delete_column('poster_postercache', 'status')
# Adding field 'poster.upload'
db.add_column('poster_poster', 'upload', self.gf('django.db.models.fields.files.ImageField')(default=None, max_length=255), keep_default=False)
# Adding field 'poster.oxdb'
db.add_column('poster_poster', 'oxdb', self.gf('django.db.models.fields.files.ImageField')(default=None, max_length=255), keep_default=False)
# Renaming column for 'Poster.poster' to match new field type.
db.rename_column('poster_poster', 'poster', 'poster_id')
# Changing field 'Poster.poster'
db.alter_column('poster_poster', 'poster_id', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['poster.PosterCache'], blank=True))
# Adding index on 'poster', fields ['poster']
db.create_index('poster_poster', ['poster_id'])
models = {
'lookup.movieid': {
'Meta': {'object_name': 'MovieId'},
'amg_id': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'criterion_id': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'director': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
'episode': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
'episode_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'imdb_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '7', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'impawards_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'oxdb_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '42', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'season': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
'series_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
'title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
'wikipedia_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
'year': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '4', 'blank': 'True'})
},
'poster.poster': {
'Meta': {'object_name': 'Poster'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'movie_id': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'poster'", 'to': "orm['lookup.MovieId']"}),
'poster': ('django.db.models.fields.files.ImageField', [], {'max_length': '255'})
},
'poster.postercache': {
'Meta': {'object_name': 'PosterCache'},
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'failed': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'image': ('django.db.models.fields.files.ImageField', [], {'max_length': '255'}),
'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
'movie_id': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'postercache'", 'to': "orm['lookup.MovieId']"}),
'site': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'site_id': ('django.db.models.fields.CharField', [], {'max_length': '42'}),
'status': ('django.db.models.fields.CharField', [], {'default': "'200'", 'max_length': '1024'}),
'url': ('django.db.models.fields.CharField', [], {'max_length': '1024'})
}
}
complete_apps = ['poster']

49
oxdata/poster/models.py
View file

@ -3,6 +3,7 @@
import os.path
import hashlib
from django.conf import settings
from django.db import models
from django.db.models import Q
from django.contrib.auth.models import User
@ -15,19 +16,20 @@ import ox.web.impawards
from oxdata.lookup.models import MovieId
def getPosters(movie_id):
def getPosters(movie_id, url_prefix='', limit=lambda x, y: x<y):
getPosterUrls(movie_id)
posters = {}
if url_prefix.endswith('/'): url_prefix = url_prefix[:-1]
for p in PosterCache.objects.all().filter(movie_id=movie_id, failed=False).order_by('id'):
if p.site not in posters:
posters[p.site] = []
poster = p.get()
if poster:
pjson = {}
pjson['url'] = poster._get_url()
pjson['url'] = url_prefix + poster._get_url()
pjson['width'] = poster.width
pjson['height'] = poster.height
if poster.width < poster.height:
if limit(poster.width, poster.height):
posters[p.site].append(pjson)
for p in posters.keys():
if not posters[p]:
@ -37,32 +39,13 @@ def getPosters(movie_id):
posters['local'] = [p.poster.url]
return posters
def getMovieStills(movie_id):
getPosterUrls(movie_id)
stills = {}
for p in PosterCache.objects.all().filter(movie_id=movie_id, failed=False).order_by('id'):
if p.site not in stills:
stills[p.site] = []
still = p.get()
if still:
pjson = {}
pjson['url'] = still._get_url()
pjson['width'] = still.width
pjson['height'] = still.height
if still.width > still.height:
stills[p.site].append(pjson)
for p in stills.keys():
if not stills[p]:
del stills[p]
return stills
def poster_path(url, filename):
url_hash = hashlib.sha1(url).hexdigest()
h = hashlib.sha1(url).hexdigest()
ext = 'jpg'
if filename.endswith('.png'):
ext = 'png'
name = "%s.%s" % (url_hash, ext)
return os.path.join('posters', url_hash[:2], url_hash[2:4], url_hash[4:6], name)
name = "%s.%s" % (h, ext)
return os.path.join('posters', h[:2], h[2:4], h[4:6], name)
class PosterCache(models.Model):
created = models.DateTimeField(auto_now_add=True)
@ -112,26 +95,36 @@ def getPosterUrls(m):
p.save()
if m.imdb_id:
if settings.DEBUG:
print 'imdb'
poster = ox.web.imdb.getMoviePoster(m.imdb_id)
if poster:
addPoster(poster, 'imdb.com', m.imdb_id)
for poster in ox.web.movieposterdb.getData(m.imdb_id)['posters']:
addPoster(poster, 'movieposterdb.com', m.imdb_id)
#site is sometimes down
#for poster in ox.web.movieposterdb.getData(m.imdb_id)['posters']:
# addPoster(poster, 'movieposterdb.com', m.imdb_id)
if m.criterion_id:
print 'criterion', m.criterion_id
if settings.DEBUG:
print 'criterion', m.criterion_id
for poster in ox.web.criterion.getData(m.criterion_id)['posters']:
addPoster(poster, 'criterion.com', m.criterion_id)
if m.wikipedia_id:
if settings.DEBUG:
print 'wikipedia'
poster = ox.web.wikipedia.getPosterUrl(m.wikipedia_id)
if poster:
if PosterCache.objects.all().filter(url=poster).count() == 0:
addPoster(poster, 'wikipedia.org', m.wikipedia_id)
if m.impawards_id:
if settings.DEBUG:
print 'impawards'
data = ox.web.impawards.getData(m.impawards_id)
if data and 'imdbId' in data:
for poster in data['posters']:
addPoster(poster, 'impawards.com', m.imdb_id)
print 'done'
#fixme: get 0xdb still, possibly imdb still as fallback?

26
oxdata/poster/tasks.py Normal file
View file

@ -0,0 +1,26 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from datetime import timedelta
from celery.decorators import task, periodic_task
import models
from lookup.models import getMovieIdByImdbId
'''
@periodic_task(run_every=timedelta(days=1))
def cronjob(**kwargs):
print "do some cleanup stuff once a day"
'''
@task(ignore_resulsts=True, queue='default')
def getMovieposteredb(imdb_id):
m = getMovieIdByImdbId(imdb_id)
def addPoster(url, site, site_id):
if PosterCache.objects.all().filter(url=url).count() == 0:
p = PosterCache(url=url, site=site, site_id=site_id, movie_id=m)
p.save()
for poster in ox.web.movieposterdb.getData(imdb_id)['posters']:
addPoster(poster, 'movieposterdb.com', imdb_id)

3
oxdata/poster/urls.py
View file

@ -1,7 +1,6 @@
from django.conf.urls.defaults import *
urlpatterns = patterns('oxdata.poster.views',
(r'^$', 'poster_json'),
(r'^.html$', 'poster_html'),
(r'^$', 'poster'),
)

21
oxdata/poster/views.py
View file

@ -10,30 +10,17 @@ from django.template import RequestContext
from oxdjango.shortcuts import render_to_json_response
from oxdata.lookup.models import MovieId
import models
from oxdata.lookup.views import get_movie_id
def get_movie_id(request):
if 'movieId' in request.GET:
movieId = request.GET['movieId']
if len(movieId) == 7:
movie_id = MovieId.objects.get(imdb_id=imdbId)
else:
movie_id = MovieId.objects.get(oxdb_id=oxdbId)
if 'imdb' in request.GET:
imdbId = request.GET['imdb']
movie_id = MovieId.objects.get(imdb_id=imdbId)
elif 'oxdb' in request.GET:
oxdbId = request.GET['oxdb']
movie_id = MovieId.objects.get(oxdb_id=oxdbId)
return movie_id
import models
def poster(request):
movie_id = get_movie_id(request)
json = models.getPosters(movie_id)
json = models.getPosters(movie_id, request.build_absolute_uri('/'))
return render_to_json_response(json)
def still(request):
movie_id = get_movie_id(request)
json = models.getMovieStills(movie_id)
json = models.getPosters(movie_id, request.build_absolute_uri('/'), limit=lambda x, y: x>y)
return render_to_json_response(json)

8
oxdata/settings.py
View file

@ -86,6 +86,7 @@ INSTALLED_APPS = (
'django.contrib.admin',
'django.contrib.humanize',
'south',
'djcelery',
'oxdata.lookup',
'oxdata.poster',
@ -94,6 +95,13 @@ INSTALLED_APPS = (
LOGIN_REDIRECT_URL='/'
CELERY_RESULT_BACKEND = "database"
BROKER_HOST = "127.0.0.1"
BROKER_PORT = 5672
BROKER_USER = "oxdata"
BROKER_PASSWORD = "ox"
BROKER_VHOST = "/oxdata"
#overwrite default settings with local settings
try:
from local_settings import *

5
requirements.txt
View file

@ -1,6 +1,5 @@
-e svn+http://code.djangoproject.com/svn/django/branches/releases/1.2.X/#egg=django
South==0.6.1
South
-e bzr+http://code.0x2620.org/python-oxdjango/#egg=python-oxdjango
-e bzr+http://code.0x2620.org/python-ox/#egg=python-ox
django-celery