From f6c44a68a7a45fe259b986e9634be2c4ef92dd09 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Sun, 8 Aug 2010 14:39:37 +0200 Subject: [PATCH] migrations, add celery tasks --- .bzrignore | 1 + README | 8 ++ oxdata/lookup/migrations/0002_lookdown.py | 47 ++++++++++++ oxdata/lookup/tasks.py | 19 +++++ oxdata/lookup/views.py | 35 +++++---- oxdata/poster/migrations/0002_reposter.py | 92 +++++++++++++++++++++++ oxdata/poster/models.py | 49 ++++++------ oxdata/poster/tasks.py | 26 +++++++ oxdata/poster/urls.py | 3 +- oxdata/poster/views.py | 21 +----- oxdata/settings.py | 8 ++ requirements.txt | 5 +- 12 files changed, 251 insertions(+), 63 deletions(-) create mode 100644 oxdata/lookup/migrations/0002_lookdown.py create mode 100644 oxdata/lookup/tasks.py create mode 100644 oxdata/poster/migrations/0002_reposter.py create mode 100644 oxdata/poster/tasks.py diff --git a/.bzrignore b/.bzrignore index cab8e2d..9179d34 100644 --- a/.bzrignore +++ b/.bzrignore @@ -9,3 +9,4 @@ bin lib src include +local_urls.py diff --git a/README b/README index f92000f..905efe5 100644 --- a/README +++ b/README @@ -15,3 +15,11 @@ also it uses system python moduels, so make sure you have ipython, dbmodule(psycopg2, python-mysqldb,..) installed via apt-get/pip/easy_install in your system path. +Install rabbitmq and carrot: + sudo apt-get install rabbitmq-server + + sudo rabbitmqctl add_user oxdata ox + sudo rabbitmqctl add_vhost /oxdata + sudo rabbitmqctl set_permissions -p /oxdata oxdata ".*" ".*" ".*" + + diff --git a/oxdata/lookup/migrations/0002_lookdown.py b/oxdata/lookup/migrations/0002_lookdown.py new file mode 100644 index 0000000..be66839 --- /dev/null +++ b/oxdata/lookup/migrations/0002_lookdown.py @@ -0,0 +1,47 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Deleting model 'karagarga' + db.delete_table('lookup_karagarga') + + def backwards(self, orm): + + # Adding model 'karagarga' + db.create_table('lookup_karagarga', ( + ('movie_id', self.gf('django.db.models.fields.related.ForeignKey')(default=None, related_name='karagarga_ids', to=orm['lookup.MovieId'])), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('karagarga_id', self.gf('django.db.models.fields.IntegerField')(unique=True)), + )) + db.send_create_signal('lookup', ['karagarga']) + + + models = { + 'lookup.movieid': { + 'Meta': {'object_name': 'MovieId'}, + 'amg_id': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'criterion_id': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'director': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}), + 'episode': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'episode_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'imdb_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '7', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'impawards_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'oxdb_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '42', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'season': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'series_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}), + 'wikipedia_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'year': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '4', 'blank': 'True'}) + } + } + + complete_apps = ['lookup'] diff --git a/oxdata/lookup/tasks.py b/oxdata/lookup/tasks.py new file mode 100644 index 0000000..e5cd624 --- /dev/null +++ b/oxdata/lookup/tasks.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from datetime import timedelta + +from celery.decorators import task, periodic_task + +import models +import cache + +@periodic_task(run_every=timedelta(days=1)) +def cronjob(**kwargs): + print "do some cleanup stuff once a day" + cache.getIds() + +@task(ignore_resulsts=True, queue='default') +def updateImdb(id): + movie_id = models.MovieId.objects.get(pk=id) + movie_id.updateFromImdb() + diff --git a/oxdata/lookup/views.py b/oxdata/lookup/views.py index b76d7d4..722a215 100644 --- a/oxdata/lookup/views.py +++ b/oxdata/lookup/views.py @@ -9,22 +9,31 @@ from oxdjango.shortcuts import render_to_json_response import models +def get_movie_id(request): + movie_id = None + if 'movieId' in request.GET: + movieId = request.GET['movieId'] + if len(movieId) == 7: + movie_id = models.MovieId.objects.get(imdb_id=movieId) + else: + movie_id = models.MovieId.objects.get(oxdb_id=movieId) + if 'imdb' in request.GET: + imdbId = request.GET['imdb'] + movie_id = models.MovieId.objects.get(imdb_id=imdbId) + elif 'oxdb' in request.GET: + oxdbId = request.GET['oxdb'] + movie_id = models.MovieId.objects.get(oxdb_id=oxdbId) + elif 'criterion' in request.GET: + criterion_id = request.GET['criterion'] + movie_id = models.MovieId.objects.get(criterion_id=criterion_id) + if movie_id: + movie_id.updateFromImdb() + return movie_id + def ids(request): json = {} - movie = None - if 'imdb' in request.GET: - imdb_id = request.GET['imdb'] - movie = models.getMovieIdByImdbId(imdb_id) - elif 'oxdb' in request.GET: - try: - movie = models.MovieId.objects.get(oxdb_id=request.GET['oxdb']) - except models.MovieId.DoesNotExist: - pass - if 'criterion' in request.GET: - criterion_id = request.GET['criterion'] - movie = models.MovieId.objects.get(criterion_id=criterion_id) + movie = get_movie_id(request) if movie: - movie.updateFromImdb() json = movie.json() return render_to_json_response(json) diff --git a/oxdata/poster/migrations/0002_reposter.py b/oxdata/poster/migrations/0002_reposter.py new file mode 100644 index 0000000..0236f63 --- /dev/null +++ b/oxdata/poster/migrations/0002_reposter.py @@ -0,0 +1,92 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding field 'PosterCache.status' + db.add_column('poster_postercache', 'status', self.gf('django.db.models.fields.CharField')(default='200', max_length=1024), keep_default=False) + + # Deleting field 'poster.upload' + db.delete_column('poster_poster', 'upload') + + # Deleting field 'poster.oxdb' + db.delete_column('poster_poster', 'oxdb') + + # Renaming column for 'Poster.poster' to match new field type. + db.rename_column('poster_poster', 'poster_id', 'poster') + # Changing field 'Poster.poster' + db.add_column('poster_poster', 'poster', self.gf('django.db.models.fields.files.ImageField')(max_length=255)) + + # Removing index on 'poster', fields ['poster'] + db.delete_index('poster_poster', ['poster_id']) + + + def backwards(self, orm): + + # Deleting field 'PosterCache.status' + db.delete_column('poster_postercache', 'status') + + # Adding field 'poster.upload' + db.add_column('poster_poster', 'upload', self.gf('django.db.models.fields.files.ImageField')(default=None, max_length=255), keep_default=False) + + # Adding field 'poster.oxdb' + db.add_column('poster_poster', 'oxdb', self.gf('django.db.models.fields.files.ImageField')(default=None, max_length=255), keep_default=False) + + # Renaming column for 'Poster.poster' to match new field type. + db.rename_column('poster_poster', 'poster', 'poster_id') + # Changing field 'Poster.poster' + db.alter_column('poster_poster', 'poster_id', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['poster.PosterCache'], blank=True)) + + # Adding index on 'poster', fields ['poster'] + db.create_index('poster_poster', ['poster_id']) + + + models = { + 'lookup.movieid': { + 'Meta': {'object_name': 'MovieId'}, + 'amg_id': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'criterion_id': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'director': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}), + 'episode': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'episode_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'imdb_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '7', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'impawards_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'oxdb_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '42', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'season': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'series_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}), + 'wikipedia_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'year': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '4', 'blank': 'True'}) + }, + 'poster.poster': { + 'Meta': {'object_name': 'Poster'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'movie_id': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'poster'", 'to': "orm['lookup.MovieId']"}), + 'poster': ('django.db.models.fields.files.ImageField', [], {'max_length': '255'}) + }, + 'poster.postercache': { + 'Meta': {'object_name': 'PosterCache'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'failed': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'image': ('django.db.models.fields.files.ImageField', [], {'max_length': '255'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'movie_id': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'postercache'", 'to': "orm['lookup.MovieId']"}), + 'site': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'site_id': ('django.db.models.fields.CharField', [], {'max_length': '42'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'200'", 'max_length': '1024'}), + 'url': ('django.db.models.fields.CharField', [], {'max_length': '1024'}) + } + } + + complete_apps = ['poster'] diff --git a/oxdata/poster/models.py b/oxdata/poster/models.py index ccf7885..e495a18 100644 --- a/oxdata/poster/models.py +++ b/oxdata/poster/models.py @@ -3,6 +3,7 @@ import os.path import hashlib +from django.conf import settings from django.db import models from django.db.models import Q from django.contrib.auth.models import User @@ -15,19 +16,20 @@ import ox.web.impawards from oxdata.lookup.models import MovieId -def getPosters(movie_id): +def getPosters(movie_id, url_prefix='', limit=lambda x, y: x still.height: - stills[p.site].append(pjson) - for p in stills.keys(): - if not stills[p]: - del stills[p] - return stills - def poster_path(url, filename): - url_hash = hashlib.sha1(url).hexdigest() + h = hashlib.sha1(url).hexdigest() ext = 'jpg' if filename.endswith('.png'): ext = 'png' - name = "%s.%s" % (url_hash, ext) - return os.path.join('posters', url_hash[:2], url_hash[2:4], url_hash[4:6], name) + name = "%s.%s" % (h, ext) + return os.path.join('posters', h[:2], h[2:4], h[4:6], name) class PosterCache(models.Model): created = models.DateTimeField(auto_now_add=True) @@ -112,26 +95,36 @@ def getPosterUrls(m): p.save() if m.imdb_id: + if settings.DEBUG: + print 'imdb' poster = ox.web.imdb.getMoviePoster(m.imdb_id) if poster: addPoster(poster, 'imdb.com', m.imdb_id) - for poster in ox.web.movieposterdb.getData(m.imdb_id)['posters']: - addPoster(poster, 'movieposterdb.com', m.imdb_id) + + #site is sometimes down + #for poster in ox.web.movieposterdb.getData(m.imdb_id)['posters']: + # addPoster(poster, 'movieposterdb.com', m.imdb_id) if m.criterion_id: - print 'criterion', m.criterion_id + if settings.DEBUG: + print 'criterion', m.criterion_id for poster in ox.web.criterion.getData(m.criterion_id)['posters']: addPoster(poster, 'criterion.com', m.criterion_id) if m.wikipedia_id: + if settings.DEBUG: + print 'wikipedia' poster = ox.web.wikipedia.getPosterUrl(m.wikipedia_id) if poster: if PosterCache.objects.all().filter(url=poster).count() == 0: addPoster(poster, 'wikipedia.org', m.wikipedia_id) if m.impawards_id: + if settings.DEBUG: + print 'impawards' data = ox.web.impawards.getData(m.impawards_id) if data and 'imdbId' in data: for poster in data['posters']: addPoster(poster, 'impawards.com', m.imdb_id) + print 'done' #fixme: get 0xdb still, possibly imdb still as fallback? diff --git a/oxdata/poster/tasks.py b/oxdata/poster/tasks.py new file mode 100644 index 0000000..cbd1535 --- /dev/null +++ b/oxdata/poster/tasks.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from datetime import timedelta + +from celery.decorators import task, periodic_task + +import models +from lookup.models import getMovieIdByImdbId + + +''' +@periodic_task(run_every=timedelta(days=1)) +def cronjob(**kwargs): + print "do some cleanup stuff once a day" +''' + +@task(ignore_resulsts=True, queue='default') +def getMovieposteredb(imdb_id): + m = getMovieIdByImdbId(imdb_id) + def addPoster(url, site, site_id): + if PosterCache.objects.all().filter(url=url).count() == 0: + p = PosterCache(url=url, site=site, site_id=site_id, movie_id=m) + p.save() + for poster in ox.web.movieposterdb.getData(imdb_id)['posters']: + addPoster(poster, 'movieposterdb.com', imdb_id) + diff --git a/oxdata/poster/urls.py b/oxdata/poster/urls.py index 3cb48c5..b9d0d22 100644 --- a/oxdata/poster/urls.py +++ b/oxdata/poster/urls.py @@ -1,7 +1,6 @@ from django.conf.urls.defaults import * urlpatterns = patterns('oxdata.poster.views', - (r'^$', 'poster_json'), - (r'^.html$', 'poster_html'), + (r'^$', 'poster'), ) diff --git a/oxdata/poster/views.py b/oxdata/poster/views.py index 4e1c1e3..a76c8b6 100644 --- a/oxdata/poster/views.py +++ b/oxdata/poster/views.py @@ -10,30 +10,17 @@ from django.template import RequestContext from oxdjango.shortcuts import render_to_json_response from oxdata.lookup.models import MovieId -import models +from oxdata.lookup.views import get_movie_id -def get_movie_id(request): - if 'movieId' in request.GET: - movieId = request.GET['movieId'] - if len(movieId) == 7: - movie_id = MovieId.objects.get(imdb_id=imdbId) - else: - movie_id = MovieId.objects.get(oxdb_id=oxdbId) - if 'imdb' in request.GET: - imdbId = request.GET['imdb'] - movie_id = MovieId.objects.get(imdb_id=imdbId) - elif 'oxdb' in request.GET: - oxdbId = request.GET['oxdb'] - movie_id = MovieId.objects.get(oxdb_id=oxdbId) - return movie_id +import models def poster(request): movie_id = get_movie_id(request) - json = models.getPosters(movie_id) + json = models.getPosters(movie_id, request.build_absolute_uri('/')) return render_to_json_response(json) def still(request): movie_id = get_movie_id(request) - json = models.getMovieStills(movie_id) + json = models.getPosters(movie_id, request.build_absolute_uri('/'), limit=lambda x, y: x>y) return render_to_json_response(json) diff --git a/oxdata/settings.py b/oxdata/settings.py index 90307ff..5c0d51a 100644 --- a/oxdata/settings.py +++ b/oxdata/settings.py @@ -86,6 +86,7 @@ INSTALLED_APPS = ( 'django.contrib.admin', 'django.contrib.humanize', 'south', + 'djcelery', 'oxdata.lookup', 'oxdata.poster', @@ -94,6 +95,13 @@ INSTALLED_APPS = ( LOGIN_REDIRECT_URL='/' +CELERY_RESULT_BACKEND = "database" +BROKER_HOST = "127.0.0.1" +BROKER_PORT = 5672 +BROKER_USER = "oxdata" +BROKER_PASSWORD = "ox" +BROKER_VHOST = "/oxdata" + #overwrite default settings with local settings try: from local_settings import * diff --git a/requirements.txt b/requirements.txt index be56740..d612bbf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ -e svn+http://code.djangoproject.com/svn/django/branches/releases/1.2.X/#egg=django -South==0.6.1 +South -e bzr+http://code.0x2620.org/python-oxdjango/#egg=python-oxdjango -e bzr+http://code.0x2620.org/python-ox/#egg=python-ox - - +django-celery