migrations, add celery tasks

.bzrignore

@@ -9,3 +9,4 @@ bin
 lib
 src
 include
+local_urls.py

README

@@ -15,3 +15,11 @@ also it uses system python moduels, so make sure you have
  ipython, dbmodule(psycopg2, python-mysqldb,..)
 installed via apt-get/pip/easy_install in your system path.
 
+Install rabbitmq and carrot:
+    sudo apt-get install rabbitmq-server
+    
+    sudo rabbitmqctl add_user oxdata ox
+    sudo rabbitmqctl add_vhost /oxdata
+    sudo rabbitmqctl set_permissions -p /oxdata oxdata ".*" ".*" ".*"
+
+

oxdata/lookup/migrations/0002_lookdown.py

@@ -0,0 +1,47 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Deleting model 'karagarga'
+        db.delete_table('lookup_karagarga')
+
+    def backwards(self, orm):
+        
+        # Adding model 'karagarga'
+        db.create_table('lookup_karagarga', (
+            ('movie_id', self.gf('django.db.models.fields.related.ForeignKey')(default=None, related_name='karagarga_ids', to=orm['lookup.MovieId'])),
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('karagarga_id', self.gf('django.db.models.fields.IntegerField')(unique=True)),
+        ))
+        db.send_create_signal('lookup', ['karagarga'])
+
+
+    models = {
+        'lookup.movieid': {
+            'Meta': {'object_name': 'MovieId'},
+            'amg_id': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'criterion_id': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'director': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
+            'episode': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'episode_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'imdb_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '7', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'impawards_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'oxdb_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '42', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'season': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'series_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
+            'wikipedia_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'year': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '4', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['lookup']

oxdata/lookup/tasks.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+from datetime import timedelta
+
+from celery.decorators import task, periodic_task
+
+import models
+import cache
+
+@periodic_task(run_every=timedelta(days=1))
+def cronjob(**kwargs):
+    print "do some cleanup stuff once a day"
+    cache.getIds()
+
+@task(ignore_resulsts=True, queue='default')
+def updateImdb(id):
+    movie_id = models.MovieId.objects.get(pk=id)
+    movie_id.updateFromImdb()
+

oxdata/lookup/views.py

@@ -9,22 +9,31 @@ from oxdjango.shortcuts import render_to_json_response
 
 import models
 
+def get_movie_id(request):
+    movie_id = None
+    if 'movieId' in request.GET:
+        movieId = request.GET['movieId']
+        if len(movieId) == 7:
+            movie_id = models.MovieId.objects.get(imdb_id=movieId)
+        else:
+            movie_id = models.MovieId.objects.get(oxdb_id=movieId)
+    if 'imdb' in request.GET:
+        imdbId = request.GET['imdb']
+        movie_id = models.MovieId.objects.get(imdb_id=imdbId)
+    elif 'oxdb' in request.GET:
+        oxdbId = request.GET['oxdb']
+        movie_id = models.MovieId.objects.get(oxdb_id=oxdbId)
+    elif 'criterion' in request.GET:
+        criterion_id = request.GET['criterion']
+        movie_id = models.MovieId.objects.get(criterion_id=criterion_id)
+    if movie_id:
+        movie_id.updateFromImdb()
+    return movie_id
+
 def ids(request):
     json = {}
-    movie = None
+    movie = get_movie_id(request)
-    if 'imdb' in request.GET:
-        imdb_id = request.GET['imdb']
-        movie = models.getMovieIdByImdbId(imdb_id)
-    elif 'oxdb' in request.GET:
-        try:
-            movie = models.MovieId.objects.get(oxdb_id=request.GET['oxdb'])
-        except models.MovieId.DoesNotExist:
-            pass
-    if 'criterion' in request.GET:
-        criterion_id = request.GET['criterion']
-        movie = models.MovieId.objects.get(criterion_id=criterion_id)
     if movie:
-        movie.updateFromImdb()
         json = movie.json()
     return render_to_json_response(json)
 

oxdata/poster/migrations/0002_reposter.py

@@ -0,0 +1,92 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding field 'PosterCache.status'
+        db.add_column('poster_postercache', 'status', self.gf('django.db.models.fields.CharField')(default='200', max_length=1024), keep_default=False)
+
+        # Deleting field 'poster.upload'
+        db.delete_column('poster_poster', 'upload')
+
+        # Deleting field 'poster.oxdb'
+        db.delete_column('poster_poster', 'oxdb')
+
+        # Renaming column for 'Poster.poster' to match new field type.
+        db.rename_column('poster_poster', 'poster_id', 'poster')
+        # Changing field 'Poster.poster'
+        db.add_column('poster_poster', 'poster', self.gf('django.db.models.fields.files.ImageField')(max_length=255))
+
+        # Removing index on 'poster', fields ['poster']
+        db.delete_index('poster_poster', ['poster_id'])
+
+
+    def backwards(self, orm):
+        
+        # Deleting field 'PosterCache.status'
+        db.delete_column('poster_postercache', 'status')
+
+        # Adding field 'poster.upload'
+        db.add_column('poster_poster', 'upload', self.gf('django.db.models.fields.files.ImageField')(default=None, max_length=255), keep_default=False)
+
+        # Adding field 'poster.oxdb'
+        db.add_column('poster_poster', 'oxdb', self.gf('django.db.models.fields.files.ImageField')(default=None, max_length=255), keep_default=False)
+
+        # Renaming column for 'Poster.poster' to match new field type.
+        db.rename_column('poster_poster', 'poster', 'poster_id')
+        # Changing field 'Poster.poster'
+        db.alter_column('poster_poster', 'poster_id', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['poster.PosterCache'], blank=True))
+
+        # Adding index on 'poster', fields ['poster']
+        db.create_index('poster_poster', ['poster_id'])
+
+
+    models = {
+        'lookup.movieid': {
+            'Meta': {'object_name': 'MovieId'},
+            'amg_id': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'criterion_id': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'director': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
+            'episode': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'episode_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'imdb_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '7', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'impawards_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'oxdb_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '42', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'season': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'series_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
+            'wikipedia_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'year': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '4', 'blank': 'True'})
+        },
+        'poster.poster': {
+            'Meta': {'object_name': 'Poster'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'movie_id': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'poster'", 'to': "orm['lookup.MovieId']"}),
+            'poster': ('django.db.models.fields.files.ImageField', [], {'max_length': '255'})
+        },
+        'poster.postercache': {
+            'Meta': {'object_name': 'PosterCache'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'failed': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'image': ('django.db.models.fields.files.ImageField', [], {'max_length': '255'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'movie_id': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'postercache'", 'to': "orm['lookup.MovieId']"}),
+            'site': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'site_id': ('django.db.models.fields.CharField', [], {'max_length': '42'}),
+            'status': ('django.db.models.fields.CharField', [], {'default': "'200'", 'max_length': '1024'}),
+            'url': ('django.db.models.fields.CharField', [], {'max_length': '1024'})
+        }
+    }
+
+    complete_apps = ['poster']

oxdata/poster/models.py

@@ -3,6 +3,7 @@
 import os.path
 import hashlib
 
+from django.conf import settings
 from django.db import models
 from django.db.models import Q
 from django.contrib.auth.models import User
@@ -15,19 +16,20 @@ import ox.web.impawards
 
 from oxdata.lookup.models import MovieId
 
-def getPosters(movie_id):
+def getPosters(movie_id, url_prefix='', limit=lambda x, y: x<y):
     getPosterUrls(movie_id)
     posters = {}
+    if url_prefix.endswith('/'): url_prefix = url_prefix[:-1]
     for p in PosterCache.objects.all().filter(movie_id=movie_id, failed=False).order_by('id'):
         if p.site not in posters:
             posters[p.site] = []
         poster = p.get()
         if poster:
             pjson = {}
-            pjson['url'] = poster._get_url()
+            pjson['url'] = url_prefix + poster._get_url()
             pjson['width'] = poster.width
             pjson['height'] = poster.height
-            if poster.width < poster.height:
+            if limit(poster.width, poster.height):
                 posters[p.site].append(pjson)
     for p in posters.keys():
       if not posters[p]:
@@ -37,32 +39,13 @@ def getPosters(movie_id):
         posters['local'] = [p.poster.url]
     return posters
 
-def getMovieStills(movie_id):
-    getPosterUrls(movie_id)
-    stills = {}
-    for p in PosterCache.objects.all().filter(movie_id=movie_id, failed=False).order_by('id'):
-        if p.site not in stills:
-            stills[p.site] = []
-        still = p.get()
-        if still:
-            pjson = {}
-            pjson['url'] = still._get_url()
-            pjson['width'] = still.width
-            pjson['height'] = still.height
-            if still.width > still.height:
-                stills[p.site].append(pjson)
-    for p in stills.keys():
-      if not stills[p]:
-        del stills[p]
-    return stills
-
 def poster_path(url, filename):
-    url_hash = hashlib.sha1(url).hexdigest()
+    h = hashlib.sha1(url).hexdigest()
     ext = 'jpg'
     if filename.endswith('.png'):
         ext = 'png'
-    name = "%s.%s" % (url_hash, ext)
+    name = "%s.%s" % (h, ext)
-    return os.path.join('posters', url_hash[:2], url_hash[2:4], url_hash[4:6], name)
+    return os.path.join('posters', h[:2], h[2:4], h[4:6], name)
 
 class PosterCache(models.Model):
     created = models.DateTimeField(auto_now_add=True)
@@ -112,26 +95,36 @@ def getPosterUrls(m):
             p.save()
 
     if m.imdb_id:
+        if settings.DEBUG:
+            print 'imdb'
         poster = ox.web.imdb.getMoviePoster(m.imdb_id)
         if poster:
             addPoster(poster, 'imdb.com', m.imdb_id)
-        for poster in ox.web.movieposterdb.getData(m.imdb_id)['posters']:
+        
-            addPoster(poster, 'movieposterdb.com', m.imdb_id)
+        #site is sometimes down
+        #for poster in ox.web.movieposterdb.getData(m.imdb_id)['posters']:
+        #    addPoster(poster, 'movieposterdb.com', m.imdb_id)
     if m.criterion_id:
+        if settings.DEBUG:
             print 'criterion', m.criterion_id
         for poster in ox.web.criterion.getData(m.criterion_id)['posters']:
             addPoster(poster, 'criterion.com', m.criterion_id)
     if m.wikipedia_id:
+        if settings.DEBUG:
+            print 'wikipedia'
         poster = ox.web.wikipedia.getPosterUrl(m.wikipedia_id)
         if poster:
             if PosterCache.objects.all().filter(url=poster).count() == 0:
                 addPoster(poster, 'wikipedia.org', m.wikipedia_id)
     if m.impawards_id:
+        if settings.DEBUG:
+            print 'impawards'
         data = ox.web.impawards.getData(m.impawards_id)
         if data and 'imdbId' in data:
             for poster in data['posters']:
                 addPoster(poster, 'impawards.com', m.imdb_id)
 
+    print 'done'
 
     #fixme: get 0xdb still, possibly imdb still as fallback?
 

oxdata/poster/tasks.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+from datetime import timedelta
+
+from celery.decorators import task, periodic_task
+
+import models
+from lookup.models import getMovieIdByImdbId
+
+
+'''
+@periodic_task(run_every=timedelta(days=1))
+def cronjob(**kwargs):
+    print "do some cleanup stuff once a day"
+'''
+
+@task(ignore_resulsts=True, queue='default')
+def getMovieposteredb(imdb_id):
+    m = getMovieIdByImdbId(imdb_id)
+    def addPoster(url, site, site_id):
+        if PosterCache.objects.all().filter(url=url).count() == 0:
+            p = PosterCache(url=url, site=site, site_id=site_id, movie_id=m)
+            p.save()
+    for poster in ox.web.movieposterdb.getData(imdb_id)['posters']:
+        addPoster(poster, 'movieposterdb.com', imdb_id)
+

oxdata/poster/urls.py

@@ -1,7 +1,6 @@
 from django.conf.urls.defaults import *
 
 urlpatterns = patterns('oxdata.poster.views',
-    (r'^$', 'poster_json'),
+    (r'^$', 'poster'),
-    (r'^.html$', 'poster_html'),
 )
 

oxdata/poster/views.py

@@ -10,30 +10,17 @@ from django.template import RequestContext
 from oxdjango.shortcuts import render_to_json_response
 from oxdata.lookup.models import MovieId
 
-import models
+from oxdata.lookup.views import get_movie_id
 
-def get_movie_id(request):
+import models
-    if 'movieId' in request.GET:
-        movieId = request.GET['movieId']
-        if len(movieId) == 7:
-            movie_id = MovieId.objects.get(imdb_id=imdbId)
-        else:
-            movie_id = MovieId.objects.get(oxdb_id=oxdbId)
-    if 'imdb' in request.GET:
-        imdbId = request.GET['imdb']
-        movie_id = MovieId.objects.get(imdb_id=imdbId)
-    elif 'oxdb' in request.GET:
-        oxdbId = request.GET['oxdb']
-        movie_id = MovieId.objects.get(oxdb_id=oxdbId)
-    return movie_id
 
 def poster(request):
     movie_id = get_movie_id(request)
-    json = models.getPosters(movie_id)
+    json = models.getPosters(movie_id, request.build_absolute_uri('/'))
     return render_to_json_response(json)
 
 def still(request):
     movie_id = get_movie_id(request)
-    json = models.getMovieStills(movie_id)
+    json = models.getPosters(movie_id, request.build_absolute_uri('/'), limit=lambda x, y: x>y)
     return render_to_json_response(json)
 

oxdata/settings.py

@@ -86,6 +86,7 @@ INSTALLED_APPS = (
     'django.contrib.admin',
     'django.contrib.humanize',
     'south',
+    'djcelery',
 
     'oxdata.lookup',
     'oxdata.poster',
@@ -94,6 +95,13 @@ INSTALLED_APPS = (
 LOGIN_REDIRECT_URL='/'
 
 
+CELERY_RESULT_BACKEND = "database"
+BROKER_HOST = "127.0.0.1"
+BROKER_PORT = 5672
+BROKER_USER = "oxdata"
+BROKER_PASSWORD = "ox"
+BROKER_VHOST = "/oxdata"
+
 #overwrite default settings with local settings
 try:
     from local_settings import *

requirements.txt

@@ -1,6 +1,5 @@
 -e svn+http://code.djangoproject.com/svn/django/branches/releases/1.2.X/#egg=django
-South==0.6.1
+South
 -e bzr+http://code.0x2620.org/python-oxdjango/#egg=python-oxdjango
 -e bzr+http://code.0x2620.org/python-ox/#egg=python-ox
-
+django-celery
-