posters all over

.bzrignore

@@ -1 +1,3 @@
 data
+cache
+media

criterion/cache.py

@@ -1,49 +0,0 @@
-# -*- coding: UTF-8 -*-
-# vi:si:et:sw=4:sts=4:ts=4
-import os
-
-from django.conf import settings
-from oxlib.cache import getUrlUnicode
-from oxlib import findRe
-import oxlib.net
-import oxweb.criterion
-
-from oxdata.lookup.models import MovieId, getMovieIdByImdbId
-
-
-def getPoster(id, url=None):
-    dirname = os.path.join(settings.DATA_ROOT, 'criterion.com', id)
-    filename = os.path.join(dirname, 'poster.jpg')
-    filename = os.path.normpath(filename)
-    if not os.path.exists(filename):
-        if not url:
-            data = oxweb.criterion.getData(id)
-            url = data['posterUrl']
-        oxlib.net.saveUrl(url, filename)
-    return filename
-
-def archivePosters(init=False):
-    for criterionId in oxweb.criterion.getIds():
-        data = oxweb.criterion.getData(criterionId)
-        imdbId = data['imdbId']
-        try:
-            m = MovieId.objects.get(criterion_id=criterionId)
-            if imdbId and not m.imdb_id:
-                m.imdb_id = imdbId
-                m.save()
-        except MovieId.DoesNotExist:
-            if imdbId:
-                m = getMovieIdByImdbId(imdbId)
-            else:
-                m = MovieId()
-            m.criterion_id = criterionId
-            m.save()
-            url = data['posterUrl']
-            getPoster(criterionId, url)
-
-def cron():
-    archivePosters()
-
-def init():
-    archivePosters(True)
-

criterion/models.py

@@ -1,3 +0,0 @@
-from django.db import models
-
-# Create your models here.

criterion/views.py

@@ -1 +0,0 @@
-# Create your views here.

impawards/cache.py

@@ -1,47 +0,0 @@
-# -*- coding: UTF-8 -*-
-# vi:si:et:sw=4:sts=4:ts=4
-import os
-
-from django.conf import settings
-from oxlib.cache import getUrlUnicode
-from oxlib import findRe
-import oxlib.net
-import oxweb.impawards
-
-
-def getPosterFilename(id, url):
-    dirname = os.path.join(settings.DATA_ROOT, 'impawards.com', id[:1], id[:4], id)
-    filename = os.path.join(dirname, os.path.split(url)[1])
-    filename = os.path.normpath(filename)
-    return filename
-
-def getPoster(id, url):
-    filename = getPosterFilename(id, url)
-    oxlib.net.saveUrl(url, filename)
-    return filename
-
-def archivePosters(init=False):
-    html = getUrlUnicode('http://impawards.com/archives/latest.html', timeout = 0)
-    pages = int(findRe(html, '<a href = page(.*?).html>'))
-    for page in range(pages + 1, 0, -1):
-        if page <= pages:
-            html = getUrlUnicode('http://impawards.com/archives/page%s.html' % page, timeout = -1)
-        urls = oxweb.impawards.parseArchivePage(html)
-        for url in urls:
-            html = getUrlUnicode(url, timeout = -1)
-            data = oxweb.impawards.parseMoviePage(html)
-            service = 'impawards'
-            url = data['posterUrl']
-            imdbId = data['imdbId']
-            filename = getPosterFilename(imdbId, url)
-            if not os.path.exists(filename):
-                getPoster(imdbId, url)
-            elif not init:
-                return
-
-def cron():
-    archivePosters()
-
-def init():
-    archivePosters(True)
-

impawards/models.py

@@ -1,3 +0,0 @@
-from django.db import models
-
-# Create your models here.

impawards/views.py

@@ -1 +0,0 @@
-# Create your views here.

karagarga/cache.py

@@ -1,33 +0,0 @@
-# -*- coding: UTF-8 -*-
-# vi:si:et:sw=4:sts=4:ts=4
-import os
-import hashlib
-from django.conf import settings
-from oxlib.cache import getUrlUnicode
-from oxlib import findRe
-import oxlib.net
-import oxweb.movieposterdb
-
-
-def getPosterFilename(id, url):
-    id = str(id)
-    dirname = os.path.join(settings.DATA_ROOT, 'karagarga.net', id[:1], id)
-    url_hash = hashlib.sha1(url).hexdigest()
-    filename = os.path.join(dirname, '%s.jpg' % url_hash)
-    filename = os.path.normpath(filename)
-    return filename
-
-def getPoster(id, url):
-    filename = getPosterFilename(id, url)
-    oxlib.net.saveUrl(url, filename)
-    return filename
-
-def archivePosters(init=False):
-    return
-
-def cron():
-    archivePosters()
-
-def init():
-    archivePosters(True)
-

karagarga/models.py

@@ -1,3 +0,0 @@
-from django.db import models
-
-# Create your models here.

karagarga/views.py

@@ -1 +0,0 @@
-# Create your views here.

lookup/cache.py

@@ -0,0 +1,59 @@
+# -*- coding: UTF-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+import os
+
+from django.conf import settings
+from oxlib.cache import getUrlUnicode
+from oxlib import findRe
+import oxlib.net
+import oxweb.criterion
+import oxweb.karagarga
+import oxweb.imdb
+import oxweb.impawards
+
+import models
+from oxdata.poster.models import PosterCache
+
+def addPoster(m, url, site, site_id):
+    if PosterCache.objects.all().filter(url=url).count() == 0:
+        p = PosterCache(url=url, site=site, site_id=site_id, movie_id=m)
+        p.save()
+
+def getIds():
+    for id in oxweb.impawards.getIds():
+        if models.MovieId.objects.all().filter(impawards_id=id).count() == 0:
+            data = oxweb.impawards.getData(id)
+            if data:
+                m = models.getMovieIdByImdbId(data['imdbId'])
+                if not m.impawards_id:
+                    m.impawards_id = id
+                    m.save()
+                for poster in data['posters']:
+                        addPoster(m, poster, 'impawards.com', m.imdb_id)
+
+    for criterionId in oxweb.criterion.getIds():
+        if models.MovieId.objects.all().filter(criterion_id=id).count() == 0:
+            data = oxweb.criterion.getData(criterionId)
+            if data:
+                m = models.getMovieIdByImdbId(data['imdbId'])
+                if not m.criterion_id:
+                    m.criterion_id = id
+                    m.save()
+                addPoster(m, poster, 'criterion.com', m.criterion_id)
+
+    #kg
+    lastId = models.Karagarga.maxId()
+    for karagargaId in oxweb.karagarga.getIds(lastId):
+        if models.Karagarga.objects.filter(karagarga_id=karagargaId).count() == 0:
+            data = oxweb.karagarga.getData(karagargaId)
+            if data:
+                if 'imdbId' in data:
+                    m = models.getMovieIdByImdbId(data['imdbId'])
+                    kg = models.Karagarga()
+                    kg.movie_id = m
+                    kg.karagarga_id = karagargaId
+                    kg.save()
+                #fixme, what to do else?
+                for poster in data['posters']:
+                    addPoster(poster, 'karagarga.net', kg.karagarga_id)
+

lookup/models.py

@@ -4,10 +4,10 @@ import os
 import hashlib
 
 from django.db import models
-from django.db.models import Q
+from django.db.models import Q, Max
 from django.contrib.auth.models import User
+from django.utils import simplejson
 
-import simplejson
 import oxweb.wikipedia
 import oxweb.imdb
 from oxlib import stripTags
@@ -20,6 +20,7 @@ def getMovieIdByImdbId(imdb_id):
         m = MovieId()
         m.imdb_id = imdb_id
         m.save()
+        #m.updateFromImdb()
     return m
 
 class MovieId(models.Model):
@@ -79,6 +80,8 @@ class MovieId(models.Model):
             if 'directors' in credits:
                 directors = credits['directors']
             self.director = ', '.join([stripTags(d[0]) for d in directors])
+            if not self.wikipedia_id:
+                self.wikipedia_id = oxweb.wikipedia.getId(oxweb.wikipedia.getUrlByImdb(self.imdb_id))
             if not self.oxdb_id:
                 self.gen_oxdb_id()
             self.save()
@@ -118,3 +121,7 @@ class Karagarga(models.Model):
     movie_id = models.ForeignKey(MovieId, related_name='karagarga_ids', default=None)
     karagarga_id = models.IntegerField(unique=True)
 
+    @classmethod
+    def maxId(cls):
+        return cls.objects.aggregate(Max('karagarga_id'))['karagarga_id__max']
+    

movieposterdb/cache.py

@@ -1,34 +0,0 @@
-# -*- coding: UTF-8 -*-
-# vi:si:et:sw=4:sts=4:ts=4
-import os
-import hashlib
-from django.conf import settings
-from oxlib.cache import getUrlUnicode
-from oxlib import findRe
-import oxlib.net
-import oxweb.movieposterdb
-
-
-def getPoster(id, url=''):
-    if not url:
-        urls = oxweb.movieposterdb.getPosterUrls(id)
-        if urls:
-            url = urls[0]
-        else:
-            return 
-    dirname = os.path.join(settings.DATA_ROOT, 'movieposterdb.com', id[:1], id[:4], id)
-    url_hash = hashlib.sha1(url).hexdigest()
-    filename = os.path.join(dirname, '%s.jpg' % url_hash)
-    filename = os.path.normpath(filename)
-    oxlib.net.saveUrl(url, filename)
-    return filename
-
-def archivePosters(init=False):
-    return
-
-def cron():
-    archivePosters()
-
-def init():
-    archivePosters(True)
-

movieposterdb/models.py

@@ -1,3 +0,0 @@
-from django.db import models
-
-# Create your models here.

movieposterdb/views.py

@@ -1 +0,0 @@
-# Create your views here.

poster/models.py

@@ -15,17 +15,31 @@ import oxweb.imdb
 
 from oxdata.lookup.models import MovieId, Karagarga
 
-
+def getPosters(movie_id):
-class Poster(models.Model):
+    getPosterUrls(movie_id)
-    created = models.DateTimeField(auto_now_add=True)
+    posters = {}
-    modified = models.DateTimeField(auto_now=True)
+    for p in PosterCache.objects.all().filter(movie_id=movie_id):
-
+        if p.site not in posters:
-    movie_id = models.ForeignKey(MovieId, related_name='poster')
+            posters[p.site] = []
-    url = models.CharField(max_length=255)
+        poster = p.get()
+        if poster:
+            pjson = {}
+            pjson['url'] = poster._get_url()
+            pjson['width'] = poster.width
+            pjson['height'] = poster.height
+            posters[p.site].append(pjson)
+    for p in posters:
+      if not posters[p]:
+        del posters[p]
+    return posters
 
 def poster_path(instance, filename):
-    id = instance.service_id
+    url_hash = hashlib.sha1(instance.url).hexdigest()
-    return os.path.join('poster', instance.service, id[:1], id[:4], id, filename)
+    ext = 'jpg'
+    if filename.endswith('.png'):
+        ext = 'png'
+    name = "%s.%s" % (url_hash, ext)
+    return os.path.join('posters', url_hash[:2], url_hash[2:4], url_hash[4:6], name)
 
 class PosterCache(models.Model):
     created = models.DateTimeField(auto_now_add=True)
@@ -33,26 +47,50 @@ class PosterCache(models.Model):
 
     movie_id = models.ForeignKey(MovieId, related_name='postercache')
     url = models.CharField(max_length=1024)
-    service = models.CharField(max_length=255)
+    site = models.CharField(max_length=255)
-    service_id = models.CharField(max_length=42)
+    site_id = models.CharField(max_length=42)
     image = models.ImageField(max_length=255, upload_to=poster_path)
+    failed = models.BooleanField(default=False)
 
     def get(self):
-        if not self.image:
+        if not self.image and not self.failed:
+            try:
                 import oxlib.net
-            ext = 'jpg'
+                name = hashlib.sha1(self.url).hexdigest()
-            if self.url.endswith('.png'):
-                ext = 'png'
-            name = "%s.%s" % (hashlib.sha1(self.url).hexdigest(), ext)
                 data = oxlib.net.getUrl(self.url)
-            
                 self.image.save(name, ContentFile(data))
+            except:
+                self.failed = True
+                self.save()
         return self.image
 
+def manual_poster_path(instance, filename):
+    url_hash = hashlib.sha1('%s' % instance.id).hexdigest()
+    ext = 'jpg'
+    if filename.endswith('.png'):
+        ext = 'png'
+    name = "%s.%s" % (url_hash, ext)
+    return os.path.join('posters', url_hash[:2], url_hash[2:4], url_hash[4:6], name)
+
+class Poster(models.Model):
+    created = models.DateTimeField(auto_now_add=True)
+    modified = models.DateTimeField(auto_now=True)
+
+    movie_id = models.ForeignKey(MovieId, related_name='poster')
+    poster = models.ForeignKey(PosterCache, default=None, blank=True)
+    image = models.ImageField(max_length=255, upload_to=manual_poster_path)
+    
+    def get(self):
+        if self.image:
+            return self.image
+        if self.poster:
+            return self.poster.get()
+        #FIXME: select poster
+        return None       
 
 def getPosterUrls(m):
-    def addPoster(url, service, service_id):
+    def addPoster(url, site, site_id):
-        p = PosterCache(url=url, service=service, service_id=service_id, movie_id=m)
+        p = PosterCache(url=url, site=site, site_id=site_id, movie_id=m)
         p.save()
 
     if m.imdb_id:
@@ -75,4 +113,6 @@ def getPosterUrls(m):
             for poster in data['posters']:
                 if PosterCache.objects.all().filter(url=poster).count() == 0:
                     addPoster(poster, 'karagarga.net', kg.karagarga_id)
+        else:
+            kg.delete()
 

poster/urls.py

@@ -2,5 +2,6 @@ from django.conf.urls.defaults import *
 
 urlpatterns = patterns('oxdata.poster.views',
     (r'^$', 'poster'),
+    (r'^json$', 'poster_json'),
 )
 

poster/views.py

@@ -4,11 +4,33 @@ import os.path
 from django.db import models
 from django.db.models import Q
 from django.contrib.auth.models import User
+from django.shortcuts import render_to_response, get_object_or_404, get_list_or_404
+from django.template import RequestContext
 
 from oxdata.utils.shortcuts import render_to_json_response
+from oxdata.lookup.models import MovieId
 
+import models
 
 def poster(request):
     #FIXME: get MovieId from provided id and redirect to poster url
-    return
+    if 'imdb' in request.GET:
+        imdbId = request.GET['imdb']
+        movie_id = MovieId.objects.get(imdb_id=imdbId)
+    elif 'oxdb' in request.GET:
+        oxdbId = request.GET['oxdb']
+        movie_id = MovieId.objects.get(oxdb_id=oxdbId)
+    context = RequestContext(request, {'imdb': imdbId})
+    return render_to_response('poster.html', context)
+
+def poster_json(request):
+    #FIXME: get MovieId from provided id and redirect to poster url
+    if 'imdb' in request.GET:
+        imdbId = request.GET['imdb']
+        movie_id = MovieId.objects.get(imdb_id=imdbId)
+    elif 'oxdb' in request.GET:
+        oxdbId = request.GET['oxdb']
+        movie_id = MovieId.objects.get(oxdb_id=oxdbId)
+    json = models.getPosters(movie_id)
+    return render_to_json_response(json)
 

settings.py

@@ -4,9 +4,9 @@
 import os
 from os.path import join
 
-PROJECT_PATH = os.path.dirname(__file__)
+PROJECT_PATH = os.path.normpath(os.path.dirname(__file__))
 
-DEBUG = False
+DEBUG = True
 TEMPLATE_DEBUG = DEBUG
 
 ADMINS = (
@@ -44,6 +44,9 @@ USE_I18N = True
 MEDIA_ROOT = join(PROJECT_PATH, 'media')
 STATIC_ROOT = join(PROJECT_PATH, 'static')
 DATA_ROOT = join(PROJECT_PATH, 'data')
+CACHE_ROOT = join(PROJECT_PATH, 'cache')
+
+os.environ['oxCACHE'] = CACHE_ROOT
 
 
 # URL that handles the media served from MEDIA_ROOT. Make sure to use a
@@ -85,11 +88,6 @@ INSTALLED_APPS = (
     'django.contrib.sites',
     'django.contrib.admin',
     'django.contrib.humanize',
-    'oxdata.criterion',
-    'oxdata.impawards',
-    'oxdata.karagarga',
-    'oxdata.impawards',
-    'oxdata.movieposterdb',
 
     'oxdata.lookup',
     'oxdata.poster',

templates/poster.html

@@ -0,0 +1,41 @@
+<!DOCTYPE html>
+<html>
+<head>
+<style>
+    .hide {
+        display: none;
+    }
+</style>
+<script src="/static/js/jquery.js"></script>
+<script>
+$(function(){
+    $.getJSON('/poster/json?imdb={{ imdb }}', function(posters) {
+        $("#loading").hide();
+        $.each(posters, function(key, urls) {
+            var id = 'row' + key.replace('.', '_');
+            var row = '<tr id="'+id+'">';
+            row += '<td id="'+id+'_title">' + key + '</td>';
+            $.each(urls, function(i, poster) {
+            row += '<td>';
+            row += '<img src="'+poster.url+'" height="256">';
+            row += '<div>' + poster.width + 'x'+ poster.height + '</div>';
+            row += '</td>';
+            });
+            row += '</tr>';
+            $("#posters").append(row);
+            $('#'+id+'_title').click(function() {
+                $('#'+id+' img, #'+id+' div').toggleClass("hide");
+            });
+        });
+    });
+});
+</script>
+</head>
+<body>
+<div id="loading">loading...</div>
+<table id="posters">
+
+</table>
+</body>
+</html>
+

urls.py

@@ -13,3 +13,12 @@ urlpatterns = patterns('',
     (r'^admin/(.*)', admin.site.root),
 )
 
+if settings.DEBUG:
+    urlpatterns += patterns('',
+        (r'^media/(?P<path>.*)$', 'django.views.static.serve',
+            {'document_root': settings.MEDIA_ROOT}),
+        (r'^static/(?P<path>.*)$', 'django.views.static.serve',
+            {'document_root': settings.STATIC_ROOT}),
+    )
+
+