diff --git a/.bzrignore b/.bzrignore index 1269488..66f3aa5 100644 --- a/.bzrignore +++ b/.bzrignore @@ -1 +1,3 @@ data +cache +media diff --git a/criterion/__init__.py b/criterion/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/criterion/cache.py b/criterion/cache.py deleted file mode 100644 index 9a79230..0000000 --- a/criterion/cache.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: UTF-8 -*- -# vi:si:et:sw=4:sts=4:ts=4 -import os - -from django.conf import settings -from oxlib.cache import getUrlUnicode -from oxlib import findRe -import oxlib.net -import oxweb.criterion - -from oxdata.lookup.models import MovieId, getMovieIdByImdbId - - -def getPoster(id, url=None): - dirname = os.path.join(settings.DATA_ROOT, 'criterion.com', id) - filename = os.path.join(dirname, 'poster.jpg') - filename = os.path.normpath(filename) - if not os.path.exists(filename): - if not url: - data = oxweb.criterion.getData(id) - url = data['posterUrl'] - oxlib.net.saveUrl(url, filename) - return filename - -def archivePosters(init=False): - for criterionId in oxweb.criterion.getIds(): - data = oxweb.criterion.getData(criterionId) - imdbId = data['imdbId'] - try: - m = MovieId.objects.get(criterion_id=criterionId) - if imdbId and not m.imdb_id: - m.imdb_id = imdbId - m.save() - except MovieId.DoesNotExist: - if imdbId: - m = getMovieIdByImdbId(imdbId) - else: - m = MovieId() - m.criterion_id = criterionId - m.save() - url = data['posterUrl'] - getPoster(criterionId, url) - -def cron(): - archivePosters() - -def init(): - archivePosters(True) - diff --git a/criterion/models.py b/criterion/models.py deleted file mode 100644 index 71a8362..0000000 --- a/criterion/models.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.db import models - -# Create your models here. diff --git a/criterion/views.py b/criterion/views.py deleted file mode 100644 index 60f00ef..0000000 --- a/criterion/views.py +++ /dev/null @@ -1 +0,0 @@ -# Create your views here. diff --git a/impawards/__init__.py b/impawards/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/impawards/cache.py b/impawards/cache.py deleted file mode 100644 index e3e17c2..0000000 --- a/impawards/cache.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: UTF-8 -*- -# vi:si:et:sw=4:sts=4:ts=4 -import os - -from django.conf import settings -from oxlib.cache import getUrlUnicode -from oxlib import findRe -import oxlib.net -import oxweb.impawards - - -def getPosterFilename(id, url): - dirname = os.path.join(settings.DATA_ROOT, 'impawards.com', id[:1], id[:4], id) - filename = os.path.join(dirname, os.path.split(url)[1]) - filename = os.path.normpath(filename) - return filename - -def getPoster(id, url): - filename = getPosterFilename(id, url) - oxlib.net.saveUrl(url, filename) - return filename - -def archivePosters(init=False): - html = getUrlUnicode('http://impawards.com/archives/latest.html', timeout = 0) - pages = int(findRe(html, '')) - for page in range(pages + 1, 0, -1): - if page <= pages: - html = getUrlUnicode('http://impawards.com/archives/page%s.html' % page, timeout = -1) - urls = oxweb.impawards.parseArchivePage(html) - for url in urls: - html = getUrlUnicode(url, timeout = -1) - data = oxweb.impawards.parseMoviePage(html) - service = 'impawards' - url = data['posterUrl'] - imdbId = data['imdbId'] - filename = getPosterFilename(imdbId, url) - if not os.path.exists(filename): - getPoster(imdbId, url) - elif not init: - return - -def cron(): - archivePosters() - -def init(): - archivePosters(True) - diff --git a/impawards/models.py b/impawards/models.py deleted file mode 100644 index 71a8362..0000000 --- a/impawards/models.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.db import models - -# Create your models here. diff --git a/impawards/views.py b/impawards/views.py deleted file mode 100644 index 60f00ef..0000000 --- a/impawards/views.py +++ /dev/null @@ -1 +0,0 @@ -# Create your views here. diff --git a/karagarga/__init__.py b/karagarga/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/karagarga/cache.py b/karagarga/cache.py deleted file mode 100644 index 0c41cc0..0000000 --- a/karagarga/cache.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: UTF-8 -*- -# vi:si:et:sw=4:sts=4:ts=4 -import os -import hashlib -from django.conf import settings -from oxlib.cache import getUrlUnicode -from oxlib import findRe -import oxlib.net -import oxweb.movieposterdb - - -def getPosterFilename(id, url): - id = str(id) - dirname = os.path.join(settings.DATA_ROOT, 'karagarga.net', id[:1], id) - url_hash = hashlib.sha1(url).hexdigest() - filename = os.path.join(dirname, '%s.jpg' % url_hash) - filename = os.path.normpath(filename) - return filename - -def getPoster(id, url): - filename = getPosterFilename(id, url) - oxlib.net.saveUrl(url, filename) - return filename - -def archivePosters(init=False): - return - -def cron(): - archivePosters() - -def init(): - archivePosters(True) - diff --git a/karagarga/models.py b/karagarga/models.py deleted file mode 100644 index 71a8362..0000000 --- a/karagarga/models.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.db import models - -# Create your models here. diff --git a/karagarga/views.py b/karagarga/views.py deleted file mode 100644 index 60f00ef..0000000 --- a/karagarga/views.py +++ /dev/null @@ -1 +0,0 @@ -# Create your views here. diff --git a/lookup/cache.py b/lookup/cache.py new file mode 100644 index 0000000..dd809ea --- /dev/null +++ b/lookup/cache.py @@ -0,0 +1,59 @@ +# -*- coding: UTF-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +import os + +from django.conf import settings +from oxlib.cache import getUrlUnicode +from oxlib import findRe +import oxlib.net +import oxweb.criterion +import oxweb.karagarga +import oxweb.imdb +import oxweb.impawards + +import models +from oxdata.poster.models import PosterCache + +def addPoster(m, url, site, site_id): + if PosterCache.objects.all().filter(url=url).count() == 0: + p = PosterCache(url=url, site=site, site_id=site_id, movie_id=m) + p.save() + +def getIds(): + for id in oxweb.impawards.getIds(): + if models.MovieId.objects.all().filter(impawards_id=id).count() == 0: + data = oxweb.impawards.getData(id) + if data: + m = models.getMovieIdByImdbId(data['imdbId']) + if not m.impawards_id: + m.impawards_id = id + m.save() + for poster in data['posters']: + addPoster(m, poster, 'impawards.com', m.imdb_id) + + for criterionId in oxweb.criterion.getIds(): + if models.MovieId.objects.all().filter(criterion_id=id).count() == 0: + data = oxweb.criterion.getData(criterionId) + if data: + m = models.getMovieIdByImdbId(data['imdbId']) + if not m.criterion_id: + m.criterion_id = id + m.save() + addPoster(m, poster, 'criterion.com', m.criterion_id) + + #kg + lastId = models.Karagarga.maxId() + for karagargaId in oxweb.karagarga.getIds(lastId): + if models.Karagarga.objects.filter(karagarga_id=karagargaId).count() == 0: + data = oxweb.karagarga.getData(karagargaId) + if data: + if 'imdbId' in data: + m = models.getMovieIdByImdbId(data['imdbId']) + kg = models.Karagarga() + kg.movie_id = m + kg.karagarga_id = karagargaId + kg.save() + #fixme, what to do else? + for poster in data['posters']: + addPoster(poster, 'karagarga.net', kg.karagarga_id) + diff --git a/lookup/models.py b/lookup/models.py index a6ce658..3a08004 100644 --- a/lookup/models.py +++ b/lookup/models.py @@ -4,10 +4,10 @@ import os import hashlib from django.db import models -from django.db.models import Q +from django.db.models import Q, Max from django.contrib.auth.models import User +from django.utils import simplejson -import simplejson import oxweb.wikipedia import oxweb.imdb from oxlib import stripTags @@ -20,6 +20,7 @@ def getMovieIdByImdbId(imdb_id): m = MovieId() m.imdb_id = imdb_id m.save() + #m.updateFromImdb() return m class MovieId(models.Model): @@ -79,6 +80,8 @@ class MovieId(models.Model): if 'directors' in credits: directors = credits['directors'] self.director = ', '.join([stripTags(d[0]) for d in directors]) + if not self.wikipedia_id: + self.wikipedia_id = oxweb.wikipedia.getId(oxweb.wikipedia.getUrlByImdb(self.imdb_id)) if not self.oxdb_id: self.gen_oxdb_id() self.save() @@ -118,3 +121,7 @@ class Karagarga(models.Model): movie_id = models.ForeignKey(MovieId, related_name='karagarga_ids', default=None) karagarga_id = models.IntegerField(unique=True) + @classmethod + def maxId(cls): + return cls.objects.aggregate(Max('karagarga_id'))['karagarga_id__max'] + diff --git a/movieposterdb/__init__.py b/movieposterdb/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/movieposterdb/cache.py b/movieposterdb/cache.py deleted file mode 100644 index 36d5711..0000000 --- a/movieposterdb/cache.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: UTF-8 -*- -# vi:si:et:sw=4:sts=4:ts=4 -import os -import hashlib -from django.conf import settings -from oxlib.cache import getUrlUnicode -from oxlib import findRe -import oxlib.net -import oxweb.movieposterdb - - -def getPoster(id, url=''): - if not url: - urls = oxweb.movieposterdb.getPosterUrls(id) - if urls: - url = urls[0] - else: - return - dirname = os.path.join(settings.DATA_ROOT, 'movieposterdb.com', id[:1], id[:4], id) - url_hash = hashlib.sha1(url).hexdigest() - filename = os.path.join(dirname, '%s.jpg' % url_hash) - filename = os.path.normpath(filename) - oxlib.net.saveUrl(url, filename) - return filename - -def archivePosters(init=False): - return - -def cron(): - archivePosters() - -def init(): - archivePosters(True) - diff --git a/movieposterdb/models.py b/movieposterdb/models.py deleted file mode 100644 index 71a8362..0000000 --- a/movieposterdb/models.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.db import models - -# Create your models here. diff --git a/movieposterdb/views.py b/movieposterdb/views.py deleted file mode 100644 index 60f00ef..0000000 --- a/movieposterdb/views.py +++ /dev/null @@ -1 +0,0 @@ -# Create your views here. diff --git a/poster/models.py b/poster/models.py index 713f10f..71bdb46 100644 --- a/poster/models.py +++ b/poster/models.py @@ -15,17 +15,31 @@ import oxweb.imdb from oxdata.lookup.models import MovieId, Karagarga - -class Poster(models.Model): - created = models.DateTimeField(auto_now_add=True) - modified = models.DateTimeField(auto_now=True) - - movie_id = models.ForeignKey(MovieId, related_name='poster') - url = models.CharField(max_length=255) +def getPosters(movie_id): + getPosterUrls(movie_id) + posters = {} + for p in PosterCache.objects.all().filter(movie_id=movie_id): + if p.site not in posters: + posters[p.site] = [] + poster = p.get() + if poster: + pjson = {} + pjson['url'] = poster._get_url() + pjson['width'] = poster.width + pjson['height'] = poster.height + posters[p.site].append(pjson) + for p in posters: + if not posters[p]: + del posters[p] + return posters def poster_path(instance, filename): - id = instance.service_id - return os.path.join('poster', instance.service, id[:1], id[:4], id, filename) + url_hash = hashlib.sha1(instance.url).hexdigest() + ext = 'jpg' + if filename.endswith('.png'): + ext = 'png' + name = "%s.%s" % (url_hash, ext) + return os.path.join('posters', url_hash[:2], url_hash[2:4], url_hash[4:6], name) class PosterCache(models.Model): created = models.DateTimeField(auto_now_add=True) @@ -33,26 +47,50 @@ class PosterCache(models.Model): movie_id = models.ForeignKey(MovieId, related_name='postercache') url = models.CharField(max_length=1024) - service = models.CharField(max_length=255) - service_id = models.CharField(max_length=42) + site = models.CharField(max_length=255) + site_id = models.CharField(max_length=42) image = models.ImageField(max_length=255, upload_to=poster_path) + failed = models.BooleanField(default=False) def get(self): - if not self.image: - import oxlib.net - ext = 'jpg' - if self.url.endswith('.png'): - ext = 'png' - name = "%s.%s" % (hashlib.sha1(self.url).hexdigest(), ext) - data = oxlib.net.getUrl(self.url) - - self.image.save(name, ContentFile(data)) + if not self.image and not self.failed: + try: + import oxlib.net + name = hashlib.sha1(self.url).hexdigest() + data = oxlib.net.getUrl(self.url) + self.image.save(name, ContentFile(data)) + except: + self.failed = True + self.save() return self.image +def manual_poster_path(instance, filename): + url_hash = hashlib.sha1('%s' % instance.id).hexdigest() + ext = 'jpg' + if filename.endswith('.png'): + ext = 'png' + name = "%s.%s" % (url_hash, ext) + return os.path.join('posters', url_hash[:2], url_hash[2:4], url_hash[4:6], name) + +class Poster(models.Model): + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + + movie_id = models.ForeignKey(MovieId, related_name='poster') + poster = models.ForeignKey(PosterCache, default=None, blank=True) + image = models.ImageField(max_length=255, upload_to=manual_poster_path) + + def get(self): + if self.image: + return self.image + if self.poster: + return self.poster.get() + #FIXME: select poster + return None def getPosterUrls(m): - def addPoster(url, service, service_id): - p = PosterCache(url=url, service=service, service_id=service_id, movie_id=m) + def addPoster(url, site, site_id): + p = PosterCache(url=url, site=site, site_id=site_id, movie_id=m) p.save() if m.imdb_id: @@ -75,4 +113,6 @@ def getPosterUrls(m): for poster in data['posters']: if PosterCache.objects.all().filter(url=poster).count() == 0: addPoster(poster, 'karagarga.net', kg.karagarga_id) + else: + kg.delete() diff --git a/poster/urls.py b/poster/urls.py index b9d0d22..a87a951 100644 --- a/poster/urls.py +++ b/poster/urls.py @@ -2,5 +2,6 @@ from django.conf.urls.defaults import * urlpatterns = patterns('oxdata.poster.views', (r'^$', 'poster'), + (r'^json$', 'poster_json'), ) diff --git a/poster/views.py b/poster/views.py index 169724d..74a6c04 100644 --- a/poster/views.py +++ b/poster/views.py @@ -4,11 +4,33 @@ import os.path from django.db import models from django.db.models import Q from django.contrib.auth.models import User +from django.shortcuts import render_to_response, get_object_or_404, get_list_or_404 +from django.template import RequestContext from oxdata.utils.shortcuts import render_to_json_response +from oxdata.lookup.models import MovieId +import models def poster(request): #FIXME: get MovieId from provided id and redirect to poster url - return + if 'imdb' in request.GET: + imdbId = request.GET['imdb'] + movie_id = MovieId.objects.get(imdb_id=imdbId) + elif 'oxdb' in request.GET: + oxdbId = request.GET['oxdb'] + movie_id = MovieId.objects.get(oxdb_id=oxdbId) + context = RequestContext(request, {'imdb': imdbId}) + return render_to_response('poster.html', context) + +def poster_json(request): + #FIXME: get MovieId from provided id and redirect to poster url + if 'imdb' in request.GET: + imdbId = request.GET['imdb'] + movie_id = MovieId.objects.get(imdb_id=imdbId) + elif 'oxdb' in request.GET: + oxdbId = request.GET['oxdb'] + movie_id = MovieId.objects.get(oxdb_id=oxdbId) + json = models.getPosters(movie_id) + return render_to_json_response(json) diff --git a/settings.py b/settings.py index ce4dc54..869d4db 100644 --- a/settings.py +++ b/settings.py @@ -4,9 +4,9 @@ import os from os.path import join -PROJECT_PATH = os.path.dirname(__file__) +PROJECT_PATH = os.path.normpath(os.path.dirname(__file__)) -DEBUG = False +DEBUG = True TEMPLATE_DEBUG = DEBUG ADMINS = ( @@ -44,6 +44,9 @@ USE_I18N = True MEDIA_ROOT = join(PROJECT_PATH, 'media') STATIC_ROOT = join(PROJECT_PATH, 'static') DATA_ROOT = join(PROJECT_PATH, 'data') +CACHE_ROOT = join(PROJECT_PATH, 'cache') + +os.environ['oxCACHE'] = CACHE_ROOT # URL that handles the media served from MEDIA_ROOT. Make sure to use a @@ -85,11 +88,6 @@ INSTALLED_APPS = ( 'django.contrib.sites', 'django.contrib.admin', 'django.contrib.humanize', - 'oxdata.criterion', - 'oxdata.impawards', - 'oxdata.karagarga', - 'oxdata.impawards', - 'oxdata.movieposterdb', 'oxdata.lookup', 'oxdata.poster', diff --git a/templates/poster.html b/templates/poster.html new file mode 100644 index 0000000..a4cf5d0 --- /dev/null +++ b/templates/poster.html @@ -0,0 +1,41 @@ + + + + + + + + +
loading...
+ + +
+ + + diff --git a/urls.py b/urls.py index a038af9..1bf0816 100644 --- a/urls.py +++ b/urls.py @@ -13,3 +13,12 @@ urlpatterns = patterns('', (r'^admin/(.*)', admin.site.root), ) +if settings.DEBUG: + urlpatterns += patterns('', + (r'^media/(?P.*)$', 'django.views.static.serve', + {'document_root': settings.MEDIA_ROOT}), + (r'^static/(?P.*)$', 'django.views.static.serve', + {'document_root': settings.STATIC_ROOT}), + ) + +