From b817a103017986a99286636c3bab622a4b7d137d Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Mon, 19 Jul 2010 13:11:34 +0200 Subject: [PATCH] use python-ox --- oxdata/lookup/cache.py | 24 ++++++++++----------- oxdata/lookup/models.py | 41 ++++++++++++++++++------------------ oxdata/poster/models.py | 28 ++++++++++++------------ oxdata/poster/views.py | 4 +++- oxdata/templates/poster.html | 7 +++++- requirements.txt | 6 +++--- 6 files changed, 60 insertions(+), 50 deletions(-) diff --git a/oxdata/lookup/cache.py b/oxdata/lookup/cache.py index 283de7a..1cc0423 100644 --- a/oxdata/lookup/cache.py +++ b/oxdata/lookup/cache.py @@ -3,12 +3,12 @@ import os from django.conf import settings -from oxlib.cache import readUrlUnicode -from oxlib import findRe -import oxweb.criterion -import oxweb.karagarga -import oxweb.imdb -import oxweb.impawards +from ox.cache import readUrlUnicode +from ox import findRe +import ox.web.criterion +import ox.web.karagarga +import ox.web.imdb +import ox.web.impawards import models from oxdata.poster.models import PosterCache @@ -19,10 +19,10 @@ def addPoster(m, url, site, site_id): p.save() def getIds(): - for id in oxweb.impawards.getIds(): + for id in ox.web.impawards.getIds(): if models.MovieId.objects.all().filter(impawards_id=id).count() == 0: print 'impawards', id - data = oxweb.impawards.getData(id) + data = ox.web.impawards.getData(id) if data and 'imdbId' in data: m = models.getMovieIdByImdbId(data['imdbId']) if not m.impawards_id: @@ -31,10 +31,10 @@ def getIds(): for poster in data['posters']: addPoster(m, poster, 'impawards.com', m.imdb_id) - for id in oxweb.criterion.getIds(): + for id in ox.web.criterion.getIds(): if models.MovieId.objects.all().filter(criterion_id=id).count() == 0: print 'criterion', id - data = oxweb.criterion.getData(id) + data = ox.web.criterion.getData(id) if data and 'imdbId' in data: m = models.getMovieIdByImdbId(data['imdbId']) if not m.criterion_id: @@ -46,10 +46,10 @@ def getIds(): #kg lastId = models.Karagarga.maxId() - for id in oxweb.karagarga.getIds(lastId): + for id in ox.web.karagarga.getIds(lastId): if models.Karagarga.objects.filter(karagarga_id=id).count() == 0: print 'kg', id - data = oxweb.karagarga.getData(id) + data = ox.web.karagarga.getData(id) if data and 'imdbId' in data: m = models.getMovieIdByImdbId(data['imdbId']) kg = models.Karagarga() diff --git a/oxdata/lookup/models.py b/oxdata/lookup/models.py index cbb14c8..b42505d 100644 --- a/oxdata/lookup/models.py +++ b/oxdata/lookup/models.py @@ -8,9 +8,10 @@ from django.db.models import Q, Max from django.contrib.auth.models import User from django.utils import simplejson -import oxweb.wikipedia -import oxweb.imdb -from oxlib import stripTags +import ox.web.imdb +import ox.web.wikipedia + +from ox import stripTags def getMovieIdByImdbId(imdb_id): @@ -56,8 +57,8 @@ class MovieId(models.Model): def updateFromWikipedia(self): if self.wikipedia_id: - wikipedia_url = oxweb.wikipedia.getUrl(self.wikipedia_id) - data = oxweb.wikipedia.getMovieData(wikipedia_url) + wikipedia_url = ox.web.wikipedia.getUrl(self.wikipedia_id) + data = ox.web.wikipedia.getMovieData(wikipedia_url) _key = {} for key in ('imdb_id', 'amg_id'): if key in data: @@ -67,32 +68,32 @@ class MovieId(models.Model): def updateFromImdb(self): if self.imdb_id: - data = oxweb.imdb.getMovieInfo(self.imdb_id) - _key = { - 'episode title': 'episode_title', - 'series title': 'series_title', - } - for key in ('title', 'year', 'series title', 'episode title', 'season', 'episode'): + data = ox.web.imdb.ImdbCombined(self.imdb_id) + for key in ('title', 'year', 'series_title', 'episode_title', 'season', 'episode'): if key in data and data[key]: - setattr(self, _key.get(key, key), data[key]) - directors = [] - credits = oxweb.imdb.getMovieCredits(self.imdb_id) - if 'directors' in credits: - directors = credits['directors'] - self.director = ', '.join([stripTags(d[0]) for d in directors]) + setattr(self, key, data.get(key, '')) + + directors = data.get('directors', []) + self.director = ', '.join(directors) if not self.wikipedia_id: - self.wikipedia_id = oxweb.wikipedia.getId(oxweb.wikipedia.getUrlByImdb(self.imdb_id)) + self.wikipedia_id = ox.web.wikipedia.getId(ox.web.wikipedia.getUrlByImdb(self.imdb_id)) + if not self.wikipedia_id: + self.wikipedia_id=None + #ignore wikipedia id if already used by another movie, + #its most likely wrong for both in that case + elif MovieId.objects.filter(wikipedia_id=self.wikipedia_id).count() >= 1: + self.wikipedia_id=None if not self.oxdb_id: self.gen_oxdb_id() self.save() def gen_oxdb_id(self): - oxid_value = u"\n".join([self.title, self.director, self.year]) + oxid_value = u"\n".join([self.title, self.director, str(self.year)]) oxid = hashlib.sha1(oxid_value.encode('utf-8')).hexdigest() if self.episode > -1: oxid_value = u"\n".join([self.series_title, "%02d" % self.season]) oxid = hashlib.sha1(oxid_value.encode('utf-8')).hexdigest()[:20] - oxid_value = u"\n".join(["%02d" % self.episode, self.episode_title, self.director, self.year]) + oxid_value = u"\n".join(["%02d" % self.episode, self.episode_title, self.director, str(self.year)]) oxid += hashlib.sha1(oxid_value.encode('utf-8')).hexdigest()[:20] self.oxdb_id = u"0x" + oxid self.save() diff --git a/oxdata/poster/models.py b/oxdata/poster/models.py index b46a690..cdc3528 100644 --- a/oxdata/poster/models.py +++ b/oxdata/poster/models.py @@ -8,11 +8,11 @@ from django.db.models import Q from django.contrib.auth.models import User from django.core.files.base import ContentFile -import oxweb.criterion -import oxweb.movieposterdb -import oxweb.karagarga -import oxweb.imdb -import oxweb.impawards +import ox.web.criterion +import ox.web.movieposterdb +import ox.web.karagarga +import ox.web.imdb +import ox.web.impawards from oxdata.lookup.models import MovieId, Karagarga @@ -56,9 +56,9 @@ class PosterCache(models.Model): def get(self): if not self.image and not self.failed: try: - import oxlib.net + import ox.net name = hashlib.sha1(self.url).hexdigest() - data = oxlib.net.readUrl(self.url) + data = ox.net.readUrl(self.url) self.image.save(name, ContentFile(data)) except: self.failed = True @@ -90,31 +90,33 @@ def getPosterUrls(m): p.save() if m.imdb_id: - poster = oxweb.imdb.getMoviePoster(m.imdb_id) + poster = ox.web.imdb.getMoviePoster(m.imdb_id) if poster: addPoster(poster, 'imdb.com', m.imdb_id) - for poster in oxweb.movieposterdb.getData(m.imdb_id)['posters']: + for poster in ox.web.movieposterdb.getData(m.imdb_id)['posters']: addPoster(poster, 'movieposterdb.com', m.imdb_id) if m.criterion_id: print 'criterion', m.criterion_id - for poster in oxweb.criterion.getData(m.criterion_id)['posters']: + for poster in ox.web.criterion.getData(m.criterion_id)['posters']: addPoster(poster, 'criterion.com', m.criterion_id) if m.wikipedia_id: - poster = oxweb.wikipedia.getPosterUrl(m.wikipedia_id) + poster = ox.web.wikipedia.getPosterUrl(m.wikipedia_id) if poster: if PosterCache.objects.all().filter(url=poster).count() == 0: addPoster(poster, 'wikipedia.org', m.wikipedia_id) if m.impawards_id: - data = oxweb.impawards.getData(m.impawards_id) + data = ox.web.impawards.getData(m.impawards_id) if data and 'imdbId' in data: for poster in data['posters']: addPoster(poster, 'impawards.com', m.imdb_id) for kg in Karagarga.objects.all().filter(movie_id=m): - data = oxweb.karagarga.getData(kg.karagarga_id) + data = ox.web.karagarga.getData(kg.karagarga_id) if data: for poster in data['posters']: addPoster(poster, 'karagarga.net', kg.karagarga_id) else: kg.delete() + #fixme: get 0xdb still, possibly use kg or imdb still as fallback? + diff --git a/oxdata/poster/views.py b/oxdata/poster/views.py index 3416488..2f530ed 100644 --- a/oxdata/poster/views.py +++ b/oxdata/poster/views.py @@ -14,13 +14,15 @@ import models def poster(request): #FIXME: get MovieId from provided id and redirect to poster url + imdbId = None + oxdbId = None if 'imdb' in request.GET: imdbId = request.GET['imdb'] movie_id = MovieId.objects.get(imdb_id=imdbId) elif 'oxdb' in request.GET: oxdbId = request.GET['oxdb'] movie_id = MovieId.objects.get(oxdb_id=oxdbId) - context = RequestContext(request, {'imdb': imdbId}) + context = RequestContext(request, {'imdb': imdbId, 'oxdb': oxdbId}) return render_to_response('poster.html', context) def poster_json(request): diff --git a/oxdata/templates/poster.html b/oxdata/templates/poster.html index 279be87..35d4831 100644 --- a/oxdata/templates/poster.html +++ b/oxdata/templates/poster.html @@ -9,7 +9,12 @@