0x2620/oxdata
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

use python-ox

Browse source
This commit is contained in:
j 2010-07-19 13:11:34 +02:00
parent 0f232e01cb
commit b817a10301
6 changed files with 60 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
oxdata/lookup/cache.py
View file

@ -3,12 +3,12 @@
import os import os
from django.conf import settings from django.conf import settings
from oxlib.cache import readUrlUnicode from ox.cache import readUrlUnicode
from oxlib import findRe from ox import findRe
import oxweb.criterion import ox.web.criterion
import oxweb.karagarga import ox.web.karagarga
import oxweb.imdb import ox.web.imdb
import oxweb.impawards import ox.web.impawards
import models import models
from oxdata.poster.models import PosterCache from oxdata.poster.models import PosterCache
@ -19,10 +19,10 @@ def addPoster(m, url, site, site_id):
p.save() p.save()
def getIds(): def getIds():
for id in oxweb.impawards.getIds(): for id in ox.web.impawards.getIds():
if models.MovieId.objects.all().filter(impawards_id=id).count() == 0: if models.MovieId.objects.all().filter(impawards_id=id).count() == 0:
print 'impawards', id print 'impawards', id
data = oxweb.impawards.getData(id) data = ox.web.impawards.getData(id)
if data and 'imdbId' in data: if data and 'imdbId' in data:
m = models.getMovieIdByImdbId(data['imdbId']) m = models.getMovieIdByImdbId(data['imdbId'])
if not m.impawards_id: if not m.impawards_id:
@ -31,10 +31,10 @@ def getIds():
for poster in data['posters']: for poster in data['posters']:
addPoster(m, poster, 'impawards.com', m.imdb_id) addPoster(m, poster, 'impawards.com', m.imdb_id)
for id in oxweb.criterion.getIds(): for id in ox.web.criterion.getIds():
if models.MovieId.objects.all().filter(criterion_id=id).count() == 0: if models.MovieId.objects.all().filter(criterion_id=id).count() == 0:
print 'criterion', id print 'criterion', id
data = oxweb.criterion.getData(id) data = ox.web.criterion.getData(id)
if data and 'imdbId' in data: if data and 'imdbId' in data:
m = models.getMovieIdByImdbId(data['imdbId']) m = models.getMovieIdByImdbId(data['imdbId'])
if not m.criterion_id: if not m.criterion_id:
@ -46,10 +46,10 @@ def getIds():
#kg #kg
lastId = models.Karagarga.maxId() lastId = models.Karagarga.maxId()
for id in oxweb.karagarga.getIds(lastId): for id in ox.web.karagarga.getIds(lastId):
if models.Karagarga.objects.filter(karagarga_id=id).count() == 0: if models.Karagarga.objects.filter(karagarga_id=id).count() == 0:
print 'kg', id print 'kg', id
data = oxweb.karagarga.getData(id) data = ox.web.karagarga.getData(id)
if data and 'imdbId' in data: if data and 'imdbId' in data:
m = models.getMovieIdByImdbId(data['imdbId']) m = models.getMovieIdByImdbId(data['imdbId'])
kg = models.Karagarga() kg = models.Karagarga()

41
oxdata/lookup/models.py
View file

@ -8,9 +8,10 @@ from django.db.models import Q, Max
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.utils import simplejson from django.utils import simplejson
import oxweb.wikipedia import ox.web.imdb
import oxweb.imdb import ox.web.wikipedia
from oxlib import stripTags
from ox import stripTags
def getMovieIdByImdbId(imdb_id): def getMovieIdByImdbId(imdb_id):
@ -56,8 +57,8 @@ class MovieId(models.Model):
def updateFromWikipedia(self): def updateFromWikipedia(self):
if self.wikipedia_id: if self.wikipedia_id:
wikipedia_url = oxweb.wikipedia.getUrl(self.wikipedia_id) wikipedia_url = ox.web.wikipedia.getUrl(self.wikipedia_id)
data = oxweb.wikipedia.getMovieData(wikipedia_url) data = ox.web.wikipedia.getMovieData(wikipedia_url)
_key = {} _key = {}
for key in ('imdb_id', 'amg_id'): for key in ('imdb_id', 'amg_id'):
if key in data: if key in data:
@ -67,32 +68,32 @@ class MovieId(models.Model):
def updateFromImdb(self): def updateFromImdb(self):
if self.imdb_id: if self.imdb_id:
data = oxweb.imdb.getMovieInfo(self.imdb_id) data = ox.web.imdb.ImdbCombined(self.imdb_id)
_key = { for key in ('title', 'year', 'series_title', 'episode_title', 'season', 'episode'):
'episode title': 'episode_title',
'series title': 'series_title',
}
for key in ('title', 'year', 'series title', 'episode title', 'season', 'episode'):
if key in data and data[key]: if key in data and data[key]:
setattr(self, _key.get(key, key), data[key]) setattr(self, key, data.get(key, ''))
directors = []
credits = oxweb.imdb.getMovieCredits(self.imdb_id) directors = data.get('directors', [])
if 'directors' in credits: self.director = ', '.join(directors)
directors = credits['directors']
self.director = ', '.join([stripTags(d[0]) for d in directors])
if not self.wikipedia_id: if not self.wikipedia_id:
self.wikipedia_id = oxweb.wikipedia.getId(oxweb.wikipedia.getUrlByImdb(self.imdb_id)) self.wikipedia_id = ox.web.wikipedia.getId(ox.web.wikipedia.getUrlByImdb(self.imdb_id))
if not self.wikipedia_id:
self.wikipedia_id=None
#ignore wikipedia id if already used by another movie,
#its most likely wrong for both in that case
elif MovieId.objects.filter(wikipedia_id=self.wikipedia_id).count() >= 1:
self.wikipedia_id=None
if not self.oxdb_id: if not self.oxdb_id:
self.gen_oxdb_id() self.gen_oxdb_id()
self.save() self.save()
def gen_oxdb_id(self): def gen_oxdb_id(self):
oxid_value = u"\n".join([self.title, self.director, self.year]) oxid_value = u"\n".join([self.title, self.director, str(self.year)])
oxid = hashlib.sha1(oxid_value.encode('utf-8')).hexdigest() oxid = hashlib.sha1(oxid_value.encode('utf-8')).hexdigest()
if self.episode > -1: if self.episode > -1:
oxid_value = u"\n".join([self.series_title, "%02d" % self.season]) oxid_value = u"\n".join([self.series_title, "%02d" % self.season])
oxid = hashlib.sha1(oxid_value.encode('utf-8')).hexdigest()[:20] oxid = hashlib.sha1(oxid_value.encode('utf-8')).hexdigest()[:20]
oxid_value = u"\n".join(["%02d" % self.episode, self.episode_title, self.director, self.year]) oxid_value = u"\n".join(["%02d" % self.episode, self.episode_title, self.director, str(self.year)])
oxid += hashlib.sha1(oxid_value.encode('utf-8')).hexdigest()[:20] oxid += hashlib.sha1(oxid_value.encode('utf-8')).hexdigest()[:20]
self.oxdb_id = u"0x" + oxid self.oxdb_id = u"0x" + oxid
self.save() self.save()

28
oxdata/poster/models.py
View file

@ -8,11 +8,11 @@ from django.db.models import Q
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.core.files.base import ContentFile from django.core.files.base import ContentFile
import oxweb.criterion import ox.web.criterion
import oxweb.movieposterdb import ox.web.movieposterdb
import oxweb.karagarga import ox.web.karagarga
import oxweb.imdb import ox.web.imdb
import oxweb.impawards import ox.web.impawards
from oxdata.lookup.models import MovieId, Karagarga from oxdata.lookup.models import MovieId, Karagarga
@ -56,9 +56,9 @@ class PosterCache(models.Model):
def get(self): def get(self):
if not self.image and not self.failed: if not self.image and not self.failed:
try: try:
import oxlib.net import ox.net
name = hashlib.sha1(self.url).hexdigest() name = hashlib.sha1(self.url).hexdigest()
data = oxlib.net.readUrl(self.url) data = ox.net.readUrl(self.url)
self.image.save(name, ContentFile(data)) self.image.save(name, ContentFile(data))
except: except:
self.failed = True self.failed = True
@ -90,31 +90,33 @@ def getPosterUrls(m):
p.save() p.save()
if m.imdb_id: if m.imdb_id:
poster = oxweb.imdb.getMoviePoster(m.imdb_id) poster = ox.web.imdb.getMoviePoster(m.imdb_id)
if poster: if poster:
addPoster(poster, 'imdb.com', m.imdb_id) addPoster(poster, 'imdb.com', m.imdb_id)
for poster in oxweb.movieposterdb.getData(m.imdb_id)['posters']: for poster in ox.web.movieposterdb.getData(m.imdb_id)['posters']:
addPoster(poster, 'movieposterdb.com', m.imdb_id) addPoster(poster, 'movieposterdb.com', m.imdb_id)
if m.criterion_id: if m.criterion_id:
print 'criterion', m.criterion_id print 'criterion', m.criterion_id
for poster in oxweb.criterion.getData(m.criterion_id)['posters']: for poster in ox.web.criterion.getData(m.criterion_id)['posters']:
addPoster(poster, 'criterion.com', m.criterion_id) addPoster(poster, 'criterion.com', m.criterion_id)
if m.wikipedia_id: if m.wikipedia_id:
poster = oxweb.wikipedia.getPosterUrl(m.wikipedia_id) poster = ox.web.wikipedia.getPosterUrl(m.wikipedia_id)
if poster: if poster:
if PosterCache.objects.all().filter(url=poster).count() == 0: if PosterCache.objects.all().filter(url=poster).count() == 0:
addPoster(poster, 'wikipedia.org', m.wikipedia_id) addPoster(poster, 'wikipedia.org', m.wikipedia_id)
if m.impawards_id: if m.impawards_id:
data = oxweb.impawards.getData(m.impawards_id) data = ox.web.impawards.getData(m.impawards_id)
if data and 'imdbId' in data: if data and 'imdbId' in data:
for poster in data['posters']: for poster in data['posters']:
addPoster(poster, 'impawards.com', m.imdb_id) addPoster(poster, 'impawards.com', m.imdb_id)
for kg in Karagarga.objects.all().filter(movie_id=m): for kg in Karagarga.objects.all().filter(movie_id=m):
data = oxweb.karagarga.getData(kg.karagarga_id) data = ox.web.karagarga.getData(kg.karagarga_id)
if data: if data:
for poster in data['posters']: for poster in data['posters']:
addPoster(poster, 'karagarga.net', kg.karagarga_id) addPoster(poster, 'karagarga.net', kg.karagarga_id)
else: else:
kg.delete() kg.delete()
#fixme: get 0xdb still, possibly use kg or imdb still as fallback?

4
oxdata/poster/views.py
View file

@ -14,13 +14,15 @@ import models
def poster(request): def poster(request):
#FIXME: get MovieId from provided id and redirect to poster url #FIXME: get MovieId from provided id and redirect to poster url
imdbId = None
oxdbId = None
if 'imdb' in request.GET: if 'imdb' in request.GET:
imdbId = request.GET['imdb'] imdbId = request.GET['imdb']
movie_id = MovieId.objects.get(imdb_id=imdbId) movie_id = MovieId.objects.get(imdb_id=imdbId)
elif 'oxdb' in request.GET: elif 'oxdb' in request.GET:
oxdbId = request.GET['oxdb'] oxdbId = request.GET['oxdb']
movie_id = MovieId.objects.get(oxdb_id=oxdbId) movie_id = MovieId.objects.get(oxdb_id=oxdbId)
context = RequestContext(request, {'imdb': imdbId}) context = RequestContext(request, {'imdb': imdbId, 'oxdb': oxdbId})
return render_to_response('poster.html', context) return render_to_response('poster.html', context)
def poster_json(request): def poster_json(request):

7
oxdata/templates/poster.html
View file

@ -9,7 +9,12 @@
<script src="/static/js/jquery.js"></script> <script src="/static/js/jquery.js"></script>
<script> <script>
$(function(){ $(function(){
$.getJSON('/poster/json?imdb={{ imdb }}', function(posters) { {% if imdb %}
var url= '/poster/json?imdb={{ imdb }}';
{% else %}
var url= '/poster/json?oxdb={{ oxdb }}';
{% endif %}
$.getJSON(url, function(posters) {
$("#loading").hide(); $("#loading").hide();
$.each(posters, function(key, urls) { $.each(posters, function(key, urls) {
var id = 'row' + key.replace('.', '_'); var id = 'row' + key.replace('.', '_');

6
requirements.txt
View file

@ -1,6 +1,6 @@
-e svn+http://code.djangoproject.com/svn/django/branches/releases/1.1.X/#egg=django -e svn+http://code.djangoproject.com/svn/django/branches/releases/1.1.X/#egg=django
South==0.6.1 South==0.6.1
-e bzr+http://code.0xdb.org/python-oxlib/#egg=python-oxlib -e bzr+http://code.0x2620.org/python-oxdjango/#egg=python-oxdjango
-e bzr+file:///home/j/Projects/python-oxweb/#egg=python-oxweb -e bzr+http://code.0x2620.org/python-ox/#egg=python-ox
-e bzr+http://code.0xdb.org/python-oxdjango/#egg=python-oxdjango