0x2620/oxdata
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

use python-ox

Browse source
This commit is contained in:
j 2010-07-19 13:11:34 +02:00
parent 0f232e01cb
commit b817a10301
6 changed files with 60 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
oxdata/lookup/cache.py
View file

@ -3,12 +3,12 @@
import os
from django.conf import settings
from oxlib.cache import readUrlUnicode
from oxlib import findRe
import oxweb.criterion
import oxweb.karagarga
import oxweb.imdb
import oxweb.impawards
from ox.cache import readUrlUnicode
from ox import findRe
import ox.web.criterion
import ox.web.karagarga
import ox.web.imdb
import ox.web.impawards
import models
from oxdata.poster.models import PosterCache
@ -19,10 +19,10 @@ def addPoster(m, url, site, site_id):
p.save()
def getIds():
for id in oxweb.impawards.getIds():
for id in ox.web.impawards.getIds():
if models.MovieId.objects.all().filter(impawards_id=id).count() == 0:
print 'impawards', id
data = oxweb.impawards.getData(id)
data = ox.web.impawards.getData(id)
if data and 'imdbId' in data:
m = models.getMovieIdByImdbId(data['imdbId'])
if not m.impawards_id:
@ -31,10 +31,10 @@ def getIds():
for poster in data['posters']:
addPoster(m, poster, 'impawards.com', m.imdb_id)
for id in oxweb.criterion.getIds():
for id in ox.web.criterion.getIds():
if models.MovieId.objects.all().filter(criterion_id=id).count() == 0:
print 'criterion', id
data = oxweb.criterion.getData(id)
data = ox.web.criterion.getData(id)
if data and 'imdbId' in data:
m = models.getMovieIdByImdbId(data['imdbId'])
if not m.criterion_id:
@ -46,10 +46,10 @@ def getIds():
#kg
lastId = models.Karagarga.maxId()
for id in oxweb.karagarga.getIds(lastId):
for id in ox.web.karagarga.getIds(lastId):
if models.Karagarga.objects.filter(karagarga_id=id).count() == 0:
print 'kg', id
data = oxweb.karagarga.getData(id)
data = ox.web.karagarga.getData(id)
if data and 'imdbId' in data:
m = models.getMovieIdByImdbId(data['imdbId'])
kg = models.Karagarga()

41
oxdata/lookup/models.py
View file

@ -8,9 +8,10 @@ from django.db.models import Q, Max
from django.contrib.auth.models import User
from django.utils import simplejson
import oxweb.wikipedia
import oxweb.imdb
from oxlib import stripTags
import ox.web.imdb
import ox.web.wikipedia
from ox import stripTags
def getMovieIdByImdbId(imdb_id):
@ -56,8 +57,8 @@ class MovieId(models.Model):
def updateFromWikipedia(self):
if self.wikipedia_id:
wikipedia_url = oxweb.wikipedia.getUrl(self.wikipedia_id)
data = oxweb.wikipedia.getMovieData(wikipedia_url)
wikipedia_url = ox.web.wikipedia.getUrl(self.wikipedia_id)
data = ox.web.wikipedia.getMovieData(wikipedia_url)
_key = {}
for key in ('imdb_id', 'amg_id'):
if key in data:
@ -67,32 +68,32 @@ class MovieId(models.Model):
def updateFromImdb(self):
if self.imdb_id:
data = oxweb.imdb.getMovieInfo(self.imdb_id)
_key = {
'episode title': 'episode_title',
'series title': 'series_title',
}
for key in ('title', 'year', 'series title', 'episode title', 'season', 'episode'):
data = ox.web.imdb.ImdbCombined(self.imdb_id)
for key in ('title', 'year', 'series_title', 'episode_title', 'season', 'episode'):
if key in data and data[key]:
setattr(self, _key.get(key, key), data[key])
directors = []
credits = oxweb.imdb.getMovieCredits(self.imdb_id)
if 'directors' in credits:
directors = credits['directors']
self.director = ', '.join([stripTags(d[0]) for d in directors])
setattr(self, key, data.get(key, ''))
directors = data.get('directors', [])
self.director = ', '.join(directors)
if not self.wikipedia_id:
self.wikipedia_id = oxweb.wikipedia.getId(oxweb.wikipedia.getUrlByImdb(self.imdb_id))
self.wikipedia_id = ox.web.wikipedia.getId(ox.web.wikipedia.getUrlByImdb(self.imdb_id))
if not self.wikipedia_id:
self.wikipedia_id=None
#ignore wikipedia id if already used by another movie,
#its most likely wrong for both in that case
elif MovieId.objects.filter(wikipedia_id=self.wikipedia_id).count() >= 1:
self.wikipedia_id=None
if not self.oxdb_id:
self.gen_oxdb_id()
self.save()
def gen_oxdb_id(self):
oxid_value = u"\n".join([self.title, self.director, self.year])
oxid_value = u"\n".join([self.title, self.director, str(self.year)])
oxid = hashlib.sha1(oxid_value.encode('utf-8')).hexdigest()
if self.episode > -1:
oxid_value = u"\n".join([self.series_title, "%02d" % self.season])
oxid = hashlib.sha1(oxid_value.encode('utf-8')).hexdigest()[:20]
oxid_value = u"\n".join(["%02d" % self.episode, self.episode_title, self.director, self.year])
oxid_value = u"\n".join(["%02d" % self.episode, self.episode_title, self.director, str(self.year)])
oxid += hashlib.sha1(oxid_value.encode('utf-8')).hexdigest()[:20]
self.oxdb_id = u"0x" + oxid
self.save()

28
oxdata/poster/models.py
View file

@ -8,11 +8,11 @@ from django.db.models import Q
from django.contrib.auth.models import User
from django.core.files.base import ContentFile
import oxweb.criterion
import oxweb.movieposterdb
import oxweb.karagarga
import oxweb.imdb
import oxweb.impawards
import ox.web.criterion
import ox.web.movieposterdb
import ox.web.karagarga
import ox.web.imdb
import ox.web.impawards
from oxdata.lookup.models import MovieId, Karagarga
@ -56,9 +56,9 @@ class PosterCache(models.Model):
def get(self):
if not self.image and not self.failed:
try:
import oxlib.net
import ox.net
name = hashlib.sha1(self.url).hexdigest()
data = oxlib.net.readUrl(self.url)
data = ox.net.readUrl(self.url)
self.image.save(name, ContentFile(data))
except:
self.failed = True
@ -90,31 +90,33 @@ def getPosterUrls(m):
p.save()
if m.imdb_id:
poster = oxweb.imdb.getMoviePoster(m.imdb_id)
poster = ox.web.imdb.getMoviePoster(m.imdb_id)
if poster:
addPoster(poster, 'imdb.com', m.imdb_id)
for poster in oxweb.movieposterdb.getData(m.imdb_id)['posters']:
for poster in ox.web.movieposterdb.getData(m.imdb_id)['posters']:
addPoster(poster, 'movieposterdb.com', m.imdb_id)
if m.criterion_id:
print 'criterion', m.criterion_id
for poster in oxweb.criterion.getData(m.criterion_id)['posters']:
for poster in ox.web.criterion.getData(m.criterion_id)['posters']:
addPoster(poster, 'criterion.com', m.criterion_id)
if m.wikipedia_id:
poster = oxweb.wikipedia.getPosterUrl(m.wikipedia_id)
poster = ox.web.wikipedia.getPosterUrl(m.wikipedia_id)
if poster:
if PosterCache.objects.all().filter(url=poster).count() == 0:
addPoster(poster, 'wikipedia.org', m.wikipedia_id)
if m.impawards_id:
data = oxweb.impawards.getData(m.impawards_id)
data = ox.web.impawards.getData(m.impawards_id)
if data and 'imdbId' in data:
for poster in data['posters']:
addPoster(poster, 'impawards.com', m.imdb_id)
for kg in Karagarga.objects.all().filter(movie_id=m):
data = oxweb.karagarga.getData(kg.karagarga_id)
data = ox.web.karagarga.getData(kg.karagarga_id)
if data:
for poster in data['posters']:
addPoster(poster, 'karagarga.net', kg.karagarga_id)
else:
kg.delete()
#fixme: get 0xdb still, possibly use kg or imdb still as fallback?

4
oxdata/poster/views.py
View file

@ -14,13 +14,15 @@ import models
def poster(request):
#FIXME: get MovieId from provided id and redirect to poster url
imdbId = None
oxdbId = None
if 'imdb' in request.GET:
imdbId = request.GET['imdb']
movie_id = MovieId.objects.get(imdb_id=imdbId)
elif 'oxdb' in request.GET:
oxdbId = request.GET['oxdb']
movie_id = MovieId.objects.get(oxdb_id=oxdbId)
context = RequestContext(request, {'imdb': imdbId})
context = RequestContext(request, {'imdb': imdbId, 'oxdb': oxdbId})
return render_to_response('poster.html', context)
def poster_json(request):

7
oxdata/templates/poster.html
View file

@ -9,7 +9,12 @@
<script src="/static/js/jquery.js"></script>
<script>
$(function(){
$.getJSON('/poster/json?imdb={{ imdb }}', function(posters) {
{% if imdb %}
var url= '/poster/json?imdb={{ imdb }}';
{% else %}
var url= '/poster/json?oxdb={{ oxdb }}';
{% endif %}
$.getJSON(url, function(posters) {
$("#loading").hide();
$.each(posters, function(key, urls) {
var id = 'row' + key.replace('.', '_');

6
requirements.txt
View file

@ -1,6 +1,6 @@
-e svn+http://code.djangoproject.com/svn/django/branches/releases/1.1.X/#egg=django
South==0.6.1
-e bzr+http://code.0xdb.org/python-oxlib/#egg=python-oxlib
-e bzr+file:///home/j/Projects/python-oxweb/#egg=python-oxweb
-e bzr+http://code.0xdb.org/python-oxdjango/#egg=python-oxdjango
-e bzr+http://code.0x2620.org/python-oxdjango/#egg=python-oxdjango
-e bzr+http://code.0x2620.org/python-ox/#egg=python-ox