0x2620/oxdata
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

use python-ox

Browse source
This commit is contained in:
j 2010-07-19 13:11:34 +02:00
commit b817a10301
6 changed files with 60 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

24
oxdata/lookup/cache.py
View file

@ -3,12 +3,12 @@
import os
from django.conf import settings
from oxlib.cache import readUrlUnicode
from oxlib import findRe
import oxweb.criterion
import oxweb.karagarga
import oxweb.imdb
import oxweb.impawards
from ox.cache import readUrlUnicode
from ox import findRe
import ox.web.criterion
import ox.web.karagarga
import ox.web.imdb
import ox.web.impawards
import models
from oxdata.poster.models import PosterCache
@ -19,10 +19,10 @@ def addPoster(m, url, site, site_id):
p.save()
def getIds():
for id in oxweb.impawards.getIds():
for id in ox.web.impawards.getIds():
if models.MovieId.objects.all().filter(impawards_id=id).count() == 0:
print 'impawards', id
data = oxweb.impawards.getData(id)
data = ox.web.impawards.getData(id)
if data and 'imdbId' in data:
m = models.getMovieIdByImdbId(data['imdbId'])
if not m.impawards_id:
@ -31,10 +31,10 @@ def getIds():
for poster in data['posters']:
addPoster(m, poster, 'impawards.com', m.imdb_id)
for id in oxweb.criterion.getIds():
for id in ox.web.criterion.getIds():
if models.MovieId.objects.all().filter(criterion_id=id).count() == 0:
print 'criterion', id
data = oxweb.criterion.getData(id)
data = ox.web.criterion.getData(id)
if data and 'imdbId' in data:
m = models.getMovieIdByImdbId(data['imdbId'])
if not m.criterion_id:
@ -46,10 +46,10 @@ def getIds():
#kg
lastId = models.Karagarga.maxId()
for id in oxweb.karagarga.getIds(lastId):
for id in ox.web.karagarga.getIds(lastId):
if models.Karagarga.objects.filter(karagarga_id=id).count() == 0:
print 'kg', id
data = oxweb.karagarga.getData(id)
data = ox.web.karagarga.getData(id)
if data and 'imdbId' in data:
m = models.getMovieIdByImdbId(data['imdbId'])
kg = models.Karagarga()

41
oxdata/lookup/models.py
View file

@ -8,9 +8,10 @@ from django.db.models import Q, Max
from django.contrib.auth.models import User
from django.utils import simplejson
import oxweb.wikipedia
import oxweb.imdb
from oxlib import stripTags
import ox.web.imdb
import ox.web.wikipedia
from ox import stripTags
def getMovieIdByImdbId(imdb_id):
@ -56,8 +57,8 @@ class MovieId(models.Model):
def updateFromWikipedia(self):
if self.wikipedia_id:
wikipedia_url = oxweb.wikipedia.getUrl(self.wikipedia_id)
data = oxweb.wikipedia.getMovieData(wikipedia_url)
wikipedia_url = ox.web.wikipedia.getUrl(self.wikipedia_id)
data = ox.web.wikipedia.getMovieData(wikipedia_url)
_key = {}
for key in ('imdb_id', 'amg_id'):
if key in data:
@ -67,32 +68,32 @@ class MovieId(models.Model):
def updateFromImdb(self):
if self.imdb_id:
data = oxweb.imdb.getMovieInfo(self.imdb_id)
_key = {
'episode title': 'episode_title',
'series title': 'series_title',
}
for key in ('title', 'year', 'series title', 'episode title', 'season', 'episode'):
data = ox.web.imdb.ImdbCombined(self.imdb_id)
for key in ('title', 'year', 'series_title', 'episode_title', 'season', 'episode'):
if key in data and data[key]:
setattr(self, _key.get(key, key), data[key])
directors = []
credits = oxweb.imdb.getMovieCredits(self.imdb_id)
if 'directors' in credits:
directors = credits['directors']
self.director = ', '.join([stripTags(d[0]) for d in directors])
setattr(self, key, data.get(key, ''))
directors = data.get('directors', [])
self.director = ', '.join(directors)
if not self.wikipedia_id:
self.wikipedia_id = oxweb.wikipedia.getId(oxweb.wikipedia.getUrlByImdb(self.imdb_id))
self.wikipedia_id = ox.web.wikipedia.getId(ox.web.wikipedia.getUrlByImdb(self.imdb_id))
if not self.wikipedia_id:
self.wikipedia_id=None
#ignore wikipedia id if already used by another movie,
#its most likely wrong for both in that case
elif MovieId.objects.filter(wikipedia_id=self.wikipedia_id).count() >= 1:
self.wikipedia_id=None
if not self.oxdb_id:
self.gen_oxdb_id()
self.save()
def gen_oxdb_id(self):
oxid_value = u"\n".join([self.title, self.director, self.year])
oxid_value = u"\n".join([self.title, self.director, str(self.year)])
oxid = hashlib.sha1(oxid_value.encode('utf-8')).hexdigest()
if self.episode > -1:
oxid_value = u"\n".join([self.series_title, "%02d" % self.season])
oxid = hashlib.sha1(oxid_value.encode('utf-8')).hexdigest()[:20]
oxid_value = u"\n".join(["%02d" % self.episode, self.episode_title, self.director, self.year])
oxid_value = u"\n".join(["%02d" % self.episode, self.episode_title, self.director, str(self.year)])
oxid += hashlib.sha1(oxid_value.encode('utf-8')).hexdigest()[:20]
self.oxdb_id = u"0x" + oxid
self.save()