more imdb refinement

This commit is contained in:
j 2010-07-10 10:24:56 +02:00
parent 6afafa7355
commit 18ce4cd92d
2 changed files with 35 additions and 19 deletions

View file

@ -7,12 +7,13 @@ import os
import time
import ox
from ox import findRe
from ox import findRe, stripTags
from ox.normalize import normalizeTitle, normalizeImdbId
from siteparser import SiteParser
import google
class Imdb(SiteParser):
regex = {
'alternative_titles': {
@ -25,13 +26,17 @@ class Imdb(SiteParser):
},
'cast': {
'page': 'combined',
're': '<td class="nm">.*?>(.*?)</a>.*?<td class="char">(.*?)</td>',
'page': 'fullcredits',
're': [
'<td class="nm">.*?>(.*?)</a>.*?<td class="char">(.*?)</td>',
lambda ll: [stripTags(l) for l in ll]
],
'type': 'list'
},
'cinematographers': {
'page': 'combined',
'page': 'fullcredits',
're': [
lambda data: data.split('Series Crew')[0],
'Cinematography by</a>(.*?)</table>',
'<a href="/name/.*?/">(.*?)</a>'
],
@ -48,16 +53,18 @@ class Imdb(SiteParser):
'type': 'list'
},
'directors': {
'page': 'combined',
'page': 'fullcredits',
're': [
lambda data: data.split('Series Crew')[0],
'Directed by</a>(.*?)</table>',
'<a href="/name/.*?/">(.*?)</a>'
],
'type': 'list'
},
'editors': {
'page': 'combined',
'page': 'fullcredits',
're': [
lambda data: data.split('Series Crew')[0],
'Film Editing by</a>(.*?)</table>',
'<a href="/name/.*?/">(.*?)</a>'
],
@ -108,7 +115,7 @@ class Imdb(SiteParser):
},
'rating': {
'page': 'combined',
're': '<div class="starbar-meta">.*?<b>(.*?)/10</b>',
're': '<div class="starbar-meta">.*?<b>([\d,.]?)/10</b>',
'type': 'float'
},
'release_date': {
@ -141,12 +148,13 @@ class Imdb(SiteParser):
},
'votes': {
'page': 'combined',
're': '<a href="ratings" class="tn15more">(.*?) votes</a>',
're': '<a href="ratings" class="tn15more">([\d,]*?) votes</a>',
'type': 'string'
},
'writers': {
'page': 'combined',
'page': 'fullcredits',
're': [
lambda data: data.split('Series Crew')[0],
'Writing credits</a>(.*?)</table>',
'<a href="/name/.*?/">(.*?)</a>'
],
@ -167,8 +175,8 @@ class Imdb(SiteParser):
if 'min' in self['runtime']: base=60
else: base=1
self['runtime'] = int(findRe(self['runtime'], '([0-9]+)')) * base
else:
self['runtime'] = 0
if 'runtime' in self and not self['runtime']:
del self['runtime']
if 'votes' in self: self['votes'] = self['votes'].replace(',', '')
if 'connections' in self:
cc={}
@ -179,6 +187,7 @@ class Imdb(SiteParser):
self['connections'] = cc
for key in ('countries', 'genres'):
if key in self:
self[key] = filter(lambda x: x.lower() != 'home', self[key])
@ -192,7 +201,7 @@ def guess(title, director='', timeout=google.DEFAULT_TIMEOUT):
return_url = ''
#lest first try google
#i.e. site:imdb.com Michael Stevens Sin
#i.e. site:imdb.com Michael Stevens "Sin"
if director:
search = 'site:imdb.com %s "%s"' % (director, title)
else:

View file

@ -15,7 +15,7 @@ def cleanup(key, data, data_type):
data = [decodeHtml(p).strip() for p in data]
elif isinstance(data[0], list) or isinstance(data[0], tuple):
data = [cleanup(key, p, data_type) for p in data]
while len(data) == 1:
while len(data) == 1 and not isinstance(data, basestring):
data = data[0]
if data_type == 'list' and isinstance(data, basestring):
data = [data, ]
@ -37,12 +37,18 @@ class SiteParser(dict):
if isinstance(self.regex[key]['re'], basestring):
data = re.compile(self.regex[key]['re'], re.DOTALL).findall(data)
data = cleanup(key, data, self.regex[key]['type'])
elif callable(self.regex[key]['re']):
data = self.regex[key]['re'](data)
else:
for r in self.regex[key]['re']:
if isinstance(data, basestring):
data = re.compile(r, re.DOTALL).findall(data)
if callable(r):
f = r
else:
data = [re.compile(r, re.DOTALL).findall(d) for d in data]
f = re.compile(r, re.DOTALL).findall
if isinstance(data, basestring):
data = f(data)
else:
data = [f(d) for d in data]
data = cleanup(key, data, self.regex[key]['type'])
def apply_f(f, data):
if data and isinstance(data[0], list):
@ -50,12 +56,13 @@ class SiteParser(dict):
else:
data = f(data)
return data
if self.regex[key]['type'] == 'float':
if self.regex[key]['type'] == 'float' and data:
data = apply_f(float, data)
elif self.regex[key]['type'] == 'int':
data = apply_f(int, data)
elif self.regex[key]['type'] == 'date':
parse_date = lambda d: d and datetime.strptime('-'.join(d), '%m-%d-%Y').strftime('%Y-%m-%d')
data = apply_f(parse_date, data)
if data:
self[key] = data