0x2620/python-ox
2
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

cleanup metacritic

Browse source
This commit is contained in:
j 2012-07-08 13:16:59 +02:00
parent e74dc77ab1
commit d860742aea
1 changed files with 15 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
ox/web/metacritic.py
View file

@ -7,6 +7,12 @@ from lxml.html import document_fromstring
from ox.cache import readUrl, readUrlUnicode from ox.cache import readUrl, readUrlUnicode
from ox import findRe, stripTags from ox import findRe, stripTags
def getUrl(id):
return 'http://www.metacritic.com/movie/%s' % id
def getId(url):
return url.split('/')[-1]
def getUrlByImdb(imdb): def getUrlByImdb(imdb):
url = "http://www.imdb.com/title/tt%s/criticreviews" % imdb url = "http://www.imdb.com/title/tt%s/criticreviews" % imdb
data = readUrl(url) data = readUrl(url)
@ -30,28 +36,29 @@ def getData(url):
score = -1 score = -1
authors = [a.text authors = [a.text
for a in doc.xpath('//div[@class="review_content"]//div[@class="author"]//a')] for a in doc.xpath('//div[@class="review_content"]//div[@class="author"]//a')]
publications = [d.text sources = [d.text
for d in doc.xpath('//div[@class="review_content"]//div[@class="source"]/a')] for d in doc.xpath('//div[@class="review_content"]//div[@class="source"]/a')]
reviews = [d.text reviews = [d.text
for d in doc.xpath('//div[@class="review_content"]//div[@class="review_body"]')] for d in doc.xpath('//div[@class="review_content"]//div[@class="review_body"]')]
scores = [int(d.text.strip()) scores = [int(d.text.strip())
for d in doc.xpath('//div[@class="review_content"]//div[contains(@class, "critscore")]')] for d in doc.xpath('//div[@class="review_content"]//div[contains(@class, "critscore")]')]
links = [a.attrib['href'] urls = [a.attrib['href']
for a in doc.xpath('//div[@class="review_content"]//a[contains(@class, "external")]')] for a in doc.xpath('//div[@class="review_content"]//a[contains(@class, "external")]')]
metacritics = [] metacritics = []
for i in range(len(authors)): for i in range(len(authors)):
metacritics.append({ metacritics.append({
'score': scores[i],
'publication': publications[i],
'critic': authors[i], 'critic': authors[i],
'url': urls[i],
'source': sources[i],
'quote': stripTags(reviews[i]).strip(), 'quote': stripTags(reviews[i]).strip(),
'link': links[i], 'score': scores[i],
}) })
return { return {
'score': score, 'critics': metacritics,
'critics': metacritics, 'id': getId(url),
'url': url 'score': score,
'url': url,
} }