python-oxweb/web/criterion.py

91 lines
3.2 KiB
Python
Raw Normal View History

# -*- coding: UTF-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import re
import ox.cache
from ox.cache import readUrlUnicode
from ox.html import stripTags
from ox.text import findRe, removeSpecialCharacters
2008-07-03 09:21:18 +00:00
import imdb
2009-07-15 13:29:22 +00:00
def getId(url):
return url.split("/")[-1]
def getUrl(id):
return "http://www.criterion.com/films/%s" % id
2008-06-19 09:47:02 +00:00
2009-07-04 10:25:24 +00:00
def getData(id):
'''
2009-07-04 10:25:24 +00:00
>>> getData('1333')['imdbId']
'0060304'
2009-07-13 18:40:59 +00:00
>>> getData('236')['posters'][0]
2009-07-04 10:25:24 +00:00
'http://criterion_production.s3.amazonaws.com/release_images/1586/ThirdManReplace.jpg'
2009-07-13 18:40:59 +00:00
>>> getData('786')['posters'][0]
2009-07-04 10:25:24 +00:00
'http://criterion_production.s3.amazonaws.com/product_images/185/343_box_348x490.jpg'
'''
2009-07-13 18:40:59 +00:00
data = {
"url": getUrl(id)
}
2009-07-04 10:25:24 +00:00
try:
html = readUrlUnicode(data["url"])
2009-07-04 10:25:24 +00:00
except:
html = ox.cache.getUrl(data["url"])
2009-07-04 10:25:24 +00:00
data["number"] = findRe(html, "<p class=\"spinenumber\">(.*?)</p>")
data["title"] = findRe(html, "<h2 class=\"movietitle\">(.*?)</h2>")
data["director"] = findRe(html, "<h2 class=\"director\">(.*?)</h2>")
results = re.compile("<p><strong>(.*?)</strong></p>").findall(html)
data["country"] = results[0]
data["year"] = results[1]
result = findRe(html, "<div class=\"synopsis contentbox lightgray\">(.*?)</div>")
data["synopsis"] = findRe(result, "<p>(.*?)</p>")
result = findRe(html, "<div class=\"editioninfo\">(.*?)</div>")
if 'Blu-Ray' in result or 'Essential Art House DVD' in result:
result = re.compile("<div class=\"editioninfo\">(.*?)</div>", re.DOTALL).findall(html)[1]
result = findRe(result, "<a href=\"(.*?)\">")
if not "/boxsets/" in result:
2009-07-13 18:40:59 +00:00
data["posters"] = [result]
2009-07-04 10:25:24 +00:00
else:
html_ = readUrlUnicode(result)
2009-07-04 10:25:24 +00:00
result = findRe(html_, "<a href=\"http://www.criterion.com/films/%s\">(.*?)</a>" % id)
result = findRe(result, "src=\"(.*?)\"")
2009-07-13 18:40:59 +00:00
data["posters"] = [result.replace("_w100", "")]
2009-07-04 10:25:24 +00:00
result = findRe(html, "<img alt=\"Film Still\" height=\"252\" src=\"(.*?)\"")
if result:
2009-07-13 18:40:59 +00:00
data["stills"] = [result]
data["trailers"] = []
2009-07-04 10:25:24 +00:00
else:
2009-07-13 18:40:59 +00:00
data["stills"] = [findRe(html, "\"thumbnailURL\", \"(.*?)\"")]
data["trailers"] = [findRe(html, "\"videoURL\", \"(.*?)\"")]
2009-07-04 10:25:24 +00:00
data['imdbId'] = imdb.getMovieId(data['title'], data['director'], data['year'])
return data
2009-07-13 18:40:59 +00:00
def getIds():
ids = []
html = readUrlUnicode("http://www.criterion.com/library/dvd")
2009-07-13 18:40:59 +00:00
results = re.compile("page=(.*?)\"").findall(html)
pages = int(results[len(results) - 2])
for page in range(pages, 0, -1):
for id in getIdsByPage(page):
ids.append(id)
return map(lambda id: str(id), sorted(map(lambda id: int(id), set(ids))))
def getIdsByPage(page):
ids = []
html = readUrlUnicode("http://www.criterion.com/library/dvd?page=%s" % page)
2009-07-13 18:40:59 +00:00
results = re.compile("films/(.*?)\"").findall(html)
for result in results:
ids.append(result)
results = re.compile("boxsets/(.*?)\"").findall(html)
for result in results:
html = readUrlUnicode("http://www.criterion.com/boxsets/" + result)
2009-07-13 18:40:59 +00:00
results = re.compile("films/(.*?)\"").findall(html)
for result in results:
ids.append(result)
return set(ids)
2009-07-13 18:40:59 +00:00
if __name__ == '__main__':
2009-07-15 13:29:22 +00:00
print getIds()