python-ox/ox/web/criterion.py

118 lines
4.6 KiB
Python
Raw Normal View History

2010-07-07 23:25:57 +00:00
# -*- coding: UTF-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
2014-09-30 19:27:26 +00:00
from __future__ import print_function
2010-07-07 23:25:57 +00:00
import re
import ox.cache
from ox.cache import read_url
2015-08-02 13:58:59 +00:00
from ox.html import strip_tags, decode_html
2014-09-30 19:04:46 +00:00
from ox.text import find_re
2010-07-07 23:25:57 +00:00
2016-09-07 12:44:42 +00:00
from . import imdb
2010-07-07 23:25:57 +00:00
2012-08-15 15:15:40 +00:00
def get_id(url):
2010-07-07 23:25:57 +00:00
return url.split("/")[-1]
2012-08-15 15:15:40 +00:00
def get_url(id):
2018-07-09 13:20:00 +00:00
return "https://www.criterion.com/films/%s" % id
2010-07-07 23:25:57 +00:00
2012-08-15 15:15:40 +00:00
def get_data(id, timeout=ox.cache.cache_timeout, get_imdb=False):
2010-07-07 23:25:57 +00:00
'''
2012-09-09 17:28:11 +00:00
>>> get_data('1333').get('imdbId')
2010-09-03 21:19:19 +00:00
u'0060304'
2010-07-07 23:25:57 +00:00
2012-08-15 15:15:40 +00:00
>>> get_data('236')['posters'][0]
2012-09-09 17:28:11 +00:00
u'http://s3.amazonaws.com/criterion-production/release_images/1586/ThirdManReplace.jpg'
2010-07-07 23:25:57 +00:00
2012-08-15 15:15:40 +00:00
>>> get_data('786')['posters'][0]
2012-09-09 17:28:11 +00:00
u'http://s3.amazonaws.com/criterion-production/product_images/185/343_box_348x490.jpg'
2010-07-07 23:25:57 +00:00
'''
data = {
2018-05-23 08:43:32 +00:00
"id": id,
2012-08-15 15:15:40 +00:00
"url": get_url(id)
2010-07-07 23:25:57 +00:00
}
try:
html = read_url(data["url"], timeout=timeout, unicode=True)
2010-07-07 23:25:57 +00:00
except:
2018-05-09 11:09:13 +00:00
html = read_url(data["url"], timeout=timeout).decode('utf-8', 'ignore')
2018-05-07 08:48:14 +00:00
data["number"] = find_re(html, "<b>Spine #(\d+)")
2011-02-27 15:33:25 +00:00
2018-05-07 08:42:15 +00:00
data["title"] = decode_html(find_re(html, "<h1 class=\"header__primarytitle\".*?>(.*?)</h1>"))
2015-08-02 13:58:59 +00:00
data["title"] = data["title"].split(u' \u2014 The Television Version')[0].strip()
2018-05-07 08:42:15 +00:00
results = find_re(html, '<ul class="film-meta-list">(.*?)</ul>')
info = re.compile('<li itemprop="(.*?)".*?>(.*?)</li>', re.DOTALL).findall(results)
info = {k: strip_tags(v).strip() for k, v in info}
2021-08-07 09:30:23 +00:00
meta = re.compile('<meta.*? name="(.*?)".*? content="(.*?)"', re.DOTALL).findall(html)
meta = {k: v.strip() for k, v in meta}
2018-05-07 08:42:15 +00:00
if 'director' in info:
2018-05-09 11:09:13 +00:00
data['director'] = info['director']
2021-08-07 09:30:23 +00:00
elif 'director' in meta:
data['director'] = meta['director']
2018-05-07 08:42:15 +00:00
if 'countryOfOrigin' in info:
2018-05-09 11:09:13 +00:00
data['country'] = [c.strip() for c in decode_html(info['countryOfOrigin']).split(', ')]
2018-05-07 08:42:15 +00:00
if 'inLanguage' in info:
2018-05-09 11:09:13 +00:00
data['language'] = [l.strip() for l in decode_html(info['inLanguage']).split(', ')]
2018-05-07 08:42:15 +00:00
for v in re.compile('<li>(.*?)</li>', re.DOTALL).findall(results):
if 'datePublished' in v:
data['year'] = strip_tags(v).strip()
elif 'duration' in v:
data['duration'] = strip_tags(v).strip()
2015-08-02 13:58:59 +00:00
data["synopsis"] = decode_html(strip_tags(find_re(html,
2018-05-07 08:42:15 +00:00
"<div class=\"product-summary\".*?>.*?<p>(.*?)</p>")))
2010-07-18 18:57:22 +00:00
result = find_re(html, "<div class=\"purchase\">(.*?)</div>")
2010-07-07 23:25:57 +00:00
if 'Blu-Ray' in result or 'Essential Art House DVD' in result:
r = re.compile('<h3 class="section_title first">Other Editions</h3>(.*?)</div>', re.DOTALL).findall(html)
if r:
result = r[0]
result = find_re(result, "<a href=\"(.*?)\"")
2010-07-07 23:25:57 +00:00
if not "/boxsets/" in result:
data["posters"] = [result]
else:
html_ = read_url(result, unicode=True)
2018-07-09 13:20:00 +00:00
result = find_re(html_, '//www.criterion.com/films/%s.*?">(.*?)</a>' % id)
result = find_re(result, "src=\"(.*?)\"")
if result:
data["posters"] = [result.replace("_w100", "")]
else:
data["posters"] = []
2012-09-09 17:28:11 +00:00
data['posters'] = [re.sub('(\?\d+)$', '', p) for p in data['posters']]
2018-05-07 08:42:15 +00:00
data['posters'] = [p for p in data['posters'] if p]
posters = find_re(html, '<div class="product-box-art".*?>(.*?)</div>')
for poster in re.compile('<img src="(.*?)"').findall(posters):
data['posters'].append(poster)
2021-08-07 09:30:23 +00:00
result = re.compile('<div class="gallery-item ">.*?src="(.*?)"', re.DOTALL).findall(html)
#result = find_re(html, "<img alt=\"Film Still\" height=\"252\" src=\"(.*?)\"")
2010-07-07 23:25:57 +00:00
if result:
data["stills"] = [result]
data["trailers"] = []
else:
2016-09-18 11:39:22 +00:00
data["stills"] = list(filter(lambda x: x, [find_re(html, "\"thumbnailURL\", \"(.*?)\"")]))
data["trailers"] = list(filter(lambda x: x, [find_re(html, "\"videoURL\", \"(.*?)\"")]))
2010-09-03 21:19:19 +00:00
if timeout == ox.cache.cache_timeout:
timeout = -1
if get_imdb and 'title' in data and 'director' in data:
2013-06-09 14:48:58 +00:00
# removed year, as "title (year)" may fail to match
data['imdbId'] = imdb.get_movie_id(data['title'], data['director'], timeout=timeout)
2010-07-07 23:25:57 +00:00
return data
def get_ids(page=None):
2010-07-07 23:25:57 +00:00
ids = []
2018-05-21 08:59:50 +00:00
html = read_url("https://www.criterion.com/shop/browse/list?sort=spine_number", unicode=True)
results = re.compile("films/(\d+)-").findall(html)
ids += results
results = re.compile("boxsets/(.*?)\"").findall(html)
for result in results:
2018-07-09 13:20:00 +00:00
html = read_url("https://www.criterion.com/boxsets/" + result, unicode=True)
2018-05-21 08:59:50 +00:00
results = re.compile("films/(\d+)-").findall(html)
ids += results
return sorted(set(ids), key=int)
2010-07-07 23:25:57 +00:00
2018-05-21 08:59:50 +00:00
2010-07-07 23:25:57 +00:00
if __name__ == '__main__':
2014-09-30 19:27:26 +00:00
print(get_ids())