diff --git a/oxweb/criterion.py b/oxweb/criterion.py index 3f2c1c3..ea983e4 100644 --- a/oxweb/criterion.py +++ b/oxweb/criterion.py @@ -10,41 +10,24 @@ from oxlib.text import findRe, removeSpecialCharacters import imdb -def getIds(): - ids = [] - html = getUrlUnicode("http://www.criterion.com/library/dvd") - results = re.compile("page=(.*?)\"").findall(html) - pages = int(results[len(results) - 2]) - for page in range(1, pages + 1): - html = getUrlUnicode("http://www.criterion.com/library/dvd?page=" + str(page)) - results = re.compile("films/(.*?)\"").findall(html) - for result in results: - ids.append(result) - results = re.compile("boxsets/(.*?)\"").findall(html) - for result in results: - html = getUrlUnicode("http://www.criterion.com/boxsets/" + result) - results = re.compile("films/(.*?)\"").findall(html) - for result in results: - ids.append(result) - return map(lambda id: str(id), sorted(map(lambda id: int(id), set(ids)))) - def getData(id): ''' >>> getData('1333')['imdbId'] '0060304' - >>> getData('236')['posterUrl'] + >>> getData('236')['posters'][0] 'http://criterion_production.s3.amazonaws.com/release_images/1586/ThirdManReplace.jpg' - >>> getData('786')['posterUrl'] + >>> getData('786')['posters'][0] 'http://criterion_production.s3.amazonaws.com/product_images/185/343_box_348x490.jpg' ''' - data = {} - data['id'] = id + data = { + "url": getUrl(id) + } try: - html = getUrlUnicode("http://www.criterion.com/films/" + id) + html = getUrlUnicode(data["url"]) except: - html = getUrl("http://www.criterion.com/films/" + id) + html = getUrl(data["url"]) data["number"] = findRe(html, "

(.*?)

") data["title"] = findRe(html, "

(.*?)

") data["director"] = findRe(html, "

(.*?)

") @@ -58,48 +41,51 @@ def getData(id): result = re.compile("
(.*?)
", re.DOTALL).findall(html)[1] result = findRe(result, "") if not "/boxsets/" in result: - data["posterUrl"] = result + data["posters"] = [result] else: html_ = getUrlUnicode(result) result = findRe(html_, "(.*?)" % id) result = findRe(result, "src=\"(.*?)\"") - data["posterUrl"] = result.replace("_w100", "") + data["posters"] = [result.replace("_w100", "")] result = findRe(html, "\"Film>> getMovieData('Pierrot le fou', 'Jean-Luc Godard', '1965')['id'] - '149' - ''' - data = {} - if not imdbId: - imdbId = imdb.getMovieId(title, director, year) - id = getMovieId(imdbId = imdbId) - if id: - data_ = getData(id) - data['id'] = data_['id'] - data['posterUrl'] = data_['posterUrl'] - data['synopsis'] = data_['synopsis'] - return data +def getIdsByPage(page): + ids = [] + html = getUrlUnicode("http://www.criterion.com/library/dvd?page=%s" % page) + results = re.compile("films/(.*?)\"").findall(html) + for result in results: + ids.append(result) + results = re.compile("boxsets/(.*?)\"").findall(html) + for result in results: + html = getUrlUnicode("http://www.criterion.com/boxsets/" + result) + results = re.compile("films/(.*?)\"").findall(html) + for result in results: + ids.append(result) + return set(ids) + +def getUrl(id): + return "http://www.criterion.com/films/%s" % id + +if __name__ == '__main__': + print getIds() \ No newline at end of file diff --git a/oxweb/impawards.py b/oxweb/impawards.py index f90bbb0..4a912a0 100644 --- a/oxweb/impawards.py +++ b/oxweb/impawards.py @@ -57,11 +57,12 @@ def getId(url): def getIds(): ids = [] html = getUrlUnicode('http://www.impawards.com/archives/latest.html', timeout = 0) - pages = int(findRe(html, '')) - for page in range(pages + 1, 0, -1): + pages = int(findRe(html, '')) + 1 + for page in range(pages, 0, -1): for id in getIdsByPage(page): if not id in ids: ids.append(id) + print sorted(ids), len(ids), "%d/%d" % (page, pages) return ids def getIdsByPage(page): @@ -78,4 +79,7 @@ def getUrl(id): html = getUrlUnicode(url) if findRe(html, "No Movie Posters on This Page"): url = "http://www.impawards.com/%s_ver1.html" % id - return url \ No newline at end of file + return url + +if __name__ == '__main__': + getIds()