diff --git a/oxweb/criterion.py b/oxweb/criterion.py
index 3f2c1c3..ea983e4 100644
--- a/oxweb/criterion.py
+++ b/oxweb/criterion.py
@@ -10,41 +10,24 @@ from oxlib.text import findRe, removeSpecialCharacters
import imdb
-def getIds():
- ids = []
- html = getUrlUnicode("http://www.criterion.com/library/dvd")
- results = re.compile("page=(.*?)\"").findall(html)
- pages = int(results[len(results) - 2])
- for page in range(1, pages + 1):
- html = getUrlUnicode("http://www.criterion.com/library/dvd?page=" + str(page))
- results = re.compile("films/(.*?)\"").findall(html)
- for result in results:
- ids.append(result)
- results = re.compile("boxsets/(.*?)\"").findall(html)
- for result in results:
- html = getUrlUnicode("http://www.criterion.com/boxsets/" + result)
- results = re.compile("films/(.*?)\"").findall(html)
- for result in results:
- ids.append(result)
- return map(lambda id: str(id), sorted(map(lambda id: int(id), set(ids))))
-
def getData(id):
'''
>>> getData('1333')['imdbId']
'0060304'
- >>> getData('236')['posterUrl']
+ >>> getData('236')['posters'][0]
'http://criterion_production.s3.amazonaws.com/release_images/1586/ThirdManReplace.jpg'
- >>> getData('786')['posterUrl']
+ >>> getData('786')['posters'][0]
'http://criterion_production.s3.amazonaws.com/product_images/185/343_box_348x490.jpg'
'''
- data = {}
- data['id'] = id
+ data = {
+ "url": getUrl(id)
+ }
try:
- html = getUrlUnicode("http://www.criterion.com/films/" + id)
+ html = getUrlUnicode(data["url"])
except:
- html = getUrl("http://www.criterion.com/films/" + id)
+ html = getUrl(data["url"])
data["number"] = findRe(html, "
(.*?)
")
data["title"] = findRe(html, "(.*?)
")
data["director"] = findRe(html, "(.*?)
")
@@ -58,48 +41,51 @@ def getData(id):
result = re.compile("(.*?)
", re.DOTALL).findall(html)[1]
result = findRe(result, "")
if not "/boxsets/" in result:
- data["posterUrl"] = result
+ data["posters"] = [result]
else:
html_ = getUrlUnicode(result)
result = findRe(html_, "(.*?)" % id)
result = findRe(result, "src=\"(.*?)\"")
- data["posterUrl"] = result.replace("_w100", "")
+ data["posters"] = [result.replace("_w100", "")]
result = findRe(html, ">> getMovieData('Pierrot le fou', 'Jean-Luc Godard', '1965')['id']
- '149'
- '''
- data = {}
- if not imdbId:
- imdbId = imdb.getMovieId(title, director, year)
- id = getMovieId(imdbId = imdbId)
- if id:
- data_ = getData(id)
- data['id'] = data_['id']
- data['posterUrl'] = data_['posterUrl']
- data['synopsis'] = data_['synopsis']
- return data
+def getIdsByPage(page):
+ ids = []
+ html = getUrlUnicode("http://www.criterion.com/library/dvd?page=%s" % page)
+ results = re.compile("films/(.*?)\"").findall(html)
+ for result in results:
+ ids.append(result)
+ results = re.compile("boxsets/(.*?)\"").findall(html)
+ for result in results:
+ html = getUrlUnicode("http://www.criterion.com/boxsets/" + result)
+ results = re.compile("films/(.*?)\"").findall(html)
+ for result in results:
+ ids.append(result)
+ return set(ids)
+
+def getUrl(id):
+ return "http://www.criterion.com/films/%s" % id
+
+if __name__ == '__main__':
+ print getIds()
\ No newline at end of file
diff --git a/oxweb/impawards.py b/oxweb/impawards.py
index f90bbb0..4a912a0 100644
--- a/oxweb/impawards.py
+++ b/oxweb/impawards.py
@@ -57,11 +57,12 @@ def getId(url):
def getIds():
ids = []
html = getUrlUnicode('http://www.impawards.com/archives/latest.html', timeout = 0)
- pages = int(findRe(html, ''))
- for page in range(pages + 1, 0, -1):
+ pages = int(findRe(html, '')) + 1
+ for page in range(pages, 0, -1):
for id in getIdsByPage(page):
if not id in ids:
ids.append(id)
+ print sorted(ids), len(ids), "%d/%d" % (page, pages)
return ids
def getIdsByPage(page):
@@ -78,4 +79,7 @@ def getUrl(id):
html = getUrlUnicode(url)
if findRe(html, "No Movie Posters on This Page"):
url = "http://www.impawards.com/%s_ver1.html" % id
- return url
\ No newline at end of file
+ return url
+
+if __name__ == '__main__':
+ getIds()