From 25997b68ddb573852f23ce932745cede1e796408 Mon Sep 17 00:00:00 2001
From: Rolux <rolux@Rolux.local>
Date: Mon, 13 Jul 2009 20:40:59 +0200
Subject: [PATCH] updating criterion

---
 oxweb/criterion.py | 100 +++++++++++++++++++--------------------------
 oxweb/impawards.py |  10 +++--
 2 files changed, 50 insertions(+), 60 deletions(-)

diff --git a/oxweb/criterion.py b/oxweb/criterion.py
index 3f2c1c3..ea983e4 100644
--- a/oxweb/criterion.py
+++ b/oxweb/criterion.py
@@ -10,41 +10,24 @@ from oxlib.text import findRe, removeSpecialCharacters
 import imdb
 
 
-def getIds():
-    ids = []
-    html = getUrlUnicode("http://www.criterion.com/library/dvd")
-    results = re.compile("page=(.*?)\"").findall(html)
-    pages = int(results[len(results) - 2])
-    for page in range(1, pages + 1):
-        html = getUrlUnicode("http://www.criterion.com/library/dvd?page=" + str(page))
-        results = re.compile("films/(.*?)\"").findall(html)
-        for result in results:
-            ids.append(result)
-        results = re.compile("boxsets/(.*?)\"").findall(html)
-        for result in results:
-            html = getUrlUnicode("http://www.criterion.com/boxsets/" + result)
-            results = re.compile("films/(.*?)\"").findall(html)
-            for result in results:
-                ids.append(result)
-    return map(lambda id: str(id), sorted(map(lambda id: int(id), set(ids))))
-
 def getData(id):
     '''
     >>> getData('1333')['imdbId']
     '0060304'
 
-    >>> getData('236')['posterUrl']
+    >>> getData('236')['posters'][0]
     'http://criterion_production.s3.amazonaws.com/release_images/1586/ThirdManReplace.jpg'
 
-    >>> getData('786')['posterUrl']
+    >>> getData('786')['posters'][0]
     'http://criterion_production.s3.amazonaws.com/product_images/185/343_box_348x490.jpg'
     '''
-    data = {}
-    data['id'] = id
+    data = {
+        "url": getUrl(id)
+    }
     try:
-        html = getUrlUnicode("http://www.criterion.com/films/" + id)
+        html = getUrlUnicode(data["url"])
     except:
-        html = getUrl("http://www.criterion.com/films/" + id)
+        html = getUrl(data["url"])
     data["number"] = findRe(html, "<p class=\"spinenumber\">(.*?)</p>")
     data["title"] = findRe(html, "<h2 class=\"movietitle\">(.*?)</h2>")
     data["director"] = findRe(html, "<h2 class=\"director\">(.*?)</h2>")
@@ -58,48 +41,51 @@ def getData(id):
         result = re.compile("<div class=\"editioninfo\">(.*?)</div>", re.DOTALL).findall(html)[1]
     result = findRe(result, "<a href=\"(.*?)\">")
     if not "/boxsets/" in result:
-        data["posterUrl"] = result
+        data["posters"] = [result]
     else:
         html_ = getUrlUnicode(result)
         result = findRe(html_, "<a href=\"http://www.criterion.com/films/%s\">(.*?)</a>" % id)
         result = findRe(result, "src=\"(.*?)\"")
-        data["posterUrl"] = result.replace("_w100", "")
+        data["posters"] = [result.replace("_w100", "")]
     result = findRe(html, "<img alt=\"Film Still\" height=\"252\" src=\"(.*?)\"")
     if result:
-        data["stillUrl"] = result
-        data["trailerUrl"] = ""
+        data["stills"] = [result]
+        data["trailers"] = []
     else:
-        data["stillUrl"] = findRe(html, "\"thumbnailURL\", \"(.*?)\"")
-        data["trailerUrl"] = findRe(html, "\"videoURL\", \"(.*?)\"")
+        data["stills"] = [findRe(html, "\"thumbnailURL\", \"(.*?)\"")]
+        data["trailers"] = [findRe(html, "\"videoURL\", \"(.*?)\"")]
     data['imdbId'] = imdb.getMovieId(data['title'], data['director'], data['year'])
     return data
 
-def getPosterUrl(id):
-    data = getData(id)
-    return data['posterUrl']
+def getId(url):
+    return url.split("/")[-1]
 
-def getMovieId(title = '', director = '', year = '', imdbId = ''):
-    if not imdbId:
-        imdbId = imdb.getMovieId(title, director, year)
-    ids = getIds()
-    for id in ids:
-        data = getData(id)
-        if imdb.getMovieId(data['title'], data['director'], data['year'] == imdbId):
-            return id
-    return ''
+def getIds():
+    ids = []
+    html = getUrlUnicode("http://www.criterion.com/library/dvd")
+    results = re.compile("page=(.*?)\"").findall(html)
+    pages = int(results[len(results) - 2])
+    for page in range(pages, 0, -1):
+        for id in getIdsByPage(page):
+            ids.append(id)
+    return map(lambda id: str(id), sorted(map(lambda id: int(id), set(ids))))
 
-def getMovieData(title = '', director = '', year = '', imdbId = ''):
-    '''
-    >>> getMovieData('Pierrot le fou', 'Jean-Luc Godard', '1965')['id']
-    '149'
-    '''
-    data = {}
-    if not imdbId:
-        imdbId = imdb.getMovieId(title, director, year)
-    id = getMovieId(imdbId = imdbId)
-    if id:
-        data_ = getData(id)
-        data['id'] = data_['id']
-        data['posterUrl'] = data_['posterUrl']
-        data['synopsis'] = data_['synopsis']
-    return data
+def getIdsByPage(page):
+    ids = []
+    html = getUrlUnicode("http://www.criterion.com/library/dvd?page=%s" % page)
+    results = re.compile("films/(.*?)\"").findall(html)
+    for result in results:
+        ids.append(result)
+    results = re.compile("boxsets/(.*?)\"").findall(html)
+    for result in results:
+        html = getUrlUnicode("http://www.criterion.com/boxsets/" + result)
+        results = re.compile("films/(.*?)\"").findall(html)
+        for result in results:
+            ids.append(result)
+    return set(ids)
+
+def getUrl(id):
+    return "http://www.criterion.com/films/%s" % id
+
+if __name__ == '__main__':
+    print getIds()
\ No newline at end of file
diff --git a/oxweb/impawards.py b/oxweb/impawards.py
index f90bbb0..4a912a0 100644
--- a/oxweb/impawards.py
+++ b/oxweb/impawards.py
@@ -57,11 +57,12 @@ def getId(url):
 def getIds():
     ids = []
     html = getUrlUnicode('http://www.impawards.com/archives/latest.html', timeout = 0)
-    pages = int(findRe(html, '<a href= page(.*?).html>'))
-    for page in range(pages + 1, 0, -1):
+    pages = int(findRe(html, '<a href= page(.*?).html>')) + 1
+    for page in range(pages, 0, -1):
         for id in getIdsByPage(page):
             if not id in ids:
                 ids.append(id)
+        print sorted(ids), len(ids), "%d/%d" % (page, pages)
     return ids
 
 def getIdsByPage(page):
@@ -78,4 +79,7 @@ def getUrl(id):
     html = getUrlUnicode(url)
     if findRe(html, "No Movie Posters on This Page"):
         url = "http://www.impawards.com/%s_ver1.html" % id
-    return url
\ No newline at end of file
+    return url
+
+if __name__ == '__main__':
+    getIds()