updating criterion
This commit is contained in:
parent
c78864d3ab
commit
25997b68dd
2 changed files with 50 additions and 60 deletions
|
@ -10,41 +10,24 @@ from oxlib.text import findRe, removeSpecialCharacters
|
|||
import imdb
|
||||
|
||||
|
||||
def getIds():
|
||||
ids = []
|
||||
html = getUrlUnicode("http://www.criterion.com/library/dvd")
|
||||
results = re.compile("page=(.*?)\"").findall(html)
|
||||
pages = int(results[len(results) - 2])
|
||||
for page in range(1, pages + 1):
|
||||
html = getUrlUnicode("http://www.criterion.com/library/dvd?page=" + str(page))
|
||||
results = re.compile("films/(.*?)\"").findall(html)
|
||||
for result in results:
|
||||
ids.append(result)
|
||||
results = re.compile("boxsets/(.*?)\"").findall(html)
|
||||
for result in results:
|
||||
html = getUrlUnicode("http://www.criterion.com/boxsets/" + result)
|
||||
results = re.compile("films/(.*?)\"").findall(html)
|
||||
for result in results:
|
||||
ids.append(result)
|
||||
return map(lambda id: str(id), sorted(map(lambda id: int(id), set(ids))))
|
||||
|
||||
def getData(id):
|
||||
'''
|
||||
>>> getData('1333')['imdbId']
|
||||
'0060304'
|
||||
|
||||
>>> getData('236')['posterUrl']
|
||||
>>> getData('236')['posters'][0]
|
||||
'http://criterion_production.s3.amazonaws.com/release_images/1586/ThirdManReplace.jpg'
|
||||
|
||||
>>> getData('786')['posterUrl']
|
||||
>>> getData('786')['posters'][0]
|
||||
'http://criterion_production.s3.amazonaws.com/product_images/185/343_box_348x490.jpg'
|
||||
'''
|
||||
data = {}
|
||||
data['id'] = id
|
||||
data = {
|
||||
"url": getUrl(id)
|
||||
}
|
||||
try:
|
||||
html = getUrlUnicode("http://www.criterion.com/films/" + id)
|
||||
html = getUrlUnicode(data["url"])
|
||||
except:
|
||||
html = getUrl("http://www.criterion.com/films/" + id)
|
||||
html = getUrl(data["url"])
|
||||
data["number"] = findRe(html, "<p class=\"spinenumber\">(.*?)</p>")
|
||||
data["title"] = findRe(html, "<h2 class=\"movietitle\">(.*?)</h2>")
|
||||
data["director"] = findRe(html, "<h2 class=\"director\">(.*?)</h2>")
|
||||
|
@ -58,48 +41,51 @@ def getData(id):
|
|||
result = re.compile("<div class=\"editioninfo\">(.*?)</div>", re.DOTALL).findall(html)[1]
|
||||
result = findRe(result, "<a href=\"(.*?)\">")
|
||||
if not "/boxsets/" in result:
|
||||
data["posterUrl"] = result
|
||||
data["posters"] = [result]
|
||||
else:
|
||||
html_ = getUrlUnicode(result)
|
||||
result = findRe(html_, "<a href=\"http://www.criterion.com/films/%s\">(.*?)</a>" % id)
|
||||
result = findRe(result, "src=\"(.*?)\"")
|
||||
data["posterUrl"] = result.replace("_w100", "")
|
||||
data["posters"] = [result.replace("_w100", "")]
|
||||
result = findRe(html, "<img alt=\"Film Still\" height=\"252\" src=\"(.*?)\"")
|
||||
if result:
|
||||
data["stillUrl"] = result
|
||||
data["trailerUrl"] = ""
|
||||
data["stills"] = [result]
|
||||
data["trailers"] = []
|
||||
else:
|
||||
data["stillUrl"] = findRe(html, "\"thumbnailURL\", \"(.*?)\"")
|
||||
data["trailerUrl"] = findRe(html, "\"videoURL\", \"(.*?)\"")
|
||||
data["stills"] = [findRe(html, "\"thumbnailURL\", \"(.*?)\"")]
|
||||
data["trailers"] = [findRe(html, "\"videoURL\", \"(.*?)\"")]
|
||||
data['imdbId'] = imdb.getMovieId(data['title'], data['director'], data['year'])
|
||||
return data
|
||||
|
||||
def getPosterUrl(id):
|
||||
data = getData(id)
|
||||
return data['posterUrl']
|
||||
def getId(url):
|
||||
return url.split("/")[-1]
|
||||
|
||||
def getMovieId(title = '', director = '', year = '', imdbId = ''):
|
||||
if not imdbId:
|
||||
imdbId = imdb.getMovieId(title, director, year)
|
||||
ids = getIds()
|
||||
for id in ids:
|
||||
data = getData(id)
|
||||
if imdb.getMovieId(data['title'], data['director'], data['year'] == imdbId):
|
||||
return id
|
||||
return ''
|
||||
def getIds():
|
||||
ids = []
|
||||
html = getUrlUnicode("http://www.criterion.com/library/dvd")
|
||||
results = re.compile("page=(.*?)\"").findall(html)
|
||||
pages = int(results[len(results) - 2])
|
||||
for page in range(pages, 0, -1):
|
||||
for id in getIdsByPage(page):
|
||||
ids.append(id)
|
||||
return map(lambda id: str(id), sorted(map(lambda id: int(id), set(ids))))
|
||||
|
||||
def getMovieData(title = '', director = '', year = '', imdbId = ''):
|
||||
'''
|
||||
>>> getMovieData('Pierrot le fou', 'Jean-Luc Godard', '1965')['id']
|
||||
'149'
|
||||
'''
|
||||
data = {}
|
||||
if not imdbId:
|
||||
imdbId = imdb.getMovieId(title, director, year)
|
||||
id = getMovieId(imdbId = imdbId)
|
||||
if id:
|
||||
data_ = getData(id)
|
||||
data['id'] = data_['id']
|
||||
data['posterUrl'] = data_['posterUrl']
|
||||
data['synopsis'] = data_['synopsis']
|
||||
return data
|
||||
def getIdsByPage(page):
|
||||
ids = []
|
||||
html = getUrlUnicode("http://www.criterion.com/library/dvd?page=%s" % page)
|
||||
results = re.compile("films/(.*?)\"").findall(html)
|
||||
for result in results:
|
||||
ids.append(result)
|
||||
results = re.compile("boxsets/(.*?)\"").findall(html)
|
||||
for result in results:
|
||||
html = getUrlUnicode("http://www.criterion.com/boxsets/" + result)
|
||||
results = re.compile("films/(.*?)\"").findall(html)
|
||||
for result in results:
|
||||
ids.append(result)
|
||||
return set(ids)
|
||||
|
||||
def getUrl(id):
|
||||
return "http://www.criterion.com/films/%s" % id
|
||||
|
||||
if __name__ == '__main__':
|
||||
print getIds()
|
|
@ -57,11 +57,12 @@ def getId(url):
|
|||
def getIds():
|
||||
ids = []
|
||||
html = getUrlUnicode('http://www.impawards.com/archives/latest.html', timeout = 0)
|
||||
pages = int(findRe(html, '<a href= page(.*?).html>'))
|
||||
for page in range(pages + 1, 0, -1):
|
||||
pages = int(findRe(html, '<a href= page(.*?).html>')) + 1
|
||||
for page in range(pages, 0, -1):
|
||||
for id in getIdsByPage(page):
|
||||
if not id in ids:
|
||||
ids.append(id)
|
||||
print sorted(ids), len(ids), "%d/%d" % (page, pages)
|
||||
return ids
|
||||
|
||||
def getIdsByPage(page):
|
||||
|
@ -78,4 +79,7 @@ def getUrl(id):
|
|||
html = getUrlUnicode(url)
|
||||
if findRe(html, "No Movie Posters on This Page"):
|
||||
url = "http://www.impawards.com/%s_ver1.html" % id
|
||||
return url
|
||||
return url
|
||||
|
||||
if __name__ == '__main__':
|
||||
getIds()
|
||||
|
|
Loading…
Reference in a new issue