0x2620/python-ox
2
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

speedup imdb parser

Browse source
This commit is contained in:
j 2012-08-21 09:06:29 +02:00
parent 03dbc4d045
commit f38c7c4ea2
2 changed files with 7 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
ox/web/imdb.py
View file

@ -264,7 +264,9 @@ class Imdb(SiteParser):
} }
def read_url(self, url, timeout): def read_url(self, url, timeout):
return read_url(url, timeout=timeout, unicode=True) if not url in self._cache:
self._cache[url] = read_url(url, timeout=timeout, unicode=True)
return self._cache[url]
def __init__(self, id, timeout=-1): def __init__(self, id, timeout=-1):
#use akas.imdb.com to always get original title: #use akas.imdb.com to always get original title:

5
ox/web/siteparser.py
View file

@ -31,9 +31,12 @@ class SiteParser(dict):
return "%s%s" % (self.baseUrl, page) return "%s%s" % (self.baseUrl, page)
def read_url(self, url, timeout): def read_url(self, url, timeout):
return read_url(url, timeout=timeout, unicode=True) if not url in self._cache:
self._cache[url] = read_url(url, timeout=timeout, unicode=True)
return self._cache[url]
def __init__(self, timeout=-1): def __init__(self, timeout=-1):
self._cache = {}
for key in self.regex: for key in self.regex:
url = self.get_url(self.regex[key]['page']) url = self.get_url(self.regex[key]['page'])
data = self.read_url(url, timeout) data = self.read_url(url, timeout)