0x2620/python-ox
2
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

change api, getUrl -> readUrl

Browse source
This commit is contained in:
j 2009-10-11 15:03:00 +02:00
parent d14f13faaf
commit 8d14cf2291
2 changed files with 10 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
ox/cache.py
View file

@ -43,7 +43,7 @@ def exists(url, data=None, headers=DEFAULT_HEADERS, timeout=cache_timeout):
return False return False
def getHeaders(url, data=None, headers=DEFAULT_HEADERS, timeout=cache_timeout): def getHeaders(url, data=None, headers=DEFAULT_HEADERS, timeout=cache_timeout):
url_headers = _getUrlCache(url, data, headers, timeout, "headers") url_headers = _readUrlCache(url, data, headers, timeout, "headers")
if url_headers: if url_headers:
url_headers = simplejson.loads(url_headers) url_headers = simplejson.loads(url_headers)
else: else:
@ -57,7 +57,7 @@ class InvalidResult(Exception):
self.result = result self.result = result
self.headers = headers self.headers = headers
def getUrl(url, data=None, headers=DEFAULT_HEADERS, timeout=cache_timeout, valid=None): def readUrl(url, data=None, headers=DEFAULT_HEADERS, timeout=cache_timeout, valid=None):
''' '''
url - url to load url - url to load
data - possible post data data - possible post data
@ -69,11 +69,11 @@ def getUrl(url, data=None, headers=DEFAULT_HEADERS, timeout=cache_timeout, valid
#FIXME: send last-modified / etag from cache and only update if needed #FIXME: send last-modified / etag from cache and only update if needed
if isinstance(url, unicode): if isinstance(url, unicode):
url = url.encode('utf-8') url = url.encode('utf-8')
result = _getUrlCache(url, data, headers, timeout) result = _readUrlCache(url, data, headers, timeout)
if not result: if not result:
#print "get data", url #print "get data", url
try: try:
url_headers, result = net.getUrl(url, data, headers, returnHeaders=True) url_headers, result = net.readUrl(url, data, headers, returnHeaders=True)
except urllib2.HTTPError, e: except urllib2.HTTPError, e:
e.headers['Status'] = "%s" % e.code e.headers['Status'] = "%s" % e.code
url_headers = dict(e.headers) url_headers = dict(e.headers)
@ -86,8 +86,8 @@ def getUrl(url, data=None, headers=DEFAULT_HEADERS, timeout=cache_timeout, valid
raise InvalidResult(result, url_headers) raise InvalidResult(result, url_headers)
return result return result
def getUrlUnicode(url, data=None, headers=DEFAULT_HEADERS, timeout=cache_timeout, _getUrl=getUrl, valid=None): def readUrlUnicode(url, data=None, headers=DEFAULT_HEADERS, timeout=cache_timeout, _readUrl=readUrl, valid=None):
data = _getUrl(url, data, headers, timeout, valid) data = _readUrl(url, data, headers, timeout, valid)
encoding = getEncoding(data) encoding = getEncoding(data)
if not encoding: if not encoding:
encoding = 'latin-1' encoding = 'latin-1'
@ -114,7 +114,7 @@ def _createDb(c):
c.execute('''CREATE INDEX IF NOT EXISTS cache_url_hash ON cache (url_hash)''') c.execute('''CREATE INDEX IF NOT EXISTS cache_url_hash ON cache (url_hash)''')
def _getUrlCache(url, data, headers=DEFAULT_HEADERS, timeout=-1, value="data"): def _readUrlCache(url, data, headers=DEFAULT_HEADERS, timeout=-1, value="data"):
r = None r = None
if timeout == 0: if timeout == 0:
return r return r

6
ox/net.py
View file

@ -46,7 +46,7 @@ def openUrl(url, data=None, headers=DEFAULT_HEADERS):
req = urllib2.Request(url, data, headers) req = urllib2.Request(url, data, headers)
return urllib2.urlopen(req) return urllib2.urlopen(req)
def getUrl(url, data=None, headers=DEFAULT_HEADERS, returnHeaders=False): def readUrl(url, data=None, headers=DEFAULT_HEADERS, returnHeaders=False):
f = openUrl(url, data, headers) f = openUrl(url, data, headers)
data = f.read() data = f.read()
f.close() f.close()
@ -57,8 +57,8 @@ def getUrl(url, data=None, headers=DEFAULT_HEADERS, returnHeaders=False):
return dict(f.headers), data return dict(f.headers), data
return data return data
def getUrlUnicode(url): def readUrlUnicode(url):
data = getUrl(url) data = readUrl(url)
encoding = getEncoding(data) encoding = getEncoding(data)
if not encoding: if not encoding:
encoding = 'latin-1' encoding = 'latin-1'