add read_url debug output

This commit is contained in:
j 2012-08-21 08:41:25 +02:00
parent 21dd1c7903
commit 61b14e9537
2 changed files with 5 additions and 0 deletions

View file

@ -82,6 +82,8 @@ def read_url(url, data=None, headers=DEFAULT_HEADERS, timeout=cache_timeout, val
valid - function to check if result is ok, its passed result and headers
if this function fails, InvalidResult will be raised deal with it in your code
'''
if net.DEBUG:
print 'ox.cache.read_url', url
#FIXME: send last-modified / etag from cache and only update if needed
url = _fix_unicode_url(url)
result = store.get(url, data, headers, timeout)

View file

@ -11,6 +11,7 @@ import urllib2
from chardet.universaldetector import UniversalDetector
DEBUG = False
# Default headers for HTTP requests.
DEFAULT_HEADERS = {
'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:7.0.1) Gecko/20100101 Firefox/7.0.1',
@ -51,6 +52,8 @@ def open_url(url, data=None, headers=DEFAULT_HEADERS):
return urllib2.urlopen(req)
def read_url(url, data=None, headers=DEFAULT_HEADERS, return_headers=False, unicode=False):
if DEBUG:
print 'ox.net.read_url', url
f = open_url(url, data, headers)
result = f.read()
f.close()