0x2620/python-ox
2
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

use requests if available

Browse source
This commit is contained in:
j 2016-06-16 15:30:27 +02:00
parent fa7f83e8c6
commit ce50a626dc
1 changed files with 28 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
ox/cache.py
View file

@ -13,6 +13,11 @@ import zlib
from six import BytesIO from six import BytesIO
from six.moves import urllib from six.moves import urllib
from six import PY2 from six import PY2
try:
import requests
USE_REQUESTS = True
except:
USE_REQUESTS = False
from .utils import json from .utils import json
from .file import makedirs from .file import makedirs
@ -94,19 +99,30 @@ def read_url(url, data=None, headers=None, timeout=cache_timeout, valid=None, un
result = store.get(url, data, headers, timeout) result = store.get(url, data, headers, timeout)
url_headers = {} url_headers = {}
if not result: if not result:
try: if USE_REQUESTS:
url_headers, result = net.read_url(url, data, headers, return_headers=True) r = requests.get(url, headers=headers)
except urllib.error.HTTPError as e: for key in r.headers:
e.headers['Status'] = "%s" % e.code url_headers[key.lower()] = r.headers[key]
for key in e.headers: result = r.content
url_headers[key.lower()] = e.headers[key] url_headers['Status'] = "%s" % r.status_code
result = e.read() if not valid or valid(result, url_headers):
if url_headers.get('content-encoding', None) == 'gzip': store.set(url, post_data=data, data=result, headers=url_headers)
result = gzip.GzipFile(fileobj=BytesIO(result)).read() else:
if not valid or valid(result, url_headers): raise InvalidResult(result, url_headers)
store.set(url, post_data=data, data=result, headers=url_headers)
else: else:
raise InvalidResult(result, url_headers) try:
url_headers, result = net.read_url(url, data, headers, return_headers=True)
except urllib.error.HTTPError as e:
e.headers['Status'] = "%s" % e.code
for key in e.headers:
url_headers[key.lower()] = e.headers[key]
result = e.read()
if url_headers.get('content-encoding', None) == 'gzip':
result = gzip.GzipFile(fileobj=BytesIO(result)).read()
if not valid or valid(result, url_headers):
store.set(url, post_data=data, data=result, headers=url_headers)
else:
raise InvalidResult(result, url_headers)
if unicode: if unicode:
ctype = url_headers.get('content-type', '').lower() ctype = url_headers.get('content-type', '').lower()
if 'charset' in ctype: if 'charset' in ctype: