0x2620/python-ox
2
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

add ox.cache.get_json/ox.net.get_json, fixes #2451

Browse source
This commit is contained in:
j 2014-10-05 13:24:14 +02:00
parent 9b860d0d33
commit 865e94da22
2 changed files with 14 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
ox/cache.py
View file

@ -62,6 +62,9 @@ def get_headers(url, data=None, headers=DEFAULT_HEADERS, timeout=cache_timeout):
store.set(url, data, -1, url_headers)
return url_headers
def get_json(url, data=None, headers=DEFAULT_HEADERS, timeout=cache_timeout):
return json.loads(read_url(url, data, headers, timeout).decode('utf-8'))
class InvalidResult(Exception):
"""Base class for exceptions in this module."""
def __init__(self, result, headers):
@ -113,6 +116,8 @@ def read_url(url, data=None, headers=DEFAULT_HEADERS, timeout=cache_timeout, val
result = result.decode(encoding)
return result
get_url=read_url
def save_url(url, filename, overwrite=False):
if not os.path.exists(filename) or overwrite:
dirname = os.path.dirname(filename)

12
ox/net.py
View file

@ -2,13 +2,14 @@
# vi:si:et:sw=4:sts=4:ts=4
# GPL 2008
from __future__ import with_statement, print_function
import os
import gzip
import json
import os
import re
from six import BytesIO, PY3
import struct
from six.moves import urllib
from six import BytesIO, PY3
from six.moves import urllib
from chardet.universaldetector import UniversalDetector
@ -47,6 +48,9 @@ def get_headers(url, data=None, headers=DEFAULT_HEADERS):
headers = e.headers
return dict(headers)
def get_json(url, data=None, headers=DEFAULT_HEADERS):
return json.loads(read_url(url, data, headers).decode('utf-8'))
def open_url(url, data=None, headers=DEFAULT_HEADERS):
if isinstance(url, bytes):
url = url.decode('utf-8')
@ -100,6 +104,8 @@ def detect_encoding(data):
detector.close()
return detector.result['encoding']
get_url=read_url
def save_url(url, filename, overwrite=False):
if not os.path.exists(filename) or overwrite:
dirname = os.path.dirname(filename)