0x2620/python-ox
2
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

add ox.cache.get_json/ox.net.get_json, fixes #2451

Browse source
This commit is contained in:
j 2014-10-05 13:24:14 +02:00
parent 9b860d0d33
commit 865e94da22
2 changed files with 14 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
ox/cache.py
View file

@ -62,6 +62,9 @@ def get_headers(url, data=None, headers=DEFAULT_HEADERS, timeout=cache_timeout):
store.set(url, data, -1, url_headers) store.set(url, data, -1, url_headers)
return url_headers return url_headers
def get_json(url, data=None, headers=DEFAULT_HEADERS, timeout=cache_timeout):
return json.loads(read_url(url, data, headers, timeout).decode('utf-8'))
class InvalidResult(Exception): class InvalidResult(Exception):
"""Base class for exceptions in this module.""" """Base class for exceptions in this module."""
def __init__(self, result, headers): def __init__(self, result, headers):
@ -113,6 +116,8 @@ def read_url(url, data=None, headers=DEFAULT_HEADERS, timeout=cache_timeout, val
result = result.decode(encoding) result = result.decode(encoding)
return result return result
get_url=read_url
def save_url(url, filename, overwrite=False): def save_url(url, filename, overwrite=False):
if not os.path.exists(filename) or overwrite: if not os.path.exists(filename) or overwrite:
dirname = os.path.dirname(filename) dirname = os.path.dirname(filename)

12
ox/net.py
View file

@ -2,13 +2,14 @@
# vi:si:et:sw=4:sts=4:ts=4 # vi:si:et:sw=4:sts=4:ts=4
# GPL 2008 # GPL 2008
from __future__ import with_statement, print_function from __future__ import with_statement, print_function
import os
import gzip import gzip
import json
import os
import re import re
from six import BytesIO, PY3
import struct import struct
from six.moves import urllib
from six import BytesIO, PY3
from six.moves import urllib
from chardet.universaldetector import UniversalDetector from chardet.universaldetector import UniversalDetector
@ -47,6 +48,9 @@ def get_headers(url, data=None, headers=DEFAULT_HEADERS):
headers = e.headers headers = e.headers
return dict(headers) return dict(headers)
def get_json(url, data=None, headers=DEFAULT_HEADERS):
return json.loads(read_url(url, data, headers).decode('utf-8'))
def open_url(url, data=None, headers=DEFAULT_HEADERS): def open_url(url, data=None, headers=DEFAULT_HEADERS):
if isinstance(url, bytes): if isinstance(url, bytes):
url = url.decode('utf-8') url = url.decode('utf-8')
@ -100,6 +104,8 @@ def detect_encoding(data):
detector.close() detector.close()
return detector.result['encoding'] return detector.result['encoding']
get_url=read_url
def save_url(url, filename, overwrite=False): def save_url(url, filename, overwrite=False):
if not os.path.exists(filename) or overwrite: if not os.path.exists(filename) or overwrite:
dirname = os.path.dirname(filename) dirname = os.path.dirname(filename)