0x2620/openmedialibrary_platform
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

update Shared

Browse source
This commit is contained in:
j 2015-11-04 13:01:55 +01:00
commit 6881f3471a
184 changed files with 13080 additions and 13691 deletions
Download patch file Download diff file Expand all files Collapse all files

29
Shared/lib/python3.4/site-packages/ox/net.py
View file

@ -2,20 +2,21 @@
# vi:si:et:sw=4:sts=4:ts=4
# GPL 2008
from __future__ import with_statement, print_function
import os
import gzip
import json
import os
import re
from six import BytesIO, PY3
import struct
from six.moves import urllib
from six import BytesIO, PY3
from six.moves import urllib
from chardet.universaldetector import UniversalDetector
DEBUG = False
# Default headers for HTTP requests.
DEFAULT_HEADERS = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0',
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-us,en;q=0.5',
@ -47,9 +48,16 @@ def get_headers(url, data=None, headers=DEFAULT_HEADERS):
headers = e.headers
return dict(headers)
def get_json(url, data=None, headers=DEFAULT_HEADERS):
return json.loads(read_url(url, data, headers).decode('utf-8'))
def open_url(url, data=None, headers=DEFAULT_HEADERS):
if isinstance(url, bytes):
url = url.decode('utf-8')
if PY3:
if isinstance(url, bytes):
url = url.decode('utf-8')
else:
if not isinstance(url, bytes):
url = url.encode('utf-8')
url = url.replace(' ', '%20')
if data and PY3 and not isinstance(data, bytes):
data = data.encode('utf-8')
@ -100,10 +108,12 @@ def detect_encoding(data):
detector.close()
return detector.result['encoding']
get_url=read_url
def save_url(url, filename, overwrite=False):
if not os.path.exists(filename) or overwrite:
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
data = read_url(url)
with open(filename, 'wb') as f:
@ -135,8 +145,9 @@ def oshash(url):
if filesize > 65536:
tail = get_range(url, filesize-65536, filesize)
if filesize < 65536:
for offset in range(0, filesize, bytesize):
buffer = head[offset:offset+bytesize]
f = BytesIO(head)
for x in range(int(filesize/bytesize)):
buffer = f.read(bytesize)
(l_value,)= struct.unpack(longlongformat, buffer)
hash += l_value
hash = hash & 0xFFFFFFFFFFFFFFFF #cut off 64bit overflow