python-ox/ox/net.py

173 lines
5.6 KiB
Python
Raw Normal View History

2008-04-27 16:54:37 +00:00
# -*- coding: utf-8 -*-
2008-06-19 09:21:21 +00:00
# vi:si:et:sw=4:sts=4:ts=4
2008-07-06 13:00:06 +00:00
# GPL 2008
from __future__ import print_function
2008-04-28 09:55:30 +00:00
import gzip
import json
import os
import re
2012-04-30 08:40:10 +00:00
import struct
2008-04-27 16:54:37 +00:00
2023-07-27 16:07:49 +00:00
import requests
2023-07-27 11:07:13 +00:00
from io import BytesIO
import urllib
from chardet.universaldetector import UniversalDetector
2008-04-27 16:54:37 +00:00
2012-08-21 06:41:25 +00:00
DEBUG = False
2008-04-27 16:54:37 +00:00
# Default headers for HTTP requests.
DEFAULT_HEADERS = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0',
2010-08-12 21:35:31 +00:00
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
2017-08-02 14:48:22 +00:00
'Accept-Language': 'en-US,en;q=0.8,fr;q=0.6,de;q=0.4',
'Accept-Encoding': 'gzip',
}
2008-04-27 16:54:37 +00:00
2016-06-08 13:30:25 +00:00
def status(url, data=None, headers=None):
2008-06-19 09:21:21 +00:00
try:
f = open_url(url, data, headers)
2008-06-19 09:21:21 +00:00
s = f.code
2014-09-30 19:04:46 +00:00
except urllib.error.HTTPError as e:
2008-06-19 09:21:21 +00:00
s = e.code
return s
2016-06-08 13:30:25 +00:00
def exists(url, data=None, headers=None):
2008-06-19 09:21:21 +00:00
s = status(url, data, headers)
if s >= 200 and s < 400:
return True
return False
2016-06-08 13:30:25 +00:00
def get_headers(url, data=None, headers=None):
2008-06-19 09:21:21 +00:00
try:
f = open_url(url, data, headers)
2008-06-19 09:21:21 +00:00
f.headers['Status'] = "%s" % f.code
headers = f.headers
f.close()
2014-09-30 19:04:46 +00:00
except urllib.error.HTTPError as e:
2008-06-19 09:21:21 +00:00
e.headers['Status'] = "%s" % e.code
headers = e.headers
return dict(headers)
2016-06-08 13:30:25 +00:00
def get_json(url, data=None, headers=None):
return json.loads(read_url(url, data, headers).decode('utf-8')) # pylint: disable=no-member
2016-06-08 13:30:25 +00:00
def open_url(url, data=None, headers=None):
if headers is None:
headers = DEFAULT_HEADERS.copy()
2023-07-27 11:07:13 +00:00
if isinstance(url, bytes):
url = url.decode('utf-8')
2008-06-19 09:21:21 +00:00
url = url.replace(' ', '%20')
2023-07-27 11:07:13 +00:00
if data and not isinstance(data, bytes):
2014-10-04 19:04:55 +00:00
data = data.encode('utf-8')
2014-09-30 19:04:46 +00:00
req = urllib.request.Request(url, data, headers)
return urllib.request.urlopen(req)
2008-04-27 16:54:37 +00:00
2016-06-08 13:30:25 +00:00
def read_url(url, data=None, headers=None, return_headers=False, unicode=False):
2012-08-21 06:41:25 +00:00
if DEBUG:
2014-09-30 19:04:46 +00:00
print('ox.net.read_url', url)
f = open_url(url, data, headers)
2012-08-17 20:20:35 +00:00
result = f.read()
2008-06-19 09:21:21 +00:00
f.close()
if f.headers.get('content-encoding', None) == 'gzip':
2014-09-30 19:04:46 +00:00
result = gzip.GzipFile(fileobj=BytesIO(result)).read()
if unicode:
2014-09-30 19:04:46 +00:00
ctype = f.headers.get('content-type', '').lower()
if 'charset' in ctype:
encoding = ctype.split('charset=')[-1]
else:
encoding = detect_encoding(result)
if not encoding:
encoding = 'latin-1'
2012-08-17 20:20:35 +00:00
result = result.decode(encoding)
if return_headers:
2008-06-19 09:21:21 +00:00
f.headers['Status'] = "%s" % f.code
2014-09-30 19:04:46 +00:00
headers = {}
for key in f.headers:
headers[key.lower()] = f.headers[key]
return headers, result
2012-08-17 20:20:35 +00:00
return result
2008-04-27 16:54:37 +00:00
def detect_encoding(data):
2014-09-30 19:04:46 +00:00
data_lower = data.lower().decode('utf-8', 'ignore')
charset = re.compile('content="text/html; charset=(.*?)"').findall(data_lower)
if not charset:
2014-09-30 19:04:46 +00:00
charset = re.compile('meta charset="(.*?)"').findall(data_lower)
if charset:
return charset[0].lower()
2008-06-19 09:21:21 +00:00
detector = UniversalDetector()
2014-09-30 19:04:46 +00:00
p = 0
l = len(data)
s = 1024
while p < l:
detector.feed(data[p:p+s])
2008-06-19 09:21:21 +00:00
if detector.done:
break
2014-09-30 19:04:46 +00:00
p += s
2008-06-19 09:21:21 +00:00
detector.close()
return detector.result['encoding']
2016-06-08 13:30:25 +00:00
get_url = read_url
def save_url(url, filename, overwrite=False):
2009-07-15 13:53:40 +00:00
if not os.path.exists(filename) or overwrite:
dirname = os.path.dirname(filename)
os.makedirs(dirname, exist_ok=True)
headers = DEFAULT_HEADERS.copy()
2023-07-27 16:07:49 +00:00
r = requests.get(url, headers=headers, stream=True)
filename_tmp = filename + '~'
with open(filename_tmp, 'wb') as f:
2023-07-27 16:07:49 +00:00
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
os.rename(filename_tmp, filename)
2009-07-15 13:53:40 +00:00
2016-06-08 13:30:25 +00:00
def _get_size(url):
req = urllib.request.Request(url, headers=DEFAULT_HEADERS.copy())
req.get_method = lambda: 'HEAD'
u = urllib.request.urlopen(req)
if u.code != 200 or 'Content-Length' not in u.headers:
raise IOError
return int(u.headers['Content-Length'])
def _get_range(url, start, end):
headers = DEFAULT_HEADERS.copy()
headers['Range'] = 'bytes=%s-%s' % (start, end)
req = urllib.request.Request(url, headers=headers)
u = urllib.request.urlopen(req)
return u.read()
2012-04-30 08:40:10 +00:00
2016-06-08 13:30:25 +00:00
def oshash(url):
2012-04-30 08:40:10 +00:00
try:
longlongformat = 'q' # long long
bytesize = struct.calcsize(longlongformat)
2016-06-08 13:30:25 +00:00
filesize = _get_size(url)
hash_ = filesize
head = _get_range(url, 0, min(filesize, 65536))
2012-04-30 08:40:10 +00:00
if filesize > 65536:
2016-06-08 13:30:25 +00:00
tail = _get_range(url, filesize-65536, filesize)
2012-04-30 08:40:10 +00:00
if filesize < 65536:
2015-03-19 13:27:50 +00:00
f = BytesIO(head)
2016-06-08 13:30:25 +00:00
for _ in range(int(filesize/bytesize)):
2015-03-19 13:27:50 +00:00
buffer = f.read(bytesize)
2016-06-08 13:30:25 +00:00
(l_value,) = struct.unpack(longlongformat, buffer)
hash_ += l_value
hash_ = hash_ & 0xFFFFFFFFFFFFFFFF # cut off 64bit overflow
2012-04-30 08:40:10 +00:00
else:
for offset in range(0, 65536, bytesize):
buffer = head[offset:offset+bytesize]
2016-06-08 13:30:25 +00:00
(l_value,) = struct.unpack(longlongformat, buffer)
hash_ += l_value
hash_ = hash_ & 0xFFFFFFFFFFFFFFFF # cut of 64bit overflow
2012-04-30 08:40:10 +00:00
for offset in range(0, 65536, bytesize):
buffer = tail[offset:offset+bytesize]
2016-06-08 13:30:25 +00:00
(l_value,) = struct.unpack(longlongformat, buffer)
hash_ += l_value
hash_ = hash_ & 0xFFFFFFFFFFFFFFFF
returnedhash = "%016x" % hash_
2012-04-30 08:40:10 +00:00
return returnedhash
2016-06-08 13:30:25 +00:00
except IOError:
2012-04-30 08:40:10 +00:00
return "IOError"