python-oxlib/oxlib/net.py

90 lines
2.4 KiB
Python
Raw Normal View History

2008-04-27 16:54:37 +00:00
# -*- coding: utf-8 -*-
2008-06-19 09:21:21 +00:00
# vi:si:et:sw=4:sts=4:ts=4
2008-07-06 13:00:06 +00:00
# GPL 2008
2009-07-13 10:30:37 +00:00
import os
2008-04-28 09:55:30 +00:00
import gzip
import StringIO
2008-04-27 16:54:37 +00:00
import urllib
import urllib2
from chardet.universaldetector import UniversalDetector
2008-04-27 16:54:37 +00:00
# Default headers for HTTP requests.
DEFAULT_HEADERS = {
'User-Agent': 'Mozilla/5.0 (X11; U; Linux i386; en-US; rv:1.9.1.1) Gecko/20090716 Firefox/3.5',
2008-06-19 09:21:21 +00:00
'Accept-Encoding': 'gzip'
}
2008-04-27 16:54:37 +00:00
def status(url, data=None, headers=DEFAULT_HEADERS):
2008-06-19 09:21:21 +00:00
try:
f = openUrl(url, data, headers)
s = f.code
except urllib2.HTTPError, e:
s = e.code
return s
def exists(url, data=None, headers=DEFAULT_HEADERS):
2008-06-19 09:21:21 +00:00
s = status(url, data, headers)
if s >= 200 and s < 400:
return True
return False
def getHeaders(url, data=None, headers=DEFAULT_HEADERS):
2008-06-19 09:21:21 +00:00
try:
f = openUrl(url, data, headers)
f.headers['Status'] = "%s" % f.code
headers = f.headers
f.close()
except urllib2.HTTPError, e:
e.headers['Status'] = "%s" % e.code
headers = e.headers
return dict(headers)
2008-04-27 16:54:37 +00:00
def openUrl(url, data=None, headers=DEFAULT_HEADERS):
2008-06-19 09:21:21 +00:00
url = url.replace(' ', '%20')
req = urllib2.Request(url, data, headers)
return urllib2.urlopen(req)
2008-04-27 16:54:37 +00:00
2009-10-11 13:03:00 +00:00
def readUrl(url, data=None, headers=DEFAULT_HEADERS, returnHeaders=False):
2008-06-19 09:21:21 +00:00
f = openUrl(url, data, headers)
data = f.read()
f.close()
if f.headers.get('content-encoding', None) == 'gzip':
data = gzip.GzipFile(fileobj=StringIO.StringIO(data)).read()
if returnHeaders:
f.headers['Status'] = "%s" % f.code
return dict(f.headers), data
return data
2008-04-27 16:54:37 +00:00
2009-10-11 13:03:00 +00:00
def readUrlUnicode(url):
data = readUrl(url)
2008-06-19 09:21:21 +00:00
encoding = getEncoding(data)
if not encoding:
encoding = 'latin-1'
return unicode(data, encoding)
2008-04-27 16:54:37 +00:00
def getEncoding(data):
2009-03-14 21:02:20 +00:00
if 'content="text/html; charset=utf-8"' in data:
return 'utf-8'
2009-07-15 13:53:40 +00:00
elif 'content="text/html; charset=iso-8859-1"' in data:
return 'iso-8859-1'
2008-06-19 09:21:21 +00:00
detector = UniversalDetector()
for line in data.split('\n'):
detector.feed(line)
if detector.done:
break
detector.close()
return detector.result['encoding']
2009-07-15 13:53:40 +00:00
def saveUrl(url, filename, overwrite=False):
if not os.path.exists(filename) or overwrite:
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
os.makedirs(dirname)
2009-10-15 12:57:07 +00:00
data = readUrl(url)
2009-07-15 13:53:40 +00:00
f = open(filename, 'w')
f.write(data)
f.close()