python-ox/ox/html.py

453 lines
16 KiB
Python
Raw Normal View History

2008-04-27 16:54:37 +00:00
# -*- coding: utf-8 -*-
2008-06-19 09:21:21 +00:00
# vi:si:et:sw=4:sts=4:ts=4
2008-07-06 13:00:06 +00:00
# GPL 2008
2008-04-27 16:54:37 +00:00
import re
import string
2023-07-27 11:07:13 +00:00
from html.entities import name2codepoint
2008-04-27 16:54:37 +00:00
2015-12-25 15:13:15 +00:00
letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
2008-04-27 16:54:37 +00:00
2012-05-27 11:50:10 +00:00
# Configuration for add_links() function
2015-12-25 15:13:15 +00:00
2016-06-08 13:32:46 +00:00
LEADING_PUNCTUATION = ['(', '<', '&lt;']
2008-04-27 16:54:37 +00:00
TRAILING_PUNCTUATION = ['.', ',', ')', '>', '\n', '&gt;', "'", '"']
# list of possible strings used for bullets in bulleted lists
DOTS = ['&middot;', '*', '\xe2\x80\xa2', '&#149;', '&bull;', '&#8226;']
unencoded_ampersands_re = re.compile(r'&(?!(\w+|#\d+);)')
2024-03-20 11:55:14 +00:00
word_split_re = re.compile(r'(\s+|<br>)')
2016-06-08 13:32:46 +00:00
punctuation_re = re.compile('^(?P<lead>(?:%s)*)(?P<middle>.*?)(?P<trail>(?:%s)*)$' % (
'|'.join([re.escape(x) for x in LEADING_PUNCTUATION]),
'|'.join([re.escape(x) for x in TRAILING_PUNCTUATION])))
2008-04-27 16:54:37 +00:00
simple_email_re = re.compile(r'^\S+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+$')
link_target_attribute_re = re.compile(r'(<a [^>]*?)target=[^\s>]+')
html_gunk_re = re.compile(r'(?:<br clear="all">|<i><\/i>|<b><\/b>|<em><\/em>|<strong><\/strong>|<\/?smallcaps>|<\/?uppercase>)', re.IGNORECASE)
hard_coded_bullets_re = re.compile(r'((?:<p>(?:%s).*?[a-zA-Z].*?</p>\s*)+)' % '|'.join([re.escape(x) for x in DOTS]), re.DOTALL)
trailing_empty_content_re = re.compile(r'(?:<p>(?:&nbsp;|\s|<br \/>)*?</p>\s*)+\Z')
2023-07-27 11:07:13 +00:00
2008-04-27 16:54:37 +00:00
def escape(html):
2008-06-19 09:21:21 +00:00
'''
Returns the given HTML with ampersands, quotes and carets encoded
2008-06-19 09:21:21 +00:00
>>> escape('html "test" & <brothers>')
'html &quot;test&quot; &amp; &lt;brothers&gt;'
'''
2023-07-27 11:07:13 +00:00
if not isinstance(html, str):
html = str(html)
2012-02-21 15:44:50 +00:00
return html.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;').replace("'", '&apos;')
2008-04-27 16:54:37 +00:00
def linebreaks(value):
2008-06-19 09:21:21 +00:00
'''
Converts newlines into <p> and <br />
'''
2016-06-08 13:32:46 +00:00
value = re.sub(r'\r\n|\r|\n', '\n', value) # normalize newlines
2008-06-19 09:21:21 +00:00
paras = re.split('\n{2,}', value)
paras = ['<p>%s</p>' % p.strip().replace('\n', '<br />') for p in paras]
return '\n\n'.join(paras)
2008-04-27 16:54:37 +00:00
2012-05-27 11:50:10 +00:00
def strip_tags(value):
2008-06-19 09:21:21 +00:00
"""
Returns the given HTML with all tags stripped
2012-05-27 11:50:10 +00:00
>>> strip_tags('some <h2>title</h2> <script>asdfasdf</script>')
2008-06-19 09:21:21 +00:00
'some title asdfasdf'
"""
return re.sub(r'<[^>]*?>', '', value)
2012-05-27 11:50:10 +00:00
stripTags = strip_tags
def strip_spaces_between_tags(value):
2008-06-19 09:21:21 +00:00
"Returns the given HTML with spaces between tags normalized to a single space"
return re.sub(r'>\s+<', '> <', value)
2008-04-27 16:54:37 +00:00
def strip_entities(value):
2008-06-19 09:21:21 +00:00
"Returns the given HTML with all entities (&something;) stripped"
return re.sub(r'&(?:\w+|#\d);', '', value)
2008-04-27 16:54:37 +00:00
def fix_ampersands(value):
2008-06-19 09:21:21 +00:00
"Returns the given HTML with all unencoded ampersands encoded correctly"
return unencoded_ampersands_re.sub('&amp;', value)
2008-04-27 16:54:37 +00:00
2012-05-27 11:50:10 +00:00
def add_links(text, trim_url_limit=None, nofollow=False):
2008-06-19 09:21:21 +00:00
"""
Converts any URLs in text into clickable links. Works on http://, https:// and
www. links. Links can have trailing punctuation (periods, commas, close-parens)
and leading punctuation (opening parens) and it'll still do the right thing.
If trim_url_limit is not None, the URLs in link text will be limited to
trim_url_limit characters.
If nofollow is True, the URLs in link text will get a rel="nofollow" attribute.
"""
2016-06-08 13:32:46 +00:00
trim_url = lambda x, limit=trim_url_limit: limit is not None and (x[:limit] + (len(x) >= limit and '...' or '')) or x
2008-06-19 09:21:21 +00:00
words = word_split_re.split(text)
nofollow_attr = nofollow and ' rel="nofollow"' or ''
for i, word in enumerate(words):
match = punctuation_re.match(word)
if match:
lead, middle, trail = match.groups()
2016-06-08 13:32:46 +00:00
if middle.startswith('www.') or ('@' not in middle and not middle.startswith('http://') and
len(middle) > 0 and middle[0] in letters + string.digits and
(middle.endswith('.org') or
middle.endswith('.net') or
middle.endswith('.com'))):
2008-06-19 09:21:21 +00:00
middle = '<a href="http://%s"%s>%s</a>' % (middle, nofollow_attr, trim_url(middle))
if middle.startswith('http://') or middle.startswith('https://'):
middle = '<a href="%s"%s>%s</a>' % (middle, nofollow_attr, trim_url(middle))
2016-06-08 13:32:46 +00:00
if '@' in middle and not middle.startswith('www.') and ':' not in middle \
and simple_email_re.match(middle):
2008-06-19 09:21:21 +00:00
middle = '<a href="mailto:%s">%s</a>' % (middle, middle)
if lead + middle + trail != word:
words[i] = lead + middle + trail
return ''.join(words)
2008-04-27 16:54:37 +00:00
2012-05-27 11:50:10 +00:00
urlize = add_links
def clean_html(text):
2008-06-19 09:21:21 +00:00
"""
Cleans the given HTML. Specifically, it does the following:
* Converts <b> and <i> to <strong> and <em>.
* Encodes all ampersands correctly.
* Removes all "target" attributes from <a> tags.
* Removes extraneous HTML, such as presentational tags that open and
immediately close and <br clear="all">.
* Converts hard-coded bullets into HTML unordered lists.
* Removes stuff like "<p>&nbsp;&nbsp;</p>", but only if it's at the
bottom of the text.
"""
2016-01-14 11:39:10 +00:00
from .text import normalize_newlines
text = normalize_newlines(text)
2008-06-19 09:21:21 +00:00
text = re.sub(r'<(/?)\s*b\s*>', '<\\1strong>', text)
text = re.sub(r'<(/?)\s*i\s*>', '<\\1em>', text)
text = fix_ampersands(text)
2008-06-19 09:21:21 +00:00
# Remove all target="" attributes from <a> tags.
text = link_target_attribute_re.sub('\\1', text)
# Trim stupid HTML such as <br clear="all">.
text = html_gunk_re.sub('', text)
# Convert hard-coded bullets into HTML unordered lists.
2016-06-08 13:32:46 +00:00
2008-06-19 09:21:21 +00:00
def replace_p_tags(match):
s = match.group().replace('</p>', '</li>')
for d in DOTS:
s = s.replace('<p>%s' % d, '<li>')
return '<ul>\n%s\n</ul>' % s
text = hard_coded_bullets_re.sub(replace_p_tags, text)
# Remove stuff like "<p>&nbsp;&nbsp;</p>", but only if it's at the bottom of the text.
text = trailing_empty_content_re.sub('', text)
return text
2008-04-27 16:54:37 +00:00
# This pattern matches a character entity reference (a decimal numeric
# references, a hexadecimal numeric reference, or a named reference).
charrefpat = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?')
2012-05-27 11:50:10 +00:00
def decode_html(html):
2008-06-19 09:21:21 +00:00
"""
2012-05-27 11:50:10 +00:00
>>> decode_html('me &amp; you and &#36;&#38;%')
2023-07-27 16:12:13 +00:00
'me & you and $&%'
2012-05-27 11:50:10 +00:00
>>> decode_html('&#x80;')
2023-07-27 16:12:13 +00:00
'\u20ac'
2012-05-27 11:50:10 +00:00
>>> decode_html('Anniversary of Daoud&apos;s Republic')
2023-07-27 16:12:13 +00:00
"Anniversary of Daoud's Republic"
2008-06-19 09:21:21 +00:00
"""
2014-09-30 19:04:46 +00:00
if isinstance(html, bytes):
html = html.decode('utf-8')
2023-07-27 11:07:13 +00:00
uchr = chr
2016-06-08 13:32:46 +00:00
2008-06-19 09:21:21 +00:00
def entitydecode(match, uchr=uchr):
entity = match.group(1)
if entity == '#x80':
2023-07-27 16:12:13 +00:00
return '€'
elif entity.startswith('#x'):
2008-06-19 09:21:21 +00:00
return uchr(int(entity[2:], 16))
elif entity.startswith('#'):
return uchr(int(entity[1:]))
elif entity in name2codepoint:
return uchr(name2codepoint[entity])
elif entity == 'apos':
return "'"
2008-06-19 09:21:21 +00:00
else:
return match.group(0)
2023-07-27 16:12:13 +00:00
return charrefpat.sub(entitydecode, html).replace('\xa0', ' ')
2008-04-27 16:54:37 +00:00
def highlight(text, query, hlClass="hl"):
2008-06-19 09:21:21 +00:00
"""
>>> highlight('me &amp; you and &#36;&#38;%', 'and')
'me &amp; you <span class="hl">and</span> &#36;&#38;%'
"""
if query:
text = text.replace('<br />', '|')
query = re.escape(query).replace('\ ', '.')
m = re.compile("(%s)" % query, re.IGNORECASE).findall(text)
for i in m:
text = re.sub("(%s)" % re.escape(i).replace('\ ', '.'), '<span class="%s">\\1</span>' % hlClass, text)
text = text.replace('|', '<br />')
return text
2008-04-27 16:54:37 +00:00
2012-02-21 15:44:50 +00:00
def escape_html(value):
'''
2023-07-27 16:12:13 +00:00
>>> escape_html('<script> foo')
'&lt;script&gt; foo'
>>> escape_html('&lt;script&gt; foo')
'&lt;script&gt; foo'
2012-02-21 15:44:50 +00:00
'''
return escape(decode_html(value))
2012-02-21 15:44:50 +00:00
def sanitize_html(html, tags=None, global_attributes=[]):
2012-02-21 15:44:50 +00:00
'''
>>> sanitize_html('http://foo.com, bar')
2023-07-27 16:12:13 +00:00
'<a href="http://foo.com">http://foo.com</a>, bar'
>>> sanitize_html('http://foo.com/foobar?foo, bar')
2023-07-27 16:12:13 +00:00
'<a href="http://foo.com/foobar?foo">http://foo.com/foobar?foo</a>, bar'
>>> sanitize_html('(see: www.foo.com)')
2023-07-27 16:12:13 +00:00
'(see: <a href="http://www.foo.com">www.foo.com</a>)'
>>> sanitize_html('foo@bar.com')
2023-07-27 16:12:13 +00:00
'<a href="mailto:foo@bar.com">foo@bar.com</a>'
>>> sanitize_html(sanitize_html('foo@bar.com'))
2023-07-27 16:12:13 +00:00
'<a href="mailto:foo@bar.com">foo@bar.com</a>'
>>> sanitize_html('<a href="http://foo.com" onmouseover="alert()">foo</a>')
2023-07-27 16:12:13 +00:00
'<a href="http://foo.com">foo</a>'
>>> sanitize_html('<a href="javascript:alert()">foo</a>')
2023-07-27 16:12:13 +00:00
'&lt;a href="javascript:alert()"&gt;foo&lt;/a&gt;'
>>> sanitize_html('[http://foo.com foo]')
2023-07-27 16:12:13 +00:00
'<a href="http://foo.com">foo</a>'
>>> sanitize_html('<div style="direction: rtl">foo</div>')
2023-07-27 16:12:13 +00:00
'<div style="direction: rtl">foo</div>'
>>> sanitize_html('<script>alert()</script>')
2023-07-27 16:12:13 +00:00
'&lt;script&gt;alert()&lt;/script&gt;'
>>> sanitize_html("'foo' < 'bar' && \\"foo\\" > \\"bar\\"")
2023-07-27 16:12:13 +00:00
'\\'foo\\' &lt; \\'bar\\' &amp;&amp; "foo" &gt; "bar"'
>>> sanitize_html('<b>foo')
2023-07-27 16:12:13 +00:00
'<b>foo</b>'
>>> sanitize_html('<b>foo</b></b>')
2023-07-27 16:12:13 +00:00
'<b>foo</b>'
>>> sanitize_html('Anniversary of Daoud&apos;s Republic')
2023-07-27 16:12:13 +00:00
"Anniversary of Daoud's Republic"
>>> sanitize_html('')
2023-07-27 16:12:13 +00:00
''
>>> sanitize_html(' ')
2023-07-27 16:12:13 +00:00
' '
>>> sanitize_html('&nbsp;') # canonicalised to a space: okay, I suppose
' '
>>> sanitize_html('\u00a0') # also nbsp
' '
2012-02-21 15:44:50 +00:00
'''
if not tags:
valid_url = '^((https?:\/\/|\/|mailto:).*?)'
2012-02-21 15:44:50 +00:00
tags = [
# inline formatting
{'name': 'b'},
{'name': 'bdi'},
{'name': 'code'},
{'name': 'em'},
{'name': 'i'},
{'name': 'q'},
{'name': 's'},
{'name': 'span'},
{'name': 'strong'},
{'name': 'sub'},
{'name': 'sup'},
{'name': 'u'},
2012-02-21 15:44:50 +00:00
# block formatting
{'name': 'blockquote'},
{'name': 'cite'},
{
'name': 'div',
'optional': ['style'],
'validation': {
'style': '^direction: rtl$'
}
},
{'name': 'h1'},
{'name': 'h2'},
{'name': 'h3'},
{'name': 'h4'},
{'name': 'h5'},
{'name': 'h6'},
{'name': 'p'},
{'name': 'pre'},
2012-02-21 15:44:50 +00:00
# lists
{'name': 'li'},
{'name': 'ol'},
{'name': 'ul'},
# definition lists
{'name': 'dl'},
{'name': 'dt'},
{'name': 'dd'},
2012-02-21 15:44:50 +00:00
# tables
{'name': 'table'},
{'name': 'tbody'},
{'name': 'td'},
{'name': 'tfoot'},
{'name': 'th'},
{'name': 'thead'},
{'name': 'tr'},
2012-02-21 15:44:50 +00:00
# other
2016-06-08 13:32:46 +00:00
{'name': '[]'},
{
'name': 'a',
'required': ['href'],
2020-10-15 09:39:19 +00:00
'optional': ['target'],
'validation': {
2020-10-15 09:39:19 +00:00
'href': valid_url,
'target': '^_blank$',
}
},
{'name': 'br'},
{
'name': 'iframe',
'optional': ['width', 'height'],
'required': ['src'],
'validation': {
'width': '^\d+$',
'height': '^\d+$',
'src': valid_url
}
},
{
'name': 'img',
'optional': ['width', 'height'],
'required': ['src'],
'validation': {
'width': '^\d+$',
'height': '^\d+$',
'src': valid_url
},
},
{'name': 'figure'},
{'name': 'figcaption'}
2012-02-21 15:44:50 +00:00
]
tag_re = re.compile('<(/)?([^\ /]+)(.*?)(/)?>')
attr_re = re.compile('([^=\ ]+)="([^"]+)"')
escaped = {}
level = 0
non_closing_tags = ['img', 'br']
required_attributes = {}
validation = {}
valid_attributes = {}
valid_tags = set([tag['name'] for tag in tags if tag['name'] != '[]'])
2012-02-21 15:44:50 +00:00
for tag in tags:
valid_attributes[tag['name']] = tag.get('required', []) \
2016-06-08 13:32:46 +00:00
+ tag.get('optional', []) + global_attributes
required_attributes[tag['name']] = tag.get('required', [])
validation[tag['name']] = tag.get('validation', {})
if '[]' in validation:
html = re.sub(
re.compile('\[((https?:\/\/|\/).+?) (.+?)\]', re.IGNORECASE),
2016-06-08 13:32:46 +00:00
'<a href="\\1">\\3</a>', html)
parts = split_tags(html)
for i, part in enumerate(parts):
is_tag = i % 2
if is_tag:
t = tag_re.findall(part)
if not t:
parts[i] = escape_html(decode_html(part))
continue
closing, name, attributes, end = t[0]
closing = closing != ''
a = attr_re.findall(attributes)
attrs = dict(a)
2016-06-08 13:32:46 +00:00
if not closing and name not in non_closing_tags:
level += 1
2016-06-08 13:32:46 +00:00
if not attrs and attributes or name not in valid_tags:
valid = False
else:
valid = True
for key in set(attrs) - set(valid_attributes[name]):
del attrs[key]
for key in required_attributes[tag['name']]:
2016-06-08 13:32:46 +00:00
if key not in attrs:
valid = False
if valid:
for attr in attrs:
if attr in validation[name]:
if not re.compile(validation[name][attr]).findall(attrs[attr]):
valid = False
break
if valid and closing:
valid = not escaped.get(level)
else:
escaped[level] = not valid
if closing:
level -= 1
if valid:
parts[i] = '<%s%s%s>' % (
('/' if closing else ''),
name,
(' ' + ' '.join(['%s="%s"' % (key, attrs[key]) for key, value in a if key in attrs])
if not closing and attrs else '')
)
else:
parts[i] = escape_html(decode_html(part))
else:
parts[i] = escape_html(decode_html(part))
html = ''.join(parts)
2012-05-27 11:50:10 +00:00
html = add_links(html)
2024-03-20 11:55:14 +00:00
html = html.replace('\n\n', '<br><br>')
2013-10-24 16:40:04 +00:00
return sanitize_fragment(html)
2012-02-21 15:44:50 +00:00
def split_tags(string):
tags = []
2016-06-08 13:32:46 +00:00
def collect(match):
tags.append(match.group(0))
return '\0'
strings = re.sub('<[^<>]+>', collect, string).split('\0')
tags.append('')
return [item for sublist in zip(strings, tags) for item in sublist][:-1]
2012-02-21 15:44:50 +00:00
def sanitize_fragment(html):
'''
Ensures that tags are closed (or not, as appropriate), attributes
are quoted, etc. Does not strip potentially-malicious HTML: use
sanitize_html() for that.
2023-07-27 16:12:13 +00:00
>>> sanitize_fragment('<span lang="en">')
'<span lang="en"></span>'
>>> sanitize_fragment('<span lang=en></span>')
'<span lang="en"></span>'
>>> sanitize_fragment('<br><br/></br>')
'<br><br>'
>>> sanitize_fragment('<a href="javascript:alert()">foo</a>')
'<a href="javascript:alert()">foo</a>'
>>> sanitize_fragment('')
''
>>> sanitize_fragment(' ')
' '
>>> sanitize_fragment('&nbsp;')
'\\xa0'
>>> sanitize_fragment('\\u00a0') # nbsp
'\\xa0'
>>> sanitize_fragment('\\ufeff') # zero-width no-break space
'\\ufeff'
'''
2013-10-24 16:40:04 +00:00
'''
#html5lib reorders arguments, so not usable
2012-02-21 15:44:50 +00:00
import html5lib
return html5lib.parseFragment(html).toxml().decode('utf-8')
2013-10-24 16:40:04 +00:00
'''
if not html.strip():
return html
2013-10-24 16:40:04 +00:00
import lxml.html
2023-07-27 16:35:33 +00:00
try:
body = lxml.html.document_fromstring(html).find('body')
except lxml.etree.ParserError as e:
if e.args and e.args[0] == 'Document is empty':
return html
raise e
html = lxml.html.tostring(body, encoding='utf-8')[6:-7].decode('utf-8')
if html.startswith('<p>') and html.endswith('</p>'):
html = html[3:-4]
return html