0x2620/python-ox
2
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

get rid of u string literal

Browse source
This commit is contained in:
j 2023-07-27 18:12:13 +02:00
parent d03a6b120d
commit 99e221095b
7 changed files with 98 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
ox/format.py
View file

@ -224,16 +224,16 @@ def to36(q):
def from36(q): def from36(q):
return int(q, 36) return int(q, 36)
def int_value(strValue, default=u''): def int_value(strValue, default=''):
""" """
>>> int_value('abc23') >>> int_value('abc23')
u'23' '23'
>>> int_value(' abc23') >>> int_value(' abc23')
u'23' '23'
>>> int_value('ab') >>> int_value('ab')
u'' ''
""" """
try: try:
val = re.compile('(\d+)').findall(str(strValue).strip())[0] val = re.compile('(\d+)').findall(str(strValue).strip())[0]
@ -241,16 +241,16 @@ def int_value(strValue, default=u''):
val = default val = default
return val return val
def float_value(strValue, default=u''): def float_value(strValue, default=''):
""" """
>>> float_value('abc23.4') >>> float_value('abc23.4')
u'23.4' '23.4'
>>> float_value(' abc23.4') >>> float_value(' abc23.4')
u'23.4' '23.4'
>>> float_value('ab') >>> float_value('ab')
u'' ''
""" """
try: try:
val = re.compile('([\d.]+)').findall(str(strValue).strip())[0] val = re.compile('([\d.]+)').findall(str(strValue).strip())[0]

94
ox/html.py
View file

@ -145,11 +145,11 @@ charrefpat = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?')
def decode_html(html): def decode_html(html):
""" """
>>> decode_html('me & you and $&%') >>> decode_html('me & you and $&%')
u'me & you and $&%' 'me & you and $&%'
>>> decode_html('€') >>> decode_html('€')
u'\u20ac' '\u20ac'
>>> decode_html('Anniversary of Daoud's Republic') >>> decode_html('Anniversary of Daoud's Republic')
u"Anniversary of Daoud's Republic" "Anniversary of Daoud's Republic"
""" """
if isinstance(html, bytes): if isinstance(html, bytes):
html = html.decode('utf-8') html = html.decode('utf-8')
@ -158,7 +158,7 @@ def decode_html(html):
def entitydecode(match, uchr=uchr): def entitydecode(match, uchr=uchr):
entity = match.group(1) entity = match.group(1)
if entity == '#x80': if entity == '#x80':
return u'€' return '€'
elif entity.startswith('#x'): elif entity.startswith('#x'):
return uchr(int(entity[2:], 16)) return uchr(int(entity[2:], 16))
elif entity.startswith('#'): elif entity.startswith('#'):
@ -169,7 +169,7 @@ def decode_html(html):
return "'" return "'"
else: else:
return match.group(0) return match.group(0)
return charrefpat.sub(entitydecode, html).replace(u'\xa0', ' ') return charrefpat.sub(entitydecode, html).replace('\xa0', ' ')
def highlight(text, query, hlClass="hl"): def highlight(text, query, hlClass="hl"):
""" """
@ -187,51 +187,51 @@ def highlight(text, query, hlClass="hl"):
def escape_html(value): def escape_html(value):
''' '''
>>> escape_html(u'<script> foo') >>> escape_html('<script> foo')
u'&lt;script&gt; foo' '&lt;script&gt; foo'
>>> escape_html(u'&lt;script&gt; foo') >>> escape_html('&lt;script&gt; foo')
u'&lt;script&gt; foo' '&lt;script&gt; foo'
''' '''
return escape(decode_html(value)) return escape(decode_html(value))
def sanitize_html(html, tags=None, global_attributes=[]): def sanitize_html(html, tags=None, global_attributes=[]):
''' '''
>>> sanitize_html('http://foo.com, bar') >>> sanitize_html('http://foo.com, bar')
u'<a href="http://foo.com">http://foo.com</a>, bar' '<a href="http://foo.com">http://foo.com</a>, bar'
>>> sanitize_html('http://foo.com/foobar?foo, bar') >>> sanitize_html('http://foo.com/foobar?foo, bar')
u'<a href="http://foo.com/foobar?foo">http://foo.com/foobar?foo</a>, bar' '<a href="http://foo.com/foobar?foo">http://foo.com/foobar?foo</a>, bar'
>>> sanitize_html('(see: www.foo.com)') >>> sanitize_html('(see: www.foo.com)')
u'(see: <a href="http://www.foo.com">www.foo.com</a>)' '(see: <a href="http://www.foo.com">www.foo.com</a>)'
>>> sanitize_html('foo@bar.com') >>> sanitize_html('foo@bar.com')
u'<a href="mailto:foo@bar.com">foo@bar.com</a>' '<a href="mailto:foo@bar.com">foo@bar.com</a>'
>>> sanitize_html(sanitize_html('foo@bar.com')) >>> sanitize_html(sanitize_html('foo@bar.com'))
u'<a href="mailto:foo@bar.com">foo@bar.com</a>' '<a href="mailto:foo@bar.com">foo@bar.com</a>'
>>> sanitize_html('<a href="http://foo.com" onmouseover="alert()">foo</a>') >>> sanitize_html('<a href="http://foo.com" onmouseover="alert()">foo</a>')
u'<a href="http://foo.com">foo</a>' '<a href="http://foo.com">foo</a>'
>>> sanitize_html('<a href="javascript:alert()">foo</a>') >>> sanitize_html('<a href="javascript:alert()">foo</a>')
u'&lt;a href="javascript:alert()"&gt;foo&lt;/a&gt;' '&lt;a href="javascript:alert()"&gt;foo&lt;/a&gt;'
>>> sanitize_html('[http://foo.com foo]') >>> sanitize_html('[http://foo.com foo]')
u'<a href="http://foo.com">foo</a>' '<a href="http://foo.com">foo</a>'
>>> sanitize_html('<div style="direction: rtl">foo</div>') >>> sanitize_html('<div style="direction: rtl">foo</div>')
u'<div style="direction: rtl">foo</div>' '<div style="direction: rtl">foo</div>'
>>> sanitize_html('<script>alert()</script>') >>> sanitize_html('<script>alert()</script>')
u'&lt;script&gt;alert()&lt;/script&gt;' '&lt;script&gt;alert()&lt;/script&gt;'
>>> sanitize_html("'foo' < 'bar' && \\"foo\\" > \\"bar\\"") >>> sanitize_html("'foo' < 'bar' && \\"foo\\" > \\"bar\\"")
u'\\'foo\\' &lt; \\'bar\\' &amp;&amp; "foo" &gt; "bar"' '\\'foo\\' &lt; \\'bar\\' &amp;&amp; "foo" &gt; "bar"'
>>> sanitize_html('<b>foo') >>> sanitize_html('<b>foo')
u'<b>foo</b>' '<b>foo</b>'
>>> sanitize_html('<b>foo</b></b>') >>> sanitize_html('<b>foo</b></b>')
u'<b>foo</b>' '<b>foo</b>'
>>> sanitize_html('Anniversary of Daoud&apos;s Republic') >>> sanitize_html('Anniversary of Daoud&apos;s Republic')
u"Anniversary of Daoud's Republic" "Anniversary of Daoud's Republic"
>>> sanitize_html('') >>> sanitize_html('')
u'' ''
>>> sanitize_html(' ') >>> sanitize_html(' ')
u' ' ' '
>>> sanitize_html(u'&nbsp;') # canonicalised to a space: okay, I suppose >>> sanitize_html('&nbsp;') # canonicalised to a space: okay, I suppose
u' ' ' '
>>> sanitize_html(u'\u00a0') # also nbsp >>> sanitize_html('\u00a0') # also nbsp
u' ' ' '
''' '''
if not tags: if not tags:
valid_url = '^((https?:\/\/|\/|mailto:).*?)' valid_url = '^((https?:\/\/|\/|mailto:).*?)'
@ -412,24 +412,24 @@ def sanitize_fragment(html):
are quoted, etc. Does not strip potentially-malicious HTML: use are quoted, etc. Does not strip potentially-malicious HTML: use
sanitize_html() for that. sanitize_html() for that.
>>> sanitize_fragment(u'<span lang="en">') >>> sanitize_fragment('<span lang="en">')
u'<span lang="en"></span>' '<span lang="en"></span>'
>>> sanitize_fragment(u'<span lang=en></span>') >>> sanitize_fragment('<span lang=en></span>')
u'<span lang="en"></span>' '<span lang="en"></span>'
>>> sanitize_fragment(u'<br><br/></br>') >>> sanitize_fragment('<br><br/></br>')
u'<br><br>' '<br><br>'
>>> sanitize_fragment(u'<a href="javascript:alert()">foo</a>') >>> sanitize_fragment('<a href="javascript:alert()">foo</a>')
u'<a href="javascript:alert()">foo</a>' '<a href="javascript:alert()">foo</a>'
>>> sanitize_fragment(u'') >>> sanitize_fragment('')
u'' ''
>>> sanitize_fragment(u' ') >>> sanitize_fragment(' ')
u' ' ' '
>>> sanitize_fragment(u'&nbsp;') >>> sanitize_fragment('&nbsp;')
u'\\xa0' '\\xa0'
>>> sanitize_fragment(u'\\u00a0') # nbsp >>> sanitize_fragment('\\u00a0') # nbsp
u'\\xa0' '\\xa0'
>>> sanitize_fragment(u'\\ufeff') # zero-width no-break space >>> sanitize_fragment('\\ufeff') # zero-width no-break space
u'\\ufeff' '\\ufeff'
''' '''
''' '''

54
ox/text.py
View file

@ -475,10 +475,10 @@ def wrap(text, width):
def wrap_string(string, length=80, separator='\n', balance=False): def wrap_string(string, length=80, separator='\n', balance=False):
''' '''
>>> wrap_string(u"Anticonstitutionellement, Paris s'eveille", 16) >>> wrap_string("Anticonstitutionellement, Paris s'eveille", 16)
u"Anticonstitution\\nellement, Paris \\ns'eveille" "Anticonstitution\\nellement, Paris \\ns'eveille"
>>> wrap_string(u'All you can eat', 12, '\\n', True) >>> wrap_string(u'All you can eat', 12, '\\n', True)
u'All you \\ncan eat' 'All you \\ncan eat'
''' '''
words = string.split(' ') words = string.split(' ')
if balance: if balance:
@ -493,20 +493,20 @@ def wrap_string(string, length=80, separator='\n', balance=False):
break break
lines = [''] lines = ['']
for word in words: for word in words:
if len(lines[len(lines) - 1] + word + u' ') <= length + 1: if len(lines[len(lines) - 1] + word + ' ') <= length + 1:
# word fits in current line # word fits in current line
lines[len(lines) - 1] += word + u' ' lines[len(lines) - 1] += word + ' '
else: else:
if len(word) <= length: if len(word) <= length:
# word fits in next line # word fits in next line
lines.append(word + u' ') lines.append(word + ' ')
else: else:
# word is longer than line # word is longer than line
position = length - len(lines[len(lines) - 1]) position = length - len(lines[len(lines) - 1])
lines[len(lines) - 1] += word[0:position] lines[len(lines) - 1] += word[0:position]
for i in range(position, len(word), length): for i in range(position, len(word), length):
lines.append(word[i:i+length]) lines.append(word[i:i+length])
lines[len(lines) - 1] += u' ' lines[len(lines) - 1] += ' '
return separator.join(lines).strip() return separator.join(lines).strip()
def truncate_string(string, length, padding='...', position='right'): def truncate_string(string, length, padding='...', position='right'):
@ -578,14 +578,14 @@ def get_valid_filename(s):
def get_text_list(list_, last_word='or'): def get_text_list(list_, last_word='or'):
""" """
>>> get_text_list([u'a', u'b', u'c', u'd']) >>> get_text_list(['a', 'b', 'c', 'd'])
u'a, b, c or d' 'a, b, c or d'
>>> get_text_list([u'a', u'b', u'c'], 'and') >>> get_text_list(['a', 'b', 'c'], 'and')
u'a, b and c' 'a, b and c'
>>> get_text_list([u'a', u'b'], 'and') >>> get_text_list(['a', 'b'], 'and')
u'a and b' 'a and b'
>>> get_text_list([u'a']) >>> get_text_list(['a'])
u'a' 'a'
>>> get_text_list([]) >>> get_text_list([])
'' ''
""" """
@ -593,24 +593,24 @@ def get_text_list(list_, last_word='or'):
return '' return ''
if len(list_) == 1: if len(list_) == 1:
return list_[0] return list_[0]
return u'%s %s %s' % (u', '.join([i for i in list_][:-1]), last_word, list_[-1]) return '%s %s %s' % (', '.join([i for i in list_][:-1]), last_word, list_[-1])
def get_list_text(text, last_word='or'): def get_list_text(text, last_word='or'):
""" """
>>> get_list_text(u'a, b, c or d') >>> get_list_text('a, b, c or d')
[u'a', u'b', u'c', u'd'] ['a', 'b', 'c', 'd']
>>> get_list_text(u'a, b and c', u'and') >>> get_list_text('a, b and c', 'and')
[u'a', u'b', u'c'] ['a', 'b', 'c']
>>> get_list_text(u'a and b', u'and') >>> get_list_text('a and b', 'and')
[u'a', u'b'] ['a', 'b']
>>> get_list_text(u'a') >>> get_list_text('a')
[u'a'] ['a']
>>> get_list_text(u'') >>> get_list_text('')
[] []
""" """
list_ = [] list_ = []
if text: if text:
list_ = text.split(u', ') list_ = text.split(', ')
if list_: if list_:
i = len(list_)-1 i = len(list_)-1
last = list_[i].split(last_word) last = list_[i].split(last_word)
@ -682,7 +682,7 @@ def words(text):
return [re.sub("(([.!?:-_]|'s)$)", '', x) for x in text] return [re.sub("(([.!?:-_]|'s)$)", '', x) for x in text]
def sort_string(string): def sort_string(string):
string = string.replace(u'Æ', 'AE').replace(u'Ø', 'O').replace(u'Þ', 'Th') string = string.replace('Æ', 'AE').replace('Ø', 'O').replace('Þ', 'Th')
# pad numbered titles # pad numbered titles
string = re.sub('(\d),(\d{3})', '\\1\\2', string) string = re.sub('(\d),(\d{3})', '\\1\\2', string)

8
ox/web/allmovie.py
View file

@ -13,13 +13,13 @@ def get_id(url):
def get_data(id): def get_data(id):
''' '''
>>> get_data('129689')['cast'][1][1] >>> get_data('129689')['cast'][1][1]
u'Marianne' 'Marianne'
>>> get_data('129689')['credits'][0][0] >>> get_data('129689')['credits'][0][0]
u'Jean-Luc Godard' 'Jean-Luc Godard'
>>> get_data('129689')['posters'][0] >>> get_data('129689')['posters'][0]
u'http://image.allmusic.com/00/adg/cov200/dru800/u812/u81260bbffr.jpg' 'http://image.allmusic.com/00/adg/cov200/dru800/u812/u81260bbffr.jpg'
>>> get_data('129689')['rating'] >>> get_data('129689')['rating']
u'4.5' '4.5'
''' '''
if id.startswith('http'): if id.startswith('http'):
id = get_id(id) id = get_id(id)

12
ox/web/arsenalberlin.py
View file

@ -19,18 +19,18 @@ def get_data(id, language='en'):
if 'Willkommen in der Datenbank des Arsenal' in html: if 'Willkommen in der Datenbank des Arsenal' in html:
return None return None
data = {} data = {}
data[u'id'] = id data['id'] = id
data[u'url'] = url data['url'] = url
m = re.compile('<h1>(.*?)</h1>').findall(html) m = re.compile('<h1>(.*?)</h1>').findall(html)
if m: if m:
data[u'title'] = m[0] data['title'] = m[0]
m = re.compile("<b>Director: </b><a href='.*?'>(.*?)</a>").findall(html) m = re.compile("<b>Director: </b><a href='.*?'>(.*?)</a>").findall(html)
if m: if m:
data[u'director'] = m[0] data['director'] = m[0]
m = re.compile("caUI.initImageScroller\(\[\{url:'(.*?)'").findall(html) m = re.compile("caUI.initImageScroller\(\[\{url:'(.*?)'").findall(html)
if m: if m:
data[u'image'] = m[0] data['image'] = m[0]
units = re.compile("<div class='unit'>(.*?)</div>", re.DOTALL).findall(html) units = re.compile("<div class='unit'>(.*?)</div>", re.DOTALL).findall(html)
for x in map(re.compile('<b>(.*?)</b>: (.*)', re.DOTALL).findall, units): for x in map(re.compile('<b>(.*?)</b>: (.*)', re.DOTALL).findall, units):
@ -43,7 +43,7 @@ def get_data(id, language='en'):
else: else:
data[key] = strip_tags(data[key]) data[key] = strip_tags(data[key])
if "running time (minutes)" in data: if "running time (minutes)" in data:
data[u'runtime'] = float(data.pop("running time (minutes)").replace(',', '.')) * 60 data['runtime'] = float(data.pop("running time (minutes)").replace(',', '.')) * 60
for key in ('year', 'length in metres', 'forum participation year', 'number of reels'): for key in ('year', 'length in metres', 'forum participation year', 'number of reels'):
if key in data and data[key].isdigit(): if key in data and data[key].isdigit():
data[key] = int(data[key]) data[key] = int(data[key])

8
ox/web/criterion.py
View file

@ -19,13 +19,13 @@ def get_url(id):
def get_data(id, timeout=ox.cache.cache_timeout, get_imdb=False): def get_data(id, timeout=ox.cache.cache_timeout, get_imdb=False):
''' '''
>>> get_data('1333').get('imdbId') >>> get_data('1333').get('imdbId')
u'0060304' '0060304'
>>> get_data('236')['posters'][0] >>> get_data('236')['posters'][0]
u'http://s3.amazonaws.com/criterion-production/release_images/1586/ThirdManReplace.jpg' 'http://s3.amazonaws.com/criterion-production/release_images/1586/ThirdManReplace.jpg'
>>> get_data('786')['posters'][0] >>> get_data('786')['posters'][0]
u'http://s3.amazonaws.com/criterion-production/product_images/185/343_box_348x490.jpg' 'http://s3.amazonaws.com/criterion-production/product_images/185/343_box_348x490.jpg'
''' '''
data = { data = {
"id": id, "id": id,
@ -39,7 +39,7 @@ def get_data(id, timeout=ox.cache.cache_timeout, get_imdb=False):
data["number"] = find_re(html, "<b>Spine #(\d+)") data["number"] = find_re(html, "<b>Spine #(\d+)")
data["title"] = decode_html(find_re(html, "<h1 class=\"header__primarytitle\".*?>(.*?)</h1>")) data["title"] = decode_html(find_re(html, "<h1 class=\"header__primarytitle\".*?>(.*?)</h1>"))
data["title"] = data["title"].split(u' \u2014 The Television Version')[0].strip() data["title"] = data["title"].split(' \u2014 The Television Version')[0].strip()
results = find_re(html, '<ul class="film-meta-list">(.*?)</ul>') results = find_re(html, '<ul class="film-meta-list">(.*?)</ul>')
info = re.compile('<li itemprop="(.*?)".*?>(.*?)</li>', re.DOTALL).findall(results) info = re.compile('<li itemprop="(.*?)".*?>(.*?)</li>', re.DOTALL).findall(results)
info = {k: strip_tags(v).strip() for k, v in info} info = {k: strip_tags(v).strip() for k, v in info}

4
ox/web/flixter.py
View file

@ -58,10 +58,10 @@ def get_data(id, timeout=-1):
def get_id(url=None, imdb=None): def get_id(url=None, imdb=None):
''' '''
>>> get_id(imdb='0133093') >>> get_id(imdb='0133093')
u'the-matrix' 'the-matrix'
#>>> get_id(imdb='0060304') #>>> get_id(imdb='0060304')
#u'2-or-3-things-i-know-about-her' #'2-or-3-things-i-know-about-her'
''' '''
if imdb: if imdb:
i = ImdbCombined(imdb) i = ImdbCombined(imdb)