0x2620/python-ox
2
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

drop six and python2 support

Browse source
This commit is contained in:
j 2023-07-27 13:07:13 +02:00
commit adad3be419
31 changed files with 54 additions and 426 deletions
Download patch file Download diff file Expand all files Collapse all files

12
ox/web/siteparser.py
View file

@ -4,8 +4,6 @@ import re
import json
from multiprocessing.pool import ThreadPool
from six import string_types
from ..cache import read_url
from .. import decode_html
from ..utils import datetime
@ -13,15 +11,15 @@ from ..utils import datetime
def cleanup(key, data, data_type):
if data:
if isinstance(data[0], string_types):
if isinstance(data[0], str):
#FIXME: some types need strip_tags
#data = [strip_tags(decode_html(p)).strip() for p in data]
data = [decode_html(p).strip() for p in data]
elif isinstance(data[0], list) or isinstance(data[0], tuple):
data = [cleanup(key, p, data_type) for p in data]
while len(data) == 1 and not isinstance(data, string_types):
while len(data) == 1 and not isinstance(data, str):
data = data[0]
if data_type == 'list' and isinstance(data, string_types):
if data_type == 'list' and isinstance(data, str):
data = [data, ]
elif data_type != 'list':
data = ''
@ -49,7 +47,7 @@ class SiteParser(dict):
for key in self.regex:
url = self.get_url(self.regex[key]['page'])
data = self.read_url(url, timeout)
if isinstance(self.regex[key]['re'], string_types):
if isinstance(self.regex[key]['re'], str):
data = re.compile(self.regex[key]['re'], re.DOTALL).findall(data)
data = cleanup(key, data, self.regex[key]['type'])
elif callable(self.regex[key]['re']):
@ -60,7 +58,7 @@ class SiteParser(dict):
f = r
else:
f = re.compile(r, re.DOTALL).findall
if isinstance(data, string_types):
if isinstance(data, str):
data = f(data)
else:
data = [f(d) for d in data]