ox/__init__.py

@@ -5,7 +5,7 @@ try:
     from . import __version
     __version__ = __version.VERSION
 except:
-    __version__ = '3.0.x'
+    __version__ = '2.3.x'
 
 from . import cache
 from . import js
@@ -17,6 +17,7 @@ from . import vtt
 
 from .api import *
 from .file import *
+from .form import *
 from .format import *
 from .geo import *
 from .html import *

ox/api.py

@@ -4,20 +4,19 @@
 from __future__ import print_function
 from types import MethodType
 import gzip
-import mimetypes
 import os
 import shutil
 import sys
 import time
 
-from http import cookiejar as cookielib
+from six.moves import http_cookiejar as cookielib
-from io import BytesIO
+from six import BytesIO, PY2
-import urllib
+from six.moves import urllib
-from urllib.parse import urlparse
+from six.moves.urllib.parse import urlparse
-import requests
 
 from . import __version__
 from .utils import json
+from .form import MultiPartForm
 
 __all__ = ['getAPI', 'API']
 
@@ -38,13 +37,12 @@ class API(object):
             self._cj = cj
         else:
             self._cj = cookielib.CookieJar()
+        self._opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(self._cj),
+                                            urllib.request.HTTPHandler(debuglevel=self.debuglevel))
+        self._opener.addheaders = [
+            ('User-Agent', '%s/%s' % (self.__name__, self.__version__))
+        ]
 
-        self._requests_session = requests.Session()
-        self._requests_session.cookies = self._cj
-        self._requests_session.headers = {
-            'User-Agent': '%s/%s' % (self.__name__, self.__version__),
-            'Accept-Encoding': 'gzip, deflate',
-        }
         self.url = url
         r = self._request('api', {'docs': True})
         self._properties = r['data']['actions']
@@ -55,7 +53,10 @@ class API(object):
     def _add_method(self, method, name):
         if name is None:
             name = method.func_name
-        setattr(self, name, MethodType(method, self))
+        if PY2:
+            setattr(self, name, MethodType(method, self, type(self)))
+        else:
+            setattr(self, name, MethodType(method, self))
 
     def _add_action(self, action):
         def method(self, *args, **kw):
@@ -69,20 +70,37 @@ class API(object):
             return self._request(action, kw)
         if 'doc' in self._properties[action]:
             method.__doc__ = self._properties[action]['doc']
-        method.func_name = action
+        if PY2:
+            method.func_name = str(action)
+        else:
+            method.func_name = action
         self._add_method(method, action)
 
-    def _json_request(self, url, data, files=None):
+    def _json_request(self, url, form):
         result = {}
         try:
-            request = self._requests_session.post(url, data=data, files=files)
+            body = form.body()
-            result = request.json()
+            if PY2:
-            return result
+                if not isinstance(url, bytes):
+                    url = url.encode('utf-8')
+                request = urllib.request.Request(url)
+                request.add_data(body)
+            else:
+                request = urllib.request.Request(url, data=body, method='POST')
+            request.add_header('Content-Type', form.get_content_type())
+            request.add_header('Content-Length', str(len(body)))
+            request.add_header('Accept-Encoding', 'gzip, deflate')
+            f = self._opener.open(request)
+            result = f.read()
+            if f.headers.get('content-encoding', None) == 'gzip':
+                result = gzip.GzipFile(fileobj=BytesIO(result)).read()
+            result = result.decode('utf-8')
+            return json.loads(result)
         except urllib.error.HTTPError as e:
             if self.DEBUG:
                 import webbrowser
                 if e.code >= 500:
-                    with open('/tmp/error.html', 'wb') as f:
+                    with open('/tmp/error.html', 'w') as f:
                         f.write(e.read())
                     webbrowser.open_new_tab('/tmp/error.html')
 
@@ -107,15 +125,17 @@ class API(object):
             raise
 
     def _request(self, action, data=None):
-        form = {
+        form = MultiPartForm()
-            'action': action
+        form.add_field('action', action)
-        }
         if data:
-            form['data'] = json.dumps(data)
+            form.add_field('data', json.dumps(data))
         return self._json_request(self.url, form)
 
     def get_url(self, url):
-        return self._requests_session.get(url).content
+        request = urllib.request.Request(url, method='GET')
+        f = self._opener.open(request)
+        result = f.read()
+        return result
 
     def save_url(self, url, filename, overwrite=False):
         chunk_size = 16 * 1024
@@ -123,15 +143,21 @@ class API(object):
             dirname = os.path.dirname(filename)
             if dirname and not os.path.exists(dirname):
                 os.makedirs(dirname)
+            request = urllib.request.Request(url, method='GET')
             tmpname = filename + '.tmp'
             with open(tmpname, 'wb') as fd:
-                r = self._requests_session.get(url)
+                u = self._opener.open(request)
-                for chunk in iter(lambda: r.read(chunk_size), b''):
+                for chunk in iter(lambda: u.read(chunk_size), b''):
                     fd.write(chunk)
             shutil.move(tmpname, filename)
 
+
     def upload_chunks(self, url, filename, data=None, silent=False):
-        data = self._json_request(url, data)
+        form = MultiPartForm()
+        if data:
+            for key in data:
+                form.add_field(key, data[key])
+        data = self._json_request(url, form)
 
         def full_url(path):
             if path.startswith('/'):
@@ -152,20 +178,16 @@ class API(object):
                 resume_offset = 0
             chunk = f.read(CHUNK_SIZE)
             fname = os.path.basename(filename)
-            mime_type = mimetypes.guess_type(fname)[0] or 'application/octet-stream'
             if not isinstance(fname, bytes):
                 fname = fname.encode('utf-8')
             while chunk:
-                meta = {
+                form = MultiPartForm()
-                    'offset': str(done)
+                form.add_file('chunk', fname, chunk)
-                }
                 if len(chunk) < CHUNK_SIZE or f.tell() == fsize:
-                    meta['done'] = '1'
+                    form.add_field('done', '1')
-                files = [
+                form.add_field('offset', str(done))
-                    ('chunk', (fname, chunk, mime_type))
-                ]
                 try:
-                    data = self._json_request(uploadUrl, meta, files=files)
+                    data = self._json_request(uploadUrl, form)
                 except KeyboardInterrupt:
                     if not slient:
                         print("\ninterrupted by user.")

ox/cache.py

@@ -10,11 +10,15 @@ import sqlite3
 import time
 import zlib
 
-from io import BytesIO
+from six import BytesIO
-import urllib
+from six.moves import urllib
-import requests
+from six import PY2
-from requests.structures import CaseInsensitiveDict
+try:
-
+    import requests
+    USE_REQUESTS = True
+    requests_session = requests.Session()
+except:
+    USE_REQUESTS = False
 
 from .utils import json
 from .file import makedirs
@@ -24,7 +28,6 @@ from .net import DEFAULT_HEADERS, detect_encoding
 
 
 cache_timeout = 30*24*60*60  # default is 30 days
-requests_session = requests.Session()
 
 COMPRESS_TYPES = (
     'text/html',
@@ -66,7 +69,7 @@ def get_headers(url, data=None, headers=None, timeout=cache_timeout):
     if not url_headers:
         url_headers = net.get_headers(url, data, headers)
         store.set(url, data, -1, url_headers)
-    return CaseInsensitiveDict(url_headers)
+    return url_headers
 
 def get_json(url, data=None, headers=None, timeout=cache_timeout):
     return json.loads(read_url(url, data, headers, timeout).decode('utf-8'))
@@ -98,20 +101,32 @@ def read_url(url, data=None, headers=None, timeout=cache_timeout, valid=None, un
     result = store.get(url, data, headers, timeout)
     url_headers = {}
     if not result:
-        if headers is None:
+        if USE_REQUESTS:
-            headers = DEFAULT_HEADERS.copy()
+            if headers is None:
-        if data:
+                headers = DEFAULT_HEADERS.copy()
-            r = requests_session.post(url, data=data, headers=headers)
-        else:
             r = requests_session.get(url, headers=headers)
-        for key in r.headers:
+            for key in r.headers:
-            url_headers[key.lower()] = r.headers[key]
+                url_headers[key.lower()] = r.headers[key]
-        result = r.content
+            result = r.content
-        url_headers['Status'] = "%s" % r.status_code
+            url_headers['Status'] = "%s" % r.status_code
-        if not valid or valid(result, url_headers):
+            if not valid or valid(result, url_headers):
-            store.set(url, post_data=data, data=result, headers=url_headers)
+                store.set(url, post_data=data, data=result, headers=url_headers)
+            else:
+                raise InvalidResult(result, url_headers)
         else:
-            raise InvalidResult(result, url_headers)
+            try:
+                url_headers, result = net.read_url(url, data, headers, return_headers=True)
+            except urllib.error.HTTPError as e:
+                e.headers['Status'] = "%s" % e.code
+                for key in e.headers:
+                    url_headers[key.lower()] = e.headers[key]
+                result = e.read()
+                if url_headers.get('content-encoding', None) == 'gzip':
+                    result = gzip.GzipFile(fileobj=BytesIO(result)).read()
+            if not valid or valid(result, url_headers):
+                store.set(url, post_data=data, data=result, headers=url_headers)
+            else:
+                raise InvalidResult(result, url_headers)
     if unicode:
         ctype = url_headers.get('content-type', '').lower()
         if 'charset' in ctype:
@@ -224,6 +239,8 @@ class SQLiteCache(Cache):
             elif value == 'data':
                 if row[1] == 1:
                     r = zlib.decompress(r)
+                elif PY2:
+                    r = str(r)
             break
 
         c.close()
@@ -262,8 +279,6 @@ class SQLiteCache(Cache):
             data = zlib.compress(data)
         else:
             compressed = 0
-        if isinstance(data, str):
-            data = data.encode("utf-8")
         data = sqlite3.Binary(data)
 
         #fixme: this looks wrong

ox/file.py

@@ -19,8 +19,7 @@ __all__ = ['sha1sum', 'oshash', 'avinfo', 'makedirs', 'iexists']
 EXTENSIONS = {
     'audio': [
         'aac', 'aif', 'aiff', 'amr',
-        'flac', 'm4a', 'mp3', 'oga', 'ogg', 'wav', 'wma', 'opus',
+        'flac', 'm4a', 'mp3', 'oga', 'ogg', 'wav', 'wma', 'opus'
-        'ra',  # Real Audio
     ],
     'image': [
         'bmp', 'gif', 'jpeg', 'jpg', 'png', 'svg', 'webp'
@@ -30,12 +29,11 @@ EXTENSIONS = {
     ],
     'video': [
         '3gp',
-        'avi', 'divx', 'dv', 'flv', 'm2t', 'm2ts', 'm2v', 'm4v', 'mkv', 'mov', 'mp4',
+        'avi', 'divx', 'dv', 'flv', 'm2t', 'm2ts', 'm4v', 'mkv', 'mov', 'mp4',
-        'mpeg', 'mpg', 'mts', 'ogm', 'ogv', 'vob', 'webm', 'wmv', 'asf',
+        'mpeg', 'mpg', 'mts', 'ogm', 'ogv', 'rm', 'rmvb', 'vob', 'webm', 'wmv', 'asf',
         'mod', 'tod',  # http://en.wikipedia.org/wiki/MOD_and_TOD
         'mxf', 'ts',
         'dat',  # VOD files
-        'rm', 'rmvb',  # Real Media
     ],
 }
 
@@ -216,16 +214,12 @@ def ffprobe(filename):
             ]
         for s in ffinfo['streams']:
             tags = s.pop('tags', {})
-            side_data_list = s.pop('side_data_list', [])
             language = None
             for t in tags:
                 if t == 'language':
                     language = tags[t]
                 else:
                     info['metadata'][t] = tags[t]
-            for kv in side_data_list:
-                for k, v in kv.items():
-                    info['metadata'][k] = v
             if s.get('codec_type') in ('audio', 'video'):
                 stream = {}
                 if language and language != 'und':
@@ -279,15 +273,9 @@ def ffprobe(filename):
                 pass
                 # print s
         for v in info['video']:
+            if 'rotate' in info.get('metadata', {}) and int(info['metadata']['rotate']) in (-180, -90, 90, 180):
+                v['width'], v['height'] = v['height'], v['width']
             k = 'display_aspect_ratio'
-            if 'rotate' in info.get('metadata', {}) and int(info['metadata']['rotate']) in (-90, 90):
-                v['width'], v['height'] = v['height'], v['width']
-                if k in v:
-                    v[k] = ':'.join(reversed(v[k].split(':')))
-            elif 'rotation' in info.get('metadata', {}) and int(info['metadata']['rotation']) in (-90, 90):
-                v['width'], v['height'] = v['height'], v['width']
-                if k in v:
-                    v[k] = ':'.join(reversed(v[k].split(':')))
             if k not in v and 'width' in v \
                     or (k in v and v[k] == '0:1'):
                 v[k] = '%d:%d' % (v['width'], v['height'])

ox/fixunicode.py

@@ -6,6 +6,7 @@ from __future__ import print_function
 
 import unicodedata
 
+from six import unichr, text_type
 
 __all__ = ['fix_bad_unicode']
 
@@ -150,7 +151,7 @@ def text_badness(text):
     - Improbable single-byte characters, such as ƒ or ¬
     - Letters in somewhat rare scripts
     '''
-    assert isinstance(text, str)
+    assert isinstance(text, text_type)
     errors = 0
     very_weird_things = 0
     weird_things = 0
@@ -288,7 +289,7 @@ SINGLE_BYTE_WEIRDNESS = (
 # Pre-cache the Unicode data saying which of these first 256 characters are
 # letters. We'll need it often.
 SINGLE_BYTE_LETTERS = [
-    unicodedata.category(chr(i)).startswith('L')
+    unicodedata.category(unichr(i)).startswith('L')
     for i in range(256)
 ]
 

ox/form.py

@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+# GPL 2014
+from __future__ import print_function
+
+import itertools
+import mimetypes
+import os
+import hashlib
+import sys
+
+from six import PY2
+
+
+__all__ = ['MultiPartForm']
+
+# from /usr/lib/python3.4/email/generator.py
+# Helper used by Generator._make_boundary
+_width = len(repr(sys.maxsize-1))
+_fmt = '%%0%dd' % _width
+
+def _make_boundary():
+    # Craft a random boundary.
+    boundary = ('=' * 15) + hashlib.sha1(os.urandom(32)).hexdigest() + '=='
+    return boundary
+
+class MultiPartForm(object):
+    """Accumulate the data to be used when posting a form."""
+
+    def __init__(self):
+        self.form_fields = []
+        self.files = []
+        self.boundary = _make_boundary()
+        return
+    
+    def get_content_type(self):
+        return 'multipart/form-data; boundary=%s' % self.boundary
+
+    def add_field(self, name, value):
+        """Add a simple field to the form data."""
+        if isinstance(name, bytes):
+            name = name.decode('utf-8')
+        if isinstance(value, bytes):
+            value = value.decode('utf-8')
+        self.form_fields.append((name, value))
+        return
+
+    def add_file(self, fieldname, filename, fileHandle, mimetype=None):
+        """Add a file to be uploaded."""
+        if isinstance(fieldname, bytes):
+            fieldname = fieldname.decode('utf-8')
+        if isinstance(filename, bytes):
+            filename = filename.decode('utf-8')
+
+        if hasattr(fileHandle, 'read'):
+            body = fileHandle.read()
+        else:
+            body = fileHandle
+        if mimetype is None:
+            mimetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
+        self.files.append((fieldname, filename, mimetype, body))
+        return
+
+    def __str__(self):
+        body = self.body()
+        if not PY2:
+            body = body.decode('utf-8')
+        return body
+
+    def body(self):
+        """Return a byte string representing the form data, including attached files."""
+        # Build a list of lists, each containing "lines" of the
+        # request.  Each part is separated by a boundary string.
+        # Once the list is built, return a string where each
+        # line is separated by '\r\n'.  
+        parts = []
+        part_boundary = '--' + self.boundary
+
+        # Add the form fields
+        parts.extend(
+            [ part_boundary,
+              'Content-Disposition: form-data; name="%s"' % name,
+              '',
+              value,
+            ]
+            for name, value in self.form_fields
+            )
+
+        # Add the files to upload
+        parts.extend(
+            [ part_boundary,
+              'Content-Disposition: file; name="%s"; filename="%s"' % \
+                 (field_name, filename),
+              'Content-Type: %s' % content_type,
+              '',
+              body,
+            ]
+            for field_name, filename, content_type, body in self.files
+            )
+
+        # Flatten the list and add closing boundary marker,
+        # then return CR+LF separated data
+        flattened = list(itertools.chain(*parts))
+        flattened.append('--' + self.boundary + '--')
+        flattened.append('')
+        flattened = [part if isinstance(part, bytes) else part.encode('utf-8') for part in flattened]
+        return b'\r\n'.join(flattened)
+

ox/format.py

@@ -4,6 +4,8 @@ import math
 import re
 import string
 
+from six import text_type
+
 def toAZ(num):
     """
     Converts an integer to bijective base 26 string using A-Z
@@ -106,7 +108,7 @@ def to32(q):
     >>> to32(555306645)
     'GHJKMN'
 
-    >>> to32(800197332334559)
+    >>> to32(800197332334559L)
     'PQRSTVWXYZ'
 
     >>> to32(32)
@@ -224,36 +226,36 @@ def to36(q):
 def from36(q):
     return int(q, 36)
 
-def int_value(strValue, default=''):
+def int_value(strValue, default=u''):
     """
     >>> int_value('abc23')
-    '23'
+    u'23'
 
     >>> int_value(' abc23')
-    '23'
+    u'23'
 
     >>> int_value('ab')
-    ''
+    u''
     """
     try:
-        val = re.compile('(\d+)').findall(str(strValue).strip())[0]
+        val = re.compile('(\d+)').findall(text_type(strValue).strip())[0]
     except:
         val = default
     return val
 
-def float_value(strValue, default=''):
+def float_value(strValue, default=u''):
     """
     >>> float_value('abc23.4')
-    '23.4'
+    u'23.4'
 
     >>> float_value(' abc23.4')
-    '23.4'
+    u'23.4'
 
     >>> float_value('ab')
-    ''
+    u''
     """
     try:
-        val = re.compile('([\d.]+)').findall(str(strValue).strip())[0]
+        val = re.compile('([\d.]+)').findall(text_type(strValue).strip())[0]
     except:
         val = default
     return val

ox/html.py

@@ -3,7 +3,8 @@
 # GPL 2008
 import re
 import string
-from html.entities import name2codepoint
+from six.moves.html_entities import name2codepoint
+from six import unichr, PY2, string_types
 
 letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
 
@@ -25,7 +26,8 @@ link_target_attribute_re = re.compile(r'(<a [^>]*?)target=[^\s>]+')
 html_gunk_re = re.compile(r'(?:<br clear="all">|<i><\/i>|<b><\/b>|<em><\/em>|<strong><\/strong>|<\/?smallcaps>|<\/?uppercase>)', re.IGNORECASE)
 hard_coded_bullets_re = re.compile(r'((?:<p>(?:%s).*?[a-zA-Z].*?</p>\s*)+)' % '|'.join([re.escape(x) for x in DOTS]), re.DOTALL)
 trailing_empty_content_re = re.compile(r'(?:<p>(?:&nbsp;|\s|<br \/>)*?</p>\s*)+\Z')
-
+if PY2:
+    del x  # Temporary variable
 
 def escape(html):
     '''
@@ -34,7 +36,7 @@ def escape(html):
     >>> escape('html "test" & <brothers>')
     'html &quot;test&quot; &amp; &lt;brothers&gt;'
     '''
-    if not isinstance(html, str):
+    if not isinstance(html, string_types):
         html = str(html)
     return html.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;').replace("'", '&apos;')
 
@@ -145,20 +147,20 @@ charrefpat = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?')
 def decode_html(html):
     """
     >>> decode_html('me &amp; you and &#36;&#38;%')
-    'me & you and $&%'
+    u'me & you and $&%'
     >>> decode_html('&#x80;')
-    '\u20ac'
+    u'\u20ac'
     >>> decode_html('Anniversary of Daoud&apos;s Republic')
-    "Anniversary of Daoud's Republic"
+    u"Anniversary of Daoud's Republic"
     """
     if isinstance(html, bytes):
         html = html.decode('utf-8')
-    uchr = chr
+    uchr = unichr
 
     def entitydecode(match, uchr=uchr):
         entity = match.group(1)
         if entity == '#x80':
-            return '€'
+            return u'€'
         elif entity.startswith('#x'):
             return uchr(int(entity[2:], 16))
         elif entity.startswith('#'):
@@ -169,7 +171,7 @@ def decode_html(html):
             return "'"
         else:
             return match.group(0)
-    return charrefpat.sub(entitydecode, html).replace('\xa0', ' ')
+    return charrefpat.sub(entitydecode, html).replace(u'\xa0', ' ')
 
 def highlight(text, query, hlClass="hl"):
     """
@@ -187,51 +189,51 @@ def highlight(text, query, hlClass="hl"):
 
 def escape_html(value):
     '''
-    >>> escape_html('<script> foo')
+    >>> escape_html(u'<script> foo')
-    '&lt;script&gt; foo'
+    u'&lt;script&gt; foo'
-    >>> escape_html('&lt;script&gt; foo')
+    >>> escape_html(u'&lt;script&gt; foo')
-    '&lt;script&gt; foo'
+    u'&lt;script&gt; foo'
     '''
     return escape(decode_html(value))
 
 def sanitize_html(html, tags=None, global_attributes=[]):
     '''
     >>> sanitize_html('http://foo.com, bar')
-    '<a href="http://foo.com">http://foo.com</a>, bar'
+    u'<a href="http://foo.com">http://foo.com</a>, bar'
     >>> sanitize_html('http://foo.com/foobar?foo, bar')
-    '<a href="http://foo.com/foobar?foo">http://foo.com/foobar?foo</a>, bar'
+    u'<a href="http://foo.com/foobar?foo">http://foo.com/foobar?foo</a>, bar'
     >>> sanitize_html('(see: www.foo.com)')
-    '(see: <a href="http://www.foo.com">www.foo.com</a>)'
+    u'(see: <a href="http://www.foo.com">www.foo.com</a>)'
     >>> sanitize_html('foo@bar.com')
-    '<a href="mailto:foo@bar.com">foo@bar.com</a>'
+    u'<a href="mailto:foo@bar.com">foo@bar.com</a>'
     >>> sanitize_html(sanitize_html('foo@bar.com'))
-    '<a href="mailto:foo@bar.com">foo@bar.com</a>'
+    u'<a href="mailto:foo@bar.com">foo@bar.com</a>'
     >>> sanitize_html('<a href="http://foo.com" onmouseover="alert()">foo</a>')
-    '<a href="http://foo.com">foo</a>'
+    u'<a href="http://foo.com">foo</a>'
     >>> sanitize_html('<a href="javascript:alert()">foo</a>')
-    '&lt;a href="javascript:alert()"&gt;foo&lt;/a&gt;'
+    u'&lt;a href="javascript:alert()"&gt;foo&lt;/a&gt;'
     >>> sanitize_html('[http://foo.com foo]')
-    '<a href="http://foo.com">foo</a>'
+    u'<a href="http://foo.com">foo</a>'
     >>> sanitize_html('<div style="direction: rtl">foo</div>')
-    '<div style="direction: rtl">foo</div>'
+    u'<div style="direction: rtl">foo</div>'
     >>> sanitize_html('<script>alert()</script>')
-    '&lt;script&gt;alert()&lt;/script&gt;'
+    u'&lt;script&gt;alert()&lt;/script&gt;'
     >>> sanitize_html("'foo' < 'bar' && \\"foo\\" > \\"bar\\"")
-    '\\'foo\\' &lt; \\'bar\\' &amp;&amp; "foo" &gt; "bar"'
+    u'\\'foo\\' &lt; \\'bar\\' &amp;&amp; "foo" &gt; "bar"'
     >>> sanitize_html('<b>foo')
-    '<b>foo</b>'
+    u'<b>foo</b>'
     >>> sanitize_html('<b>foo</b></b>')
-    '<b>foo</b>'
+    u'<b>foo</b>'
     >>> sanitize_html('Anniversary of Daoud&apos;s Republic')
-    "Anniversary of Daoud's Republic"
+    u"Anniversary of Daoud's Republic"
     >>> sanitize_html('')
-    ''
+    u''
     >>> sanitize_html(' ')
-    ' '
+    u' '
-    >>> sanitize_html('&nbsp;')  # canonicalised to a space: okay, I suppose
+    >>> sanitize_html(u'&nbsp;')  # canonicalised to a space: okay, I suppose
-    ' '
+    u' '
-    >>> sanitize_html('\u00a0')  # also nbsp
+    >>> sanitize_html(u'\u00a0')  # also nbsp
-    ' '
+    u' '
     '''
     if not tags:
         valid_url = '^((https?:\/\/|\/|mailto:).*?)'
@@ -412,24 +414,24 @@ def sanitize_fragment(html):
     are quoted, etc. Does not strip potentially-malicious HTML: use
     sanitize_html() for that.
 
-    >>> sanitize_fragment('<span lang="en">')
+    >>> sanitize_fragment(u'<span lang="en">')
-    '<span lang="en"></span>'
+    u'<span lang="en"></span>'
-    >>> sanitize_fragment('<span lang=en></span>')
+    >>> sanitize_fragment(u'<span lang=en></span>')
-    '<span lang="en"></span>'
+    u'<span lang="en"></span>'
-    >>> sanitize_fragment('<br><br/></br>')
+    >>> sanitize_fragment(u'<br><br/></br>')
-    '<br><br>'
+    u'<br><br>'
-    >>> sanitize_fragment('<a href="javascript:alert()">foo</a>')
+    >>> sanitize_fragment(u'<a href="javascript:alert()">foo</a>')
-    '<a href="javascript:alert()">foo</a>'
+    u'<a href="javascript:alert()">foo</a>'
-    >>> sanitize_fragment('')
+    >>> sanitize_fragment(u'')
-    ''
+    u''
-    >>> sanitize_fragment(' ')
+    >>> sanitize_fragment(u' ')
-    ' '
+    u' '
-    >>> sanitize_fragment('&nbsp;')
+    >>> sanitize_fragment(u'&nbsp;')
-    '\\xa0'
+    u'\\xa0'
-    >>> sanitize_fragment('\\u00a0')  # nbsp
+    >>> sanitize_fragment(u'\\u00a0')  # nbsp
-    '\\xa0'
+    u'\\xa0'
-    >>> sanitize_fragment('\\ufeff')  # zero-width no-break space
+    >>> sanitize_fragment(u'\\ufeff')  # zero-width no-break space
-    '\\ufeff'
+    u'\\ufeff'
     '''
 
     '''
@@ -440,12 +442,7 @@ def sanitize_fragment(html):
     if not html.strip():
         return html
     import lxml.html
-    try:
+    body = lxml.html.document_fromstring(html).find('body')
-        body = lxml.html.document_fromstring(html).find('body')
-    except lxml.etree.ParserError as e:
-        if e.args and e.args[0] == 'Document is empty':
-            return html
-        raise e
     html = lxml.html.tostring(body, encoding='utf-8')[6:-7].decode('utf-8')
     if html.startswith('<p>') and html.endswith('</p>'):
         html = html[3:-4]

ox/image.py

@@ -22,17 +22,11 @@ ZONE_INDEX = [
     ]
 ]
 
-
-def textsize(draw, text, font):
-    left, top, right, bottom = draw.textbbox((0, 0), text, font=font)
-    return (right, bottom)
-
-
 def drawText(image, position, text, font_file, font_size, color):
     draw = ImageDraw.Draw(image)
     font = ImageFont.truetype(font_file, font_size, encoding='unic')
     draw.text(position, text, fill=color, font=font)
-    size = textsize(draw, text, font)
+    size = draw.textsize(text, font=font)
     version = getattr(Image, 'PILLOW_VERSION', None)
     if version and version > '2.1.0' and version < '2.6.1':
         offset = font.getoffset(text)
@@ -63,7 +57,7 @@ def getHSL(rgb):
     return tuple(hsl)
 
 def getImageHash(image_file, mode):
-    image = Image.open(image_file).convert('RGB').resize((8, 8), Image.LANCZOS)
+    image = Image.open(image_file).convert('RGB').resize((8, 8), Image.ANTIALIAS)
     image_hash = 0
     if mode == 'color':
         # divide the image into 8 zones:
@@ -105,7 +99,7 @@ def getImageHash(image_file, mode):
     return image_hash
 
 def getImageHeat(image_file):
-    image = Image.open(image_file).convert('RGB').resize((16, 16), Image.LANCZOS)
+    image = Image.open(image_file).convert('RGB').resize((16, 16), Image.ANTIALIAS)
     pixel = image.load()
     image_heat = 0
     for y in range(image.size[1]):
@@ -120,7 +114,7 @@ def getImageHeat(image_file):
     return image_heat / 256
 
 def getImageHSL(image_file):
-    image = Image.open(image_file).convert('RGB').resize((1, 1), Image.LANCZOS)
+    image = Image.open(image_file).convert('RGB').resize((1, 1), Image.ANTIALIAS)
     return getHSL(image.getpixel((0, 0)))
 
 def getRGB(hsl):
@@ -154,7 +148,7 @@ def getRGB(hsl):
 def getTextSize(image, text, font_file, font_size):
     draw = ImageDraw.Draw(image)
     font = ImageFont.truetype(font_file, font_size, encoding='unic')
-    size = textsize(draw, text, font)
+    size = draw.textsize(text, font=font)
     version = getattr(Image, 'PILLOW_VERSION', None)
     if version and version > '2.1.0' and version < '2.6.1':
         offset = font.getoffset(text)
@@ -174,7 +168,7 @@ def wrapText(text, max_width, max_lines, font_file, font_size):
         return min_width
 
     def get_width(string):
-        return textsize(draw, string, font)[0]
+        return draw.textsize(string, font=font)[0]
 
     image = Image.new('RGB', (1, 1))
     draw = ImageDraw.Draw(image)

ox/js.py

@@ -2,12 +2,19 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
 
+from six import PY2
 from .utils import json
 
 def minify(source, comment=''):
     # see https://github.com/douglascrockford/JSMin/blob/master/README
     def get_next_non_whitespace_token():
         pass
+    # python2 performance with unicode string is terrible
+    if PY2:
+        if isinstance(source, unicode):  # pylint: disable=undefined-variable
+            source = source.encode('utf-8')
+        if isinstance(comment, unicode):  # pylint: disable=undefined-variable
+            comment = comment.encode('utf-8')
     tokens = tokenize(source)
     length = len(tokens)
     minified = '/*' + comment + '*/' if comment else ''

ox/movie.py

@@ -206,7 +206,7 @@ def parse_path(path, directory_key='director'):
         string = re.sub(re.compile('(?<=\w)_(?=\w)', re.U), '/', string)
         string = re.sub(' _ ', ' / ', string)
         # 'foo_ ' is ':'
-        string = re.sub(re.compile('(?<=[\w\)\]])_ ', re.U), ': ', string)
+        string = re.sub(re.compile('(?<=\w)_ ', re.U), ': ', string)
         string = unicodedata.normalize('NFD', string)
         return string
 

ox/net.py

@@ -8,10 +8,13 @@ import os
 import re
 import struct
 
-import requests
+try:
-
+    import requests
-from io import BytesIO
+    USE_REQUESTS = True
-import urllib
+except:
+    USE_REQUESTS = False
+from six import BytesIO, PY2
+from six.moves import urllib
 from chardet.universaldetector import UniversalDetector
 
 
@@ -56,10 +59,14 @@ def get_json(url, data=None, headers=None):
 def open_url(url, data=None, headers=None):
     if headers is None:
         headers = DEFAULT_HEADERS.copy()
-    if isinstance(url, bytes):
+    if PY2:
-        url = url.decode('utf-8')
+        if not isinstance(url, bytes):
+            url = url.encode('utf-8')
+    else:
+        if isinstance(url, bytes):
+            url = url.decode('utf-8')
     url = url.replace(' ', '%20')
-    if data and not isinstance(data, bytes):
+    if data and not PY2 and not isinstance(data, bytes):
         data = data.encode('utf-8')
     req = urllib.request.Request(url, data, headers)
     return urllib.request.urlopen(req)
@@ -116,11 +123,16 @@ def save_url(url, filename, overwrite=False):
         if dirname and not os.path.exists(dirname):
             os.makedirs(dirname)
         headers = DEFAULT_HEADERS.copy()
-        r = requests.get(url, headers=headers, stream=True)
+        if USE_REQUESTS:
-        with open(filename, 'wb') as f:
+            r = requests.get(url, headers=headers, stream=True)
-            for chunk in r.iter_content(chunk_size=1024):
+            with open(filename, 'wb') as f:
-                if chunk:  # filter out keep-alive new chunks
+                for chunk in r.iter_content(chunk_size=1024):
-                    f.write(chunk)
+                    if chunk:  # filter out keep-alive new chunks
+                        f.write(chunk)
+        else:
+            data = read_url(url)
+            with open(filename, 'wb') as f:
+                f.write(data)
 
 def _get_size(url):
     req = urllib.request.Request(url, headers=DEFAULT_HEADERS.copy())

ox/normalize.py

@@ -4,6 +4,8 @@
 import re
 import unicodedata
 
+from six import string_types
+
 
 _articles = ('the', 'la', 'a', 'die', 'der', 'le', 'el',
              "l'", 'il', 'das', 'les', 'o', 'ein', 'i', 'un', 'los', 'de',
@@ -101,7 +103,7 @@ def normalize_imdbid(imdbId):
     >>> normalize_imdbid('tt0159206')
     '0159206'
     """
-    if isinstance(imdbId, str):
+    if isinstance(imdbId, string_types):
         imdbId = re.sub('.*(\d{7}).*', '\\1', imdbId)
     elif isinstance(imdbId, int):
         imdbId = "%07d" % imdbId

ox/srt.py

@@ -5,6 +5,7 @@ import codecs
 import re
 
 import chardet
+from six import PY2
 import ox
 
 
@@ -23,7 +24,10 @@ def _detect_encoding(fp):
     # go to beginning of file and get the first 4 bytes
     oldFP = fp.tell()
     fp.seek(0)
-    (byte1, byte2, byte3, byte4) = fp.read(4)
+    if PY2:
+        (byte1, byte2, byte3, byte4) = [ord(b) for b in fp.read(4)]
+    else:
+        (byte1, byte2, byte3, byte4) = fp.read(4)
 
     # try bom detection using 4 bytes, 3 bytes, or 2 bytes
     bomDetection = bomDict.get((byte1, byte2, byte3, byte4))

ox/text.py

@@ -1,13 +1,11 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
 # GPL 2008
-import gzip
 import math
 import re
 import unicodedata
-from io import BytesIO
 
-from functools import reduce
+from six.moves import reduce
 
 ARTICLES = list(set([
     # def sg, def pl, indef sg, indef pl (each m/f/n)
@@ -475,10 +473,10 @@ def wrap(text, width):
 
 def wrap_string(string, length=80, separator='\n', balance=False):
     '''
-    >>> wrap_string("Anticonstitutionellement, Paris s'eveille", 16)
+    >>> wrap_string(u"Anticonstitutionellement, Paris s'eveille", 16)
-    "Anticonstitution\\nellement, Paris \\ns'eveille"
+    u"Anticonstitution\\nellement, Paris \\ns'eveille"
     >>> wrap_string(u'All you can eat', 12, '\\n', True)
-    'All you \\ncan eat'
+    u'All you \\ncan eat'
     '''
     words = string.split(' ')
     if balance:
@@ -493,20 +491,20 @@ def wrap_string(string, length=80, separator='\n', balance=False):
                     break
     lines = ['']
     for word in words:
-        if len(lines[len(lines) - 1] + word + ' ') <= length + 1:
+        if len(lines[len(lines) - 1] + word + u' ') <= length + 1:
             # word fits in current line
-            lines[len(lines) - 1] += word + ' '
+            lines[len(lines) - 1] += word + u' '
         else:
             if len(word) <= length:
                 # word fits in next line
-                lines.append(word + ' ')
+                lines.append(word + u' ')
             else:
                 # word is longer than line
                 position = length - len(lines[len(lines) - 1])
                 lines[len(lines) - 1] += word[0:position]
                 for i in range(position, len(word), length):
                     lines.append(word[i:i+length])
-                lines[len(lines) - 1] += ' '
+                lines[len(lines) - 1] += u' '
     return separator.join(lines).strip()
 
 def truncate_string(string, length, padding='...', position='right'):
@@ -578,14 +576,14 @@ def get_valid_filename(s):
 
 def get_text_list(list_, last_word='or'):
     """
-    >>> get_text_list(['a', 'b', 'c', 'd'])
+    >>> get_text_list([u'a', u'b', u'c', u'd'])
-    'a, b, c or d'
+    u'a, b, c or d'
-    >>> get_text_list(['a', 'b', 'c'], 'and')
+    >>> get_text_list([u'a', u'b', u'c'], 'and')
-    'a, b and c'
+    u'a, b and c'
-    >>> get_text_list(['a', 'b'], 'and')
+    >>> get_text_list([u'a', u'b'], 'and')
-    'a and b'
+    u'a and b'
-    >>> get_text_list(['a'])
+    >>> get_text_list([u'a'])
-    'a'
+    u'a'
     >>> get_text_list([])
     ''
     """
@@ -593,24 +591,24 @@ def get_text_list(list_, last_word='or'):
         return ''
     if len(list_) == 1:
         return list_[0]
-    return '%s %s %s' % (', '.join([i for i in list_][:-1]), last_word, list_[-1])
+    return u'%s %s %s' % (u', '.join([i for i in list_][:-1]), last_word, list_[-1])
 
 def get_list_text(text, last_word='or'):
     """
-    >>> get_list_text('a, b, c or d')
+    >>> get_list_text(u'a, b, c or d')
-    ['a', 'b', 'c', 'd']
+    [u'a', u'b', u'c', u'd']
-    >>> get_list_text('a, b and c', 'and')
+    >>> get_list_text(u'a, b and c', u'and')
-    ['a', 'b', 'c']
+    [u'a', u'b', u'c']
-    >>> get_list_text('a and b', 'and')
+    >>> get_list_text(u'a and b', u'and')
-    ['a', 'b']
+    [u'a', u'b']
-    >>> get_list_text('a')
+    >>> get_list_text(u'a')
-    ['a']
+    [u'a']
-    >>> get_list_text('')
+    >>> get_list_text(u'')
     []
     """
     list_ = []
     if text:
-        list_ = text.split(', ')
+        list_ = text.split(u', ')
         if list_:
             i = len(list_)-1
             last = list_[i].split(last_word)
@@ -648,6 +646,8 @@ def phone2numeric(phone):
     return letters.sub(char2number, phone)
 
 def compress_string(s):
+    import gzip
+    from six import BytesIO
     zbuf = BytesIO()
     zfile = gzip.GzipFile(mode='wb', compresslevel=6, fileobj=zbuf)
     zfile.write(s)
@@ -682,7 +682,7 @@ def words(text):
     return [re.sub("(([.!?:-_]|'s)$)", '', x) for x in text]
 
 def sort_string(string):
-    string = string.replace('Æ', 'AE').replace('Ø', 'O').replace('Þ', 'Th')
+    string = string.replace(u'Æ', 'AE').replace(u'Ø', 'O').replace(u'Þ', 'Th')
 
     # pad numbered titles
     string = re.sub('(\d),(\d{3})', '\\1\\2', string)

ox/torrent/__init__.py

@@ -5,8 +5,12 @@
 from threading import Event
 from hashlib import sha1
 import os
+from six import PY2
 
-from .bencode3 import bencode, bdecode
+if PY2:
+    from .bencode import bencode, bdecode
+else:
+    from .bencode3 import bencode, bdecode
 
 __all__ = ['create_torrent', 'get_info_hash', 'get_torrent_info', 'get_files', 'get_torrent_size']
 

ox/torrent/bencode.py

@@ -0,0 +1,321 @@
+# Written by Petru Paler, Uoti Urpala, Ross Cohen and John Hoffman
+# see LICENSE.txt for license information
+from __future__ import print_function
+
+from types import IntType, LongType, StringType, ListType, TupleType, DictType
+try:
+    from types import BooleanType
+except ImportError:
+    BooleanType = None
+try:
+    from types import UnicodeType
+except ImportError:
+    UnicodeType = None
+from cStringIO import StringIO
+
+def decode_int(x, f):
+    f += 1
+    newf = x.index('e', f)
+    try:
+        n = int(x[f:newf])
+    except:
+        n = long(x[f:newf])
+    if x[f] == '-':
+        if x[f + 1] == '0':
+            raise ValueError
+    elif x[f] == '0' and newf != f+1:
+        raise ValueError
+    return (n, newf+1)
+  
+def decode_string(x, f):
+    colon = x.index(':', f)
+    try:
+        n = int(x[f:colon])
+    except (OverflowError, ValueError):
+        n = long(x[f:colon])
+    if x[f] == '0' and colon != f+1:
+        raise ValueError
+    colon += 1
+    return (x[colon:colon+n], colon+n)
+
+def decode_unicode(x, f):
+    s, f = decode_string(x, f+1)
+    return (s.decode('UTF-8'),f)
+
+def decode_list(x, f):
+    r, f = [], f+1
+    while x[f] != 'e':
+        v, f = decode_func[x[f]](x, f)
+        r.append(v)
+    return (r, f + 1)
+
+def decode_dict(x, f):
+    r, f = {}, f+1
+    lastkey = None
+    while x[f] != 'e':
+        k, f = decode_string(x, f)
+        #  why is this needed
+        #  if lastkey >= k:
+        #    raise ValueError
+        lastkey = k
+        r[k], f = decode_func[x[f]](x, f)
+    return (r, f + 1)
+
+decode_func = {}
+decode_func['l'] = decode_list
+decode_func['d'] = decode_dict
+decode_func['i'] = decode_int
+decode_func['0'] = decode_string
+decode_func['1'] = decode_string
+decode_func['2'] = decode_string
+decode_func['3'] = decode_string
+decode_func['4'] = decode_string
+decode_func['5'] = decode_string
+decode_func['6'] = decode_string
+decode_func['7'] = decode_string
+decode_func['8'] = decode_string
+decode_func['9'] = decode_string
+#decode_func['u'] = decode_unicode
+  
+def bdecode(x, sloppy = 1):
+    try:
+        r, l = decode_func[x[0]](x, 0)
+#    except (IndexError, KeyError):
+    except (IndexError, KeyError, ValueError):
+        raise ValueError("bad bencoded data")
+    if not sloppy and l != len(x):
+        raise ValueError("bad bencoded data")
+    return r
+
+def test_bdecode():
+    try:
+        bdecode('0:0:')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode('ie')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode('i341foo382e')
+        assert 0
+    except ValueError:
+        pass
+    assert bdecode('i4e') == 4
+    assert bdecode('i0e') == 0
+    assert bdecode('i123456789e') == 123456789
+    assert bdecode('i-10e') == -10
+    try:
+        bdecode('i-0e')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode('i123')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode('')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode('i6easd')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode('35208734823ljdahflajhdf')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode('2:abfdjslhfld')
+        assert 0
+    except ValueError:
+        pass
+    assert bdecode('0:') == ''
+    assert bdecode('3:abc') == 'abc'
+    assert bdecode('10:1234567890') == '1234567890'
+    try:
+        bdecode('02:xy')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode('l')
+        assert 0
+    except ValueError:
+        pass
+    assert bdecode('le') == []
+    try:
+        bdecode('leanfdldjfh')
+        assert 0
+    except ValueError:
+        pass
+    assert bdecode('l0:0:0:e') == ['', '', '']
+    try:
+        bdecode('relwjhrlewjh')
+        assert 0
+    except ValueError:
+        pass
+    assert bdecode('li1ei2ei3ee') == [1, 2, 3]
+    assert bdecode('l3:asd2:xye') == ['asd', 'xy']
+    assert bdecode('ll5:Alice3:Bobeli2ei3eee') == [['Alice', 'Bob'], [2, 3]]
+    try:
+        bdecode('d')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode('defoobar')
+        assert 0
+    except ValueError:
+        pass
+    assert bdecode('de') == {}
+    assert bdecode('d3:agei25e4:eyes4:bluee') == {'age': 25, 'eyes': 'blue'}
+    assert bdecode('d8:spam.mp3d6:author5:Alice6:lengthi100000eee') == {'spam.mp3': {'author': 'Alice', 'length': 100000}}
+    try:
+        bdecode('d3:fooe')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode('di1e0:e')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode('d1:b0:1:a0:e')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode('d1:a0:1:a0:e')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode('i03e')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode('l01:ae')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode('9999:x')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode('l0:')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode('d0:0:')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode('d0:')
+        assert 0
+    except ValueError:
+        pass
+
+bencached_marker = []
+
+class Bencached:
+    def __init__(self, s):
+        self.marker = bencached_marker
+        self.bencoded = s
+
+BencachedType = type(Bencached('')) # insufficient, but good as a filter
+
+def encode_bencached(x,r):
+    assert x.marker == bencached_marker
+    r.append(x.bencoded)
+
+def encode_int(x,r):
+    r.extend(('i',str(x),'e'))
+
+def encode_bool(x,r):
+    encode_int(int(x),r)
+
+def encode_string(x,r):    
+    r.extend((str(len(x)),':',x))
+
+def encode_unicode(x,r):
+    #r.append('u')
+    encode_string(x.encode('UTF-8'),r)
+
+def encode_list(x,r):
+        r.append('l')
+        for e in x:
+            encode_func[type(e)](e, r)
+        r.append('e')
+
+def encode_dict(x,r):
+    r.append('d')
+    ilist = x.items()
+    ilist.sort()
+    for k,v in ilist:
+        r.extend((str(len(k)),':',k))
+        encode_func[type(v)](v, r)
+    r.append('e')
+
+encode_func = {}
+encode_func[BencachedType] = encode_bencached
+encode_func[IntType] = encode_int
+encode_func[LongType] = encode_int
+encode_func[StringType] = encode_string
+encode_func[ListType] = encode_list
+encode_func[TupleType] = encode_list
+encode_func[DictType] = encode_dict
+if BooleanType:
+    encode_func[BooleanType] = encode_bool
+if UnicodeType:
+    encode_func[UnicodeType] = encode_unicode
+    
+def bencode(x):
+    r = []
+    try:
+        encode_func[type(x)](x, r)
+    except:
+        print("*** error *** could not encode type %s (value: %s)" % (type(x), x))
+        assert 0
+    return ''.join(r)
+
+def test_bencode():
+    assert bencode(4) == 'i4e'
+    assert bencode(0) == 'i0e'
+    assert bencode(-10) == 'i-10e'
+    assert bencode(12345678901234567890) == 'i12345678901234567890e'
+    assert bencode('') == '0:'
+    assert bencode('abc') == '3:abc'
+    assert bencode('1234567890') == '10:1234567890'
+    assert bencode([]) == 'le'
+    assert bencode([1, 2, 3]) == 'li1ei2ei3ee'
+    assert bencode([['Alice', 'Bob'], [2, 3]]) == 'll5:Alice3:Bobeli2ei3eee'
+    assert bencode({}) == 'de'
+    assert bencode({'age': 25, 'eyes': 'blue'}) == 'd3:agei25e4:eyes4:bluee'
+    assert bencode({'spam.mp3': {'author': 'Alice', 'length': 100000}}) == 'd8:spam.mp3d6:author5:Alice6:lengthi100000eee'
+    try:
+        bencode({1: 'foo'})
+        assert 0
+    except AssertionError:
+        pass
+
+  
+try:
+    import psyco
+    psyco.bind(bdecode)
+    psyco.bind(bencode)
+except ImportError:
+    pass

ox/torrent/makemetafile.py

@@ -8,7 +8,11 @@ from hashlib import sha1 as sha
 from copy import copy
 import re
 
-from .bencode3 import bencode
+from six import PY2
+if PY2:
+    from .bencode import bencode
+else:
+    from .bencode3 import bencode
 from threading import Event
 from time import time
 from traceback import print_exc

ox/web/allmovie.py

@@ -13,13 +13,13 @@ def get_id(url):
 def get_data(id):
     '''
     >>> get_data('129689')['cast'][1][1]
-    'Marianne'
+    u'Marianne'
     >>> get_data('129689')['credits'][0][0]
-    'Jean-Luc Godard'
+    u'Jean-Luc Godard'
     >>> get_data('129689')['posters'][0]
-    'http://image.allmusic.com/00/adg/cov200/dru800/u812/u81260bbffr.jpg'
+    u'http://image.allmusic.com/00/adg/cov200/dru800/u812/u81260bbffr.jpg'
     >>> get_data('129689')['rating']
-    '4.5'
+    u'4.5'
     '''
     if id.startswith('http'):
         id = get_id(id)

ox/web/amazon.py

@@ -2,7 +2,7 @@
 # vi:si:et:sw=4:sts=4:ts=4
 from __future__ import print_function
 import re
-from urllib.parse import quote
+from six.moves.urllib.parse import quote
 
 from ox import find_re, strip_tags, decode_html
 from ox.cache import read_url

ox/web/apple.py

@@ -2,6 +2,7 @@ from __future__ import print_function
 import json
 import re
 
+from six import text_type
 from ox.cache import read_url
 
 HEADERS = {
@@ -16,9 +17,9 @@ USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7) '
 USER_AGENT += 'AppleWebKit/534.48.3 (KHTML, like Gecko) Version/5.1 Safari/534.48.3'
 
 def get_movie_data(title, director):
-    if isinstance(title, str):
+    if isinstance(title, text_type):
         title = title.encode('utf-8')
-    if isinstance(director, str):
+    if isinstance(director, text_type):
         director = director.encode('utf-8')
     data = {}
     # itunes section (preferred source for link)

ox/web/archive.py

@@ -3,6 +3,8 @@
 from .. import cache
 from ..utils import json
 
+from six import string_types
+
 def get_id(url):
     return url.split("/")[-1]
 
@@ -19,7 +21,7 @@ def get_data(id):
             data[key] = details['metadata'][key]
             if isinstance(data[key], list):
                 data[key] = data[key][0]
-            if isinstance(data[key], str):
+            if isinstance(data[key], string_types):
                 data[key] = data[key].strip()
                 if data[key][0] == '[' and data[key][-1] == ']':
                     data[key] = data[key][1:-1] 

ox/web/arsenalberlin.py

@@ -19,18 +19,18 @@ def get_data(id, language='en'):
     if 'Willkommen in der Datenbank des Arsenal' in html:
         return None
     data = {}
-    data['id'] = id
+    data[u'id'] = id
-    data['url'] = url
+    data[u'url'] = url
     m = re.compile('<h1>(.*?)</h1>').findall(html)
     if m:
-        data['title'] = m[0]
+        data[u'title'] = m[0]
     m = re.compile("<b>Director: </b><a href='.*?'>(.*?)</a>").findall(html)
     if m:
-        data['director'] = m[0]
+        data[u'director'] = m[0]
 
     m = re.compile("caUI.initImageScroller\(\[\{url:'(.*?)'").findall(html)
     if m:
-        data['image'] = m[0]
+        data[u'image'] = m[0]
 
     units = re.compile("<div class='unit'>(.*?)</div>", re.DOTALL).findall(html)
     for x in map(re.compile('<b>(.*?)</b>: (.*)', re.DOTALL).findall, units):
@@ -43,7 +43,7 @@ def get_data(id, language='en'):
             else:
                 data[key] = strip_tags(data[key])
     if "running time (minutes)" in data:
-        data['runtime'] = float(data.pop("running time (minutes)").replace(',', '.')) * 60
+        data[u'runtime'] = float(data.pop("running time (minutes)").replace(',', '.')) * 60
     for key in ('year', 'length in metres', 'forum participation year', 'number of reels'):
         if key in data and data[key].isdigit():
             data[key] = int(data[key])

ox/web/criterion.py

@@ -19,13 +19,13 @@ def get_url(id):
 def get_data(id, timeout=ox.cache.cache_timeout, get_imdb=False):
     '''
     >>> get_data('1333').get('imdbId')
-    '0060304'
+    u'0060304'
 
     >>> get_data('236')['posters'][0]
-    'http://s3.amazonaws.com/criterion-production/release_images/1586/ThirdManReplace.jpg'
+    u'http://s3.amazonaws.com/criterion-production/release_images/1586/ThirdManReplace.jpg'
 
     >>> get_data('786')['posters'][0]
-    'http://s3.amazonaws.com/criterion-production/product_images/185/343_box_348x490.jpg'
+    u'http://s3.amazonaws.com/criterion-production/product_images/185/343_box_348x490.jpg'
     '''
     data = {
         "id": id,
@@ -39,16 +39,12 @@ def get_data(id, timeout=ox.cache.cache_timeout, get_imdb=False):
     data["number"] = find_re(html, "<b>Spine #(\d+)")
 
     data["title"] = decode_html(find_re(html, "<h1 class=\"header__primarytitle\".*?>(.*?)</h1>"))
-    data["title"] = data["title"].split(' \u2014 The Television Version')[0].strip()
+    data["title"] = data["title"].split(u' \u2014 The Television Version')[0].strip()
     results = find_re(html, '<ul class="film-meta-list">(.*?)</ul>')
     info = re.compile('<li itemprop="(.*?)".*?>(.*?)</li>', re.DOTALL).findall(results)
     info = {k: strip_tags(v).strip() for k, v in info}
-    meta = re.compile('<meta.*? name="(.*?)".*? content="(.*?)"', re.DOTALL).findall(html)
-    meta = {k: v.strip() for k, v in meta}
     if 'director' in info:
         data['director'] = info['director']
-    elif 'director' in meta:
-        data['director'] = meta['director']
     if 'countryOfOrigin' in info:
         data['country'] = [c.strip() for c in decode_html(info['countryOfOrigin']).split(', ')]
     if 'inLanguage' in info:
@@ -84,8 +80,7 @@ def get_data(id, timeout=ox.cache.cache_timeout, get_imdb=False):
     for poster in re.compile('<img src="(.*?)"').findall(posters):
         data['posters'].append(poster)
 
-    result = re.compile('<div class="gallery-item ">.*?src="(.*?)"', re.DOTALL).findall(html)
+    result = find_re(html, "<img alt=\"Film Still\" height=\"252\" src=\"(.*?)\"")
-    #result = find_re(html, "<img alt=\"Film Still\" height=\"252\" src=\"(.*?)\"")
     if result:
         data["stills"] = [result]
         data["trailers"] = []

ox/web/dailymotion.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
 import re
-from urllib.parse import unquote
+from six.moves.urllib.parse import unquote
 from ox.cache import read_url
 
 

ox/web/duckduckgo.py

@@ -2,7 +2,7 @@
 # vi:si:et:sw=4:sts=4:ts=4
 import re
 
-import urllib
+from six.moves import urllib
 import ox
 from ox import strip_tags, decode_html
 from ox.cache import read_url

ox/web/flixter.py

@@ -58,10 +58,10 @@ def get_data(id, timeout=-1):
 def get_id(url=None, imdb=None):
     '''
     >>> get_id(imdb='0133093')
-    'the-matrix'
+    u'the-matrix'
 
     #>>> get_id(imdb='0060304')
-    #'2-or-3-things-i-know-about-her'
+    #u'2-or-3-things-i-know-about-her'
     '''
     if imdb:
         i = ImdbCombined(imdb)

ox/web/google.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
 import re
-import urllib
+from six.moves import urllib
 
 import ox
 from ox import strip_tags, decode_html
@@ -17,31 +17,6 @@ def quote_plus(s):
         s = s.encode('utf-8')
     return urllib.parse.quote_plus(s)
 
-
-def infobox(query, timeout=DEFAULT_TIMEOUT):
-    import lxml.html
-    data = read_url(url, timeout=timeout)
-    doc = lxml.html.document_fromstring(data)
-    k = 'kp-wholepage'
-    wholepage = doc.cssselect('.' + k)
-    infobox = {}
-    if wholepage:
-        page = wholepage[0]
-        for a in page.cssselect('a'):
-            if a.attrib.get('href', '').startswith('http'):
-                domain = '.'.join(a.attrib['href'].split('/')[2].split('.')[-2:])
-                infobox[domain] = a.attrib['href']
-        for e in page.cssselect('*[data-attrid]'):
-            key = e.attrib['data-attrid']
-            value = e.text_content()
-            if value and key not in (
-                'kc:/film/film:media_actions_wholepage',
-                'action:watch_film'
-            ):
-                infobox[key] = value
-    return infobox
-
-
 def find(query, max_results=DEFAULT_MAX_RESULTS, timeout=DEFAULT_TIMEOUT):
     """
     Return max_results tuples with title, url, description 

ox/web/imdb.py

@@ -1,13 +1,13 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
-from collections import defaultdict
+from __future__ import print_function
 
-import json
 import re
 import time
 import unicodedata
 
-from urllib.parse import urlencode
+from six.moves.urllib.parse import urlencode
+from six import text_type, string_types
 
 from .. import find_re, strip_tags, decode_html
 from .. import cache
@@ -16,13 +16,13 @@ from .. import cache
 from . siteparser import SiteParser
 from . import duckduckgo
 from ..utils import datetime
-from ..geo import normalize_country_name, get_country_name
+from ..geo import normalize_country_name
 
 
 def prepare_url(url, data=None, headers=cache.DEFAULT_HEADERS, timeout=cache.cache_timeout, valid=None, unicode=False):
     headers = headers.copy()
     # https://webapps.stackexchange.com/questions/11003/how-can-i-disable-reconfigure-imdbs-automatic-geo-location-so-it-does-not-defau
-    #headers['X-Forwarded-For'] = '72.21.206.80'
+    headers['X-Forwarded-For'] = '72.21.206.80'
     headers['Accept-Language'] = 'en'
 
     return url, data, headers, timeout, unicode
@@ -106,131 +106,6 @@ def technical(label):
     }
 
 
-def tech_spec(metadata):
-    tech = {}
-    for row in metadata['props']['pageProps']['contentData']['section']['items']:
-        title = {
-            'aspect ratio': 'aspectratio',
-            'sound mix': 'sound',
-        }.get(row['rowTitle'].lower(), row['rowTitle'].lower())
-        tech[title] = []
-        for content in row['listContent']:
-            value = content['text']
-            tech[title].append(value)
-    return tech
-
-
-def movie_connections(metadata):
-    
-    connections = {}
-    if 'props' not in metadata:
-        return connections
-    for row in metadata['props']['pageProps']['contentData']['categories']:
-        title = {
-        }.get(row['name'], row['name'])
-        if title not in connections:
-            connections[title] = []
-
-        for item in row['section']['items']:
-            item_ = {
-                'id': item['id'][2:],
-            }
-
-            item_['title'] = re.compile('<a.*?>(.*?)</a>').findall(item['listContent'][0]['html'])[0]
-            if len(item['listContent']) >=2:
-                item_['description'] = strip_tags(item['listContent'][1]['html'])
-            connections[title].append(item_)
-    return connections
-
-
-def get_category_by_id(metadata, id):
-    for category in metadata['props']['pageProps']['contentData']['categories']:
-        if category['id'] == id:
-            return category
-
-
-def get_release_date(metadata):
-    releases = get_category_by_id(metadata, 'releases')
-    def parse_date(d):
-        parsed = None
-        for fmt in (
-            '%B %d, %Y',
-            '%d %B %Y',
-            '%B %Y',
-        ):
-            try:
-                parsed = datetime.strptime(d, fmt)
-                break
-            except:
-                pass
-        if not parsed:
-            return None
-        return '%d-%02d-%02d' % (parsed.year, parsed.month, parsed.day)
-
-    dates = []
-    for item in releases['section']['items']:
-        content = item['listContent'][0]
-        date = parse_date(content['text'])
-        if date:
-            dates.append(date)
-
-    if dates:
-        return min(dates)
-
-def get_locations(metadata):
-    try:
-        locations = [
-            row['cardText']
-            for row in metadata['props']['pageProps']['contentData']['categories'][0]['section']['items']
-        ]
-    except:
-        locations = []
-    return locations
-
-
-def get_keywords(metadata):
-    try:
-        keywords = [
-            row['rowTitle']
-            for row in metadata['props']['pageProps']['contentData']['section']['items']
-        ]
-    except:
-        keywords = []
-    return keywords
-
-
-def get_entity_metadata(metadata):
-    data = {}
-    entity = metadata['props']['pageProps']['contentData']['entityMetadata']
-    data['title'] = entity['titleText']['text']
-    data['originalTitle'] = entity['originalTitleText']['text']
-    data['year'] = entity['releaseYear']['year']
-    data['plot'] = entity['plot']['plotText']['plainText']
-    data['country'] = [get_country_name(c['id']) for c in entity['countriesOfOrigin']['countries']]
-    data['poster'] = metadata['props']['pageProps']['contentData']['posterData']['image']['url']
-    return data
-
-
-def alternative_titles(metadata):
-    titles = defaultdict(list)
-    akas = get_category_by_id(metadata, 'akas')
-
-    skip = [
-        metadata['props']['pageProps']['contentData']['entityMetadata']['titleText']['text'],
-        metadata['props']['pageProps']['contentData']['entityMetadata']['originalTitleText']['text']
-    ]
-    for row in akas['section']['items']:
-        content = row['listContent'][0]
-        title = content['text']
-        country = row['rowTitle']
-        if title in skip:
-            continue
-        titles[title].append(country)
-        #if content.get('subText'):
-        #    titles[-1]['subText'] = content['subText']
-    return [kv for kv in titles.items()]
-
-
 '''
 'posterIds': {
     'page': 'posters',
@@ -241,17 +116,18 @@ def alternative_titles(metadata):
 
 class Imdb(SiteParser):
     '''
-    >>> Imdb('0068646')['title'] == 'The Godfather'
+    >>> Imdb('0068646')['title'] == text_type(u'The Godfather')
     True
 
-    >>> Imdb('0133093')['title'] == 'The Matrix'
+    >>> Imdb('0133093')['title'] == text_type(u'The Matrix')
     True
     '''
     regex = {
         'alternativeTitles': {
             'page': 'releaseinfo',
             're': [
-                '<li role="presentation" class="ipc-metadata-list__item" data-testid="list-item"><button class="ipc-metadata-list-item__label" role="button" tabindex="0" aria-disabled="false">([^>]+)</button.*?<li role="presentation" class="ipc-inline-list__item"><label class="ipc-metadata-list-item__list-content-item"[^>]*?>([^<]+)</label>',
+                '<h4[^>]*?id="akas"[^>]*?>(.*?)</table>',
+                "td[^>]*?>(.*?)</td>.*?<td[^>]*?>(.*?)</td>"
             ],
             'type': 'list'
         },
@@ -276,7 +152,21 @@ class Imdb(SiteParser):
             'type': 'list'
         },
         'cinematographer': reference_section('cinematographers'),
+        'connections': {
+            'page': 'movieconnections',
+            're': '<h4 class="li_group">(.*?)</h4>(.*?)(<\/div>\n  <a|<script)',
+            'type': 'list'
+        },
         'country': zebra_list('Country', more=['<a.*?>(.*?)</a>']),
+        'creator': {
+            'page': '',
+            're': [
+                '<div class="credit_summary_item">.*?<h4.*?>Creator.?:</h4>(.*?)</div>',
+                '<a href="/name/.*?>(.*?)</a>',
+                lambda ll: strip_tags(ll)
+            ],
+            'type': 'list'
+        },
         'director': reference_section('directors'),
         'editor': reference_section('editors'),
         'composer': reference_section('composers'),
@@ -285,14 +175,27 @@ class Imdb(SiteParser):
             're': '<h3 itemprop="name">(.*?)<',
             'type': 'string'
         },
+        'filmingLocations': {
+            'page': 'locations',
+            're': [
+                '<a href="/search/title\?locations=.*?".*?>(.*?)</a>',
+                lambda data: data.strip(),
+            ],
+            'type': 'list'
+        },
         'genre': zebra_list('Genres', more=['<a.*?>(.*?)</a>']),
         'gross': zebra_table('Cumulative Worldwide Gross', more=[
             lambda data: find_re(decode_html(data).replace(',', ''), '\d+')
         ], type='int'),
+        'keyword': {
+            'page': 'keywords',
+            're': 'data-item-keyword="(.*?)"',
+            'type': 'list'
+        },
         'language': zebra_list('Language', more=['<a.*?>(.*?)</a>']),
         'originalTitle': {
             'page': 'releaseinfo',
-            're': '<li role="presentation" class="ipc-metadata-list__item" data-testid="list-item"><button class="ipc-metadata-list-item__label" role="button" tabindex="0" aria-disabled="false">\(original title\)</button.*?<li role="presentation" class="ipc-inline-list__item"><label class="ipc-metadata-list-item__list-content-item"[^>]*?>([^<]+)</label>',
+            're': '<td.*?>\s*?\(original title\)\s*?</td>\s*<td.*?>(.*?)</td>',
             'type': 'string'
         },
         'summary': zebra_table('Plot Summary', more=[
@@ -325,6 +228,14 @@ class Imdb(SiteParser):
             ],
             'type': 'float'
         },
+        'releasedate': {
+            'page': 'releaseinfo',
+            're': [
+                '<td class="release-date-item__date".*?>(.*?)</td>',
+                strip_tags,
+            ],
+            'type': 'list'
+        },
         #FIXME using some /offsite/ redirect now
         #'reviews': {
         #    'page': 'externalreviews',
@@ -340,6 +251,11 @@ class Imdb(SiteParser):
             lambda r: r[0] if isinstance(r, list) else r,
             strip_tags
         ]),
+        'sound': zebra_list('Sound Mix', more=[
+            '<a.*?>([^(<]+)',
+            lambda r: r[0] if isinstance(r, list) else r,
+            strip_tags
+        ]),
         'season': {
             'page': 'reference',
             're': [
@@ -368,7 +284,7 @@ class Imdb(SiteParser):
         },
         'title': {
             'page': 'releaseinfo',
-            're': '<h2.*?>(.*?)</h2>',
+            're': 'h3 itemprop="name">.*?>(.*?)</a>',
             'type': 'string'
         },
         'trivia': {
@@ -407,34 +323,19 @@ class Imdb(SiteParser):
         },
         'laboratory': technical('Laboratory'),
         'camera': technical('Camera'),
+        'negative format': technical('Negative Format'),
+        'cinematographic process': technical('Cinematographic Process'),
+        'printed film format': technical('Printed Film Format'),
     }
 
     def read_url(self, url, timeout):
-        if self.debug:
-            print(url)
         if url not in self._cache:
             self._cache[url] = read_url(url, timeout=timeout, unicode=True)
         return self._cache[url]
 
-    def get_page_data(self, page, timeout=-1):
-        url = self.get_url(page)
-        data = self.read_url(url, timeout)
-        pdata = re.compile('<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', re.DOTALL).findall(data)
-        if pdata:
-            pdata = pdata[0]
-            return json.loads(pdata)
-        return {}
-
     def __init__(self, id, timeout=-1):
         # http://www.imdb.com/help/show_leaf?titlelanguagedisplay
         self.baseUrl = "http://www.imdb.com/title/tt%s/" % id
-        self._id = id
-        if timeout != 0:
-            self._cache = {}
-            url = self.baseUrl + 'releaseinfo'
-            page = self.read_url(url, timeout=-1)
-            if '<h2>See also</h2>' in page:
-                timeout = 0
         super(Imdb, self).__init__(timeout)
 
         url = self.baseUrl + 'reference'
@@ -448,7 +349,7 @@ class Imdb(SiteParser):
 
         if 'alternativeTitles' in self:
             if len(self['alternativeTitles']) == 2 and \
-               isinstance(self['alternativeTitles'][0], str):
+               isinstance(self['alternativeTitles'][0], string_types):
                self['alternativeTitles'] = [self['alternativeTitles']]
 
         for key in ('country', 'genre', 'language', 'sound', 'color'):
@@ -513,7 +414,7 @@ class Imdb(SiteParser):
             self['sound'] = list(sorted(set(self['sound'])))
 
         if 'cast' in self:
-            if isinstance(self['cast'][0], str):
+            if isinstance(self['cast'][0], string_types):
                 self['cast'] = [self['cast']]
             self['actor'] = [c[0] for c in self['cast']]
             def cleanup_character(c):
@@ -523,6 +424,26 @@ class Imdb(SiteParser):
             self['cast'] = [{'actor': x[0], 'character': cleanup_character(x[1])}
                             for x in self['cast']]
 
+        if 'connections' in self:
+            cc={}
+            if len(self['connections']) == 3 and isinstance(self['connections'][0], string_types):
+                self['connections'] = [self['connections']]
+            for rel, data, _ in self['connections']:
+                if isinstance(rel, bytes):
+                    rel = rel.decode('utf-8')
+                #cc[rel] = re.compile('<a href="/title/tt(\d+)/">(.*?)</a>').findall(data)
+                def get_conn(c):
+                    r = {
+                        'id': c[0],
+                        'title': cleanup_title(c[1]),
+                    }
+                    description = c[2].split('<br />')
+                    if len(description) == 2 and description[-1].strip() != '-':
+                        r['description'] = description[-1].strip()
+                    return r
+                cc[rel] = list(map(get_conn, re.compile('<a href="/title/tt(\d+)/?">(.*?)</a>(.*?)<\/div', re.DOTALL).findall(data)))
+
+            self['connections'] = cc
 
         if 'isSeries' in self:
             del self['isSeries']
@@ -530,6 +451,49 @@ class Imdb(SiteParser):
         if 'episodeTitle' in self:
             self['episodeTitle'] = re.sub('Episode \#\d+\.\d+', '', self['episodeTitle'])
 
+        if 'series' in self:
+            series = Imdb(self['series'], timeout=timeout)
+            self['seriesTitle'] = series['title']
+            if 'episodeTitle' in self:
+                self['seriesTitle'] = series['title']
+                if 'season' in self and 'episode' in self:
+                    self['title'] = "%s (S%02dE%02d) %s" % (
+                        self['seriesTitle'], self['season'], self['episode'], self['episodeTitle'])
+                else:
+                    self['title'] = "%s (S01) %s" % (self['seriesTitle'], self['episodeTitle'])
+                    self['season'] = 1
+                self['title'] = self['title'].strip()
+            if 'director' in self:
+                self['episodeDirector'] = self['director']
+
+            if 'creator' not in series and 'director' in series:
+                series['creator'] = series['director']
+                if len(series['creator']) > 10:
+                    series['creator'] = series['director'][:1]
+
+            for key in ['creator', 'country']:
+                if key in series:
+                    self[key] = series[key]
+
+            if 'year' in series:
+                self['seriesYear'] = series['year']
+                if 'year' not in self:
+                    self['year'] = series['year']
+
+            if 'year' in self:
+                self['episodeYear'] = self['year']
+            if 'creator' in self:
+                self['seriesDirector'] = self['creator']
+            if 'originalTitle' in self:
+                del self['originalTitle']
+        else:
+            for key in ('seriesTitle', 'episodeTitle', 'season', 'episode'):
+                if key in self:
+                    del self[key]
+        if 'creator' in self:
+            if 'director' in self:
+                self['episodeDirector'] = self['director']
+            self['director'] = self['creator']
 
         #make lists unique but keep order
         for key in ('director', 'language'):
@@ -547,20 +511,21 @@ class Imdb(SiteParser):
         if 'budget' in self and 'gross' in self:
             self['profit'] = self['gross'] - self['budget']
 
-        metadata = self.get_page_data('releaseinfo')
+        if 'releasedate' in self:
-        releasedate = get_release_date(metadata)
+            def parse_date(d):
-        if releasedate:
+                try:
-            self['releasedate'] = releasedate
+                    d = datetime.strptime(d, '%d %B %Y')
-
+                except:
-        metadata = self.get_page_data('keywords')
+                    try:
-        keywords = get_keywords(metadata)
+                        d = datetime.strptime(d, '%B %Y')
-        if keywords:
+                    except:
-            self['keyword'] = keywords
+                        return 'x'
-
+                return '%d-%02d-%02d' % (d.year, d.month, d.day)
-        metadata = self.get_page_data('locations')
+            self['releasedate'] = min([
-        locations = get_locations(metadata)
+                parse_date(d) for d in self['releasedate']
-        if locations:
+            ])
-            self['filmingLocations'] = locations
+            if self['releasedate'] == 'x':
+                del self['releasedate']
 
         if 'summary' not in self and 'storyline' in self:
             self['summary'] = self.pop('storyline')
@@ -568,22 +533,6 @@ class Imdb(SiteParser):
             if isinstance(self['summary'], list):
                 self['summary'] = self['summary'][0]
             self['summary'] = strip_tags(self['summary'].split('</p')[0]).split('  Written by\n')[0].strip()
-        else:
-
-            try:
-                summary = metadata['props']['pageProps']['contentData']['entityMetadata']['plot']['plotText']['plainText']
-                self['summary'] = summary
-
-            except:
-                pass
-
-        #self['connections'] = movie_connections(self.get_page_data('movieconnections'))
-        self['connections'] = self._get_connections()
-
-        spec = tech_spec(self.get_page_data('technical'))
-        for key in spec:
-            if not self.get(key):
-                self[key] = spec[key]
 
         if 'credits' in self:
             credits = [
@@ -632,115 +581,6 @@ class Imdb(SiteParser):
                     series_credit = [c for c in self['credits'] if c.get('deparment') == deparment]
                     if series_credit:
                         self[key] = [c['name'] for c in series_credit]
-        creator = []
-        for c in self.get('credits', []):
-            if '(created by)' in c['roles'] and c['name'] not in creator:
-                creator.append(c['name'])
-            if '(creator)' in c['roles'] and c['name'] not in creator:
-                creator.append(c['name'])
-        if creator:
-            self['creator'] = creator
-
-        if 'series' in self:
-            series = Imdb(self['series'], timeout=timeout)
-            self['seriesTitle'] = series['title']
-            if 'episodeTitle' in self:
-                self['seriesTitle'] = series['title']
-                if 'season' in self and 'episode' in self:
-                    self['title'] = "%s (S%02dE%02d) %s" % (
-                        self['seriesTitle'], self['season'], self['episode'], self['episodeTitle'])
-                else:
-                    self['title'] = "%s (S01) %s" % (self['seriesTitle'], self['episodeTitle'])
-                    self['season'] = 1
-                self['title'] = self['title'].strip()
-            if 'director' in self:
-                self['episodeDirector'] = self['director']
-
-            if 'creator' not in series and 'director' in series:
-                series['creator'] = series['director']
-                if len(series['creator']) > 10:
-                    series['creator'] = series['director'][:1]
-
-            for key in ['creator', 'country']:
-                if key in series:
-                    self[key] = series[key]
-
-            if 'year' in series:
-                self['seriesYear'] = series['year']
-                if 'year' not in self:
-                    self['year'] = series['year']
-
-            if 'year' in self:
-                self['episodeYear'] = self['year']
-            if 'creator' in self:
-                self['seriesDirector'] = self['creator']
-            if 'originalTitle' in self:
-                del self['originalTitle']
-        else:
-            for key in ('seriesTitle', 'episodeTitle', 'season', 'episode'):
-                if key in self:
-                    del self[key]
-        if 'creator' in self:
-            if 'director' in self:
-                self['episodeDirector'] = self['director']
-            self['director'] = self['creator']
-
-    def _get_connections(self):
-        query = '''query {
-    title(id: "tt%s") {
-        id
-        titleText {
-           text
-        }
-        connections(first: 5000) {
-            edges {
-                node {
-                    associatedTitle {
-                        id
-                        titleText {
-                            text
-                        }
-                    }
-                    category {
-                        text
-                    }
-                    text
-                }
-            }
-        }
-    }
-}
-''' % self._id
-        url = 'https://caching.graphql.imdb.com/'
-        headers = cache.DEFAULT_HEADERS.copy()
-        headers.update({
-             'Accept': 'application/graphql+json, application/json',
-             'Origin': 'https://www.imdb.com',
-             'Referer': 'https://www.imdb.com',
-             'x-imdb-user-country': 'US',
-             'x-imdb-user-language': 'en-US',
-             'content-type': 'application/json',
-             'Accept-Language': 'en,en-US;q=0.5'
-        })
-        #response = requests.post(url, json=
-        response = json.loads(read_url(url, data=json.dumps({
-            "query": query
-        }), headers=headers))
-        connections = {}
-        for c in response['data']['title']['connections']['edges']:
-            cat = c['node']['category']['text']
-            if cat not in connections:
-                connections[cat] = []
-            connection = {
-                'id': c['node']['associatedTitle']['id'][2:],
-                'title': c['node']['associatedTitle']['titleText']['text'],
-            }
-            description = c['node'].get('text', '')
-            if description:
-                connection['description'] = description
-            connections[cat].append(connection)
-        return connections
-
 
 class ImdbCombined(Imdb):
     def __init__(self, id, timeout=-1):

ox/web/itunes.py

@@ -2,7 +2,7 @@
 # encoding: utf-8
 from __future__ import print_function
 import re
-from urllib.parse import urlencode
+from six.moves.urllib.parse import urlencode
 
 from ox.cache import read_url
 from ox.html import decode_html, strip_tags

ox/web/metacritic.py

@@ -2,7 +2,7 @@
 # vi:si:et:sw=4:sts=4:ts=4
 import re
 
-from urllib.parse import quote
+from six.moves.urllib.parse import quote
 from lxml.html import document_fromstring
 
 from ox.cache import read_url

ox/web/siteparser.py

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
 import re
-import json
 from multiprocessing.pool import ThreadPool
 
+from six import string_types
+
 from ..cache import read_url
 from .. import decode_html
 from ..utils import datetime
@@ -11,15 +12,15 @@ from ..utils import datetime
 
 def cleanup(key, data, data_type):
     if data:
-        if isinstance(data[0], str):
+        if isinstance(data[0], string_types):
             #FIXME: some types need strip_tags
             #data = [strip_tags(decode_html(p)).strip() for p in data]
             data = [decode_html(p).strip() for p in data]
         elif isinstance(data[0], list) or isinstance(data[0], tuple):
             data = [cleanup(key, p, data_type) for p in data]
-        while len(data) == 1 and not isinstance(data, str):
+        while len(data) == 1 and not isinstance(data, string_types):
             data = data[0]
-        if data_type == 'list' and isinstance(data, str):
+        if data_type == 'list' and isinstance(data, string_types):
             data = [data, ]
     elif data_type != 'list':
         data = ''
@@ -29,7 +30,6 @@ class SiteParser(dict):
     baseUrl = ''
     regex = {}
     pool = ThreadPool(8)
-    debug = False
 
     def get_url(self, page):
         return "%s%s" % (self.baseUrl, page)
@@ -47,7 +47,7 @@ class SiteParser(dict):
         for key in self.regex:
             url = self.get_url(self.regex[key]['page'])
             data = self.read_url(url, timeout)
-            if isinstance(self.regex[key]['re'], str):
+            if isinstance(self.regex[key]['re'], string_types):
                 data = re.compile(self.regex[key]['re'], re.DOTALL).findall(data)
                 data = cleanup(key, data, self.regex[key]['type'])
             elif callable(self.regex[key]['re']):
@@ -58,7 +58,7 @@ class SiteParser(dict):
                         f = r
                     else:
                         f = re.compile(r, re.DOTALL).findall
-                    if isinstance(data, str):
+                    if isinstance(data, string_types):
                         data = f(data)
                     else:
                         data = [f(d) for d in data]
@@ -76,10 +76,6 @@ class SiteParser(dict):
             elif self.regex[key]['type'] == 'date':
                 parse_date = lambda d: d and datetime.strptime('-'.join(d), '%m-%d-%Y').strftime('%Y-%m-%d')
                 data = apply_f(parse_date, data)
-            elif self.regex[key]['type'] == 'json':
-                if isinstance(data, list) and len(data) == 1:
-                    data = data[0]
-                    data = json.loads(data)
             if data:
                 self[key] = data
 

ox/web/startpage.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
-import urllib
+from six.moves import urllib
 
 import lxml.html
 import ox

ox/web/thepiratebay.py

@@ -3,7 +3,7 @@
 from datetime import datetime
 import re
 
-from urllib.parse import quote
+from six.moves.urllib.parse import quote
 
 from ox import find_re, cache, strip_tags, decode_html, get_torrent_info, normalize_newlines
 from ox.normalize import normalize_imdbid

ox/web/twitter.py

@@ -2,7 +2,7 @@
 # vi:si:et:sw=4:sts=4:ts=4
 import re
 from datetime import datetime
-from urllib.parse import quote
+from six.moves.urllib.parse import quote
 
 import lxml.html
 import ox

ox/web/wikipedia.py

@@ -4,7 +4,8 @@ from __future__ import print_function
 
 import re
 
-import urllib
+from six.moves import urllib
+from six import string_types
 
 from ox.utils import json
 from ox.cache import read_url
@@ -68,7 +69,7 @@ def get_movie_data(wikipedia_url):
                     value = value.split('<br>')
                 if value:
                     if key in filmbox:
-                        if isinstance(value, list) and isinstance(filmbox[key], str):
+                        if isinstance(value, list) and isinstance(filmbox[key], string_types):
                             filmbox[key] = [filmbox[key]] + value
                         else:
                             filmbox[key] += value

ox/web/youtube.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
-from urllib.parse import quote, unquote_plus
+from six.moves.urllib.parse import quote, unquote_plus
-import urllib
+from six.moves import urllib
-from http import cookiejar as cookielib
+from six.moves import http_cookiejar as cookielib
 import re
 from xml.dom.minidom import parseString
 import json

requirements.txt

@@ -1,4 +1,3 @@
 chardet
+six>=1.5.2
 lxml
-requests
-Pillow>=10

setup.py

@@ -2,12 +2,14 @@
 # vi:si:et:sw=4:sts=4:ts=4
 # encoding: utf-8
 
-from setuptools import setup
+try:
+    from setuptools import setup
+except:
+    from distutils.core import setup
 
-def get_git_version():
+def get_revision():
     import subprocess
-    version = subprocess.check_output(['git', 'describe', '--tags']).decode().strip().replace('-', '.')
+    return subprocess.check_output(['git', 'rev-list', 'HEAD', '--count']).decode().strip()
-    return '.'.join((version.split('.') + ['0'])[:3])
 
 def get_version():
     import os
@@ -16,8 +18,9 @@ def get_version():
     __version = os.path.join(os.path.dirname(__file__), 'ox/__version.py')
     changelog = os.path.join(os.path.dirname(__file__), 'debian/changelog')
     if os.path.exists(_git):
-        version = get_git_version()
+        rev = get_revision()
-        if version:
+        if rev:
+            version = "2.3.%s" % rev
             with open(__version, 'w') as fd:
                 fd.write('VERSION="%s"' % version)
             return version
@@ -34,8 +37,8 @@ def get_version():
         f.close()
         rev = re.compile('\d+\.\d+\.(\d+)').findall(head)
         if rev:
-            return '3.0.%s' % rev[0]
+            return '2.3.%s' % rev[0]
-    return '3.0.x'
+    return '2.3.x'
 
 
 setup(
@@ -47,13 +50,17 @@ setup(
     url="https://code.0x2620.org/0x2620/python-ox",
     license="GPLv3",
     packages=['ox', 'ox.torrent', 'ox.web'],
-    install_requires=['chardet', 'requests'],
+    install_requires=['six>=1.5.2', 'chardet'],
     keywords=[
     ],
     classifiers=[
         'Operating System :: OS Independent',
         'Programming Language :: Python',
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
 )