py3 support

oxdata/cover/admin.py

@@ -1,9 +1,12 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
+from __future__ import division, print_function, absolute_import
 
-import models
 from django.contrib import admin
 
+from . import models
+
+
 class CoverAdmin(admin.ModelAdmin):
     search_fields = ['url', 'isbn']
 admin.site.register(models.Cover, CoverAdmin)

oxdata/cover/models.py

@@ -1,10 +1,11 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
-from __future__ import division
+from __future__ import division, print_function, absolute_import
 
 import os.path
 import hashlib
 
+from six.moves import urllib
 from django.db import models
 from django.core.files.base import ContentFile
 
@@ -12,7 +13,8 @@ from lookup.models import MovieId
 
 def getCovers(isbn, url_prefix='', limit=lambda x, y: 0.3 < x/y < 1):
     covers = {}
-    if url_prefix.endswith('/'): url_prefix = url_prefix[:-1]
+    if url_prefix.endswith('/'):
+        url_prefix = url_prefix[:-1]
     for p in CoverCache.objects.all().filter(isbn=isbn, failed=False).order_by('id'):
         if p.site not in covers:
             covers[p.site] = []
@@ -24,7 +26,7 @@ def getCovers(isbn, url_prefix='', limit=lambda x, y: 0.3 < x/y < 1):
             pjson['height'] = cover.height
             if p.site not in ['other', 'wikipedia.org'] or limit(cover.width, cover.height):
                 covers[p.site].append(pjson)
-    for p in covers.keys():
+    for p in list(covers):
         if not covers[p]:
             del covers[p]
     covers = Cover.objects.filter(isbn=isbn).exclude(cover='')
@@ -33,6 +35,8 @@ def getCovers(isbn, url_prefix='', limit=lambda x, y: 0.3 < x/y < 1):
     return covers
 
 def cover_path(url, filename):
+    if not isinstance(url, bytes):
+        url = url.encode('utf-8')
     h = hashlib.sha1(url).hexdigest()
     ext = 'jpg'
     if filename.endswith('.png'):
@@ -66,13 +70,13 @@ class CoverCache(models.Model):
             try:
                 data = ox.net.read_url(url)
                 self.image.save(name, ContentFile(data))
-            except ox.net.urllib2.HTTPError, e:
+            except urllib.error.HTTPError as e:
                 #import traceback
                 #print traceback.print_exc()
                 self.status = e.code
                 self.failed = True
                 self.save()
-            except ox.net.urllib2.URLError, e:
+            except urllib.error.URLError as e:
                 #import traceback
                 #print traceback.print_exc()
                 self.status = e.reason

oxdata/cover/urls.py

@@ -1,6 +1,7 @@
+from __future__ import division, print_function, absolute_import
 from django.conf.urls import url
 
-import views
+from . import views
 
 urlpatterns = [
     url(r'^$', views.cover),

oxdata/cover/views.py

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
+from __future__ import division, print_function, absolute_import
 
 from oxdjango.shortcuts import render_to_json_response
 
-import models
+from . import models
 
 def cover(request):
     isbn = request.GET['isbn']

oxdata/lookup/admin.py

@@ -1,8 +1,9 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
+from __future__ import division, print_function, absolute_import
 
-import models
 from django.contrib import admin
+from . import models
 
 class MovieIdAdmin(admin.ModelAdmin):
     search_fields = ['title', 'imdb_id']

oxdata/lookup/cache.py

@@ -1,13 +1,15 @@
 # -*- coding: UTF-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
+from __future__ import division, print_function, absolute_import
 
 import ox.web.criterion
 import ox.web.imdb
 import ox.web.impawards
 
-import models
 from poster.models import PosterCache
-import modules
+
+from . import models
+from . import modules
 
 def addPoster(m, url, site, site_id):
     if PosterCache.objects.all().filter(url=url).count() == 0:
@@ -32,19 +34,19 @@ def get_ids():
                 m = models.get_movie_id(data['imdbId'])
                 if m:
                     if not m.impawards_id:
-                        print 'impawards', ox.web.impawards.get_url(id)
+                        print('impawards', ox.web.impawards.get_url(id))
                         m.impawards_id = id
                         m.save()
                     for poster in data['posters']:
                             addPoster(m, poster, 'impawards.com', m.imdb_id)
             else:
-                print 'missing impawards', ox.web.impawards.get_url(id)
+                print('missing impawards', ox.web.impawards.get_url(id))
 
     for id in ox.web.criterion.get_ids():
         if id in ('626', '835'):
             continue
         if models.MovieId.objects.all().filter(criterion_id=id).count() == 0:
-            print 'criterion', id
+            print('criterion', id)
             data = ox.web.criterion.get_data(id, get_imdb=True)
             if data and 'imdbId' in data:
                 m = models.get_movie_id(data['imdbId'])
@@ -53,7 +55,7 @@ def get_ids():
                     m.save()
                 addPoster(m, poster, 'criterion.com', m.criterion_id)
             else:
-                print data['title'], "no imdbId"
+                print(data['title'], "no imdbId")
 
     modules.get_ids.run()
 

oxdata/lookup/models.py

@@ -1,7 +1,9 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
+from __future__ import division, print_function, absolute_import
+
 import os
-from urllib import quote
+from six.moves.urllib.parse import quote
 
 from django.db import models
 
@@ -12,7 +14,7 @@ import ox.web.imdb
 import ox.web.wikipedia
 import ox.web.allmovie
 
-from modules import get_info
+from .modules import get_info
 
 oxdb_api = ox.API('https://0xdb.org/api/')
 
@@ -93,11 +95,11 @@ class MovieId(models.Model):
             if not self.wikipedia_id:
                 self.wikipedia_id = ox.web.wikipedia.get_id(ox.web.wikipedia.get_url(imdb=self.imdb_id))
                 if not self.wikipedia_id:
-                    self.wikipedia_id=None
+                    self.wikipedia_id = None
                 #ignore wikipedia id if already used by another movie,
                 #its most likely wrong for both in that case
                 elif MovieId.objects.filter(wikipedia_id=self.wikipedia_id).count() >= 1:
-                    self.wikipedia_id=None
+                    self.wikipedia_id = None
             #if not self.oxdb_id:
             #    self.gen_oxdb_id()
         self.save()
@@ -234,7 +236,7 @@ class MovieId(models.Model):
 
         posters = getPosters(self, prefix)
         for key in posters:
-            if not key in json:
+            if key not in json:
                 json[key] = {}
             json[key]['posters'] = posters[key]
         return json

oxdata/lookup/modules.py

@@ -1,3 +1,6 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+from __future__ import division, print_function, absolute_import
 
 class IdModules(object):
     def __init__(self):

oxdata/lookup/tasks.py

@@ -1,13 +1,16 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
+from __future__ import division, print_function, absolute_import
+
 from datetime import timedelta, datetime
 
 from celery.decorators import task, periodic_task
 
-import models
-import cache
 import poster.models
 
+from . import models
+from . import cache
+
 
 @periodic_task(run_every=timedelta(days=1))
 def cronjob(**kwargs):

oxdata/lookup/urls.py

@@ -1,6 +1,8 @@
+from __future__ import division, print_function, absolute_import
+
 from django.conf.urls import url
 
-import views
+from . import views
 
 urlpatterns = [
     url(r'^$', views.ids),

oxdata/lookup/views.py

@@ -1,10 +1,12 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
+from __future__ import division, print_function, absolute_import
+
 from oxdjango.shortcuts import render_to_json_response, json_response
 from oxdjango.api import actions
 
 
-import models
+from . import models
 
 def get_movie_id(request):
     movie_id = None

oxdata/manage.py

@@ -5,9 +5,12 @@ import sys
 root_dir = os.path.normpath(os.path.abspath(os.path.dirname(__file__)))
 os.chdir(root_dir)
 
-#using virtualenv's activate_this.py to reorder sys.path
+# using virtualenv's activate_this.py to reorder sys.path
 activate_this = os.path.join(root_dir, '..', 'bin', 'activate_this.py')
-execfile(activate_this, dict(__file__=activate_this))
+with open(activate_this) as f:
+    code = compile(f.read(), activate_this, 'exec')
+    exec(code, dict(__file__=activate_this))
+# execfile(activate_this, dict(__file__=activate_this))
 
 
 if __name__ == "__main__":

oxdata/movie/models.py

@@ -1,12 +1,14 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
-from __future__ import division
+from __future__ import division, print_function, absolute_import
 
 import re
 import unicodedata
-from urllib import quote
+from six.moves.urllib.parse import quote
+from six import string_types
 import hashlib
 import base64
+import binascii
 
 from django.db import models
 from django.conf import settings
@@ -17,9 +19,9 @@ from lookup.models import get_movie_id
 from poster.models import getPosters
 
 def normalize_value(value):
-    if isinstance(value, str):
+    if isinstance(value, bytes):
         value = value.decode('utf-8')
-    if isinstance(value, unicode):
+    if isinstance(value, string_types):
         value = unicodedata.normalize('NFD', value)
     return value
 
@@ -222,7 +224,7 @@ class Imdb(models.Model):
                 j[key] = j[key].strip().split('\n')
             else:
                 del j[key]
-        for key in j.keys():
+        for key in list(j):
             if not j[key]:
                 del j[key]
         if 'year' in j and isinstance(j['year'], basestring) and j['year'].isdigit():
@@ -242,11 +244,11 @@ def get_new_ids(timeout=-1):
         for i in frozenset(ids) - known_ids:
             m, created = Imdb.objects.get_or_create(imdb=i)
             m.update()
-            print m
+            print(m)
             if created:
                 added += 1
         if added:
-            print url, added
+            print(url, added)
 
 class Match(models.Model):
     keys = [
@@ -268,7 +270,8 @@ class Match(models.Model):
         return self.item.json()
 
     def hexdigest(self):
-        return base64.b64decode(self.key).encode('hex')
+        key = self.key.encode()
+        return binascii.hexlify(base64.b64decode(key)).decode()
 
     @classmethod
     def get_keys(cls, data):
@@ -285,11 +288,9 @@ class Match(models.Model):
         for k in _keys:
             key = '\0'.join(k)
             value = '\0'.join([data[v] for v in k])
-            if isinstance(value, unicode):
+            key = key.encode('utf-8')
             value = value.encode('utf-8')
-            value = str(value)
+            key = base64.b64encode(hashlib.sha1(key + b'\n' + value).digest()).decode()
-            key = str(key)
-            key = base64.b64encode(hashlib.sha1(key + '\n' + value).digest())
             keys.append(key)
         return keys
 
@@ -308,7 +309,7 @@ class Match(models.Model):
     def update_item(cls, item):
         info = item.json()
         #ignore values without title, must be invalid
-        if not 'title' in info:
+        if 'title' not in info:
             return
         data = []
         if 'originalTitle' in info:

oxdata/movie/views.py

@@ -1,13 +1,13 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
-from __future__ import division
+from __future__ import division, print_function, absolute_import
 
 
 from oxdjango.shortcuts import render_to_json_response, json_response
 import ox.web.imdb
 
 from oxdjango.api import actions
-import models
+from . import models
 
 def getId(request, data):
     response = json_response()

oxdata/oxdjango/api/__init__.py

@@ -1 +1,3 @@
-from actions import actions
+from __future__ import absolute_import
+
+from .actions import actions

oxdata/oxdjango/api/actions.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
-from __future__ import division, with_statement
+from __future__ import division, absolute_import
+
 import inspect
 import sys
 
@@ -8,7 +9,7 @@ from django.conf import settings
 
 from ..shortcuts import render_to_json_response, json_response
 
-def autodiscover():
+def autodiscover(self=None):
     # Register api actions from all installed apps
     from importlib import import_module
     from django.utils.module_loading import module_has_submodule
@@ -28,14 +29,14 @@ def trim(docstring):
     # and split into a list of lines:
     lines = docstring.expandtabs().splitlines()
     # Determine minimum indentation (first line doesn't count):
-    indent = sys.maxint
+    indent = sys.maxsize
     for line in lines[1:]:
         stripped = line.lstrip()
         if stripped:
             indent = min(indent, len(line) - len(stripped))
     # Remove indentation (first line is special):
     trimmed = [lines[0].strip()]
-    if indent < sys.maxint:
+    if indent < sys.maxsize:
         for line in lines[1:]:
             trimmed.append(line[indent:].rstrip())
     # Strip off trailing and leading blank lines:
@@ -50,6 +51,9 @@ def trim(docstring):
 class ApiActions(dict):
     properties = {}
     versions = {}
+
+    autodiscover = autodiscover
+
     def __init__(self):
 
         def api(request, data):
@@ -74,10 +78,10 @@ class ApiActions(dict):
             code = data.get('code', False)
             version = getattr(request, 'version', None)
             if version:
-                _actions = self.versions.get(version, {}).keys()
+                _actions = list(self.versions.get(version, {}))
-                _actions = list(set(_actions + self.keys()))
+                _actions = list(set(_actions + list(self)))
             else:
-                _actions = self.keys()
+                _actions = list(self)
             _actions.sort()
             actions = {}
             for a in _actions:
@@ -111,7 +115,10 @@ class ApiActions(dict):
 
     def register(self, method, action=None, cache=True, version=None):
         if not action:
+            if hasattr(method, 'func_name'):
                 action = method.func_name
+            else:
+                action = method.__name__
         if version:
             if not version in self.versions:
                 self.versions[version] = {}

oxdata/oxdjango/api/urls.py

@@ -1,11 +1,12 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
+from __future__ import absolute_import
 
 from django.conf.urls import url
 
-import views
+from . import views
 
-import actions
+from . import actions
 actions.autodiscover()
 
 urlpatterns = [

oxdata/oxdjango/api/views.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
-from __future__ import division, with_statement
+from __future__ import division, absolute_import
 
 import json
 
@@ -8,9 +8,9 @@ from django.shortcuts import render_to_response
 from django.template import RequestContext
 from django.conf import settings
 
-from ..shortcuts import render_to_json_response, json_response
+from ..shortcuts import render_to_json_response, json_response, HttpErrorJson
 
-from actions import actions
+from .actions import actions
 
 def api(request):
     if request.META['REQUEST_METHOD'] == "OPTIONS":
@@ -21,7 +21,7 @@ def api(request):
     if request.META['REQUEST_METHOD'] != "POST" or (
         not 'action' in request.POST and request.META.get('CONTENT_TYPE') != 'application/json'
     ):
-        methods = actions.keys()
+        methods = list(actions)
         api = []
         for f in sorted(methods):
             api.append({'name': f,
@@ -31,7 +31,9 @@ def api(request):
             'settings': settings,
             'sitename': settings.SITENAME
         })
-        return render_to_response('api.html', context)
+        response = render_to_response('api.html', context)
+        response['Access-Control-Allow-Origin'] = '*'
+        return response
     if request.META.get('CONTENT_TYPE') == 'application/json':
         r = json.loads(request.body)
         action = r['action']
@@ -45,7 +47,10 @@ def api(request):
     else:
         f = actions.get(action)
     if f:
+        try:
             response = f(request, data)
+        except HttpErrorJson as e:
+            response = render_to_json_response(e.response)
     else:
         response = render_to_json_response(json_response(status=400,
                                 text='Unknown action %s' % action))

oxdata/oxdjango/decorators.py

@@ -1,11 +1,12 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
+from __future__ import absolute_import
 
 try:
     from django.contrib.auth.decorators import wraps
 except:
     from django.utils.functional import wraps
-from shortcuts import render_to_json_response
+from .shortcuts import render_to_json_response
 
 def login_required_json(function=None):
     """

oxdata/oxdjango/fields.py

@@ -48,10 +48,12 @@ def from_json(json_object):
 class DictField(models.TextField):
     _type = dict
 
-    def loads(self, value):
+    @classmethod
+    def loads(cls, value):
         return json.loads(value, object_hook=from_json)
 
-    def dumps(self, obj):
+    @classmethod
+    def dumps(cls, obj):
         return json.dumps(obj, default=to_json, ensure_ascii=False)
 
     def from_db_value(self, value, expression, connection, context):
@@ -85,8 +87,9 @@ class DictField(models.TextField):
 class TupleField(DictField):
     _type = (tuple, list)
 
+    @classmethod
     def loads(self, value):
-        value = DictField.loads(self, value)
+        value = DictField.loads(value)
         if isinstance(value, list):
             value = tuple(value)
         return value

oxdata/oxdjango/shortcuts.py

@@ -27,15 +27,16 @@ def _to_json(python_object):
     raise TypeError(u'%s %s is not JSON serializable' % (repr(python_object), type(python_object)))
 
 def render_to_json_response(dictionary, content_type="text/json", status=200):
-    indent=None
+    indent = None
     if settings.DEBUG:
         content_type = "text/javascript"
         indent = 2
     if getattr(settings, 'JSON_DEBUG', False):
         print(json.dumps(dictionary, indent=2, default=_to_json, ensure_ascii=False).encode('utf-8'))
-
+    response = json.dumps(dictionary, indent=indent, default=_to_json, ensure_ascii=False)
-    return HttpResponse(json.dumps(dictionary, indent=indent, default=_to_json,
+    if not isinstance(response, bytes):
-        ensure_ascii=False).encode('utf-8'), content_type=content_type, status=status)
+        response = response.encode('utf-8')
+    return HttpResponse(response, content_type=content_type, status=status)
 
 def get_object_or_404_json(klass, *args, **kwargs):
     from django.shortcuts import _get_queryset

oxdata/poster/admin.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
+from __future__ import division, print_function, absolute_import
 
-import models
 from django.contrib import admin
 
+from . import models
+

oxdata/poster/models.py

@@ -1,12 +1,12 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
-from __future__ import division
+from __future__ import division, print_function, absolute_import
 
 import os.path
 import hashlib
 import socket
-import urllib2
 
+from six.moves import urllib
 from django.db import models
 
 import ox
@@ -24,7 +24,8 @@ def getPosters(movie_id, url_prefix='', limit=lambda x, y: 0.3 < x/y < 1):
         return {}
     get_poster_urls(movie_id)
     posters = {}
-    if url_prefix.endswith('/'): url_prefix = url_prefix[:-1]
+    if url_prefix.endswith('/'):
+        url_prefix = url_prefix[:-1]
     for p in PosterCache.objects.all().filter(movie_id=movie_id, failed=False).order_by('-id'):
         if p.site not in posters:
             posters[p.site] = []
@@ -42,13 +43,13 @@ def getPosters(movie_id, url_prefix='', limit=lambda x, y: 0.3 < x/y < 1):
             pjson['height'] = poster.height
             if p.site not in ['other', 'wikipedia.org'] or limit(poster.width, poster.height):
                 posters[p.site].append(pjson)
-    for p in posters.keys():
+    for p in list(posters):
         if not posters[p]:
             del posters[p]
     return posters
 
 def poster_path(url, filename):
-    if isinstance(url, unicode):
+    if not isinstance(url, bytes):
         url = url.encode('utf-8')
     h = hashlib.sha1(url).hexdigest()
     ext = 'jpg'
@@ -90,19 +91,19 @@ class PosterCache(models.Model):
                 with open(self.image.path, 'w') as f:
                     f.write(data)
                 self.save()
-            except urllib2.HTTPError, e:
+            except urllib.error.HTTPError as e:
                 #import traceback
                 #print traceback.print_exc()
                 self.status = e.code
                 self.failed = True
                 self.save()
-            except urllib2.URLError, e:
+            except urllib.error.URLError as e:
                 #import traceback
                 #print traceback.print_exc()
                 self.status = e.reason
                 self.failed = True
                 self.save()
-            except socket.error, e:
+            except socket.error as e:
                 self.status = str(e)
                 self.failed = True
                 self.save()

oxdata/poster/tasks.py

@@ -1,12 +1,15 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
+from __future__ import division, print_function, absolute_import
+
 from datetime import timedelta
 
 from celery.decorators import task, periodic_task
 
-import models
 from lookup.models import get_movie_id
 
+from . import models
+
 
 '''
 @periodic_task(run_every=timedelta(days=1))
@@ -17,10 +20,9 @@ def cronjob(**kwargs):
 @task(ignore_resulsts=True, queue='default')
 def getMovieposteredb(imdb_id):
     m = get_movie_id(imdb_id)
-    def addPoster(url, site, site_id):
+    site = 'movieposterdb.com'
+
+    for url in ox.web.movieposterdb.get_data(imdb_id)['posters']:
         if PosterCache.objects.all().filter(url=url, movie_id=m).count() == 0:
             p = PosterCache(url=url, site=site, site_id=site_id, movie_id=m)
             p.save()
-    for poster in ox.web.movieposterdb.get_data(imdb_id)['posters']:
-        addPoster(poster, 'movieposterdb.com', imdb_id)
-

oxdata/poster/urls.py

@@ -1,6 +1,8 @@
+from __future__ import division, print_function, absolute_import
+
 from django.conf.urls import url
 
-import views
+from . import views
 
 
 urlpatterns = [

oxdata/poster/views.py

@@ -1,10 +1,12 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
+from __future__ import division, print_function, absolute_import
+
 from oxdjango.shortcuts import render_to_json_response
 
 from lookup.views import get_movie_id
 
-import models
+from . import models
 
 
 def poster(request):