py3 support

This commit is contained in:
j 2016-09-07 14:38:08 +02:00
parent 98795d3a25
commit 0b80bebf15
26 changed files with 168 additions and 111 deletions

View file

@ -1,9 +1,12 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, print_function, absolute_import
import models
from django.contrib import admin
from . import models
class CoverAdmin(admin.ModelAdmin):
search_fields = ['url', 'isbn']
admin.site.register(models.Cover, CoverAdmin)

View file

@ -1,10 +1,11 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division
from __future__ import division, print_function, absolute_import
import os.path
import hashlib
from six.moves import urllib
from django.db import models
from django.core.files.base import ContentFile
@ -12,7 +13,8 @@ from lookup.models import MovieId
def getCovers(isbn, url_prefix='', limit=lambda x, y: 0.3 < x/y < 1):
covers = {}
if url_prefix.endswith('/'): url_prefix = url_prefix[:-1]
if url_prefix.endswith('/'):
url_prefix = url_prefix[:-1]
for p in CoverCache.objects.all().filter(isbn=isbn, failed=False).order_by('id'):
if p.site not in covers:
covers[p.site] = []
@ -24,15 +26,17 @@ def getCovers(isbn, url_prefix='', limit=lambda x, y: 0.3 < x/y < 1):
pjson['height'] = cover.height
if p.site not in ['other', 'wikipedia.org'] or limit(cover.width, cover.height):
covers[p.site].append(pjson)
for p in covers.keys():
if not covers[p]:
del covers[p]
for p in list(covers):
if not covers[p]:
del covers[p]
covers = Cover.objects.filter(isbn=isbn).exclude(cover='')
if cover.count() > 0:
covers['local'] = [p.cover.url]
return covers
def cover_path(url, filename):
if not isinstance(url, bytes):
url = url.encode('utf-8')
h = hashlib.sha1(url).hexdigest()
ext = 'jpg'
if filename.endswith('.png'):
@ -66,13 +70,13 @@ class CoverCache(models.Model):
try:
data = ox.net.read_url(url)
self.image.save(name, ContentFile(data))
except ox.net.urllib2.HTTPError, e:
except urllib.error.HTTPError as e:
#import traceback
#print traceback.print_exc()
self.status = e.code
self.failed = True
self.save()
except ox.net.urllib2.URLError, e:
except urllib.error.URLError as e:
#import traceback
#print traceback.print_exc()
self.status = e.reason

View file

@ -1,6 +1,7 @@
from __future__ import division, print_function, absolute_import
from django.conf.urls import url
import views
from . import views
urlpatterns = [
url(r'^$', views.cover),

View file

@ -1,9 +1,10 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, print_function, absolute_import
from oxdjango.shortcuts import render_to_json_response
import models
from . import models
def cover(request):
isbn = request.GET['isbn']

View file

@ -1,8 +1,9 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, print_function, absolute_import
import models
from django.contrib import admin
from . import models
class MovieIdAdmin(admin.ModelAdmin):
search_fields = ['title', 'imdb_id']

View file

@ -1,13 +1,15 @@
# -*- coding: UTF-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, print_function, absolute_import
import ox.web.criterion
import ox.web.imdb
import ox.web.impawards
import models
from poster.models import PosterCache
import modules
from . import models
from . import modules
def addPoster(m, url, site, site_id):
if PosterCache.objects.all().filter(url=url).count() == 0:
@ -32,19 +34,19 @@ def get_ids():
m = models.get_movie_id(data['imdbId'])
if m:
if not m.impawards_id:
print 'impawards', ox.web.impawards.get_url(id)
print('impawards', ox.web.impawards.get_url(id))
m.impawards_id = id
m.save()
for poster in data['posters']:
addPoster(m, poster, 'impawards.com', m.imdb_id)
else:
print 'missing impawards', ox.web.impawards.get_url(id)
print('missing impawards', ox.web.impawards.get_url(id))
for id in ox.web.criterion.get_ids():
if id in ('626', '835'):
continue
if models.MovieId.objects.all().filter(criterion_id=id).count() == 0:
print 'criterion', id
print('criterion', id)
data = ox.web.criterion.get_data(id, get_imdb=True)
if data and 'imdbId' in data:
m = models.get_movie_id(data['imdbId'])
@ -53,7 +55,7 @@ def get_ids():
m.save()
addPoster(m, poster, 'criterion.com', m.criterion_id)
else:
print data['title'], "no imdbId"
print(data['title'], "no imdbId")
modules.get_ids.run()

View file

@ -1,7 +1,9 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, print_function, absolute_import
import os
from urllib import quote
from six.moves.urllib.parse import quote
from django.db import models
@ -12,7 +14,7 @@ import ox.web.imdb
import ox.web.wikipedia
import ox.web.allmovie
from modules import get_info
from .modules import get_info
oxdb_api = ox.API('https://0xdb.org/api/')
@ -31,7 +33,7 @@ def get_movie_id(imdb_id):
}.get(imdb_id, imdb_id)
m, created = MovieId.objects.get_or_create(imdb_id=imdb_id)
if created:
m = MovieId.objects.get(imdb_id=imdb_id)
m = MovieId.objects.get(imdb_id=imdb_id)
m.updateFromImdb()
return m
@ -47,14 +49,14 @@ class MovieId(models.Model):
season = models.IntegerField(default=-1)
episode = models.IntegerField(default=-1)
oxdb_id = models.CharField(max_length=42, unique=True, blank=True, null=True, default=None)
imdb_id = models.CharField(max_length=7, unique=True, blank=True, null=True, default=None)
oxdb_id = models.CharField(max_length=42, unique=True, blank=True, null=True, default=None)
imdb_id = models.CharField(max_length=7, unique=True, blank=True, null=True, default=None)
amg_id = models.IntegerField(unique=True, blank=True, null=True, default=None)
archiveorg_id = models.CharField(unique=True, max_length=255, blank=True, null=True, default=None)
wikipedia_id = models.CharField(unique=True, max_length=255, blank=True, null=True, default=None)
criterion_id = models.IntegerField(unique=True, blank=True, null=True, default=None)
impawards_id = models.CharField(max_length=255, unique=True, blank=True, null=True, default=None)
amg_id = models.IntegerField(unique=True, blank=True, null=True, default=None)
archiveorg_id = models.CharField(unique=True, max_length=255, blank=True, null=True, default=None)
wikipedia_id = models.CharField(unique=True, max_length=255, blank=True, null=True, default=None)
criterion_id = models.IntegerField(unique=True, blank=True, null=True, default=None)
impawards_id = models.CharField(max_length=255, unique=True, blank=True, null=True, default=None)
#FIXME: look into other ids
#what about tv.com ids/urls for tv episodes
@ -93,11 +95,11 @@ class MovieId(models.Model):
if not self.wikipedia_id:
self.wikipedia_id = ox.web.wikipedia.get_id(ox.web.wikipedia.get_url(imdb=self.imdb_id))
if not self.wikipedia_id:
self.wikipedia_id=None
self.wikipedia_id = None
#ignore wikipedia id if already used by another movie,
#its most likely wrong for both in that case
elif MovieId.objects.filter(wikipedia_id=self.wikipedia_id).count() >= 1:
self.wikipedia_id=None
self.wikipedia_id = None
#if not self.oxdb_id:
# self.gen_oxdb_id()
self.save()
@ -234,7 +236,7 @@ class MovieId(models.Model):
posters = getPosters(self, prefix)
for key in posters:
if not key in json:
if key not in json:
json[key] = {}
json[key]['posters'] = posters[key]
return json

View file

@ -1,3 +1,6 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, print_function, absolute_import
class IdModules(object):
def __init__(self):

View file

@ -1,13 +1,16 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, print_function, absolute_import
from datetime import timedelta, datetime
from celery.decorators import task, periodic_task
import models
import cache
import poster.models
from . import models
from . import cache
@periodic_task(run_every=timedelta(days=1))
def cronjob(**kwargs):

View file

@ -1,6 +1,8 @@
from __future__ import division, print_function, absolute_import
from django.conf.urls import url
import views
from . import views
urlpatterns = [
url(r'^$', views.ids),

View file

@ -1,10 +1,12 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, print_function, absolute_import
from oxdjango.shortcuts import render_to_json_response, json_response
from oxdjango.api import actions
import models
from . import models
def get_movie_id(request):
movie_id = None

View file

@ -5,9 +5,12 @@ import sys
root_dir = os.path.normpath(os.path.abspath(os.path.dirname(__file__)))
os.chdir(root_dir)
#using virtualenv's activate_this.py to reorder sys.path
# using virtualenv's activate_this.py to reorder sys.path
activate_this = os.path.join(root_dir, '..', 'bin', 'activate_this.py')
execfile(activate_this, dict(__file__=activate_this))
with open(activate_this) as f:
code = compile(f.read(), activate_this, 'exec')
exec(code, dict(__file__=activate_this))
# execfile(activate_this, dict(__file__=activate_this))
if __name__ == "__main__":

View file

@ -1,12 +1,14 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division
from __future__ import division, print_function, absolute_import
import re
import unicodedata
from urllib import quote
from six.moves.urllib.parse import quote
from six import string_types
import hashlib
import base64
import binascii
from django.db import models
from django.conf import settings
@ -17,9 +19,9 @@ from lookup.models import get_movie_id
from poster.models import getPosters
def normalize_value(value):
if isinstance(value, str):
if isinstance(value, bytes):
value = value.decode('utf-8')
if isinstance(value, unicode):
if isinstance(value, string_types):
value = unicodedata.normalize('NFD', value)
return value
@ -222,7 +224,7 @@ class Imdb(models.Model):
j[key] = j[key].strip().split('\n')
else:
del j[key]
for key in j.keys():
for key in list(j):
if not j[key]:
del j[key]
if 'year' in j and isinstance(j['year'], basestring) and j['year'].isdigit():
@ -242,11 +244,11 @@ def get_new_ids(timeout=-1):
for i in frozenset(ids) - known_ids:
m, created = Imdb.objects.get_or_create(imdb=i)
m.update()
print m
print(m)
if created:
added += 1
if added:
print url, added
print(url, added)
class Match(models.Model):
keys = [
@ -268,14 +270,15 @@ class Match(models.Model):
return self.item.json()
def hexdigest(self):
return base64.b64decode(self.key).encode('hex')
key = self.key.encode()
return binascii.hexlify(base64.b64decode(key)).decode()
@classmethod
def get_keys(cls, data):
data = {
'title': normalize_value(data['title'].lower()),
'year': str(data.get('year', '')),
'director': normalize_value(';'.join(sorted(data.get('director', []))))
'title': normalize_value(data['title'].lower()),
'year': str(data.get('year', '')),
'director': normalize_value(';'.join(sorted(data.get('director', []))))
}
keys = []
if not data['director']:
@ -285,11 +288,9 @@ class Match(models.Model):
for k in _keys:
key = '\0'.join(k)
value = '\0'.join([data[v] for v in k])
if isinstance(value, unicode):
value = value.encode('utf-8')
value = str(value)
key = str(key)
key = base64.b64encode(hashlib.sha1(key + '\n' + value).digest())
key = key.encode('utf-8')
value = value.encode('utf-8')
key = base64.b64encode(hashlib.sha1(key + b'\n' + value).digest()).decode()
keys.append(key)
return keys
@ -308,14 +309,14 @@ class Match(models.Model):
def update_item(cls, item):
info = item.json()
#ignore values without title, must be invalid
if not 'title' in info:
if 'title' not in info:
return
data = []
if 'originalTitle' in info:
data.append({
'title': info['originalTitle'],
'year': info.get('year', ''),
'director': info.get('director', [])
'title': info['originalTitle'],
'year': info.get('year', ''),
'director': info.get('director', [])
})
data.append(info)
existing_keys = [m.key for m in Match.objects.filter(item=item)]

View file

@ -1,13 +1,13 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division
from __future__ import division, print_function, absolute_import
from oxdjango.shortcuts import render_to_json_response, json_response
import ox.web.imdb
from oxdjango.api import actions
import models
from . import models
def getId(request, data):
response = json_response()

View file

@ -1 +1,3 @@
from actions import actions
from __future__ import absolute_import
from .actions import actions

View file

@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, with_statement
from __future__ import division, absolute_import
import inspect
import sys
@ -8,7 +9,7 @@ from django.conf import settings
from ..shortcuts import render_to_json_response, json_response
def autodiscover():
def autodiscover(self=None):
# Register api actions from all installed apps
from importlib import import_module
from django.utils.module_loading import module_has_submodule
@ -28,14 +29,14 @@ def trim(docstring):
# and split into a list of lines:
lines = docstring.expandtabs().splitlines()
# Determine minimum indentation (first line doesn't count):
indent = sys.maxint
indent = sys.maxsize
for line in lines[1:]:
stripped = line.lstrip()
if stripped:
indent = min(indent, len(line) - len(stripped))
# Remove indentation (first line is special):
trimmed = [lines[0].strip()]
if indent < sys.maxint:
if indent < sys.maxsize:
for line in lines[1:]:
trimmed.append(line[indent:].rstrip())
# Strip off trailing and leading blank lines:
@ -50,6 +51,9 @@ def trim(docstring):
class ApiActions(dict):
properties = {}
versions = {}
autodiscover = autodiscover
def __init__(self):
def api(request, data):
@ -74,10 +78,10 @@ class ApiActions(dict):
code = data.get('code', False)
version = getattr(request, 'version', None)
if version:
_actions = self.versions.get(version, {}).keys()
_actions = list(set(_actions + self.keys()))
_actions = list(self.versions.get(version, {}))
_actions = list(set(_actions + list(self)))
else:
_actions = self.keys()
_actions = list(self)
_actions.sort()
actions = {}
for a in _actions:
@ -111,7 +115,10 @@ class ApiActions(dict):
def register(self, method, action=None, cache=True, version=None):
if not action:
action = method.func_name
if hasattr(method, 'func_name'):
action = method.func_name
else:
action = method.__name__
if version:
if not version in self.versions:
self.versions[version] = {}

View file

@ -1,11 +1,12 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import absolute_import
from django.conf.urls import url
import views
from . import views
import actions
from . import actions
actions.autodiscover()
urlpatterns = [

View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, with_statement
from __future__ import division, absolute_import
import json
@ -8,9 +8,9 @@ from django.shortcuts import render_to_response
from django.template import RequestContext
from django.conf import settings
from ..shortcuts import render_to_json_response, json_response
from ..shortcuts import render_to_json_response, json_response, HttpErrorJson
from actions import actions
from .actions import actions
def api(request):
if request.META['REQUEST_METHOD'] == "OPTIONS":
@ -21,7 +21,7 @@ def api(request):
if request.META['REQUEST_METHOD'] != "POST" or (
not 'action' in request.POST and request.META.get('CONTENT_TYPE') != 'application/json'
):
methods = actions.keys()
methods = list(actions)
api = []
for f in sorted(methods):
api.append({'name': f,
@ -31,7 +31,9 @@ def api(request):
'settings': settings,
'sitename': settings.SITENAME
})
return render_to_response('api.html', context)
response = render_to_response('api.html', context)
response['Access-Control-Allow-Origin'] = '*'
return response
if request.META.get('CONTENT_TYPE') == 'application/json':
r = json.loads(request.body)
action = r['action']
@ -45,7 +47,10 @@ def api(request):
else:
f = actions.get(action)
if f:
response = f(request, data)
try:
response = f(request, data)
except HttpErrorJson as e:
response = render_to_json_response(e.response)
else:
response = render_to_json_response(json_response(status=400,
text='Unknown action %s' % action))

View file

@ -1,11 +1,12 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import absolute_import
try:
from django.contrib.auth.decorators import wraps
except:
from django.utils.functional import wraps
from shortcuts import render_to_json_response
from .shortcuts import render_to_json_response
def login_required_json(function=None):
"""
@ -13,11 +14,11 @@ def login_required_json(function=None):
return json error if not logged in.
"""
def _wrapped_view(request, *args, **kwargs):
if request.user.is_authenticated():
return function(request, *args, **kwargs)
return render_to_json_response({'status': {'code': 401, 'text': 'login required'}})
return wraps(function)(_wrapped_view)
def _wrapped_view(request, *args, **kwargs):
if request.user.is_authenticated():
return function(request, *args, **kwargs)
return render_to_json_response({'status': {'code': 401, 'text': 'login required'}})
return wraps(function)(_wrapped_view)
def admin_required_json(function=None):
"""
@ -25,8 +26,8 @@ def admin_required_json(function=None):
return json error if not logged in.
"""
def _wrapped_view(request, *args, **kwargs):
if request.user.is_authenticated() and request.user.profile.get_level() == 'admin':
return function(request, *args, **kwargs)
return render_to_json_response({'status': {'code': 403, 'text': 'permission denied'}})
return wraps(function)(_wrapped_view)
def _wrapped_view(request, *args, **kwargs):
if request.user.is_authenticated() and request.user.profile.get_level() == 'admin':
return function(request, *args, **kwargs)
return render_to_json_response({'status': {'code': 403, 'text': 'permission denied'}})
return wraps(function)(_wrapped_view)

View file

@ -48,10 +48,12 @@ def from_json(json_object):
class DictField(models.TextField):
_type = dict
def loads(self, value):
@classmethod
def loads(cls, value):
return json.loads(value, object_hook=from_json)
def dumps(self, obj):
@classmethod
def dumps(cls, obj):
return json.dumps(obj, default=to_json, ensure_ascii=False)
def from_db_value(self, value, expression, connection, context):
@ -85,8 +87,9 @@ class DictField(models.TextField):
class TupleField(DictField):
_type = (tuple, list)
@classmethod
def loads(self, value):
value = DictField.loads(self, value)
value = DictField.loads(value)
if isinstance(value, list):
value = tuple(value)
return value

View file

@ -27,15 +27,16 @@ def _to_json(python_object):
raise TypeError(u'%s %s is not JSON serializable' % (repr(python_object), type(python_object)))
def render_to_json_response(dictionary, content_type="text/json", status=200):
indent=None
indent = None
if settings.DEBUG:
content_type = "text/javascript"
indent = 2
if getattr(settings, 'JSON_DEBUG', False):
print(json.dumps(dictionary, indent=2, default=_to_json, ensure_ascii=False).encode('utf-8'))
return HttpResponse(json.dumps(dictionary, indent=indent, default=_to_json,
ensure_ascii=False).encode('utf-8'), content_type=content_type, status=status)
response = json.dumps(dictionary, indent=indent, default=_to_json, ensure_ascii=False)
if not isinstance(response, bytes):
response = response.encode('utf-8')
return HttpResponse(response, content_type=content_type, status=status)
def get_object_or_404_json(klass, *args, **kwargs):
from django.shortcuts import _get_queryset

View file

@ -1,6 +1,8 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, print_function, absolute_import
import models
from django.contrib import admin
from . import models

View file

@ -1,12 +1,12 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division
from __future__ import division, print_function, absolute_import
import os.path
import hashlib
import socket
import urllib2
from six.moves import urllib
from django.db import models
import ox
@ -24,7 +24,8 @@ def getPosters(movie_id, url_prefix='', limit=lambda x, y: 0.3 < x/y < 1):
return {}
get_poster_urls(movie_id)
posters = {}
if url_prefix.endswith('/'): url_prefix = url_prefix[:-1]
if url_prefix.endswith('/'):
url_prefix = url_prefix[:-1]
for p in PosterCache.objects.all().filter(movie_id=movie_id, failed=False).order_by('-id'):
if p.site not in posters:
posters[p.site] = []
@ -42,13 +43,13 @@ def getPosters(movie_id, url_prefix='', limit=lambda x, y: 0.3 < x/y < 1):
pjson['height'] = poster.height
if p.site not in ['other', 'wikipedia.org'] or limit(poster.width, poster.height):
posters[p.site].append(pjson)
for p in posters.keys():
if not posters[p]:
del posters[p]
for p in list(posters):
if not posters[p]:
del posters[p]
return posters
def poster_path(url, filename):
if isinstance(url, unicode):
if not isinstance(url, bytes):
url = url.encode('utf-8')
h = hashlib.sha1(url).hexdigest()
ext = 'jpg'
@ -90,19 +91,19 @@ class PosterCache(models.Model):
with open(self.image.path, 'w') as f:
f.write(data)
self.save()
except urllib2.HTTPError, e:
except urllib.error.HTTPError as e:
#import traceback
#print traceback.print_exc()
self.status = e.code
self.failed = True
self.save()
except urllib2.URLError, e:
except urllib.error.URLError as e:
#import traceback
#print traceback.print_exc()
self.status = e.reason
self.failed = True
self.save()
except socket.error, e:
except socket.error as e:
self.status = str(e)
self.failed = True
self.save()

View file

@ -1,12 +1,15 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, print_function, absolute_import
from datetime import timedelta
from celery.decorators import task, periodic_task
import models
from lookup.models import get_movie_id
from . import models
'''
@periodic_task(run_every=timedelta(days=1))
@ -17,10 +20,9 @@ def cronjob(**kwargs):
@task(ignore_resulsts=True, queue='default')
def getMovieposteredb(imdb_id):
m = get_movie_id(imdb_id)
def addPoster(url, site, site_id):
site = 'movieposterdb.com'
for url in ox.web.movieposterdb.get_data(imdb_id)['posters']:
if PosterCache.objects.all().filter(url=url, movie_id=m).count() == 0:
p = PosterCache(url=url, site=site, site_id=site_id, movie_id=m)
p.save()
for poster in ox.web.movieposterdb.get_data(imdb_id)['posters']:
addPoster(poster, 'movieposterdb.com', imdb_id)

View file

@ -1,6 +1,8 @@
from __future__ import division, print_function, absolute_import
from django.conf.urls import url
import views
from . import views
urlpatterns = [

View file

@ -1,10 +1,12 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, print_function, absolute_import
from oxdjango.shortcuts import render_to_json_response
from lookup.views import get_movie_id
import models
from . import models
def poster(request):