diff --git a/README.md b/README.md index dcc684b..76bf370 100644 --- a/README.md +++ b/README.md @@ -4,15 +4,15 @@ django based api server ## setup - apt-get install virtualenv git ipython3 + apt-get install git ipython3 python3-venv apt-get install python3-pil python3-psycopg2 \ - python3-lxml python3-requests python3-six + python3-lxml python3-requests python3-six python3-anyjson git clone https://git.0x2620.org/oxdata.git cd oxdata - virtualenv -p /usr/bin/python3 --system-site-packages . - ./bin/pip install -r requirements.txt + python3 -m venv --system-site-packages venv + ./venv/bin/pip install -r requirements.txt Install rabbitmq and carrot: @@ -26,4 +26,3 @@ django based api server ## deploy: adduser --disabled-password --disabled-login --home /srv/oxdata/ oxdata - diff --git a/oxdata/cover/models.py b/oxdata/cover/models.py index 8470809..ad4cc8c 100644 --- a/oxdata/cover/models.py +++ b/oxdata/cover/models.py @@ -99,6 +99,6 @@ class Cover(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) - isbn = models.ForeignKey(MovieId, related_name='cover') + isbn = models.ForeignKey(MovieId, related_name='cover', on_delete=models.CASCADE) cover = models.ImageField(max_length=255, upload_to=cover_image_path) diff --git a/oxdata/cover/urls.py b/oxdata/cover/urls.py index 2168cbd..42001a2 100644 --- a/oxdata/cover/urls.py +++ b/oxdata/cover/urls.py @@ -1,9 +1,8 @@ -from __future__ import division, print_function, absolute_import -from django.conf.urls import url +from django.urls import path, re_path from . import views urlpatterns = [ - url(r'^$', views.cover), + re_path(r'^$', views.cover), ] diff --git a/oxdata/lookup/urls.py b/oxdata/lookup/urls.py index 29a5484..af50d54 100644 --- a/oxdata/lookup/urls.py +++ b/oxdata/lookup/urls.py @@ -1,10 +1,20 @@ -from __future__ import division, print_function, absolute_import - -from django.conf.urls import url +from django.urls import path, re_path from . import views -urlpatterns = [ - url(r'^$', views.ids), - url(r'^urls$', views.urls), +urls = [ + [ + path(r'', views.ids), + path(r'urls', views.urls), + ], + 'lookup', + 'lookup' +] +urls2 = [ + [ + path(r'', views.ids), + path(r'urls', views.urls), + ], + 'lookup2', + 'lookup2' ] diff --git a/oxdata/manage.py b/oxdata/manage.py index 057af0b..12301ea 100755 --- a/oxdata/manage.py +++ b/oxdata/manage.py @@ -1,19 +1,46 @@ #!/usr/bin/env python3 import os import sys +import argparse -root_dir = os.path.normpath(os.path.abspath(os.path.dirname(__file__))) -os.chdir(root_dir) -# using virtualenv's activate_this.py to reorder sys.path -activate_this = os.path.join(root_dir, '..', 'bin', 'activate_this.py') -with open(activate_this) as f: - code = compile(f.read(), activate_this, 'exec') - exec(code, dict(__file__=activate_this)) -# execfile(activate_this, dict(__file__=activate_this)) +def activate_venv(base): + if os.path.exists(base): + old_os_path = os.environ.get("PATH", "") + os.environ["PATH"] = os.path.join(base, "bin") + os.pathsep + old_os_path + version = "%s.%s" % (sys.version_info.major, sys.version_info.minor) + site_packages = os.path.join(base, "lib", "python%s" % version, "site-packages") + prev_sys_path = list(sys.path) + import site + + site.addsitedir(site_packages) + sys.real_prefix = sys.prefix + sys.prefix = base + # Move the added items to the front of the path: + new_sys_path = [] + for item in list(sys.path): + if item not in prev_sys_path: + new_sys_path.append(item) + sys.path.remove(item) + sys.path[:0] = new_sys_path + + +def main(): + """Run administrative tasks.""" + root_dir = os.path.normpath(os.path.abspath(os.path.dirname(__file__))) + activate_venv(os.path.normpath(os.path.join(root_dir, "..", "venv"))) + + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") - from django.core.management import execute_from_command_line - execute_from_command_line(sys.argv) + main() diff --git a/oxdata/movie/models.py b/oxdata/movie/models.py index 74a7ba1..8b9ac87 100644 --- a/oxdata/movie/models.py +++ b/oxdata/movie/models.py @@ -13,7 +13,6 @@ import json from django.db import models from django.conf import settings import ox -from oxdjango.fields import DictField from lookup.models import get_movie_id from poster.models import getPosters @@ -102,7 +101,7 @@ class Imdb(models.Model): seriesTitle = models.CharField(max_length=1000, blank=True, default='') invalid = models.BooleanField(default=False) - patch = DictField(default=None, blank=True, null=True) + patch = models.JSONField(default=None, blank=True, null=True) def __str__(self): return "[%s] %s%s" % (self.imdb, self.title, self.year and ' (%s)' % self.year or '') @@ -185,7 +184,8 @@ class Imdb(models.Model): if 'votes' in data: max_votes = ox.web.imdb.max_votes() - data['votes'] = 100 * float(data['votes']) / max_votes + if max_votes: + data['votes'] = 100 * float(data['votes']) / max_votes else: data['votes'] = 0 @@ -299,7 +299,7 @@ class Match(models.Model): ] key = models.CharField(max_length=28, db_index=True) - item = models.ForeignKey(Imdb, related_name='matches') + item = models.ForeignKey(Imdb, related_name='matches', on_delete=models.CASCADE) def __str__(self): return '%s(%s)' % (self.hexdigest(), self.item.imdb) diff --git a/oxdata/oxdjango/api/urls.py b/oxdata/oxdjango/api/urls.py index 159c4ff..a188704 100644 --- a/oxdata/oxdjango/api/urls.py +++ b/oxdata/oxdjango/api/urls.py @@ -1,13 +1,17 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -from django.conf.urls import url +from django.urls import path, re_path from . import views from . import actions actions.autodiscover() -urlpatterns = [ - url(r'^$', views.api), +urls = [ + [ + path(r'', views.api), + ], + 'api', + 'api' ] diff --git a/oxdata/poster/models.py b/oxdata/poster/models.py index 33e3e18..4e88c0e 100644 --- a/oxdata/poster/models.py +++ b/oxdata/poster/models.py @@ -69,7 +69,7 @@ class PosterCache(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) - movie_id = models.ForeignKey(MovieId, related_name='postercache') + movie_id = models.ForeignKey(MovieId, related_name='postercache', on_delete=models.CASCADE) url = models.CharField(max_length=1024) site = models.CharField(max_length=255) site_id = models.CharField(max_length=1024) @@ -144,10 +144,12 @@ def get_poster_urls(m): #for poster in ox.web.movieposterdb.get_data(m.imdb_id)['posters']: # addPoster(poster, 'movieposterdb.com', m.imdb_id) + ''' poster = ox.web.piratecinema.get_poster_url(m.imdb_id) if poster: m.postercache.filter(url__contains='piratecinema.org', failed=True).delete() addPoster(poster, 'piratecinema.org', m.imdb_id) + ''' if m.criterion_id: #if settings.DEBUG: diff --git a/oxdata/poster/urls.py b/oxdata/poster/urls.py index a4d8863..210ac68 100644 --- a/oxdata/poster/urls.py +++ b/oxdata/poster/urls.py @@ -1,10 +1,12 @@ -from __future__ import division, print_function, absolute_import - -from django.conf.urls import url +from django.urls import path, re_path from . import views -urlpatterns = [ - url(r'^$', views.poster), +urls = [ + [ + path(r'', views.poster), + ], + 'poster', + 'poster' ] diff --git a/oxdata/settings.py b/oxdata/settings.py index ab0246f..101625e 100644 --- a/oxdata/settings.py +++ b/oxdata/settings.py @@ -2,8 +2,6 @@ # Django settings for oxdata project. import os from os.path import join, normpath, dirname -import djcelery -djcelery.setup_loader() SITENAME = 'oxdata' @@ -26,6 +24,8 @@ DATABASES = { } } +DEFAULT_AUTO_FIELD="django.db.models.BigAutoField" + # Local time zone for this installation. Choices can be found here: # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name # although not all choices may be available on all operating systems. @@ -107,7 +107,7 @@ TEMPLATES = [ }, ] -MIDDLEWARE_CLASSES = ( +MIDDLEWARE = ( 'django.middleware.common.CommonMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', @@ -126,7 +126,6 @@ INSTALLED_APPS = ( 'django.contrib.admin', 'django.contrib.humanize', 'django_extensions', - 'djcelery', 'lookup', 'movie', diff --git a/oxdata/urls.py b/oxdata/urls.py index c67fd6d..780c8a0 100644 --- a/oxdata/urls.py +++ b/oxdata/urls.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import os -from django.conf.urls import url, include +from django.urls import path, re_path from oxdjango.http import HttpFileResponse from django.conf import settings import django.views.static @@ -21,28 +21,28 @@ def serve_static_file(path, location, content_type): urlpatterns = [ - url(r'^$', views.index), - url(r'^api/?', include(oxdjango.api.urls)), - url(r'^poster/', include(poster.urls)), - url(r'^still/$', poster.views.still), - url(r'^id/', include(lookup.urls)), - url(r'^get/', include(lookup.urls)), + re_path(r'^$', views.index), + re_path(r'api/?', oxdjango.api.urls.urls), + path(r'poster/', poster.urls.urls), + re_path(r'still/', poster.views.still), + path(r'id/', lookup.urls.urls2), + path(r'get/', lookup.urls.urls), - url(r'^robots.txt$', serve_static_file, { + path(r'robots.txt', serve_static_file, { 'location': os.path.join(settings.STATIC_ROOT, 'robots.txt'), 'content_type': 'text/plain' }), - url(r'^favicon.ico$', serve_static_file, { + path(r'favicon.ico', serve_static_file, { 'location': os.path.join(settings.STATIC_ROOT, 'favicon.ico'), 'content_type': 'image/x-icon'}), - url(r'^admin/', include(admin.site.urls)), + path(r'admin/', admin.site.urls), ] if settings.DEBUG: urlpatterns += [ - url(r'^media/(?P.*)$', django.views.static.serve, + re_path(r'^media/(?P.*)$', django.views.static.serve, {'document_root': settings.MEDIA_ROOT}), - url(r'^static/(?P.*)$', django.views.static.serve, + re_path(r'^static/(?P.*)$', django.views.static.serve, {'document_root': settings.STATIC_ROOT}), ] diff --git a/requirements.txt b/requirements.txt index 8c439b2..c561a10 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -Django<2 -celery==3.1.23 -django-celery==3.1.17 -django-extensions==1.7.4 +Django==4.2.7 +celery==5.3.5 +django-celery-results==2.5.1 +django-extensions==3.2.3 gunicorn --e git+http://git.0x2620.org/python-ox.git#egg=python-ox +ox