From 62fe578f380e78235a4993425e0cdb132974492f Mon Sep 17 00:00:00 2001 From: j Date: Tue, 16 Jul 2019 12:05:40 +0100 Subject: [PATCH 1/9] update celery to a version that works in python3.7 --- pandora/archive/views.py | 9 ++++++--- pandora/settings.py | 4 +--- pandora/taskqueue/models.py | 1 - requirements.txt | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pandora/archive/views.py b/pandora/archive/views.py index 3112d193..91f700a2 100644 --- a/pandora/archive/views.py +++ b/pandora/archive/views.py @@ -10,7 +10,7 @@ from django.db.models import Count, Q from six import string_types from celery.utils import get_full_cls_name -from celery.backends import default_backend +from celery._state import current_app import ox from oxdjango.decorators import login_required_json from oxdjango.shortcuts import render_to_json_response, get_object_or_404_json, json_response @@ -390,8 +390,11 @@ def getTaskStatus(request, data): else: task_id = data['task_id'] response = json_response(status=200, text='ok') - status = default_backend.get_status(task_id) - res = default_backend.get_result(task_id) + + backend = current_app.backend + status = backend.get_status(task_id) + res = backend.get_result(task_id) + response['data'] = { 'id': task_id, 'status': status diff --git a/pandora/settings.py b/pandora/settings.py index b57d0d31..d654f948 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -6,8 +6,6 @@ from __future__ import absolute_import import os from os.path import join, normpath, dirname -import djcelery -djcelery.setup_loader() BASE_DIR = PROJECT_ROOT = normpath(dirname(__file__)) BIN_DIR = normpath(join(PROJECT_ROOT, '..', 'bin')) @@ -122,7 +120,7 @@ INSTALLED_APPS = ( 'django.contrib.humanize', 'django_extensions', - 'djcelery', + 'django_celery_results', 'app', 'log', 'annotation', diff --git a/pandora/taskqueue/models.py b/pandora/taskqueue/models.py index 839aff39..6a36e3bf 100644 --- a/pandora/taskqueue/models.py +++ b/pandora/taskqueue/models.py @@ -4,7 +4,6 @@ from __future__ import division, print_function, absolute_import from datetime import datetime, timedelta from time import time -from celery.backends import default_backend from celery.utils import get_full_cls_name from django.contrib.auth import get_user_model from django.conf import settings diff --git a/requirements.txt b/requirements.txt index 1881524a..d26e9bd9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ Django==1.11.22 simplejson chardet -celery==3.1.26.post2 -django-celery==3.2.2 +celery>4 +django-celery-results django-extensions==2.0.7 gunicorn==19.8.1 html5lib From 46621522b10ce3c57ecf09906ce18dfaec83e7a5 Mon Sep 17 00:00:00 2001 From: j Date: Tue, 16 Jul 2019 12:22:22 +0100 Subject: [PATCH 2/9] use new celery --- etc/systemd/system/pandora-encoding.service | 6 +++--- etc/systemd/system/pandora-tasks.service | 4 ++-- pandora/celery.py | 20 ++++++++++++++++++++ pandora/settings.py | 7 ++++++- 4 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 pandora/celery.py diff --git a/etc/systemd/system/pandora-encoding.service b/etc/systemd/system/pandora-encoding.service index f56f0eb2..8d5e3bda 100644 --- a/etc/systemd/system/pandora-encoding.service +++ b/etc/systemd/system/pandora-encoding.service @@ -8,9 +8,9 @@ Restart=always User=pandora Group=pandora PIDFile=/run/pandora/encoding.pid -WorkingDirectory=/srv/pandora/pandora -ExecStart=/srv/pandora/bin/python /srv/pandora/pandora/manage.py \ - celery worker \ +WorkingDirectory=/srv/pandora +ExecStart=/srv/pandora/bin/celery \ + -A pandora worker \ -Q encoding -n pandora-encoding \ --pidfile /run/pandora/encoding.pid \ --maxtasksperchild 500 \ diff --git a/etc/systemd/system/pandora-tasks.service b/etc/systemd/system/pandora-tasks.service index 5972e8b3..277a0072 100644 --- a/etc/systemd/system/pandora-tasks.service +++ b/etc/systemd/system/pandora-tasks.service @@ -9,8 +9,8 @@ User=pandora Group=pandora PIDFile=/run/pandora/tasks.pid WorkingDirectory=/srv/pandora/pandora -ExecStart=/srv/pandora/bin/python /srv/pandora/pandora/manage.py \ - celery worker \ +ExecStart=/srv/pandora/bin/celery \ + -A pandora worker \ -Q default,celery -n pandora-default \ --pidfile /run/pandora/tasks.pid \ --maxtasksperchild 1000 \ diff --git a/pandora/celery.py b/pandora/celery.py new file mode 100644 index 00000000..f53044cd --- /dev/null +++ b/pandora/celery.py @@ -0,0 +1,20 @@ +import os + +from celery import Celery + +root_dir = os.path.normpath(os.path.abspath(os.path.dirname(__file__))) +os.chdir(root_dir) + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings') + +app = Celery('pandora') + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django app configs. +app.autodiscover_tasks() diff --git a/pandora/settings.py b/pandora/settings.py index d654f948..72a48859 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -195,7 +195,7 @@ DATABASES = { } #rabbitmq connection settings -CELERY_RESULT_BACKEND = 'database' +CELERY_RESULT_BACKEND = 'django-db' CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' CELERY_ACCEPT_CONTENT = ['json'] @@ -297,3 +297,8 @@ ALLOWED_HOSTS = ['*'] SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') DATA_UPLOAD_MAX_MEMORY_SIZE = 32 * 1024 * 1024 +try: + CELERY_BROKER_URL +except NameError: + CELERY_BROKER_URL = BROKER_URL + From ee86c9ab9f7329b10179d28550bd570349f38335 Mon Sep 17 00:00:00 2001 From: j Date: Tue, 16 Jul 2019 12:27:35 +0100 Subject: [PATCH 3/9] move celery into app --- etc/systemd/system/pandora-encoding.service | 4 ++-- etc/systemd/system/pandora-tasks.service | 2 +- pandora/{ => app}/celery.py | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) rename pandora/{ => app}/celery.py (94%) diff --git a/etc/systemd/system/pandora-encoding.service b/etc/systemd/system/pandora-encoding.service index 8d5e3bda..b2a48626 100644 --- a/etc/systemd/system/pandora-encoding.service +++ b/etc/systemd/system/pandora-encoding.service @@ -8,9 +8,9 @@ Restart=always User=pandora Group=pandora PIDFile=/run/pandora/encoding.pid -WorkingDirectory=/srv/pandora +WorkingDirectory=/srv/pandora/pandora ExecStart=/srv/pandora/bin/celery \ - -A pandora worker \ + -A app worker \ -Q encoding -n pandora-encoding \ --pidfile /run/pandora/encoding.pid \ --maxtasksperchild 500 \ diff --git a/etc/systemd/system/pandora-tasks.service b/etc/systemd/system/pandora-tasks.service index 277a0072..19cf04af 100644 --- a/etc/systemd/system/pandora-tasks.service +++ b/etc/systemd/system/pandora-tasks.service @@ -10,7 +10,7 @@ Group=pandora PIDFile=/run/pandora/tasks.pid WorkingDirectory=/srv/pandora/pandora ExecStart=/srv/pandora/bin/celery \ - -A pandora worker \ + -A app worker \ -Q default,celery -n pandora-default \ --pidfile /run/pandora/tasks.pid \ --maxtasksperchild 1000 \ diff --git a/pandora/celery.py b/pandora/app/celery.py similarity index 94% rename from pandora/celery.py rename to pandora/app/celery.py index f53044cd..710d0d0e 100644 --- a/pandora/celery.py +++ b/pandora/app/celery.py @@ -3,6 +3,7 @@ import os from celery import Celery root_dir = os.path.normpath(os.path.abspath(os.path.dirname(__file__))) +root_dir = os.path.dirname(root_dir) os.chdir(root_dir) # set the default Django settings module for the 'celery' program. From d39ea08ffb416217914fbf790dd6afaf904051ed Mon Sep 17 00:00:00 2001 From: j Date: Tue, 16 Jul 2019 12:33:24 +0100 Subject: [PATCH 4/9] ignore djcelery/celery tables --- update.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/update.py b/update.py index 86dbe643..41f322e8 100755 --- a/update.py +++ b/update.py @@ -312,9 +312,22 @@ if __name__ == "__main__": run('./manage.py', 'compile_pyc', '-p', '.') os.chdir(join(base, 'pandora')) diff = get('./manage.py', 'sqldiff', '-a').strip() + for row in [ + '-- Model missing for table: djcelery_periodictasks\n', + '-- Model missing for table: celery_taskmeta\n', + '-- Model missing for table: celery_tasksetmeta\n', + '-- Model missing for table: djcelery_crontabschedule\n', + '-- Model missing for table: djcelery_periodictask\n', + '-- Model missing for table: djcelery_intervalschedule\n', + '-- Model missing for table: djcelery_workerstate\n', + '-- Model missing for table: djcelery_taskstate\n', + '-- Model missing for table: cache\n', + ]: + if row in diff: + diff = diff.replace(row, '') if diff not in [ '-- No differences', - 'BEGIN;\n-- Model missing for table: cache\nCOMMIT;' + 'BEGIN;\nCOMMIT;' ]: print('Database has changed, please make a backup and run %s db' % sys.argv[0]) elif branch != 'master': From bcf569bd68bc4a15969d5c4e3416a0e2a2f7edf0 Mon Sep 17 00:00:00 2001 From: j Date: Tue, 16 Jul 2019 12:35:39 +0100 Subject: [PATCH 5/9] update beat --- etc/systemd/system/pandora-cron.service | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/etc/systemd/system/pandora-cron.service b/etc/systemd/system/pandora-cron.service index 778a821d..9f1ef157 100644 --- a/etc/systemd/system/pandora-cron.service +++ b/etc/systemd/system/pandora-cron.service @@ -9,8 +9,9 @@ User=pandora Group=pandora PIDFile=/run/pandora/cron.pid WorkingDirectory=/srv/pandora/pandora -ExecStart=/srv/pandora/bin/python /srv/pandora/pandora/manage.py \ - celerybeat -s /run/pandora/celerybeat-schedule \ +ExecStart=/srv/pandora/bin/celery \ + -A app beat \ + -s /run/pandora/celerybeat-schedule \ --pidfile /run/pandora/cron.pid \ -l INFO ExecReload=/bin/kill -HUP $MAINPID From 68b56f0c9e402f70a86a16d075395c3aa1929a89 Mon Sep 17 00:00:00 2001 From: j Date: Tue, 16 Jul 2019 17:36:41 +0100 Subject: [PATCH 6/9] import celery for @shared_tasks --- pandora/app/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pandora/app/__init__.py b/pandora/app/__init__.py index e69de29b..62c4f9ac 100644 --- a/pandora/app/__init__.py +++ b/pandora/app/__init__.py @@ -0,0 +1,4 @@ + +from .celery import app as celery_app + +__all__ = ('celery_app',) From 2c41b17bc47ddf05661b31b46ce8285f66bbc086 Mon Sep 17 00:00:00 2001 From: j Date: Tue, 16 Jul 2019 20:58:10 +0100 Subject: [PATCH 7/9] migrate from BROKER_URL to CELERY_BROKER_URL --- docker/install.sh | 2 +- pandora/settings.py | 14 +++++--------- pandora/websocket/worker.py | 2 +- update.py | 11 +++++++++++ vm/pandora_install.sh | 6 +++--- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/docker/install.sh b/docker/install.sh index d1be570f..e1a87181 100755 --- a/docker/install.sh +++ b/docker/install.sh @@ -24,7 +24,7 @@ DATABASES = { 'PORT': 5432, } } -BROKER_URL = "amqp://{0}:{1}@rabbitmq:5672//".format(os.environ.get('RABBITMQ_DEFAULT_USER'), os.environ.get('RABBITMQ_DEFAULT_PASS')) +CELERY_BROKER_URL = "amqp://{0}:{1}@rabbitmq:5672//".format(os.environ.get('RABBITMQ_DEFAULT_USER'), os.environ.get('RABBITMQ_DEFAULT_PASS')) XACCELREDIRECT = True DEBUG = False diff --git a/pandora/settings.py b/pandora/settings.py index 72a48859..1e8f1d56 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -200,7 +200,7 @@ CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' CELERY_ACCEPT_CONTENT = ['json'] -BROKER_URL = 'amqp://pandora:box@localhost:5672//pandora' +CELERY_BROKER_URL = 'amqp://pandora:box@localhost:5672//pandora' SEND_CELERY_ERROR_EMAILS = False @@ -262,6 +262,10 @@ COLLECTION_ICON = join(SCRIPT_ROOT, 'list_icon.py') DB_GIN_TRGM = False +ALLOWED_HOSTS = ['*'] +SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') + +DATA_UPLOAD_MAX_MEMORY_SIZE = 32 * 1024 * 1024 RELOADER_RUNNING = False #you can ignore things below this line @@ -293,12 +297,4 @@ except NameError: INSTALLED_APPS = tuple(list(INSTALLED_APPS) + LOCAL_APPS) -ALLOWED_HOSTS = ['*'] -SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') - -DATA_UPLOAD_MAX_MEMORY_SIZE = 32 * 1024 * 1024 -try: - CELERY_BROKER_URL -except NameError: - CELERY_BROKER_URL = BROKER_URL diff --git a/pandora/websocket/worker.py b/pandora/websocket/worker.py index afe0634c..72c35e5a 100644 --- a/pandora/websocket/worker.py +++ b/pandora/websocket/worker.py @@ -27,7 +27,7 @@ class Worker(ConsumerMixin): message.ack() def run(): - with Connection(settings.BROKER_URL) as conn: + with Connection(settings.CELERY_BROKER_URL) as conn: try: worker = Worker(conn) worker.run() diff --git a/update.py b/update.py index 41f322e8..41283454 100755 --- a/update.py +++ b/update.py @@ -261,6 +261,17 @@ if __name__ == "__main__": run('./pandora/manage.py', 'createcachetable') if old <= 6108: run('./bin/pip', 'install', '-r', 'requirements.txt') + if old <= 6160: + run('./bin/pip', 'install', '-r', 'requirements.txt') + with open('pandora/local_settings.py', 'r') as f: + local_settings = f.read() + if 'BROKER_URL' in local_settings and 'CELERY_BROKER_URL' not in local_settings: + local_settings = [ + 'CELERY_' + l if l.startswith('BROKER_URL') else l + for l in local_settings.split('\n') + ] + with open('pandora/local_settings.py', 'w') as f: + f.write('\n'.join(local_settings)) else: if len(sys.argv) == 1: branch = get_branch() diff --git a/vm/pandora_install.sh b/vm/pandora_install.sh index 834060c5..3a62e90d 100755 --- a/vm/pandora_install.sh +++ b/vm/pandora_install.sh @@ -117,9 +117,9 @@ if [ "$RABBITMQ" == "local" ]; then rabbitmqctl add_user pandora $RABBITPWD rabbitmqctl add_vhost /pandora rabbitmqctl set_permissions -p /pandora pandora ".*" ".*" ".*" - BROKER_URL="amqp://pandora:$RABBITPWD@localhost:5672//pandora" + CELERY_BROKER_URL="amqp://pandora:$RABBITPWD@localhost:5672//pandora" else - BROKER_URL="$RABBITMQ" + CELERY_BROKER_URL="$RABBITMQ" fi # checkout pandora from git @@ -145,7 +145,7 @@ DATABASES = { 'PASSWORD': '', } } -BROKER_URL = '$BROKER_URL' +CELERY_BROKER_URL = '$CELERY_BROKER_URL' XACCELREDIRECT = True DEBUG = False From 65891f5455cea3688eed794e2427c54eccd395a2 Mon Sep 17 00:00:00 2001 From: j Date: Wed, 17 Jul 2019 11:47:57 +0200 Subject: [PATCH 8/9] fix user length --- pandora/app/monkey_patch.py | 15 +++++++++++++++ pandora/item/management/commands/sqlfindindex.py | 2 ++ 2 files changed, 17 insertions(+) diff --git a/pandora/app/monkey_patch.py b/pandora/app/monkey_patch.py index b106e224..8f48af00 100644 --- a/pandora/app/monkey_patch.py +++ b/pandora/app/monkey_patch.py @@ -32,4 +32,19 @@ def monkey_patch_username(): if isinstance(v, MaxLengthValidator): v.limit_value = 255 +def apply_patch(): + from django.db import connection, transaction + cursor = connection.cursor() + table = connection.introspection.get_table_description(cursor, User._meta.db_table) + sql = [] + for row in table: + if row.name in NEW_LENGTH and row.internal_size != NEW_LENGTH[row.name]: + sql.append('ALTER TABLE "%s" ALTER "%s" TYPE varchar(%d)' % (User._meta.db_table, row.name, NEW_LENGTH[row.name])) + + for q in sql: + cursor.execute(q) + if sql: + transaction.commit() + + monkey_patch_username() diff --git a/pandora/item/management/commands/sqlfindindex.py b/pandora/item/management/commands/sqlfindindex.py index c651e18f..3ecba6a8 100644 --- a/pandora/item/management/commands/sqlfindindex.py +++ b/pandora/item/management/commands/sqlfindindex.py @@ -31,6 +31,8 @@ class Command(BaseCommand): print(sql) cursor.execute(sql) + app.monkey_patch.apply_patch() + if settings.DB_GIN_TRGM: import entity.models import document.models From 0b44b3b66b5d52c6d506b00b650520b21860c1f4 Mon Sep 17 00:00:00 2001 From: j Date: Wed, 17 Jul 2019 11:53:21 +0200 Subject: [PATCH 9/9] fix backend --- pandora/archive/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandora/archive/views.py b/pandora/archive/views.py index 91f700a2..62731fc8 100644 --- a/pandora/archive/views.py +++ b/pandora/archive/views.py @@ -403,8 +403,8 @@ def getTaskStatus(request, data): response['data'].update(res) else: response['data']['result'] = res - if status in default_backend.EXCEPTION_STATES: - traceback = default_backend.get_traceback(task_id) + if status in backend.EXCEPTION_STATES: + traceback = backend.get_traceback(task_id) response['data'].update({ 'result': str(res), 'exc': get_full_cls_name(res.__class__),