diff --git a/docker/install.sh b/docker/install.sh index e1a87181..d1be570f 100755 --- a/docker/install.sh +++ b/docker/install.sh @@ -24,7 +24,7 @@ DATABASES = { 'PORT': 5432, } } -CELERY_BROKER_URL = "amqp://{0}:{1}@rabbitmq:5672//".format(os.environ.get('RABBITMQ_DEFAULT_USER'), os.environ.get('RABBITMQ_DEFAULT_PASS')) +BROKER_URL = "amqp://{0}:{1}@rabbitmq:5672//".format(os.environ.get('RABBITMQ_DEFAULT_USER'), os.environ.get('RABBITMQ_DEFAULT_PASS')) XACCELREDIRECT = True DEBUG = False diff --git a/etc/systemd/system/pandora-cron.service b/etc/systemd/system/pandora-cron.service index 9f1ef157..778a821d 100644 --- a/etc/systemd/system/pandora-cron.service +++ b/etc/systemd/system/pandora-cron.service @@ -9,9 +9,8 @@ User=pandora Group=pandora PIDFile=/run/pandora/cron.pid WorkingDirectory=/srv/pandora/pandora -ExecStart=/srv/pandora/bin/celery \ - -A app beat \ - -s /run/pandora/celerybeat-schedule \ +ExecStart=/srv/pandora/bin/python /srv/pandora/pandora/manage.py \ + celerybeat -s /run/pandora/celerybeat-schedule \ --pidfile /run/pandora/cron.pid \ -l INFO ExecReload=/bin/kill -HUP $MAINPID diff --git a/etc/systemd/system/pandora-encoding.service b/etc/systemd/system/pandora-encoding.service index b2a48626..f56f0eb2 100644 --- a/etc/systemd/system/pandora-encoding.service +++ b/etc/systemd/system/pandora-encoding.service @@ -9,8 +9,8 @@ User=pandora Group=pandora PIDFile=/run/pandora/encoding.pid WorkingDirectory=/srv/pandora/pandora -ExecStart=/srv/pandora/bin/celery \ - -A app worker \ +ExecStart=/srv/pandora/bin/python /srv/pandora/pandora/manage.py \ + celery worker \ -Q encoding -n pandora-encoding \ --pidfile /run/pandora/encoding.pid \ --maxtasksperchild 500 \ diff --git a/etc/systemd/system/pandora-tasks.service b/etc/systemd/system/pandora-tasks.service index 19cf04af..5972e8b3 100644 --- a/etc/systemd/system/pandora-tasks.service +++ b/etc/systemd/system/pandora-tasks.service @@ -9,8 +9,8 @@ User=pandora Group=pandora PIDFile=/run/pandora/tasks.pid WorkingDirectory=/srv/pandora/pandora -ExecStart=/srv/pandora/bin/celery \ - -A app worker \ +ExecStart=/srv/pandora/bin/python /srv/pandora/pandora/manage.py \ + celery worker \ -Q default,celery -n pandora-default \ --pidfile /run/pandora/tasks.pid \ --maxtasksperchild 1000 \ diff --git a/pandora/app/__init__.py b/pandora/app/__init__.py index 62c4f9ac..e69de29b 100644 --- a/pandora/app/__init__.py +++ b/pandora/app/__init__.py @@ -1,4 +0,0 @@ - -from .celery import app as celery_app - -__all__ = ('celery_app',) diff --git a/pandora/app/celery.py b/pandora/app/celery.py deleted file mode 100644 index 710d0d0e..00000000 --- a/pandora/app/celery.py +++ /dev/null @@ -1,21 +0,0 @@ -import os - -from celery import Celery - -root_dir = os.path.normpath(os.path.abspath(os.path.dirname(__file__))) -root_dir = os.path.dirname(root_dir) -os.chdir(root_dir) - -# set the default Django settings module for the 'celery' program. -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings') - -app = Celery('pandora') - -# Using a string here means the worker doesn't have to serialize -# the configuration object to child processes. -# - namespace='CELERY' means all celery-related configuration keys -# should have a `CELERY_` prefix. -app.config_from_object('django.conf:settings', namespace='CELERY') - -# Load task modules from all registered Django app configs. -app.autodiscover_tasks() diff --git a/pandora/app/monkey_patch.py b/pandora/app/monkey_patch.py index 8f48af00..b106e224 100644 --- a/pandora/app/monkey_patch.py +++ b/pandora/app/monkey_patch.py @@ -32,19 +32,4 @@ def monkey_patch_username(): if isinstance(v, MaxLengthValidator): v.limit_value = 255 -def apply_patch(): - from django.db import connection, transaction - cursor = connection.cursor() - table = connection.introspection.get_table_description(cursor, User._meta.db_table) - sql = [] - for row in table: - if row.name in NEW_LENGTH and row.internal_size != NEW_LENGTH[row.name]: - sql.append('ALTER TABLE "%s" ALTER "%s" TYPE varchar(%d)' % (User._meta.db_table, row.name, NEW_LENGTH[row.name])) - - for q in sql: - cursor.execute(q) - if sql: - transaction.commit() - - monkey_patch_username() diff --git a/pandora/archive/views.py b/pandora/archive/views.py index 62731fc8..3112d193 100644 --- a/pandora/archive/views.py +++ b/pandora/archive/views.py @@ -10,7 +10,7 @@ from django.db.models import Count, Q from six import string_types from celery.utils import get_full_cls_name -from celery._state import current_app +from celery.backends import default_backend import ox from oxdjango.decorators import login_required_json from oxdjango.shortcuts import render_to_json_response, get_object_or_404_json, json_response @@ -390,11 +390,8 @@ def getTaskStatus(request, data): else: task_id = data['task_id'] response = json_response(status=200, text='ok') - - backend = current_app.backend - status = backend.get_status(task_id) - res = backend.get_result(task_id) - + status = default_backend.get_status(task_id) + res = default_backend.get_result(task_id) response['data'] = { 'id': task_id, 'status': status @@ -403,8 +400,8 @@ def getTaskStatus(request, data): response['data'].update(res) else: response['data']['result'] = res - if status in backend.EXCEPTION_STATES: - traceback = backend.get_traceback(task_id) + if status in default_backend.EXCEPTION_STATES: + traceback = default_backend.get_traceback(task_id) response['data'].update({ 'result': str(res), 'exc': get_full_cls_name(res.__class__), diff --git a/pandora/item/management/commands/sqlfindindex.py b/pandora/item/management/commands/sqlfindindex.py index 3ecba6a8..c651e18f 100644 --- a/pandora/item/management/commands/sqlfindindex.py +++ b/pandora/item/management/commands/sqlfindindex.py @@ -31,8 +31,6 @@ class Command(BaseCommand): print(sql) cursor.execute(sql) - app.monkey_patch.apply_patch() - if settings.DB_GIN_TRGM: import entity.models import document.models diff --git a/pandora/settings.py b/pandora/settings.py index 1e8f1d56..b57d0d31 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -6,6 +6,8 @@ from __future__ import absolute_import import os from os.path import join, normpath, dirname +import djcelery +djcelery.setup_loader() BASE_DIR = PROJECT_ROOT = normpath(dirname(__file__)) BIN_DIR = normpath(join(PROJECT_ROOT, '..', 'bin')) @@ -120,7 +122,7 @@ INSTALLED_APPS = ( 'django.contrib.humanize', 'django_extensions', - 'django_celery_results', + 'djcelery', 'app', 'log', 'annotation', @@ -195,12 +197,12 @@ DATABASES = { } #rabbitmq connection settings -CELERY_RESULT_BACKEND = 'django-db' +CELERY_RESULT_BACKEND = 'database' CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' CELERY_ACCEPT_CONTENT = ['json'] -CELERY_BROKER_URL = 'amqp://pandora:box@localhost:5672//pandora' +BROKER_URL = 'amqp://pandora:box@localhost:5672//pandora' SEND_CELERY_ERROR_EMAILS = False @@ -262,10 +264,6 @@ COLLECTION_ICON = join(SCRIPT_ROOT, 'list_icon.py') DB_GIN_TRGM = False -ALLOWED_HOSTS = ['*'] -SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') - -DATA_UPLOAD_MAX_MEMORY_SIZE = 32 * 1024 * 1024 RELOADER_RUNNING = False #you can ignore things below this line @@ -297,4 +295,7 @@ except NameError: INSTALLED_APPS = tuple(list(INSTALLED_APPS) + LOCAL_APPS) +ALLOWED_HOSTS = ['*'] +SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') +DATA_UPLOAD_MAX_MEMORY_SIZE = 32 * 1024 * 1024 diff --git a/pandora/taskqueue/models.py b/pandora/taskqueue/models.py index 6a36e3bf..839aff39 100644 --- a/pandora/taskqueue/models.py +++ b/pandora/taskqueue/models.py @@ -4,6 +4,7 @@ from __future__ import division, print_function, absolute_import from datetime import datetime, timedelta from time import time +from celery.backends import default_backend from celery.utils import get_full_cls_name from django.contrib.auth import get_user_model from django.conf import settings diff --git a/pandora/websocket/worker.py b/pandora/websocket/worker.py index 72c35e5a..afe0634c 100644 --- a/pandora/websocket/worker.py +++ b/pandora/websocket/worker.py @@ -27,7 +27,7 @@ class Worker(ConsumerMixin): message.ack() def run(): - with Connection(settings.CELERY_BROKER_URL) as conn: + with Connection(settings.BROKER_URL) as conn: try: worker = Worker(conn) worker.run() diff --git a/requirements.txt b/requirements.txt index d26e9bd9..1881524a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ Django==1.11.22 simplejson chardet -celery>4 -django-celery-results +celery==3.1.26.post2 +django-celery==3.2.2 django-extensions==2.0.7 gunicorn==19.8.1 html5lib diff --git a/update.py b/update.py index 41283454..86dbe643 100755 --- a/update.py +++ b/update.py @@ -261,17 +261,6 @@ if __name__ == "__main__": run('./pandora/manage.py', 'createcachetable') if old <= 6108: run('./bin/pip', 'install', '-r', 'requirements.txt') - if old <= 6160: - run('./bin/pip', 'install', '-r', 'requirements.txt') - with open('pandora/local_settings.py', 'r') as f: - local_settings = f.read() - if 'BROKER_URL' in local_settings and 'CELERY_BROKER_URL' not in local_settings: - local_settings = [ - 'CELERY_' + l if l.startswith('BROKER_URL') else l - for l in local_settings.split('\n') - ] - with open('pandora/local_settings.py', 'w') as f: - f.write('\n'.join(local_settings)) else: if len(sys.argv) == 1: branch = get_branch() @@ -323,22 +312,9 @@ if __name__ == "__main__": run('./manage.py', 'compile_pyc', '-p', '.') os.chdir(join(base, 'pandora')) diff = get('./manage.py', 'sqldiff', '-a').strip() - for row in [ - '-- Model missing for table: djcelery_periodictasks\n', - '-- Model missing for table: celery_taskmeta\n', - '-- Model missing for table: celery_tasksetmeta\n', - '-- Model missing for table: djcelery_crontabschedule\n', - '-- Model missing for table: djcelery_periodictask\n', - '-- Model missing for table: djcelery_intervalschedule\n', - '-- Model missing for table: djcelery_workerstate\n', - '-- Model missing for table: djcelery_taskstate\n', - '-- Model missing for table: cache\n', - ]: - if row in diff: - diff = diff.replace(row, '') if diff not in [ '-- No differences', - 'BEGIN;\nCOMMIT;' + 'BEGIN;\n-- Model missing for table: cache\nCOMMIT;' ]: print('Database has changed, please make a backup and run %s db' % sys.argv[0]) elif branch != 'master': diff --git a/vm/pandora_install.sh b/vm/pandora_install.sh index 3a62e90d..834060c5 100755 --- a/vm/pandora_install.sh +++ b/vm/pandora_install.sh @@ -117,9 +117,9 @@ if [ "$RABBITMQ" == "local" ]; then rabbitmqctl add_user pandora $RABBITPWD rabbitmqctl add_vhost /pandora rabbitmqctl set_permissions -p /pandora pandora ".*" ".*" ".*" - CELERY_BROKER_URL="amqp://pandora:$RABBITPWD@localhost:5672//pandora" + BROKER_URL="amqp://pandora:$RABBITPWD@localhost:5672//pandora" else - CELERY_BROKER_URL="$RABBITMQ" + BROKER_URL="$RABBITMQ" fi # checkout pandora from git @@ -145,7 +145,7 @@ DATABASES = { 'PASSWORD': '', } } -CELERY_BROKER_URL = '$CELERY_BROKER_URL' +BROKER_URL = '$BROKER_URL' XACCELREDIRECT = True DEBUG = False