fix backend

docker/install.sh

@@ -24,7 +24,7 @@ DATABASES = {
         'PORT': 5432,
     }
 }
-BROKER_URL = "amqp://{0}:{1}@rabbitmq:5672//".format(os.environ.get('RABBITMQ_DEFAULT_USER'), os.environ.get('RABBITMQ_DEFAULT_PASS'))
+CELERY_BROKER_URL = "amqp://{0}:{1}@rabbitmq:5672//".format(os.environ.get('RABBITMQ_DEFAULT_USER'), os.environ.get('RABBITMQ_DEFAULT_PASS'))
 XACCELREDIRECT = True
 
 DEBUG = False

etc/systemd/system/pandora-cron.service

@@ -9,8 +9,9 @@ User=pandora
 Group=pandora
 PIDFile=/run/pandora/cron.pid
 WorkingDirectory=/srv/pandora/pandora
-ExecStart=/srv/pandora/bin/python /srv/pandora/pandora/manage.py \
-    celerybeat -s /run/pandora/celerybeat-schedule \
+ExecStart=/srv/pandora/bin/celery \
+    -A app beat \
+    -s /run/pandora/celerybeat-schedule \
     --pidfile /run/pandora/cron.pid \
     -l INFO
 ExecReload=/bin/kill -HUP $MAINPID

etc/systemd/system/pandora-encoding.service

@@ -9,8 +9,8 @@ User=pandora
 Group=pandora
 PIDFile=/run/pandora/encoding.pid
 WorkingDirectory=/srv/pandora/pandora
-ExecStart=/srv/pandora/bin/python /srv/pandora/pandora/manage.py \
-    celery worker \
+ExecStart=/srv/pandora/bin/celery \
+    -A app worker \
     -Q encoding -n pandora-encoding \
     --pidfile /run/pandora/encoding.pid \
     --maxtasksperchild 500 \

etc/systemd/system/pandora-tasks.service

@@ -9,8 +9,8 @@ User=pandora
 Group=pandora
 PIDFile=/run/pandora/tasks.pid
 WorkingDirectory=/srv/pandora/pandora
-ExecStart=/srv/pandora/bin/python /srv/pandora/pandora/manage.py \
-    celery worker \
+ExecStart=/srv/pandora/bin/celery \
+    -A app worker \
     -Q default,celery -n pandora-default \
     --pidfile /run/pandora/tasks.pid \
     --maxtasksperchild 1000 \

pandora/app/__init__.py

@@ -0,0 +1,4 @@
+
+from .celery import app as celery_app
+
+__all__ = ('celery_app',)

pandora/app/celery.py

@@ -0,0 +1,21 @@
+import os
+
+from celery import Celery
+
+root_dir = os.path.normpath(os.path.abspath(os.path.dirname(__file__)))
+root_dir = os.path.dirname(root_dir)
+os.chdir(root_dir)
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')
+
+app = Celery('pandora')
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()

pandora/app/monkey_patch.py

@@ -32,4 +32,19 @@ def monkey_patch_username():
         if isinstance(v, MaxLengthValidator):
             v.limit_value = 255
 
+def apply_patch():
+    from django.db import connection, transaction
+    cursor = connection.cursor()
+    table = connection.introspection.get_table_description(cursor, User._meta.db_table)
+    sql = []
+    for row in table:
+        if row.name in NEW_LENGTH and row.internal_size != NEW_LENGTH[row.name]:
+            sql.append('ALTER TABLE "%s" ALTER "%s" TYPE varchar(%d)' % (User._meta.db_table, row.name, NEW_LENGTH[row.name]))
+
+    for q in sql:
+        cursor.execute(q)
+    if sql:
+        transaction.commit()
+
+
 monkey_patch_username()

pandora/archive/views.py

@@ -10,7 +10,7 @@ from django.db.models import Count, Q
 
 from six import string_types
 from celery.utils import get_full_cls_name
-from celery.backends import default_backend
+from celery._state import current_app
 import ox
 from oxdjango.decorators import login_required_json
 from oxdjango.shortcuts import render_to_json_response, get_object_or_404_json, json_response
@@ -390,8 +390,11 @@ def getTaskStatus(request, data):
     else:
         task_id = data['task_id']
     response = json_response(status=200, text='ok')
-    status = default_backend.get_status(task_id)
-    res = default_backend.get_result(task_id)
+
+    backend = current_app.backend
+    status = backend.get_status(task_id)
+    res = backend.get_result(task_id)
+
     response['data'] = {
         'id': task_id,
         'status': status
@@ -400,8 +403,8 @@ def getTaskStatus(request, data):
         response['data'].update(res)
     else:
         response['data']['result'] = res
-    if status in default_backend.EXCEPTION_STATES:
-        traceback = default_backend.get_traceback(task_id)
+    if status in backend.EXCEPTION_STATES:
+        traceback = backend.get_traceback(task_id)
         response['data'].update({
             'result': str(res),
             'exc': get_full_cls_name(res.__class__),

pandora/item/management/commands/sqlfindindex.py

@@ -31,6 +31,8 @@ class Command(BaseCommand):
                 print(sql)
             cursor.execute(sql)
 
+        app.monkey_patch.apply_patch()
+
         if settings.DB_GIN_TRGM:
             import entity.models
             import document.models

pandora/settings.py

@@ -6,8 +6,6 @@ from __future__ import absolute_import
 
 import os
 from os.path import join, normpath, dirname
-import djcelery
-djcelery.setup_loader()
 
 BASE_DIR = PROJECT_ROOT = normpath(dirname(__file__))
 BIN_DIR = normpath(join(PROJECT_ROOT, '..', 'bin'))
@@ -122,7 +120,7 @@ INSTALLED_APPS = (
     'django.contrib.humanize',
 
     'django_extensions',
-    'djcelery',
+    'django_celery_results',
     'app',
     'log',
     'annotation',
@@ -197,12 +195,12 @@ DATABASES = {
 }
 
 #rabbitmq connection settings
-CELERY_RESULT_BACKEND = 'database'
+CELERY_RESULT_BACKEND = 'django-db'
 CELERY_TASK_SERIALIZER = 'json'
 CELERY_RESULT_SERIALIZER = 'json'
 CELERY_ACCEPT_CONTENT = ['json']
 
-BROKER_URL = 'amqp://pandora:box@localhost:5672//pandora'
+CELERY_BROKER_URL = 'amqp://pandora:box@localhost:5672//pandora'
 
 SEND_CELERY_ERROR_EMAILS = False
 
@@ -264,6 +262,10 @@ COLLECTION_ICON = join(SCRIPT_ROOT, 'list_icon.py')
 
 DB_GIN_TRGM = False
 
+ALLOWED_HOSTS = ['*']
+SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
+
+DATA_UPLOAD_MAX_MEMORY_SIZE = 32 * 1024 * 1024
 
 RELOADER_RUNNING = False
 #you can ignore things below this line
@@ -295,7 +297,4 @@ except NameError:
 
 INSTALLED_APPS = tuple(list(INSTALLED_APPS) + LOCAL_APPS)
 
-ALLOWED_HOSTS = ['*']
-SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
 
-DATA_UPLOAD_MAX_MEMORY_SIZE = 32 * 1024 * 1024

pandora/taskqueue/models.py

@@ -4,7 +4,6 @@ from __future__ import division, print_function, absolute_import
 from datetime import datetime, timedelta
 from time import time
 
-from celery.backends import default_backend
 from celery.utils import get_full_cls_name
 from django.contrib.auth import get_user_model
 from django.conf import settings

pandora/websocket/worker.py

@@ -27,7 +27,7 @@ class Worker(ConsumerMixin):
         message.ack()
 
 def run():
-    with Connection(settings.BROKER_URL) as conn:
+    with Connection(settings.CELERY_BROKER_URL) as conn:
         try:
             worker = Worker(conn)
             worker.run()

requirements.txt

@@ -1,8 +1,8 @@
 Django==1.11.22
 simplejson
 chardet
-celery==3.1.26.post2
-django-celery==3.2.2
+celery>4
+django-celery-results
 django-extensions==2.0.7
 gunicorn==19.8.1
 html5lib

update.py

@@ -261,6 +261,17 @@ if __name__ == "__main__":
             run('./pandora/manage.py', 'createcachetable')
         if old <= 6108:
             run('./bin/pip', 'install', '-r', 'requirements.txt')
+        if old <= 6160:
+            run('./bin/pip', 'install', '-r', 'requirements.txt')
+            with open('pandora/local_settings.py', 'r') as f:
+                local_settings = f.read()
+            if 'BROKER_URL' in local_settings and 'CELERY_BROKER_URL' not in local_settings:
+                local_settings = [
+                    'CELERY_' + l if l.startswith('BROKER_URL') else l
+                    for l in local_settings.split('\n')
+                ]
+                with open('pandora/local_settings.py', 'w') as f:
+                    f.write('\n'.join(local_settings))
     else:
         if len(sys.argv) == 1:
             branch = get_branch()
@@ -312,9 +323,22 @@ if __name__ == "__main__":
             run('./manage.py', 'compile_pyc', '-p', '.')
         os.chdir(join(base, 'pandora'))
         diff = get('./manage.py', 'sqldiff', '-a').strip()
+        for row in [
+            '-- Model missing for table: djcelery_periodictasks\n',
+            '-- Model missing for table: celery_taskmeta\n',
+            '-- Model missing for table: celery_tasksetmeta\n',
+            '-- Model missing for table: djcelery_crontabschedule\n',
+            '-- Model missing for table: djcelery_periodictask\n',
+            '-- Model missing for table: djcelery_intervalschedule\n',
+            '-- Model missing for table: djcelery_workerstate\n',
+            '-- Model missing for table: djcelery_taskstate\n',
+            '-- Model missing for table: cache\n',
+        ]:
+            if row in diff:
+                diff = diff.replace(row, '')
         if diff not in [
             '-- No differences',
-            'BEGIN;\n-- Model missing for table: cache\nCOMMIT;'
+            'BEGIN;\nCOMMIT;'
         ]:
             print('Database has changed, please make a backup and run %s db' % sys.argv[0])
         elif branch != 'master':

vm/pandora_install.sh

@@ -117,9 +117,9 @@ if [ "$RABBITMQ" == "local" ]; then
     rabbitmqctl add_user pandora $RABBITPWD
     rabbitmqctl add_vhost /pandora
     rabbitmqctl set_permissions -p /pandora pandora ".*" ".*" ".*"
-    BROKER_URL="amqp://pandora:$RABBITPWD@localhost:5672//pandora"
+    CELERY_BROKER_URL="amqp://pandora:$RABBITPWD@localhost:5672//pandora"
 else
-    BROKER_URL="$RABBITMQ"
+    CELERY_BROKER_URL="$RABBITMQ"
 fi
 
 # checkout pandora from git
@@ -145,7 +145,7 @@ DATABASES = {
         'PASSWORD': '',
     }
 }
-BROKER_URL = '$BROKER_URL'
+CELERY_BROKER_URL = '$CELERY_BROKER_URL'
 XACCELREDIRECT = True
 
 DEBUG = False