From c80f16c77af2ede1061cb619c022640dbc1d4caf Mon Sep 17 00:00:00 2001 From: j Date: Thu, 27 Jul 2023 15:35:53 +0200 Subject: [PATCH] update django/celery --- etc/systemd/system/pandora-encoding.service | 2 +- etc/systemd/system/pandora-tasks.service | 2 +- pandora/archive/queue.py | 10 ++++------ pandora/oxdjango/fields.py | 4 ++-- pandora/oxdjango/query.py | 4 ++-- pandora/taskqueue/models.py | 7 ++++--- requirements.txt | 10 +++++----- update.py | 2 ++ 8 files changed, 21 insertions(+), 20 deletions(-) diff --git a/etc/systemd/system/pandora-encoding.service b/etc/systemd/system/pandora-encoding.service index fa791febf..ee8725ce5 100644 --- a/etc/systemd/system/pandora-encoding.service +++ b/etc/systemd/system/pandora-encoding.service @@ -15,7 +15,7 @@ ExecStart=/srv/pandora/bin/celery \ -Q encoding -n pandora-encoding \ --pidfile /run/pandora/encoding.pid \ -c $CONCURRENCY \ - --maxtasksperchild $MAX_TASKS_PER_CHILD \ + --max-tasks-per-child $MAX_TASKS_PER_CHILD \ -l $LOGLEVEL ExecReload=/bin/kill -TERM $MAINPID diff --git a/etc/systemd/system/pandora-tasks.service b/etc/systemd/system/pandora-tasks.service index d8fe1b585..d1d494207 100644 --- a/etc/systemd/system/pandora-tasks.service +++ b/etc/systemd/system/pandora-tasks.service @@ -15,7 +15,7 @@ ExecStart=/srv/pandora/bin/celery \ -Q default,celery -n pandora-default \ --pidfile /run/pandora/tasks.pid \ -c $CONCURRENCY \ - --maxtasksperchild $MAX_TASKS_PER_CHILD \ + --max-tasks-per-child $MAX_TASKS_PER_CHILD \ -l $LOGLEVEL ExecReload=/bin/kill -TERM $MAINPID diff --git a/pandora/archive/queue.py b/pandora/archive/queue.py index 8a8ff4ceb..edae2c428 100644 --- a/pandora/archive/queue.py +++ b/pandora/archive/queue.py @@ -1,11 +1,9 @@ # -*- coding: utf-8 -*- from datetime import datetime -from time import time - -import celery.task.control -import kombu.five +from time import time, monotonic +from app.celery import app from .models import File @@ -18,7 +16,7 @@ def parse_job(job): 'file': f.oshash } if job['time_start']: - start_time = datetime.fromtimestamp(time() - (kombu.five.monotonic() - job['time_start'])) + start_time = datetime.fromtimestamp(time() - (monotonic() - job['time_start'])) r.update({ 'started': start_time, 'running': (datetime.now() - start_time).total_seconds() @@ -30,7 +28,7 @@ def parse_job(job): def status(): status = [] encoding_jobs = ('archive.tasks.extract_stream', 'archive.tasks.process_stream') - c = celery.task.control.inspect() + c = app.control.inspect() for job in c.active(safe=True).get('celery@pandora-encoding', []): if job['name'] in encoding_jobs: status.append(parse_job(job)) diff --git a/pandora/oxdjango/fields.py b/pandora/oxdjango/fields.py index daebd27c4..6227ffdc9 100644 --- a/pandora/oxdjango/fields.py +++ b/pandora/oxdjango/fields.py @@ -5,12 +5,12 @@ import copy from django.db import models from django.utils import datetime_safe -import django.contrib.postgres.fields from django.core.serializers.json import DjangoJSONEncoder from ox.utils import json -class JSONField(django.contrib.postgres.fields.JSONField): + +class JSONField(models.JSONField): def __init__(self, *args, **kwargs): if 'encoder' not in kwargs: diff --git a/pandora/oxdjango/query.py b/pandora/oxdjango/query.py index 465007170..474f894ae 100644 --- a/pandora/oxdjango/query.py +++ b/pandora/oxdjango/query.py @@ -37,12 +37,12 @@ class NullsLastQuery(Query): obj.nulls_last = self.nulls_last return obj - def get_compiler(self, using=None, connection=None): + def get_compiler(self, using=None, connection=None, elide_empty=True): if using is None and connection is None: raise ValueError("Need either using or connection") if using: connection = connections[using] - return NullLastSQLCompiler(self, connection, using) + return NullLastSQLCompiler(self, connection, using, elide_empty) class QuerySet(django.db.models.query.QuerySet): diff --git a/pandora/taskqueue/models.py b/pandora/taskqueue/models.py index 214f9a417..baf18a86c 100644 --- a/pandora/taskqueue/models.py +++ b/pandora/taskqueue/models.py @@ -8,10 +8,11 @@ from django.contrib.auth import get_user_model from django.conf import settings from django.db import models from django.db.models import Q -import celery.task.control -import kombu.five import ox +from app.celery import app + + User = get_user_model() def get_tasks(username): @@ -111,7 +112,7 @@ class Task(models.Model): return False def get_job(self): - c = celery.task.control.inspect() + c = app.control.inspect() active = c.active(safe=True) if active: for queue in active: diff --git a/requirements.txt b/requirements.txt index 737f77ce6..73d5d21d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ -Django==3.0.10 +Django==4.2.3 simplejson chardet -celery<5.0,>4.3 -django-celery-results<2 -django-celery-beat -django-extensions==2.2.9 +celery==5.3.1 +django-celery-results==2.5.1 +django-celery-beat==2.5.0 +django-extensions==3.2.3 libsass django-compressor django-sass-processor diff --git a/update.py b/update.py index c10ab8c13..de560ec62 100755 --- a/update.py +++ b/update.py @@ -305,6 +305,8 @@ if __name__ == "__main__": run('./bin/pip', 'install', '-r', 'requirements.txt') if old < 6500: run('./bin/pip', 'install', '-r', 'requirements.txt') + if old <= 6517: + run('./bin/pip', 'install', '-r', 'requirements.txt') else: if len(sys.argv) == 1: branch = get_branch()