update django/celery

This commit is contained in:
j 2023-07-27 15:35:53 +02:00
parent d4df903f82
commit c80f16c77a
8 changed files with 21 additions and 20 deletions

View file

@ -15,7 +15,7 @@ ExecStart=/srv/pandora/bin/celery \
-Q encoding -n pandora-encoding \ -Q encoding -n pandora-encoding \
--pidfile /run/pandora/encoding.pid \ --pidfile /run/pandora/encoding.pid \
-c $CONCURRENCY \ -c $CONCURRENCY \
--maxtasksperchild $MAX_TASKS_PER_CHILD \ --max-tasks-per-child $MAX_TASKS_PER_CHILD \
-l $LOGLEVEL -l $LOGLEVEL
ExecReload=/bin/kill -TERM $MAINPID ExecReload=/bin/kill -TERM $MAINPID

View file

@ -15,7 +15,7 @@ ExecStart=/srv/pandora/bin/celery \
-Q default,celery -n pandora-default \ -Q default,celery -n pandora-default \
--pidfile /run/pandora/tasks.pid \ --pidfile /run/pandora/tasks.pid \
-c $CONCURRENCY \ -c $CONCURRENCY \
--maxtasksperchild $MAX_TASKS_PER_CHILD \ --max-tasks-per-child $MAX_TASKS_PER_CHILD \
-l $LOGLEVEL -l $LOGLEVEL
ExecReload=/bin/kill -TERM $MAINPID ExecReload=/bin/kill -TERM $MAINPID

View file

@ -1,11 +1,9 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from datetime import datetime from datetime import datetime
from time import time from time import time, monotonic
import celery.task.control
import kombu.five
from app.celery import app
from .models import File from .models import File
@ -18,7 +16,7 @@ def parse_job(job):
'file': f.oshash 'file': f.oshash
} }
if job['time_start']: if job['time_start']:
start_time = datetime.fromtimestamp(time() - (kombu.five.monotonic() - job['time_start'])) start_time = datetime.fromtimestamp(time() - (monotonic() - job['time_start']))
r.update({ r.update({
'started': start_time, 'started': start_time,
'running': (datetime.now() - start_time).total_seconds() 'running': (datetime.now() - start_time).total_seconds()
@ -30,7 +28,7 @@ def parse_job(job):
def status(): def status():
status = [] status = []
encoding_jobs = ('archive.tasks.extract_stream', 'archive.tasks.process_stream') encoding_jobs = ('archive.tasks.extract_stream', 'archive.tasks.process_stream')
c = celery.task.control.inspect() c = app.control.inspect()
for job in c.active(safe=True).get('celery@pandora-encoding', []): for job in c.active(safe=True).get('celery@pandora-encoding', []):
if job['name'] in encoding_jobs: if job['name'] in encoding_jobs:
status.append(parse_job(job)) status.append(parse_job(job))

View file

@ -5,12 +5,12 @@ import copy
from django.db import models from django.db import models
from django.utils import datetime_safe from django.utils import datetime_safe
import django.contrib.postgres.fields
from django.core.serializers.json import DjangoJSONEncoder from django.core.serializers.json import DjangoJSONEncoder
from ox.utils import json from ox.utils import json
class JSONField(django.contrib.postgres.fields.JSONField):
class JSONField(models.JSONField):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
if 'encoder' not in kwargs: if 'encoder' not in kwargs:

View file

@ -37,12 +37,12 @@ class NullsLastQuery(Query):
obj.nulls_last = self.nulls_last obj.nulls_last = self.nulls_last
return obj return obj
def get_compiler(self, using=None, connection=None): def get_compiler(self, using=None, connection=None, elide_empty=True):
if using is None and connection is None: if using is None and connection is None:
raise ValueError("Need either using or connection") raise ValueError("Need either using or connection")
if using: if using:
connection = connections[using] connection = connections[using]
return NullLastSQLCompiler(self, connection, using) return NullLastSQLCompiler(self, connection, using, elide_empty)
class QuerySet(django.db.models.query.QuerySet): class QuerySet(django.db.models.query.QuerySet):

View file

@ -8,10 +8,11 @@ from django.contrib.auth import get_user_model
from django.conf import settings from django.conf import settings
from django.db import models from django.db import models
from django.db.models import Q from django.db.models import Q
import celery.task.control
import kombu.five
import ox import ox
from app.celery import app
User = get_user_model() User = get_user_model()
def get_tasks(username): def get_tasks(username):
@ -111,7 +112,7 @@ class Task(models.Model):
return False return False
def get_job(self): def get_job(self):
c = celery.task.control.inspect() c = app.control.inspect()
active = c.active(safe=True) active = c.active(safe=True)
if active: if active:
for queue in active: for queue in active:

View file

@ -1,10 +1,10 @@
Django==3.0.10 Django==4.2.3
simplejson simplejson
chardet chardet
celery<5.0,>4.3 celery==5.3.1
django-celery-results<2 django-celery-results==2.5.1
django-celery-beat django-celery-beat==2.5.0
django-extensions==2.2.9 django-extensions==3.2.3
libsass libsass
django-compressor django-compressor
django-sass-processor django-sass-processor

View file

@ -305,6 +305,8 @@ if __name__ == "__main__":
run('./bin/pip', 'install', '-r', 'requirements.txt') run('./bin/pip', 'install', '-r', 'requirements.txt')
if old < 6500: if old < 6500:
run('./bin/pip', 'install', '-r', 'requirements.txt') run('./bin/pip', 'install', '-r', 'requirements.txt')
if old <= 6517:
run('./bin/pip', 'install', '-r', 'requirements.txt')
else: else:
if len(sys.argv) == 1: if len(sys.argv) == 1:
branch = get_branch() branch = get_branch()