add task queue api

pandora/taskqueue/migrations/0001_initial.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2016-08-17 10:23
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('item', '0002_auto_20160219_1734'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Task',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('modified', models.DateTimeField(auto_now=True)),
+                ('status', models.CharField(default=b'unknown', max_length=32)),
+                ('started', models.DateTimeField(null=True)),
+                ('ended', models.DateTimeField(null=True)),
+                ('item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tasks', to='item.Item')),
+                ('user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='tasks', to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]

pandora/taskqueue/models.py

@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+from __future__ import division, print_function
+
+from datetime import datetime, timedelta
+from time import time
+
+from celery.backends import default_backend
+from celery.utils import get_full_cls_name
+from django.contrib.auth.models import User
+from django.db import models
+from django.db.models import Q
+import celery.task.control
+import kombu.five
+import ox
+
+
+def get_tasks(username):
+    from item.models import Item
+    tasks = []
+
+    # remove finished tasks
+    yesterday = datetime.now() - timedelta(days=1)
+    Task.objects.filter(status__in=Task.DONE, ended__lt=yesterday).delete()
+
+    # add task for that might be missing
+    for i in Item.objects.filter(rendered=False).exclude(files__id=None):
+        task, created = Task.objects.get_or_create(item=i)
+        if created:
+            task.started = i.modified
+            task.update()
+
+    qs = Task.objects.all()
+    if username:
+        qs = qs.filter(user__username=username)
+    for task in qs:
+        tasks.append(task.json())
+    return tasks
+
+class Task(models.Model):
+    DONE = ['finished', 'failed', 'cancelled']
+
+    created = models.DateTimeField(auto_now_add=True)
+    modified = models.DateTimeField(auto_now=True)
+
+    # 'queued|uploading|processing|finished|failed|cancelled',
+    status = models.CharField(default='unknown', max_length=32)
+    started = models.DateTimeField(null=True)
+    ended = models.DateTimeField(null=True)
+    item = models.ForeignKey("item.Item", related_name='tasks')
+    user = models.ForeignKey(User, related_name='tasks', null=True)
+
+    def __unicode__(self):
+        return "%s [%s]" % (self.item.public_id, self.status)
+
+    @property
+    def public_id(self):
+        return ox.toAZ(self.id)
+
+    @classmethod
+    def get(cls, id):
+        return cls.objects.get(pk=ox.fromAZ(id))
+
+    @classmethod
+    def start(cls, item, user):
+        task, created = cls.objects.get_or_create(item=item)
+        if task.update(save=False) or created:
+            task.user = user
+            task.started = datetime.now()
+            task.ended = None
+            task.save()
+
+    @classmethod
+    def finish(cls, item):
+        task, created = cls.objects.get_or_create(item=item)
+        task.update()
+        if task.status in task.DONE and not task.ended:
+            task.ended = datetime.now()
+            task.save()
+
+    def update(self, save=True):
+        if self.item.files.filter(wanted=True, available=False).count():
+            status = 'pending'
+        elif self.item.files.filter(uploading=True).count():
+            status = 'uploading'
+        elif self.item.files.filter(queued=True).count():
+            status = 'queued'
+        elif self.item.files.filter(encoding=True).count():
+            status = 'processing'
+        elif self.item.files.filter(failed=True).count():
+            status = 'failed'
+        elif self.item.rendered:
+            status = 'finished'
+        else:
+            status = 'unknown'
+        if status != self.status:
+            self.status = status
+            if save:
+                self.save()
+            return True
+        return False
+
+    def update_from_queue(self, save=True):
+        c = celery.task.control.inspect()
+        active = c.active(safe=True)
+        status = 'unknown'
+        if active:
+            for queue in active:
+                for job in active[queue]:
+                    if job.get('name') in ('item.tasks.update_timeline', ):
+                        args = job.get('args', [])
+                        if args and args[0] == self.item.public_id:
+                            if job.get('time_start'):
+                                status = 'processing'
+                            else:
+                                status = 'queued'
+        if status != self.status:
+            self.status = status
+            if save:
+                self.save()
+            return True
+        return False
+
+    def cancel(self):
+        self.state = 'cancelled'
+        self.save()
+        # FIXME: actually cancel task
+
+    def json(self):
+        if self.state != 'cancelled':
+            self.update()
+        return {
+            'started': self.started,
+            'ended': self.ended,
+            'status': self.status,
+            'title': self.item.get('title'),
+            'item': self.item.public_id,
+            'user': self.user and self.user.username or '',
+            'id': self.public_id,
+        }
+

pandora/taskqueue/views.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+from __future__ import division
+
+import ox
+from oxdjango.decorators import login_required_json
+from oxdjango.shortcuts import render_to_json_response, get_object_or_404_json, json_response
+from oxdjango.api import actions
+
+from . import models
+
+
+@login_required_json
+def getTasks(request, data):
+    '''
+    get list of tasks
+    takes {
+        user: ''
+    }
+    returns {
+        [{
+            started: 0,
+            finished: 0,
+            status: 'queued|uploading|processing|finished|failed|cancelled',
+            title: '',
+            item: 'itemID',
+            id: 'taskID'
+        }]
+    }
+    '''
+    user = data.get('user', '')
+    if user != request.user.username and not request.user.profile.capability('canSeeAllTasks'):
+        response = json_response(status=403, text='permission denied')
+    else:
+        response = json_response(status=200, text='ok')
+        response['data']['items'] = models.get_tasks(user)
+    return render_to_json_response(response)
+actions.register(getTasks, cache=False)
+
+@login_required_json
+def cancelTask(request, data):
+    response = json_response(status=200, text='ok')
+    ids = data['id']
+    if not isinstance(ids, list):
+        ids = [ids]
+    for id in ids:
+        task = models.Task.get(id)
+        if task.user != request.user and not request.user.profile.capability('canSeeAllTasks'):
+            response = json_response(status=403, text='permission denied')
+            return render_to_json_response(response)
+        else:
+            task.cancel()
+    return render_to_json_response(response)
+actions.register(cancelTask, cache=False)