pandora/pandora/item/tasks.py

# -*- coding: utf-8 -*-
from __future__ import division, print_function, absolute_import

import os
from datetime import timedelta, datetime
import gzip
import random

from six.moves.urllib.parse import quote
from celery.task import task, periodic_task
from django.conf import settings
from django.db import connection, transaction
from django.db.models import Q
from ox.utils import ET

from app.utils import limit_rate
from taskqueue.models import Task


@periodic_task(run_every=timedelta(days=1), queue='encoding')
def cronjob(**kwargs):
    if limit_rate('item.tasks.cronjob', 8 * 60 * 60):
        update_random_sort()
        update_random_clip_sort()

def update_random_sort():
    from . import models
    if list(filter(lambda f: f['id'] == 'random', settings.CONFIG['itemKeys'])):
        random.seed()
        ids = [f['item'] for f in models.ItemSort.objects.values('item')]
        random.shuffle(ids)
        n = 1
        for i in ids:
            models.ItemSort.objects.filter(pk=i).update(random=n)
            n += 1

def update_random_clip_sort():
    if list(filter(lambda f: f['id'] == 'random', settings.CONFIG['itemKeys'])):
        with transaction.atomic():
            cursor = connection.cursor()
            for row in (
                'DROP TABLE clip_cliprandom',
                'CREATE TABLE "clip_cliprandom" AS SELECT id AS clip_id, row_number() OVER (ORDER BY random()) AS id FROM "clip_clip"',
                'ALTER TABLE "clip_cliprandom" ADD UNIQUE ("clip_id")',
                'ALTER TABLE "clip_cliprandom" ADD UNIQUE ("id")',
                'ALTER TABLE "clip_cliprandom" ALTER COLUMN "id" SET NOT NULL',
                'ALTER TABLE "clip_cliprandom" ALTER COLUMN "clip_id" SET NOT NULL',

            ):
                cursor.execute(row)


@task(ignore_results=True, queue='default')
def update_clips(public_id):
    from . import models
    try:
        item = models.Item.objects.get(public_id=public_id)
    except models.Item.DoesNotExist:
        return
    item.clips.all().update(user=item.user.id)

@task(ignore_results=True, queue='default')
def update_poster(public_id):
    from . import models
    try:
        item = models.Item.objects.get(public_id=public_id)
    except models.Item.DoesNotExist:
        return
    item.remove_poster()
    item.make_poster()
    item.make_icon()
    if item.poster and os.path.exists(item.poster.path):
        models.Item.objects.filter(pk=item.id).update(
            poster=item.poster.name,
            poster_height=item.poster.height,
            poster_width=item.poster.width,
            icon=item.icon.name
        )

@task(ignore_results=True, queue='default')
def update_file_paths(public_id):
    from . import models
    try:
        item = models.Item.objects.get(public_id=public_id)
    except models.Item.DoesNotExist:
        return
    item.update_file_paths()

@task(ignore_results=True, queue='default')
def update_external(public_id):
    from . import models
    try:
        item = models.Item.objects.get(public_id=public_id)
    except models.Item.DoesNotExist:
        return
    item.update_external()

@task(queue="encoding")
def update_timeline(public_id):
    from . import models
    try:
        item = models.Item.objects.get(public_id=public_id)
    except models.Item.DoesNotExist:
        return
    item.update_timeline(async=False)
    Task.finish(item)

@task(queue="encoding")
def rebuild_timeline(public_id):
    from . import models
    i = models.Item.objects.get(public_id=public_id)
    for s in i.streams():
        s.make_timeline()
    i.update_timeline(async=False)

@task(queue="encoding")
def load_subtitles(public_id):
    from . import models
    try:
        item = models.Item.objects.get(public_id=public_id)
    except models.Item.DoesNotExist:
        return
    if item.load_subtitles():
        item.update_find()
        item.update_sort()
        item.update_facets()

@task(ignore_results=True, queue='default')
def update_sitemap(base_url):
    from . import models
    sitemap = os.path.abspath(os.path.join(settings.MEDIA_ROOT, 'sitemap.xml.gz'))

    def absolute_url(url):
        return base_url + url

    urlset = ET.Element('urlset')
    urlset.attrib['xmlns'] = "http://www.sitemaps.org/schemas/sitemap/0.9"
    urlset.attrib['xmlns:xsi'] = "http://www.w3.org/2001/XMLSchema-instance"
    urlset.attrib['xsi:schemaLocation'] = "http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"
    urlset.attrib['xmlns:video'] = "http://www.google.com/schemas/sitemap-video/1.1"

    url = ET.SubElement(urlset, "url")
    loc = ET.SubElement(url, "loc")
    loc.text = absolute_url('') 
    # always, hourly, daily, weekly, monthly, yearly, never
    changefreq = ET.SubElement(url, "changefreq")
    changefreq.text = 'daily'
    # This date should be in W3C Datetime format, can be %Y-%m-%d
    lastmod = ET.SubElement(url, "lastmod")
    lastmod.text = datetime.now().strftime("%Y-%m-%d")
    # priority of page on site values 0.1 - 1.0
    priority = ET.SubElement(url, "priority")
    priority.text = '1.0'

    for page in [s['id'] for s in settings.CONFIG['sitePages']]:
        url = ET.SubElement(urlset, "url")
        loc = ET.SubElement(url, "loc")
        loc.text = absolute_url(page)
        # always, hourly, daily, weekly, monthly, yearly, never
        changefreq = ET.SubElement(url, "changefreq")
        changefreq.text = 'monthly'
        # priority of page on site values 0.1 - 1.0
        priority = ET.SubElement(url, "priority")
        priority.text = '1.0'

    allowed_level = settings.CONFIG['capabilities']['canSeeItem']['guest']
    can_play = settings.CONFIG['capabilities']['canPlayVideo']['guest']
    for i in models.Item.objects.filter(level__lte=allowed_level):
        url = ET.SubElement(urlset, "url")
        # URL of the page. This URL must begin with the protocol (such as http)
        loc = ET.SubElement(url, "loc")
        loc.text = absolute_url("%s/info" % i.public_id)
        # This date should be in W3C Datetime format, can be %Y-%m-%d
        lastmod = ET.SubElement(url, "lastmod")
        lastmod.text = i.modified.strftime("%Y-%m-%d")
        # always, hourly, daily, weekly, monthly, yearly, never
        changefreq = ET.SubElement(url, "changefreq")
        changefreq.text = 'monthly'
        # priority of page on site values 0.1 - 1.0
        priority = ET.SubElement(url, "priority")
        priority.text = '1.0'
        if i.rendered and i.level <= can_play:
            video = ET.SubElement(url, "video:video")
            # el = ET.SubElement(video, "video:content_loc")
            # el.text = absolute_url("%s/video" % i.public_id)
            el = ET.SubElement(video, "video:player_loc")
            el.attrib['allow_embed'] = 'no'
            el.text = absolute_url("%s/player" % i.public_id)
            el = ET.SubElement(video, "video:title")
            el.text = i.get('title')
            el = ET.SubElement(video, "video:thumbnail_loc")
            el.text = absolute_url("%s/96p.jpg" % i.public_id)
            description = i.get_item_description()
            if description:
                el = ET.SubElement(video, "video:description")
                el.text = description
            el = ET.SubElement(video, "video:family_friendly")
            el.text = 'Yes'
            duration = i.sort.duration
            if duration and duration > 0:
                el = ET.SubElement(video, "video:duration")
                el.text = "%s" % int(duration)
            el = ET.SubElement(video, "video:live")
            el.text = "no"

    # Featured Lists
    from itemlist.models import List
    for l in List.objects.filter(Q(status='featured') | Q(status='public')):
        url = ET.SubElement(urlset, "url")
        # URL of the page. This URL must begin with the protocol (such as http)
        loc = ET.SubElement(url, "loc")
        loc.text = absolute_url("list==%s" % quote(l.get_id()))

        # This date should be in W3C Datetime format, can be %Y-%m-%d
        lastmod = ET.SubElement(url, "lastmod")
        lastmod.text = l.modified.strftime("%Y-%m-%d")
        # always, hourly, daily, weekly, monthly, yearly, never
        changefreq = ET.SubElement(url, "changefreq")
        changefreq.text = 'monthly'
        # priority of page on site values 0.1 - 1.0
        priority = ET.SubElement(url, "priority")
        priority.text = '1.0' if l.status == 'featured' else '0.75'
    # Featured Edits
    from edit.models import Edit
    for l in Edit.objects.filter(Q(status='featured') | Q(status='public')):
        url = ET.SubElement(urlset, "url")
        # URL of the page. This URL must begin with the protocol (such as http)
        loc = ET.SubElement(url, "loc")
        loc.text = absolute_url(l.get_absolute_url()[1:])

        # This date should be in W3C Datetime format, can be %Y-%m-%d
        lastmod = ET.SubElement(url, "lastmod")
        lastmod.text = l.modified.strftime("%Y-%m-%d")
        # always, hourly, daily, weekly, monthly, yearly, never
        changefreq = ET.SubElement(url, "changefreq")
        changefreq.text = 'monthly'
        # priority of page on site values 0.1 - 1.0
        priority = ET.SubElement(url, "priority")
        priority.text = '1.0' if l.status == 'featured' else '0.75'
    # Featured Collections
    from documentcollection.models import Collection
    for l in Collection.objects.filter(Q(status='featured') | Q(status='public')):
        url = ET.SubElement(urlset, "url")
        # URL of the page. This URL must begin with the protocol (such as http)
        loc = ET.SubElement(url, "loc")
        loc.text = absolute_url("documents/collection==%s" % quote(l.get_id()))

        # This date should be in W3C Datetime format, can be %Y-%m-%d
        lastmod = ET.SubElement(url, "lastmod")
        lastmod.text = l.modified.strftime("%Y-%m-%d")
        # always, hourly, daily, weekly, monthly, yearly, never
        changefreq = ET.SubElement(url, "changefreq")
        changefreq.text = 'monthly'
        # priority of page on site values 0.1 - 1.0
        priority = ET.SubElement(url, "priority")
        priority.text = '1.0' if l.status == 'featured' else '0.75'

    from document.models import Document
    for d in Document.objects.filter(rightslevel=0).filter(Q(extension='html') | Q(extension='pdf')):
        url = ET.SubElement(urlset, "url")
        # URL of the page. This URL must begin with the protocol (such as http)
        loc = ET.SubElement(url, "loc")
        loc.text = absolute_url(d.get_id())

        # This date should be in W3C Datetime format, can be %Y-%m-%d
        lastmod = ET.SubElement(url, "lastmod")
        lastmod.text = d.modified.strftime("%Y-%m-%d")
        # always, hourly, daily, weekly, monthly, yearly, never
        changefreq = ET.SubElement(url, "changefreq")
        changefreq.text = 'monthly'
        # priority of page on site values 0.1 - 1.0
        priority = ET.SubElement(url, "priority")
        priority.text = '0.75'
        if d.collections.filter(Q(status='featured') | Q(status='public')).count():
            priority.text = '1.0'

    data = b'<?xml version="1.0" encoding="UTF-8"?>\n' + ET.tostring(urlset)
    with open(sitemap[:-3], 'wb') as f:
        f.write(data)
    with gzip.open(sitemap, 'wb') as f:
        f.write(data)