From 16d6c2c0fe0f81d12d8b0457e7f13e17d0a3559a Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Thu, 27 Oct 2011 10:21:28 +0200 Subject: [PATCH 01/10] rebuild place/event matches once a day, this gets out of sync if new annotations are added --- pandora/event/tasks.py | 7 +++++++ pandora/place/tasks.py | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/pandora/event/tasks.py b/pandora/event/tasks.py index 37366683..b38a257b 100644 --- a/pandora/event/tasks.py +++ b/pandora/event/tasks.py @@ -7,6 +7,13 @@ from celery.decorators import task, periodic_task import models +@periodic_task(run_every=timedelta(days=1), queue='encoding') +def update_all_matches(**kwargs): + ids = [e['id'] for e in models.Event.objects.all().values('id')] + for i in ids: + e = models.Event.objects.get(pk=i) + e.update_matches() + @task(ignore_resulsts=True, queue='default') def update_matches(eventId): event = models.Event.objects.get(pk=eventId) diff --git a/pandora/place/tasks.py b/pandora/place/tasks.py index 3a276227..dc23ea7f 100644 --- a/pandora/place/tasks.py +++ b/pandora/place/tasks.py @@ -7,6 +7,13 @@ from celery.decorators import task, periodic_task import models +@periodic_task(run_every=timedelta(days=1), queue='encoding') +def update_all_matches(**kwargs): + ids = [p['id'] for p in models.Place.objects.all().values('id')] + for i in ids: + p = models.Place.objects.get(pk=i) + p.update_matches() + @task(ignore_resulsts=True, queue='default') def update_matches(id): place = models.Place.objects.get(pk=id) From dd8aee522c81e4144aebec425b3aabda3074d458 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Thu, 27 Oct 2011 10:44:05 +0200 Subject: [PATCH 02/10] place/event matching task that could be used saving annotations --- pandora/annotation/models.py | 10 ++++++++-- pandora/annotation/tasks.py | 27 +++++++++++++++++++++++++++ pandora/event/models.py | 5 ++--- pandora/event/tasks.py | 8 ++++---- pandora/place/tasks.py | 1 + 5 files changed, 42 insertions(+), 9 deletions(-) create mode 100644 pandora/annotation/tasks.py diff --git a/pandora/annotation/models.py b/pandora/annotation/models.py index b6d8cce5..d9372b56 100644 --- a/pandora/annotation/models.py +++ b/pandora/annotation/models.py @@ -9,8 +9,11 @@ import ox from archive import extract from clip.models import Clip -import utils + import managers +import utils +from tasks import update_matching_events, update_matching_places + def load_layers(layers): @@ -104,7 +107,7 @@ class Annotation(models.Model): return utils.html_parser(self.value) else: return self.value - + def set_public_id(self): public_id = Annotation.objects.filter(item=self.item, id__lt=self.id).count() self.public_id = "%s/%s" % (self.item.itemId, ox.to26(public_id)) @@ -125,6 +128,9 @@ class Annotation(models.Model): super(Annotation, self).save(*args, **kwargs) if set_public_id: self.set_public_id() + #how expensive is this? + #update_matching_events.delay(self.value) + #update_matching_places.delay(self.value) def json(self, layer=False, keys=None): j = { diff --git a/pandora/annotation/tasks.py b/pandora/annotation/tasks.py new file mode 100644 index 00000000..0a7c28e5 --- /dev/null +++ b/pandora/annotation/tasks.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from celery.decorators import task + + + +@task(ignore_resulsts=True, queue='default') +def update_matching_events(value): + from event.models import Event + ids = [e['id'] for e in Event.objects.all().values('id')] + for i in ids: + e = Event.objects.get(pk=i) + for name in [e.name] + list(e.alternativeNames): + if name in value: + e.update_matches() + break + +@task(ignore_resulsts=True, queue='default') +def update_matching_places(value): + from place.models import Place + ids = [e['id'] for e in Place.objects.all().values('id')] + for i in ids: + e = Place.objects.get(pk=i) + for name in [e.name] + list(e.alternativeNames): + if name in value: + e.update_matches() + break diff --git a/pandora/event/models.py b/pandora/event/models.py index 3c02c186..ca4ebc0c 100644 --- a/pandora/event/models.py +++ b/pandora/event/models.py @@ -1,11 +1,9 @@ # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 from __future__ import division, with_statement -import unicodedata -import string from django.db import models -from django.contrib.auth.models import User, Group +from django.contrib.auth.models import User from django.db.models import Q import ox @@ -16,6 +14,7 @@ from item.models import Item from item import utils from person.models import get_name_sort from title.models import get_title_sort + import managers diff --git a/pandora/event/tasks.py b/pandora/event/tasks.py index b38a257b..bc1fbd0d 100644 --- a/pandora/event/tasks.py +++ b/pandora/event/tasks.py @@ -4,18 +4,18 @@ from datetime import timedelta from celery.decorators import task, periodic_task -import models +from models import Event @periodic_task(run_every=timedelta(days=1), queue='encoding') def update_all_matches(**kwargs): - ids = [e['id'] for e in models.Event.objects.all().values('id')] + ids = [e['id'] for e in Event.objects.all().values('id')] for i in ids: - e = models.Event.objects.get(pk=i) + e = Event.objects.get(pk=i) e.update_matches() @task(ignore_resulsts=True, queue='default') def update_matches(eventId): - event = models.Event.objects.get(pk=eventId) + event = Event.objects.get(pk=eventId) event.update_matches() diff --git a/pandora/place/tasks.py b/pandora/place/tasks.py index dc23ea7f..7f27e380 100644 --- a/pandora/place/tasks.py +++ b/pandora/place/tasks.py @@ -18,3 +18,4 @@ def update_all_matches(**kwargs): def update_matches(id): place = models.Place.objects.get(pk=id) place.update_matches() + From 4aab40797d00df1675f7ed1fd87c118a340db1b5 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Thu, 27 Oct 2011 11:33:25 +0200 Subject: [PATCH 03/10] fix #47, director is an array --- static/js/pandora/filesView.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/static/js/pandora/filesView.js b/static/js/pandora/filesView.js index e67162cf..5f847cae 100644 --- a/static/js/pandora/filesView.js +++ b/static/js/pandora/filesView.js @@ -408,7 +408,11 @@ pandora.ui.filesView = function(options, self) { }, function(result) { ['title', 'director', 'year'].forEach(function(key) { if (result.data[key]) { - self['$' + key + 'Input'].options({value: result.data[key]}); + self['$' + key + 'Input'].options({ + value: key == 'director' + ? result.data[key].join(', ') + : result.data[key] + }); } }); updateForm(); From 8313095e532f981af2ae05950c0904c74a5fb1ea Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Thu, 27 Oct 2011 11:42:03 +0200 Subject: [PATCH 04/10] fix default clips, should be full minute + 5, fixes #45 --- pandora/item/models.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pandora/item/models.py b/pandora/item/models.py index 0d1e0e0b..49612940 100644 --- a/pandora/item/models.py +++ b/pandora/item/models.py @@ -3,6 +3,7 @@ from __future__ import division, with_statement from datetime import datetime +import math import os.path import re import subprocess @@ -1083,8 +1084,7 @@ class Item(models.Model): annotation.save() #otherwise add empty 5 seconds annotation every minute if not subtitles_added: - i = offset - while i < offset + f.duration - 5: + for i in range(int(math.ceil(offset)), int(offset + f.duration) - 5, 60): annotation = Annotation( item=self, layer=layer, @@ -1094,7 +1094,6 @@ class Item(models.Model): user=user ) annotation.save() - i += 60 offset += f.duration self.update_find() From e97a1b8e2492da53d021261301c9b9c2a86ea135 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Thu, 27 Oct 2011 11:52:07 +0200 Subject: [PATCH 05/10] really fix default clips --- pandora/item/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandora/item/models.py b/pandora/item/models.py index 49612940..0a168726 100644 --- a/pandora/item/models.py +++ b/pandora/item/models.py @@ -1084,7 +1084,9 @@ class Item(models.Model): annotation.save() #otherwise add empty 5 seconds annotation every minute if not subtitles_added: - for i in range(int(math.ceil(offset)), int(offset + f.duration) - 5, 60): + for i in range(int (offset / 60) * 60 + 60, + int(offset + f.duration) - 5, + 60): annotation = Annotation( item=self, layer=layer, From bf62056bb243a4aec1efdb5106db1cb33a489569 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Thu, 27 Oct 2011 12:07:44 +0200 Subject: [PATCH 06/10] videoprefix --- pandora/0xdb.jsonc | 3 ++- pandora/app/config.py | 1 + pandora/settings.py | 4 ++++ static/js/pandora/item.js | 2 +- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pandora/0xdb.jsonc b/pandora/0xdb.jsonc index 48350997..4f481edf 100644 --- a/pandora/0xdb.jsonc +++ b/pandora/0xdb.jsonc @@ -557,7 +557,8 @@ // fixme: there should be no magic applied to this file "id": "{{settings.SITEID}}", "name": "{{settings.SITENAME}}", - "url": "{{settings.URL}}" + "url": "{{settings.URL}}", + "videoprefix": "" }, "sitePages": [ {"id": "about", "title": "About"}, diff --git a/pandora/app/config.py b/pandora/app/config.py index 0545a085..0eded561 100644 --- a/pandora/app/config.py +++ b/pandora/app/config.py @@ -28,6 +28,7 @@ def load_config(): config['site']['name'] = settings.SITENAME config['site']['sectionName'] = settings.SITENAME config['site']['url'] = settings.URL + config['site']['videoprefix'] = settings.VIDEO_PREFIX config['keys'] = {} for key in config['itemKeys']: diff --git a/pandora/settings.py b/pandora/settings.py index 10cdd5e1..0615aba0 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -75,9 +75,13 @@ MEDIA_ROOT = normpath(join(PROJECT_ROOT, '..', 'data')) STATIC_ROOT = normpath(join(PROJECT_ROOT, '..', 'static')) TESTS_ROOT = join(PROJECT_ROOT, 'tests') +#if videos are served from another subdomain +VIDEO_PREFIX = '' + # URL that handles the media served from MEDIA_ROOT. Make sure to use a # trailing slash if there is a path component (optional in other cases). # Examples: "http://media.lawrence.com", "http://example.com/media/" + MEDIA_URL = '/data/' STATIC_URL = '/static/' diff --git a/static/js/pandora/item.js b/static/js/pandora/item.js index 6fe192c7..5f945014 100644 --- a/static/js/pandora/item.js +++ b/static/js/pandora/item.js @@ -64,7 +64,7 @@ pandora.ui.item = function() { }); pandora.site.video.resolutions.forEach(function(resolution) { video[resolution] = Ox.range(result.data.parts).map(function(i) { - return '/' + pandora.user.ui.item + '/' + return pandora.site.site.videoprefix + '/' + pandora.user.ui.item + '/' + resolution + 'p' + (i + 1) + '.' + pandora.user.videoFormat; }); }); From 910b8426dc7103e15ba8352aeac9121e38f94438 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Thu, 27 Oct 2011 13:06:13 +0200 Subject: [PATCH 07/10] run cron task in the morning --- etc/init/pandora-encoding.conf | 1 + pandora/event/tasks.py | 4 +++- pandora/place/tasks.py | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/etc/init/pandora-encoding.conf b/etc/init/pandora-encoding.conf index fdeddbc7..9129a13c 100644 --- a/etc/init/pandora-encoding.conf +++ b/etc/init/pandora-encoding.conf @@ -17,6 +17,7 @@ test -e /var/log/pandora || (mkdir -p /var/log/pandora && chown $USER:$USER /var test -e /var/run/pandora || (mkdir -p /var/run/pandora && chown $USER:$USER /var/run/pandora) cd $VENV/pandora exec /usr/bin/sudo -u $USER $VENV/bin/python $VENV/pandora/manage.py celeryd \ + -B \ -Q encoding \ -n pandora-encoding \ -p /var/run/pandora/pandora-encoding.pid \ diff --git a/pandora/event/tasks.py b/pandora/event/tasks.py index bc1fbd0d..d00c7e11 100644 --- a/pandora/event/tasks.py +++ b/pandora/event/tasks.py @@ -3,11 +3,13 @@ from datetime import timedelta from celery.decorators import task, periodic_task +from celery.task.schedules import crontab + from models import Event -@periodic_task(run_every=timedelta(days=1), queue='encoding') +@periodic_task(run_every=crontab(hour=7, minute=30), queue='encoding') def update_all_matches(**kwargs): ids = [e['id'] for e in Event.objects.all().values('id')] for i in ids: diff --git a/pandora/place/tasks.py b/pandora/place/tasks.py index 7f27e380..b8d36ee6 100644 --- a/pandora/place/tasks.py +++ b/pandora/place/tasks.py @@ -3,11 +3,12 @@ from datetime import timedelta from celery.decorators import task, periodic_task +from celery.task.schedules import crontab import models -@periodic_task(run_every=timedelta(days=1), queue='encoding') +@periodic_task(run_every=crontab(hour=6, minute=30), queue='encoding') def update_all_matches(**kwargs): ids = [p['id'] for p in models.Place.objects.all().values('id')] for i in ids: From fc1047bde13e1390bc099d66ad7c096ee5d08979 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Thu, 27 Oct 2011 13:27:03 +0200 Subject: [PATCH 08/10] run celery scheduler, compile pyc, pandora user might not have write permissions --- README | 2 +- etc/init/pandora-encoding.conf | 2 +- etc/init/pandora.conf | 1 + update.sh | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README b/README index 83116ded..a8349494 100644 --- a/README +++ b/README @@ -54,7 +54,7 @@ Running developer environment: in one terminal: ./manage.py runserver and in another one: - ./manage.py celeryd -Q default,encoding + ./manage.py celeryd -Q default,encoding -B Updating database: right now database updates are not managed, each time you update to current bzr diff --git a/etc/init/pandora-encoding.conf b/etc/init/pandora-encoding.conf index 9129a13c..0d568558 100644 --- a/etc/init/pandora-encoding.conf +++ b/etc/init/pandora-encoding.conf @@ -17,7 +17,7 @@ test -e /var/log/pandora || (mkdir -p /var/log/pandora && chown $USER:$USER /var test -e /var/run/pandora || (mkdir -p /var/run/pandora && chown $USER:$USER /var/run/pandora) cd $VENV/pandora exec /usr/bin/sudo -u $USER $VENV/bin/python $VENV/pandora/manage.py celeryd \ - -B \ + -B -s /var/run/pandora/celerybeat-schedule \ -Q encoding \ -n pandora-encoding \ -p /var/run/pandora/pandora-encoding.pid \ diff --git a/etc/init/pandora.conf b/etc/init/pandora.conf index cf624360..cfdbb8ba 100644 --- a/etc/init/pandora.conf +++ b/etc/init/pandora.conf @@ -16,6 +16,7 @@ script test -e /var/log/pandora || (mkdir -p /var/log/pandora && chown $USER:$USER /var/log/pandora) test -e /var/run/pandora || (mkdir -p /var/run/pandora && chown $USER:$USER /var/run/pandora) cd $VENV/pandora +./manage.py compile_pyc exec /usr/bin/sudo -u $USER $VENV/bin/gunicorn_django \ --bind 127.0.0.1:2620 \ --timeout 90 \ diff --git a/update.sh b/update.sh index 4572addd..45083d47 100755 --- a/update.sh +++ b/update.sh @@ -5,4 +5,4 @@ cd static/oxjs bzr pull http://code.0x2620.org/oxjs/ test -e src/python-ox && cd src/python-ox && bzr pull http://code.0x2620.org/python-ox/ cd $base -cd pandora && ./manage.py update_static +cd pandora && ./manage.py update_static && ./manage.py compile_pyc From 36d500160867fbf0a79c4328c3341dcf6ca4e754 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Thu, 27 Oct 2011 13:38:05 +0200 Subject: [PATCH 09/10] cleanup sessions once a day --- pandora/app/tasks.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 pandora/app/tasks.py diff --git a/pandora/app/tasks.py b/pandora/app/tasks.py new file mode 100644 index 00000000..3ac03816 --- /dev/null +++ b/pandora/app/tasks.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +import datetime + +from celery.decorators import task, periodic_task +from celery.task.schedules import crontab + + +@periodic_task(run_every=crontab(hour=6, minute=0), queue='encoding') +def cron(**kwargs): + from django.db import transaction + from django.contrib.sessions.models import Session + Session.objects.filter(expire_date__lt=datetime.datetime.now()).delete() + transaction.commit_unless_managed() From e458fed035cf19a924d0ff2fd9158e797c83be3f Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Thu, 27 Oct 2011 14:26:05 +0200 Subject: [PATCH 10/10] log backend errors into db --- pandora/app/config.py | 1 - pandora/app/log.py | 63 +++++++++++++++++++++++++++++++++++++++++++ pandora/app/models.py | 2 +- pandora/app/views.py | 5 ++-- pandora/settings.py | 12 +++------ 5 files changed, 71 insertions(+), 12 deletions(-) create mode 100644 pandora/app/log.py diff --git a/pandora/app/config.py b/pandora/app/config.py index 0eded561..875cb1a4 100644 --- a/pandora/app/config.py +++ b/pandora/app/config.py @@ -1,4 +1,3 @@ - # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 from __future__ import division, with_statement diff --git a/pandora/app/log.py b/pandora/app/log.py new file mode 100644 index 00000000..715e8692 --- /dev/null +++ b/pandora/app/log.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from __future__ import division, with_statement + +import logging +import sys + + +class ErrorHandler(logging.Handler): + def __init__(self): + logging.Handler.__init__(self) + + """An exception log handler that log entries into log database. + + If the request is passed as the first argument to the log record, + request data will be provided in the + """ + def emit(self, record): + import traceback + from django.views.debug import ExceptionReporter + from django.conf import settings + import models + user = None + line = 0 + text = '' + url = '' + try: + if sys.version_info < (2,5): + # A nasty workaround required because Python 2.4's logging + # module doesn't support passing in extra context. + # For this handler, the only extra data we need is the + # request, and that's in the top stack frame. + request = record.exc_info[2].tb_frame.f_locals['request'] + else: + request = record.request + + request_repr = repr(request) + if request.user.is_authenticated(): + user = request.user + url = request.META.get('PATH_INFO', '') + except: + request = None + request_repr = "%s %s\n\nRequest repr() unavailable" % (record.levelname, record.msg) + + if record.exc_info: + stack_trace = '\n'.join(traceback.format_exception(*record.exc_info)) + stack_info = traceback.extract_tb(record.exc_info[2]) + if stack_info: + url = stack_info[-1][0] + line = stack_info[-1][1] + else: + stack_trace = 'No stack trace available' + + text = "%s\n\n%s" % (stack_trace, request_repr) + if text: + l = models.Log( + text=text, + line=line, + url=url + ) + if user: + l.user = user + l.save() diff --git a/pandora/app/models.py b/pandora/app/models.py index 1f6a8ab6..1b134435 100644 --- a/pandora/app/models.py +++ b/pandora/app/models.py @@ -24,7 +24,7 @@ class Log(models.Model): text = models.TextField(blank=True) def __unicode__(self): - return self.id + return u"%s" % self.id def json(self): return { diff --git a/pandora/app/views.py b/pandora/app/views.py index c6c86a68..f07e49ff 100644 --- a/pandora/app/views.py +++ b/pandora/app/views.py @@ -108,17 +108,18 @@ def log(request): } ''' data = json.loads(request.POST['data']) - if not request.user.is_authenticated: + if request.user.is_authenticated(): user = request.user else: user = None if 'text' in data: l = models.Log( - user=user, text=data['text'], line=int(data.get('line', 0)), url=data.get('url', '') ) + if user: + l.user = user l.save() response = json_response() return render_to_json_response(response) diff --git a/pandora/settings.py b/pandora/settings.py index 0615aba0..e4cda557 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -144,23 +144,19 @@ INSTALLED_APPS = ( 'urlalias', ) -# A sample logging configuration. The only tangible logging -# performed by this configuration is to send an email to -# the site admins on every HTTP 500 error. -# See http://docs.djangoproject.com/en/dev/topics/logging for -# more details on how to customize your logging configuration. +# Log errors into db LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'handlers': { - 'mail_admins': { + 'errors': { 'level': 'ERROR', - 'class': 'django.utils.log.AdminEmailHandler' + 'class': 'pandora.app.log.ErrorHandler' } }, 'loggers': { 'django.request': { - 'handlers': ['mail_admins'], + 'handlers': ['errors'], 'level': 'ERROR', 'propagate': True, },