diff --git a/README b/README index 83116ded..a8349494 100644 --- a/README +++ b/README @@ -54,7 +54,7 @@ Running developer environment: in one terminal: ./manage.py runserver and in another one: - ./manage.py celeryd -Q default,encoding + ./manage.py celeryd -Q default,encoding -B Updating database: right now database updates are not managed, each time you update to current bzr diff --git a/etc/init/pandora-encoding.conf b/etc/init/pandora-encoding.conf index fdeddbc7..0d568558 100644 --- a/etc/init/pandora-encoding.conf +++ b/etc/init/pandora-encoding.conf @@ -17,6 +17,7 @@ test -e /var/log/pandora || (mkdir -p /var/log/pandora && chown $USER:$USER /var test -e /var/run/pandora || (mkdir -p /var/run/pandora && chown $USER:$USER /var/run/pandora) cd $VENV/pandora exec /usr/bin/sudo -u $USER $VENV/bin/python $VENV/pandora/manage.py celeryd \ + -B -s /var/run/pandora/celerybeat-schedule \ -Q encoding \ -n pandora-encoding \ -p /var/run/pandora/pandora-encoding.pid \ diff --git a/etc/init/pandora.conf b/etc/init/pandora.conf index cf624360..cfdbb8ba 100644 --- a/etc/init/pandora.conf +++ b/etc/init/pandora.conf @@ -16,6 +16,7 @@ script test -e /var/log/pandora || (mkdir -p /var/log/pandora && chown $USER:$USER /var/log/pandora) test -e /var/run/pandora || (mkdir -p /var/run/pandora && chown $USER:$USER /var/run/pandora) cd $VENV/pandora +./manage.py compile_pyc exec /usr/bin/sudo -u $USER $VENV/bin/gunicorn_django \ --bind 127.0.0.1:2620 \ --timeout 90 \ diff --git a/pandora/0xdb.jsonc b/pandora/0xdb.jsonc index 48350997..4f481edf 100644 --- a/pandora/0xdb.jsonc +++ b/pandora/0xdb.jsonc @@ -557,7 +557,8 @@ // fixme: there should be no magic applied to this file "id": "{{settings.SITEID}}", "name": "{{settings.SITENAME}}", - "url": "{{settings.URL}}" + "url": "{{settings.URL}}", + "videoprefix": "" }, "sitePages": [ {"id": "about", "title": "About"}, diff --git a/pandora/annotation/models.py b/pandora/annotation/models.py index b6d8cce5..d9372b56 100644 --- a/pandora/annotation/models.py +++ b/pandora/annotation/models.py @@ -9,8 +9,11 @@ import ox from archive import extract from clip.models import Clip -import utils + import managers +import utils +from tasks import update_matching_events, update_matching_places + def load_layers(layers): @@ -104,7 +107,7 @@ class Annotation(models.Model): return utils.html_parser(self.value) else: return self.value - + def set_public_id(self): public_id = Annotation.objects.filter(item=self.item, id__lt=self.id).count() self.public_id = "%s/%s" % (self.item.itemId, ox.to26(public_id)) @@ -125,6 +128,9 @@ class Annotation(models.Model): super(Annotation, self).save(*args, **kwargs) if set_public_id: self.set_public_id() + #how expensive is this? + #update_matching_events.delay(self.value) + #update_matching_places.delay(self.value) def json(self, layer=False, keys=None): j = { diff --git a/pandora/annotation/tasks.py b/pandora/annotation/tasks.py new file mode 100644 index 00000000..0a7c28e5 --- /dev/null +++ b/pandora/annotation/tasks.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from celery.decorators import task + + + +@task(ignore_resulsts=True, queue='default') +def update_matching_events(value): + from event.models import Event + ids = [e['id'] for e in Event.objects.all().values('id')] + for i in ids: + e = Event.objects.get(pk=i) + for name in [e.name] + list(e.alternativeNames): + if name in value: + e.update_matches() + break + +@task(ignore_resulsts=True, queue='default') +def update_matching_places(value): + from place.models import Place + ids = [e['id'] for e in Place.objects.all().values('id')] + for i in ids: + e = Place.objects.get(pk=i) + for name in [e.name] + list(e.alternativeNames): + if name in value: + e.update_matches() + break diff --git a/pandora/app/config.py b/pandora/app/config.py index 0545a085..875cb1a4 100644 --- a/pandora/app/config.py +++ b/pandora/app/config.py @@ -1,4 +1,3 @@ - # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 from __future__ import division, with_statement @@ -28,6 +27,7 @@ def load_config(): config['site']['name'] = settings.SITENAME config['site']['sectionName'] = settings.SITENAME config['site']['url'] = settings.URL + config['site']['videoprefix'] = settings.VIDEO_PREFIX config['keys'] = {} for key in config['itemKeys']: diff --git a/pandora/app/log.py b/pandora/app/log.py new file mode 100644 index 00000000..715e8692 --- /dev/null +++ b/pandora/app/log.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from __future__ import division, with_statement + +import logging +import sys + + +class ErrorHandler(logging.Handler): + def __init__(self): + logging.Handler.__init__(self) + + """An exception log handler that log entries into log database. + + If the request is passed as the first argument to the log record, + request data will be provided in the + """ + def emit(self, record): + import traceback + from django.views.debug import ExceptionReporter + from django.conf import settings + import models + user = None + line = 0 + text = '' + url = '' + try: + if sys.version_info < (2,5): + # A nasty workaround required because Python 2.4's logging + # module doesn't support passing in extra context. + # For this handler, the only extra data we need is the + # request, and that's in the top stack frame. + request = record.exc_info[2].tb_frame.f_locals['request'] + else: + request = record.request + + request_repr = repr(request) + if request.user.is_authenticated(): + user = request.user + url = request.META.get('PATH_INFO', '') + except: + request = None + request_repr = "%s %s\n\nRequest repr() unavailable" % (record.levelname, record.msg) + + if record.exc_info: + stack_trace = '\n'.join(traceback.format_exception(*record.exc_info)) + stack_info = traceback.extract_tb(record.exc_info[2]) + if stack_info: + url = stack_info[-1][0] + line = stack_info[-1][1] + else: + stack_trace = 'No stack trace available' + + text = "%s\n\n%s" % (stack_trace, request_repr) + if text: + l = models.Log( + text=text, + line=line, + url=url + ) + if user: + l.user = user + l.save() diff --git a/pandora/app/models.py b/pandora/app/models.py index 1f6a8ab6..1b134435 100644 --- a/pandora/app/models.py +++ b/pandora/app/models.py @@ -24,7 +24,7 @@ class Log(models.Model): text = models.TextField(blank=True) def __unicode__(self): - return self.id + return u"%s" % self.id def json(self): return { diff --git a/pandora/app/tasks.py b/pandora/app/tasks.py new file mode 100644 index 00000000..3ac03816 --- /dev/null +++ b/pandora/app/tasks.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +import datetime + +from celery.decorators import task, periodic_task +from celery.task.schedules import crontab + + +@periodic_task(run_every=crontab(hour=6, minute=0), queue='encoding') +def cron(**kwargs): + from django.db import transaction + from django.contrib.sessions.models import Session + Session.objects.filter(expire_date__lt=datetime.datetime.now()).delete() + transaction.commit_unless_managed() diff --git a/pandora/app/views.py b/pandora/app/views.py index c6c86a68..f07e49ff 100644 --- a/pandora/app/views.py +++ b/pandora/app/views.py @@ -108,17 +108,18 @@ def log(request): } ''' data = json.loads(request.POST['data']) - if not request.user.is_authenticated: + if request.user.is_authenticated(): user = request.user else: user = None if 'text' in data: l = models.Log( - user=user, text=data['text'], line=int(data.get('line', 0)), url=data.get('url', '') ) + if user: + l.user = user l.save() response = json_response() return render_to_json_response(response) diff --git a/pandora/event/models.py b/pandora/event/models.py index 3c02c186..ca4ebc0c 100644 --- a/pandora/event/models.py +++ b/pandora/event/models.py @@ -1,11 +1,9 @@ # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 from __future__ import division, with_statement -import unicodedata -import string from django.db import models -from django.contrib.auth.models import User, Group +from django.contrib.auth.models import User from django.db.models import Q import ox @@ -16,6 +14,7 @@ from item.models import Item from item import utils from person.models import get_name_sort from title.models import get_title_sort + import managers diff --git a/pandora/event/tasks.py b/pandora/event/tasks.py index 37366683..d00c7e11 100644 --- a/pandora/event/tasks.py +++ b/pandora/event/tasks.py @@ -3,12 +3,21 @@ from datetime import timedelta from celery.decorators import task, periodic_task +from celery.task.schedules import crontab -import models +from models import Event + + +@periodic_task(run_every=crontab(hour=7, minute=30), queue='encoding') +def update_all_matches(**kwargs): + ids = [e['id'] for e in Event.objects.all().values('id')] + for i in ids: + e = Event.objects.get(pk=i) + e.update_matches() @task(ignore_resulsts=True, queue='default') def update_matches(eventId): - event = models.Event.objects.get(pk=eventId) + event = Event.objects.get(pk=eventId) event.update_matches() diff --git a/pandora/item/models.py b/pandora/item/models.py index 0d1e0e0b..0a168726 100644 --- a/pandora/item/models.py +++ b/pandora/item/models.py @@ -3,6 +3,7 @@ from __future__ import division, with_statement from datetime import datetime +import math import os.path import re import subprocess @@ -1083,8 +1084,9 @@ class Item(models.Model): annotation.save() #otherwise add empty 5 seconds annotation every minute if not subtitles_added: - i = offset - while i < offset + f.duration - 5: + for i in range(int (offset / 60) * 60 + 60, + int(offset + f.duration) - 5, + 60): annotation = Annotation( item=self, layer=layer, @@ -1094,7 +1096,6 @@ class Item(models.Model): user=user ) annotation.save() - i += 60 offset += f.duration self.update_find() diff --git a/pandora/place/tasks.py b/pandora/place/tasks.py index 3a276227..b8d36ee6 100644 --- a/pandora/place/tasks.py +++ b/pandora/place/tasks.py @@ -3,11 +3,20 @@ from datetime import timedelta from celery.decorators import task, periodic_task +from celery.task.schedules import crontab import models +@periodic_task(run_every=crontab(hour=6, minute=30), queue='encoding') +def update_all_matches(**kwargs): + ids = [p['id'] for p in models.Place.objects.all().values('id')] + for i in ids: + p = models.Place.objects.get(pk=i) + p.update_matches() + @task(ignore_resulsts=True, queue='default') def update_matches(id): place = models.Place.objects.get(pk=id) place.update_matches() + diff --git a/pandora/settings.py b/pandora/settings.py index 10cdd5e1..e4cda557 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -75,9 +75,13 @@ MEDIA_ROOT = normpath(join(PROJECT_ROOT, '..', 'data')) STATIC_ROOT = normpath(join(PROJECT_ROOT, '..', 'static')) TESTS_ROOT = join(PROJECT_ROOT, 'tests') +#if videos are served from another subdomain +VIDEO_PREFIX = '' + # URL that handles the media served from MEDIA_ROOT. Make sure to use a # trailing slash if there is a path component (optional in other cases). # Examples: "http://media.lawrence.com", "http://example.com/media/" + MEDIA_URL = '/data/' STATIC_URL = '/static/' @@ -140,23 +144,19 @@ INSTALLED_APPS = ( 'urlalias', ) -# A sample logging configuration. The only tangible logging -# performed by this configuration is to send an email to -# the site admins on every HTTP 500 error. -# See http://docs.djangoproject.com/en/dev/topics/logging for -# more details on how to customize your logging configuration. +# Log errors into db LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'handlers': { - 'mail_admins': { + 'errors': { 'level': 'ERROR', - 'class': 'django.utils.log.AdminEmailHandler' + 'class': 'pandora.app.log.ErrorHandler' } }, 'loggers': { 'django.request': { - 'handlers': ['mail_admins'], + 'handlers': ['errors'], 'level': 'ERROR', 'propagate': True, }, diff --git a/static/js/pandora/filesView.js b/static/js/pandora/filesView.js index 1e3f67c2..0a92f0b8 100644 --- a/static/js/pandora/filesView.js +++ b/static/js/pandora/filesView.js @@ -417,7 +417,11 @@ pandora.ui.filesView = function(options, self) { }, function(result) { ['title', 'director', 'year'].forEach(function(key) { if (result.data[key]) { - self['$' + key + 'Input'].options({value: result.data[key]}); + self['$' + key + 'Input'].options({ + value: key == 'director' + ? result.data[key].join(', ') + : result.data[key] + }); } }); updateForm(); diff --git a/static/js/pandora/item.js b/static/js/pandora/item.js index 6fe192c7..5f945014 100644 --- a/static/js/pandora/item.js +++ b/static/js/pandora/item.js @@ -64,7 +64,7 @@ pandora.ui.item = function() { }); pandora.site.video.resolutions.forEach(function(resolution) { video[resolution] = Ox.range(result.data.parts).map(function(i) { - return '/' + pandora.user.ui.item + '/' + return pandora.site.site.videoprefix + '/' + pandora.user.ui.item + '/' + resolution + 'p' + (i + 1) + '.' + pandora.user.videoFormat; }); }); diff --git a/update.sh b/update.sh index 4572addd..45083d47 100755 --- a/update.sh +++ b/update.sh @@ -5,4 +5,4 @@ cd static/oxjs bzr pull http://code.0x2620.org/oxjs/ test -e src/python-ox && cd src/python-ox && bzr pull http://code.0x2620.org/python-ox/ cd $base -cd pandora && ./manage.py update_static +cd pandora && ./manage.py update_static && ./manage.py compile_pyc