From a960a7bd18c85ffb71d9dfdc2f0f84f52f51e650 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Thu, 3 Nov 2011 12:17:50 +0100 Subject: [PATCH 1/5] fix find with clips --- pandora/clip/models.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pandora/clip/models.py b/pandora/clip/models.py index aac2e10d..10c0056c 100644 --- a/pandora/clip/models.py +++ b/pandora/clip/models.py @@ -51,6 +51,10 @@ class MetaClip: for key in j.keys(): if key not in keys: del j[key] + #needed here to make item find with clips work + if 'annotations' in keys: + j['annotations'] = [a.json(keys=['value', 'id', 'layer']) + for a in self.annotations.filter(layer__in=self.layers)] for key in keys: if key not in clip_keys and key not in j: value = self.item.get(key) From 7eeb6dc1078a0587076b583f949a091eef267906 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Thu, 3 Nov 2011 12:31:46 +0100 Subject: [PATCH 2/5] add sortvalue to annotations to get better sort by text results --- pandora/annotation/models.py | 6 +++++- pandora/clip/views.py | 4 ++-- pandora/item/utils.py | 3 ++- pandora/person/models.py | 1 - 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pandora/annotation/models.py b/pandora/annotation/models.py index bcf215ce..8a7d687e 100644 --- a/pandora/annotation/models.py +++ b/pandora/annotation/models.py @@ -10,7 +10,7 @@ import ox from archive import extract from clip.models import Clip - +from item.utils import sort_string import managers import utils from tasks import update_matching_events, update_matching_places @@ -34,6 +34,7 @@ class Annotation(models.Model): layer = models.CharField(max_length=255, db_index=True) value = models.TextField() + sortvalue = models.CharField(max_length=1000, null=True, blank=True, db_index=True) def editable(self, user): if user.is_authenticated(): @@ -57,6 +58,9 @@ class Annotation(models.Model): def save(self, *args, **kwargs): set_public_id = not self.id or not self.public_id + self.sortvalue = None + if self.value.strip(): + self.sortvalue = sort_string(self.value)[:1000] #no clip or update clip def get_layer(id): diff --git a/pandora/clip/views.py b/pandora/clip/views.py index f0b8516b..39f27d69 100644 --- a/pandora/clip/views.py +++ b/pandora/clip/views.py @@ -35,14 +35,14 @@ def order_query(qs, sort): if operator != '-': operator = '' clip_keys = ('public_id', 'start', 'end', 'hue', 'saturation', 'lightness', 'volume', - 'annotations__value', 'videoRatio', + 'annotations__sortvalue', 'videoRatio', 'director', 'title') key = { 'id': 'public_id', 'in': 'start', 'out': 'end', 'position': 'start', - 'text': 'annotations__value', + 'text': 'annotations__sortvalue', 'videoRatio': 'aspect_ratio', }.get(e['key'], e['key']) if key.startswith('clip:'): diff --git a/pandora/item/utils.py b/pandora/item/utils.py index 6c89914a..fd7ae32c 100644 --- a/pandora/item/utils.py +++ b/pandora/item/utils.py @@ -21,7 +21,8 @@ def plural_key(term): def sort_string(string): - string = string.replace(u'Þ', 'Th') + string = string.replace(u'Æ', 'AE').replace(u'Ø', 'O').replace(u'Þ', 'Th') + #pad numbered titles string = re.sub('(\d+)', lambda x: '%010d' % int(x.group(0)), string) return unicodedata.normalize('NFKD', string) diff --git a/pandora/person/models.py b/pandora/person/models.py index 23b425be..275c6f28 100644 --- a/pandora/person/models.py +++ b/pandora/person/models.py @@ -48,7 +48,6 @@ class Person(models.Model): if not self.sortname: self.sortname = ox.get_sort_name(self.name) self.sortname = unicodedata.normalize('NFKD', self.sortname) - self.sortname = self.sortname.replace(u'Æ', 'AE').replace(u'Ø', 'O').replace(u'Þ', 'P') self.sortsortname = utils.sort_string(self.sortname) self.numberofnames = len(self.name.split(' ')) super(Person, self).save(*args, **kwargs) From af2c26a36a6daa8e440f5b3c564aba92a7f4fb91 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Thu, 3 Nov 2011 12:44:53 +0100 Subject: [PATCH 3/5] strip tags from annotation.sortvalue --- pandora/annotation/models.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pandora/annotation/models.py b/pandora/annotation/models.py index 8a7d687e..424ea4d9 100644 --- a/pandora/annotation/models.py +++ b/pandora/annotation/models.py @@ -58,9 +58,15 @@ class Annotation(models.Model): def save(self, *args, **kwargs): set_public_id = not self.id or not self.public_id - self.sortvalue = None - if self.value.strip(): - self.sortvalue = sort_string(self.value)[:1000] + if self.value: + sortvalue = ox.stripTags(self.value).strip() + sortvalue = sort_string(sortvalue) + if sortvalue: + self.sortvalue = sortvalue[:1000] + else: + self.sortvalue = None + else: + self.sortvalue = None #no clip or update clip def get_layer(id): From 0ed8c805c72d786f3ccc7a1fd13b80871810ca9b Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Thu, 3 Nov 2011 13:02:12 +0100 Subject: [PATCH 4/5] always serve static files, if behind nginx/apache those are not handed down --- pandora/settings.py | 1 - pandora/urls.py | 17 ++++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/pandora/settings.py b/pandora/settings.py index b9084054..a9703ead 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -42,7 +42,6 @@ APPEND_SLASH = False # Example: "/home/media/media.lawrence.com/" MEDIA_ROOT = normpath(join(PROJECT_ROOT, '..', 'data')) STATIC_ROOT = normpath(join(PROJECT_ROOT, '..', 'static')) -TESTS_ROOT = join(PROJECT_ROOT, 'tests') # URL that handles the media served from MEDIA_ROOT. Make sure to use a # trailing slash if there is a path component (optional in other cases). diff --git a/pandora/urls.py b/pandora/urls.py index cd210589..6bb4da04 100644 --- a/pandora/urls.py +++ b/pandora/urls.py @@ -30,15 +30,14 @@ urlpatterns = patterns('', (r'^robots.txt$', serve_static_file, {'location': os.path.join(settings.STATIC_ROOT, 'robots.txt'), 'content_type': 'text/plain'}), (r'^favicon.ico$', serve_static_file, {'location': os.path.join(settings.STATIC_ROOT, 'png/icon.16.png'), 'content_type': 'image/x-icon'}), ) -if settings.DEBUG: - urlpatterns += patterns('', - (r'^data/(?P.*)$', 'django.views.static.serve', - {'document_root': settings.MEDIA_ROOT}), - (r'^static/(?P.*)$', 'django.views.static.serve', - {'document_root': settings.STATIC_ROOT}), - (r'^tests/(?P.*)$', 'django.views.static.serve', - {'document_root': settings.TESTS_ROOT}), - ) +#if settings.DEBUG: +#sould this not be enabled by default? nginx should handle those +urlpatterns += patterns('', + (r'^data/(?P.*)$', 'django.views.static.serve', + {'document_root': settings.MEDIA_ROOT}), + (r'^static/(?P.*)$', 'django.views.static.serve', + {'document_root': settings.STATIC_ROOT}), +) urlpatterns += patterns('', (r'^(V[a-z0-9]*)$', 'urlalias.views.padma_video'), From f9bc5b06be32abc0a2fcfb9394f7f4129fabaa25 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Thu, 3 Nov 2011 13:32:23 +0100 Subject: [PATCH 5/5] update README, move scripts to toplevel --- README | 61 +++++++++++++----- {apache => etc/apache2}/vhost.conf.in | 0 {nginx => etc/nginx}/vhost.in | 0 fixtures/reviews.json | 59 ----------------- pandora/settings.py | 9 +-- .../data/DejaVuSansCondensedBold.ttf | Bin .../scripts => scripts}/data/iconMask.png | Bin .../data/iconTimelineInnerMask.png | Bin .../data/iconTimelineOuterMask.png | Bin {pandora/scripts => scripts}/data/logo.png | Bin {pandora/scripts => scripts}/item_icon | 2 +- {pandora/scripts => scripts}/list_icon | 2 +- {pandora/scripts => scripts}/oxdb_poster | 2 +- {pandora/scripts => scripts}/padma_poster | 2 +- 14 files changed, 53 insertions(+), 84 deletions(-) rename {apache => etc/apache2}/vhost.conf.in (100%) rename {nginx => etc/nginx}/vhost.in (100%) delete mode 100644 fixtures/reviews.json rename {pandora/scripts => scripts}/data/DejaVuSansCondensedBold.ttf (100%) rename {pandora/scripts => scripts}/data/iconMask.png (100%) rename {pandora/scripts => scripts}/data/iconTimelineInnerMask.png (100%) rename {pandora/scripts => scripts}/data/iconTimelineOuterMask.png (100%) rename {pandora/scripts => scripts}/data/logo.png (100%) rename {pandora/scripts => scripts}/item_icon (98%) rename {pandora/scripts => scripts}/list_icon (98%) rename {pandora/scripts => scripts}/oxdb_poster (99%) rename {pandora/scripts => scripts}/padma_poster (98%) diff --git a/README b/README index a8349494..602055ad 100644 --- a/README +++ b/README @@ -1,14 +1,16 @@ pan.do/ra - open media archive -== DEVELOPMENT == -To setup a development instance of pan.do/ra, -you need python, bazaar, pip and virtualenv and several other python modules: +== SETUP == +To setup pan.do/ra, you need +python, bazaar, pip and virtualenv and several other python modules: * Packages apt-get install bzr git subversion mercurial \ python-setuptools python-pip python-virtualenv ipython \ python-dev python-imaging python-numpy python-psycopg2 +* Pan.do/ra +Get code from bazzar bzr branch http://code.0x2620.org/pandora pandora cd pandora virtualenv . @@ -17,6 +19,18 @@ you need python, bazaar, pip and virtualenv and several other python modules: cd static bzr branch http://code.0x2620.org/oxjs +create settings_local.py and create site.jsonc +(use settings.py / 0xdb.jsonc / padma.jsonc as example) +create db + + ./manage.py syncdb + +create / update static files + + ./manage.py update_static + ./manage.py compile_pyc + + * Additional pandora tools: You need current versions of oxframe, oxtimeline installed. For Ubuntu we provide these packages in a ppa: @@ -30,11 +44,12 @@ you need python, bazaar, pip and virtualenv and several other python modules: (make sure you have the python bindings installed). create a postgresql database and adjust settings in local_settings.py - and run ./manage.py syncdb to populate the database - you might want to load example configurations from fixutes. + and run ./manage.py syncdb to populate the database. createdb -T template0 --locale=C --encoding=UTF8 -O pandora pandora + (setting locale to C is required to fix a bug in sort if set to UTF8) + * RabbitMQ For background tasks we use RabbitMQ, to install rabbitmq: sudo apt-get install rabbitmq-server @@ -48,7 +63,7 @@ you need python, bazaar, pip and virtualenv and several other python modules: * H264 to support h264 videos, install ffmpeg with x264 enabled, install qt-faststart from (ffmpeg/tools) - set VIDEO_H264 = True in local_settings.py + to enable add "mp4" to video.formats in your config.jsonc Running developer environment: in one terminal: @@ -56,24 +71,36 @@ Running developer environment: and in another one: ./manage.py celeryd -Q default,encoding -B -Updating database: - right now database updates are not managed, each time you update to current bzr - you might have to update db tables too. - Use - ./manage.py sqldiff appname - to check if changes exist and - ./manage.py sqldiff appname | ./manage.py dbshell - to apply them. - This has to be done for all installed apps +=== Updating === +To update a pandora installation get the latest version from bzr by running + ./update.sh +this will pull pandora/oxjs/python-ox and list possible upgrades to the db + +to update your database tables, use + ./manage.py sqldiff -a +to check if there are changes and + ./manage.py sqldiff -a | ./manage.py dbshell +to apply them. == DEPLOYMENT == +* Install upstart scripts +check etc/init for upstart scripts, adjust path and user and put into /etc/init + To run pan.do/ra in production, we use nginx, using apache2 is also possible. * nginx setup sudo apt-get install nginx + add this to local_settings.py: + XACCELREDIRECT = True + + setup nginx according to etc/nginx/vhost.in + * apache2 setup - sudo apt-get install apache2-mpm-prefork - sudo apt-get install libapache2-mod-xsendfile + apt-get install apache2-mpm-prefork libapache2-mod-xsendfile + add this to local_settings.py: + XSENDFILE = True + + setup apache according to etc/apache2/vhost.in diff --git a/apache/vhost.conf.in b/etc/apache2/vhost.conf.in similarity index 100% rename from apache/vhost.conf.in rename to etc/apache2/vhost.conf.in diff --git a/nginx/vhost.in b/etc/nginx/vhost.in similarity index 100% rename from nginx/vhost.in rename to etc/nginx/vhost.in diff --git a/fixtures/reviews.json b/fixtures/reviews.json deleted file mode 100644 index 4052c710..00000000 --- a/fixtures/reviews.json +++ /dev/null @@ -1,59 +0,0 @@ -[ - { - "pk": 1, - "model": "backend.reviewwhitelist", - "fields": { - "url": "://filmcritic.com", - "name": "Filmcritic" - } - }, - { - "pk": 2, - "model": "backend.reviewwhitelist", - "fields": { - "url": "villagevoice.com", - "name": "Village Voice" - } - }, - { - "pk": 3, - "model": "backend.reviewwhitelist", - "fields": { - "url": "salon.com", - "name": "Salon.com" - } - }, - { - "pk": 4, - "model": "backend.reviewwhitelist", - "fields": { - "url": "rottentomatoes.com", - "name": "Rotten Tomatoes" - } - }, - { - "pk": 5, - "model": "backend.reviewwhitelist", - "fields": { - "url": "nytimes.com", - "name": "New York Times" - } - }, - { - "pk": 6, - "model": "backend.reviewwhitelist", - "fields": { - "url": "metacritic.com", - "name": "Metacritic" - } - }, - { - "pk": 7, - "model": "backend.reviewwhitelist", - "fields": { - "url": "sensesofcinema.com", - "name": "Senses of Cinema" - } - } -] - diff --git a/pandora/settings.py b/pandora/settings.py index a9703ead..16315ad4 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -182,11 +182,12 @@ VIDEO_PREFIX='' #VIDEO_PREFIX='videoPART.example.com' #SESSION_COOKIE_DOMAIN=*.example.com" +SCRIPT_ROOT = normpath(join(PROJECT_ROOT, '..', 'scripts')) #copy scripts and adjust to customize -ITEM_POSTER = join('scripts', 'oxdb_poster') -#ITEM_POSTER = join('scripts', 'padma_poster') -ITEM_ICON = join('scripts', 'item_icon') -LIST_ICON = join('scripts', 'list_icon') +ITEM_POSTER = join(SCRIPT_ROOT, 'oxdb_poster') +#ITEM_POSTER = join(SCRIPT_ROOT, 'padma_poster') +ITEM_ICON = join(SCRIPT_ROOT, 'item_icon') +LIST_ICON = join(SCRIPT_ROOT, 'list_icon') #you can ignore things below this line diff --git a/pandora/scripts/data/DejaVuSansCondensedBold.ttf b/scripts/data/DejaVuSansCondensedBold.ttf similarity index 100% rename from pandora/scripts/data/DejaVuSansCondensedBold.ttf rename to scripts/data/DejaVuSansCondensedBold.ttf diff --git a/pandora/scripts/data/iconMask.png b/scripts/data/iconMask.png similarity index 100% rename from pandora/scripts/data/iconMask.png rename to scripts/data/iconMask.png diff --git a/pandora/scripts/data/iconTimelineInnerMask.png b/scripts/data/iconTimelineInnerMask.png similarity index 100% rename from pandora/scripts/data/iconTimelineInnerMask.png rename to scripts/data/iconTimelineInnerMask.png diff --git a/pandora/scripts/data/iconTimelineOuterMask.png b/scripts/data/iconTimelineOuterMask.png similarity index 100% rename from pandora/scripts/data/iconTimelineOuterMask.png rename to scripts/data/iconTimelineOuterMask.png diff --git a/pandora/scripts/data/logo.png b/scripts/data/logo.png similarity index 100% rename from pandora/scripts/data/logo.png rename to scripts/data/logo.png diff --git a/pandora/scripts/item_icon b/scripts/item_icon similarity index 98% rename from pandora/scripts/item_icon rename to scripts/item_icon index f1054ee8..7a3cafa6 100755 --- a/pandora/scripts/item_icon +++ b/scripts/item_icon @@ -4,7 +4,7 @@ from __future__ import division import os -root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) +root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) #using virtualenv's activate_this.py to reorder sys.path activate_this = os.path.join(root_dir, 'bin', 'activate_this.py') diff --git a/pandora/scripts/list_icon b/scripts/list_icon similarity index 98% rename from pandora/scripts/list_icon rename to scripts/list_icon index c2611ab8..eba0e5de 100755 --- a/pandora/scripts/list_icon +++ b/scripts/list_icon @@ -4,7 +4,7 @@ from __future__ import division import os -root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) +root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) #using virtualenv's activate_this.py to reorder sys.path activate_this = os.path.join(root_dir, 'bin', 'activate_this.py') diff --git a/pandora/scripts/oxdb_poster b/scripts/oxdb_poster similarity index 99% rename from pandora/scripts/oxdb_poster rename to scripts/oxdb_poster index 49630db1..286281b6 100755 --- a/pandora/scripts/oxdb_poster +++ b/scripts/oxdb_poster @@ -4,7 +4,7 @@ from __future__ import division import os -root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) +root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) #using virtualenv's activate_this.py to reorder sys.path activate_this = os.path.join(root_dir, 'bin', 'activate_this.py') diff --git a/pandora/scripts/padma_poster b/scripts/padma_poster similarity index 98% rename from pandora/scripts/padma_poster rename to scripts/padma_poster index 07ee23fa..8c35d647 100755 --- a/pandora/scripts/padma_poster +++ b/scripts/padma_poster @@ -4,7 +4,7 @@ from __future__ import division import os -root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) +root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) #using virtualenv's activate_this.py to reorder sys.path activate_this = os.path.join(root_dir, 'bin', 'activate_this.py')