basic text management

.bzrignore

@@ -0,0 +1,2 @@
+host_settings/*
+media/*

manage.py

@@ -0,0 +1,11 @@
+#!/usr/bin/python
+from django.core.management import execute_manager
+try:
+    import settings # Assumed to be in the same directory.
+except ImportError:
+    import sys
+    sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)
+    sys.exit(1)
+
+if __name__ == "__main__":
+    execute_manager(settings)

monitor.py

@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+import os
+import sys
+import time
+import signal
+import threading
+import atexit
+import Queue
+
+_interval = 1.0
+_times = {}
+_files = []
+
+_running = False
+_queue = Queue.Queue()
+_lock = threading.Lock()
+
+def _restart(path):
+    _queue.put(True)
+    prefix = 'monitor (pid=%d):' % os.getpid()
+    print >> sys.stderr, '%s Change detected to \'%s\'.' % (prefix, path)
+    print >> sys.stderr, '%s Triggering process restart.' % prefix
+    os.kill(os.getpid(), signal.SIGINT)
+
+def _modified(path):
+    try:
+        # If path doesn't denote a file and were previously
+        # tracking it, then it has been removed or the file type
+        # has changed so force a restart. If not previously
+        # tracking the file then we can ignore it as probably
+        # pseudo reference such as when file extracted from a
+        # collection of modules contained in a zip file.
+
+        if not os.path.isfile(path):
+            return path in _times
+
+        # Check for when file last modified.
+
+        mtime = os.stat(path).st_mtime
+        if path not in _times:
+            _times[path] = mtime
+
+        # Force restart when modification time has changed, even
+        # if time now older, as that could indicate older file
+        # has been restored.
+
+        if mtime != _times[path]:
+            return True
+    except:
+        # If any exception occured, likely that file has been
+        # been removed just before stat(), so force a restart.
+
+        return True
+
+    return False
+
+def _monitor():
+    while 1:
+        # Check modification times on all files in sys.modules.
+
+        for module in sys.modules.values():
+            if not hasattr(module, '__file__'):
+                continue
+            path = getattr(module, '__file__')
+            if not path:
+                continue
+            if os.path.splitext(path)[1] in ['.pyc', '.pyo', '.pyd']:
+                path = path[:-1]
+            if _modified(path):
+                return _restart(path)
+
+        # Check modification times on files which have
+        # specifically been registered for monitoring.
+
+        for path in _files:
+            if _modified(path):
+                return _restart(path)
+
+        # Go to sleep for specified interval.
+
+        try:
+            return _queue.get(timeout=_interval)
+        except:
+            pass
+
+_thread = threading.Thread(target=_monitor)
+_thread.setDaemon(True)
+
+def _exiting():
+    try:
+        _queue.put(True)
+    except:
+        pass
+    _thread.join()
+
+atexit.register(_exiting)
+
+def track(path):
+    if not path in _files:
+        _files.append(path)
+
+def start(interval=1.0):
+    global _interval
+    if interval < _interval:
+        _interval = interval
+
+    global _running
+    _lock.acquire()
+    if not _running:
+        _running = True
+        _thread.start()
+    _lock.release()

settings.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+# Written 2009 by j@mailb.org
+
+import os
+from os.path import join
+from django.conf import global_settings 
+
+PROJECT_PATH = os.path.normpath(os.path.dirname(__file__))
+
+DEBUG = True
+TEMPLATE_DEBUG = DEBUG
+
+ADMINS = (
+    # ('Your Name', 'your_email@domain.com'),
+)
+
+MANAGERS = ADMINS
+
+DATABASE_ENGINE = 'sqlite3'    # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
+DATABASE_NAME = 'dev.sqlite'   # Or path to database file if using sqlite3.
+DATABASE_USER = ''             # Not used with sqlite3.
+DATABASE_PASSWORD = ''         # Not used with sqlite3.
+DATABASE_HOST = ''             # Set to empty string for localhost. Not used with sqlite3.
+DATABASE_PORT = ''             # Set to empty string for default. Not used with sqlite3.
+
+# Local time zone for this installation. Choices can be found here:
+# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
+# although not all choices may be available on all operating systems.
+# If running in a Windows environment this must be set to the same as your
+# system time zone.
+TIME_ZONE = 'Europe/Berlin'
+
+# Language code for this installation. All choices can be found here:
+# http://www.i18nguy.com/unicode/language-identifiers.html
+LANGUAGE_CODE = 'en-us'
+
+SITE_ID = 1
+
+# If you set this to False, Django will make some optimizations so as not
+# to load the internationalization machinery.
+USE_I18N = True
+
+# Absolute path to the directory that holds media.
+# Example: "/home/media/media.lawrence.com/"
+MEDIA_ROOT = join(PROJECT_PATH, 'media')
+STATIC_ROOT = join(PROJECT_PATH, 'static')
+
+# URL that handles the media served from MEDIA_ROOT. Make sure to use a
+# trailing slash if there is a path component (optional in other cases).
+# Examples: "http://media.lawrence.com", "http://example.com/media/"
+MEDIA_URL = '/texts/'
+
+# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
+# trailing slash.
+# Examples: "http://foo.com/media/", "/media/".
+ADMIN_MEDIA_PREFIX = '/admin/media/'
+
+# Make this unique, and don't share it with anybody.
+SECRET_KEY = '@8+=n)(@(gv0ogqm6pnvs6ag@&qa3syb^qy8@#x7f68)cyrs(*'
+
+# List of callables that know how to import templates from various sources.
+TEMPLATE_LOADERS = (
+    'django.template.loaders.filesystem.load_template_source',
+    'django.template.loaders.app_directories.load_template_source',
+#     'django.template.loaders.eggs.load_template_source',
+)
+
+MIDDLEWARE_CLASSES = (
+    'django.middleware.common.CommonMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+)
+
+ROOT_URLCONF = 'texts.urls'
+
+TEMPLATE_DIRS = (
+    join(PROJECT_PATH, 'templates'),
+)
+
+INSTALLED_APPS = (
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.sites',
+    'django.contrib.admin',
+    'texts.text',
+)
+
+try:
+    import socket
+    # hostname = socket.gethostname().replace('.','_')
+    # exec "from host_settings.%s import *" % hostname
+    local_settings_module = socket.gethostname().split(".")[0]
+    if local_settings_module:
+        execfile(os.path.join(PROJECT_PATH, "host_settings", "%s.py" % local_settings_module))
+except ImportError, e:
+    raise e
+

text/admin.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+# Written 2009 by j@mailb.org
+
+from django.contrib import admin
+
+import models
+
+
+class TextAdmin(admin.ModelAdmin):
+    search_fields = ['title', 'authors__name']
+    list_display = ('title', 'version', 'file')
+    list_filter = ('authors', )
+admin.site.register(models.Text, TextAdmin)
+
+
+class AuthorAdmin(admin.ModelAdmin):
+    search_fields = ['name']
+    list_display = ('name', )
+admin.site.register(models.Author, AuthorAdmin)
+
+
+class LanguageAdmin(admin.ModelAdmin):
+    search_fields = ['name']
+    list_display = ('name', )
+admin.site.register(models.Language, LanguageAdmin)
+

text/management/commands/import_text.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+# Written 2009 by j@mailb.org
+
+import os
+from glob import glob
+
+import oxlib
+import oxlib.normalize
+
+from django.core.management.base import BaseCommand, CommandError
+from django.conf import settings
+
+from texts.text import models
+
+
+class Command(BaseCommand):
+    """
+    import texts from media/text.
+    """
+    help = 'import texts that are not in db.'
+    args = ''
+
+    def handle(self, **options):
+        for f in glob('%s/*/*/*' % settings.MEDIA_ROOT):
+            if os.path.isfile(f):
+                name = f[len(settings.MEDIA_ROOT)+1:]
+                name = name.decode('utf-8')
+                q = models.Text.objects.filter(file=name)
+                if q.count() == 0:
+                    print 'adding', name
+                    text = models.Text()
+                    text.file.name = name
+                    text.parsePath()
+

text/models.py

@@ -0,0 +1,130 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+# Written 2009 by j@mailb.org
+import os
+import re
+from glob import glob
+
+import oxlib
+import oxlib.normalize
+from django.db import models
+from django.conf import settings
+
+def getAuthor(name, name_sort=''):
+    try:
+        a = Author.objects.get(name=name)
+    except Author.DoesNotExist:
+        a = Author(name=name, name_sort=name_sort)
+        a.save()
+    return a
+
+class Author(models.Model):
+    name = models.CharField(blank=True, max_length=1000, unique=True)
+    name_sort = models.CharField('Sort Name', blank=True, max_length=1000)
+
+    class Meta:
+        ordering = ('name_sort', )
+
+    def __unicode__(self):
+        return self.name
+
+    def save(self, *args, **kwargs):
+        if not self.name_sort:
+            self.name_sort = oxlib.normalize.canonicalName(self.name)
+        super(Author, self).save(*args, **kwargs)
+
+class Language(models.Model):
+    name = models.CharField(blank=True, max_length=1000)
+    class Meta:
+        ordering = ('name', )
+    def __unicode__(self):
+        return self.name
+
+class Text(models.Model):
+    created = models.DateTimeField(auto_now_add=True)
+    updated = models.DateTimeField(auto_now=True)
+    title = models.CharField(blank=True, max_length=1000)
+    version = models.CharField(blank=True, max_length=1000)
+
+    file = models.FileField(upload_to='Incoming', blank=True)
+    authors = models.ManyToManyField(Author, related_name='texts', blank=True)
+    languages = models.ManyToManyField(Language, related_name='texts', blank=True)
+    compilation = models.BooleanField(default=False)
+
+    class Meta:
+        ordering = ('title', )
+
+    def __unicode__(self):
+        return self.title
+
+    def getComputedPath(self):
+        authors = [a.name_sort for a in self.authors.all()]
+        author = "; ".join(authors)
+        if not author:
+            author = "Unknown Author"
+        if self.compilation:
+            author += " (Ed.)"
+        elif author.endswith("."):
+            author = author[:-1] + "_"
+        title = self.title
+        if self.version:
+            title += ".%s" % self.version
+        extension = os.path.splitext(self.file.path)[1].lower()
+        path = u"%s/%s/%s%s" % (author[0].upper(), author, title, extension)
+        return os.path.join(settings.MEDIA_ROOT, path).encode('utf-8')
+
+    def parsePath(self):
+        if self.file:
+            match = re.compile('./(.*)/(.*)').findall(self.file.name)
+            title = os.path.splitext(match[0][1])[0]
+            compilation = False
+            #FIXME: parse version
+            version = re.compile("\.(\w+)$").findall(title)
+            if version:
+                version = version[0]
+                title = title[:-(len(version) + 1)]
+            else:
+                version = ''
+            authors = match[0][0]
+            if authors.endswith("_"):
+                authors = authors[:-1] + "."
+            if authors.endswith(' (Ed.)'):
+                authors = authors[:-len(' (Ed.)')]
+                compilation = True
+            if authors !=  'Unknown Author':
+                authors = [(oxlib.normalize.normalizeName(a), a) for a in authors.split("; ")]
+            else:
+                authors = []
+            
+            self.title = title
+            self.version = version
+            self.compilation = compilation
+            self.save(rename=False)
+            for a in authors:
+                author = getAuthor(a[0], a[1])
+                self.authors.add(author)
+            self.save()
+
+    def save(self, *args, **kwargs):
+        rename = True
+        if "rename" in kwargs:
+            rename = kwargs['rename']
+            del kwargs['rename']
+        super(Text, self).save(*args, **kwargs)
+        if rename and self.file:
+            path = self.getComputedPath()
+            if path != self.file.path:
+                #FIXME: make sure path is not used by other test in system
+                if not os.path.exists(os.path.dirname(path)):
+                    os.makedirs(os.path.dirname(path))
+                old_dir = os.path.dirname(self.file.path)
+                print 'old', self.file.path
+                print 'new', path
+                os.rename(self.file.path, path)
+                #FIXME: remove old dir if its empy
+                if not glob('%s/*' % old_dir):
+                    os.rmdir(old_dir)
+                self.file.name = path[len(settings.MEDIA_ROOT) + 1:]
+                #this could be recursive!
+                self.save()
+

text/tests.py

@@ -0,0 +1,23 @@
+"""
+This file demonstrates two different styles of tests (one doctest and one
+unittest). These will both pass when you run "manage.py test".
+
+Replace these with more appropriate tests for your application.
+"""
+
+from django.test import TestCase
+
+class SimpleTest(TestCase):
+    def test_basic_addition(self):
+        """
+        Tests that 1 + 1 always equals 2.
+        """
+        self.failUnlessEqual(1 + 1, 2)
+
+__test__ = {"doctest": """
+Another way to test that 1 + 1 is equal to 2.
+
+>>> 1 + 1 == 2
+True
+"""}
+

text/views.py

@@ -0,0 +1 @@
+# Create your views here.

urls.py

@@ -0,0 +1,30 @@
+from django.conf.urls.defaults import *
+from django.conf import settings
+
+# Uncomment the next two lines to enable the admin:
+from django.contrib import admin
+admin.autodiscover()
+
+
+urlpatterns = patterns('',
+    # Example:
+    # (r'^', include('texts.foo.urls')),
+
+    # Uncomment the admin/doc line below and add 'django.contrib.admindocs' 
+    # to INSTALLED_APPS to enable admin documentation:
+    # (r'^admin/doc/', include('django.contrib.admindocs.urls')),
+
+    # Uncomment the next line to enable the admin:
+    (r'^admin/', include(admin.site.urls)),
+)
+
+if settings.DEBUG:
+    urlpatterns += patterns('',
+        (r'^%s(?P<path>.*)$' % settings.MEDIA_URL.lstrip('/'),
+                               'django.views.static.serve',
+            {'document_root': settings.MEDIA_ROOT}),
+        (r'^static/(?P<path>.*)$', 'django.views.static.serve',
+            {'document_root': settings.STATIC_ROOT}),
+    )
+
+

wsgi/django.wsgi

@@ -0,0 +1,28 @@
+# django.wsgi
+import os
+import sys
+import site
+ 
+project_module = 'texts'
+
+root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')))
+
+#using virtualenv's activate_this.py to reorder sys.path
+activate_this = os.path.join(root_dir, 'env', 'bin', 'activate_this.py')
+execfile(activate_this, dict(__file__=activate_this))
+
+sys.path.append(root_dir)
+sys.path.append(os.path.join(root_dir, project_module))
+
+#reload if this django.wsgi gets touched
+import monitor
+monitor.start(interval=1.0)
+
+monitor.track(os.path.abspath(os.path.dirname(__file__)))
+
+os.environ['DJANGO_SETTINGS_MODULE'] = project_module + '.settings'
+ 
+import django.core.handlers.wsgi
+ 
+application = django.core.handlers.wsgi.WSGIHandler()
+