add db based translations

load translations from files
and adds option to translate
string layers (i.e. keywords)

pandora/annotation/models.py

@@ -187,6 +187,7 @@ class Annotation(models.Model):
             if not delay_matches:
                 self.update_matches()
             self.update_documents()
+            self.update_translations()
 
     def update_matches(self):
         from place.models import Place
@@ -265,6 +266,15 @@ class Annotation(models.Model):
             for document in Document.objects.filter(id__in=added):
                 self.documents.add(document)
 
+    def update_translations(self):
+        from translation.models import Translation
+        layer = self.get_layer()
+        if layer.get('translate'):
+            t, created = Translation.objects.get_or_create(lang=lang, key=self.value)
+            if created:
+                t.type = Translation.CONTENT
+                t.save()
+
     def delete(self, *args, **kwargs):
         with transaction.atomic():
             super(Annotation, self).delete(*args, **kwargs)

pandora/config.0xdb.jsonc

@@ -60,6 +60,7 @@
         "canManageHome": {},
         "canManagePlacesAndEvents": {"staff": true, "admin": true},
         "canManageTitlesAndNames": {"staff": true, "admin": true},
+        "canManageTranslations": {"admin": true},
         "canManageUsers": {"staff": true, "admin": true},
         "canPlayClips": {"guest": 2, "member": 2, "friend": 4, "staff": 4, "admin": 4},
         "canPlayVideo": {"guest": 1, "member": 1, "friend": 4, "staff": 4, "admin": 4},

pandora/config.indiancinema.jsonc

@@ -62,6 +62,7 @@
         "canManageHome": {"staff": true, "admin": true},
         "canManagePlacesAndEvents": {"member": true, "researcher": true, "staff": true, "admin": true},
         "canManageTitlesAndNames": {"member": true, "researcher": true, "staff": true, "admin": true},
+        "canManageTranslations": {"admin": true},
         "canManageUsers": {"staff": true, "admin": true},
         "canPlayClips": {"guest": 3, "member": 3, "researcher": 3, "staff": 3, "admin": 3},
         "canPlayVideo": {"guest": 1, "member": 1, "researcher": 3, "staff": 3, "admin": 3},

pandora/config.padma.jsonc

@@ -60,6 +60,7 @@
         "canManageHome": {"staff": true, "admin": true},
         "canManagePlacesAndEvents": {"member": true, "staff": true, "admin": true},
         "canManageTitlesAndNames": {"member": true, "staff": true, "admin": true},
+        "canManageTranslations": {"admin": true},
         "canManageUsers": {"staff": true, "admin": true},
         "canPlayClips": {"guest": 1, "member": 1, "staff": 4, "admin": 4},
         "canPlayVideo": {"guest": 1, "member": 1, "staff": 4, "admin": 4},

pandora/config.pandora.jsonc

@@ -67,6 +67,7 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
         "canManageHome": {"staff": true, "admin": true},
         "canManagePlacesAndEvents": {"member": true, "staff": true, "admin": true},
         "canManageTitlesAndNames": {"member": true, "staff": true, "admin": true},
+        "canManageTranslations": {"admin": true},
         "canManageUsers": {"staff": true, "admin": true},
         "canPlayClips": {"guest": 1, "member": 1, "staff": 4, "admin": 4},
         "canPlayVideo": {"guest": 1, "member": 1, "staff": 4, "admin": 4},

pandora/settings.py

@@ -141,6 +141,7 @@ INSTALLED_APPS = (
     'news',
     'user',
     'urlalias',
+    'translation',
     'tv',
     'documentcollection',
     'document',

pandora/translation/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

pandora/translation/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class TranslationConfig(AppConfig):
+    name = 'translation'

pandora/translation/managers.py

@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+from django.db.models import Q, Manager
+from django.conf import settings
+
+from oxdjango.managers import get_operator
+from oxdjango.query import QuerySet
+
+keymap = {
+    'email': 'user__email',
+    'user': 'username',
+    'group': 'user__groups__name',
+    'groups': 'user__groups__name',
+}
+default_key = 'username'
+
+def parseCondition(condition, user):
+    k = condition.get('key', default_key)
+    k = keymap.get(k, k)
+    v = condition['value']
+    op = condition.get('operator')
+    if not op:
+        op = '='
+    if op.startswith('!'):
+        op = op[1:]
+        exclude = True
+    else:
+        exclude = False
+
+    if k == 'level':
+        levels = ['robot'] + settings.CONFIG['userLevels']
+        if v in levels:
+            v = levels.index(v) - 1
+        else:
+            v = 0
+        key = k + get_operator(op, 'int')
+    else:
+        key = k + get_operator(op, 'istr')
+    key = str(key)
+    q = Q(**{key: v})
+    if exclude:
+        q = ~q
+    return q
+
+def parseConditions(conditions, operator, user):
+    '''
+    conditions: [
+    ],
+    operator: "&"
+    '''
+    conn = []
+    for condition in conditions:
+        if 'conditions' in condition:
+            q = parseConditions(condition['conditions'],
+                                condition.get('operator', '&'), user)
+            if q:
+                conn.append(q)
+            pass
+        else:
+            conn.append(parseCondition(condition, user))
+    if conn:
+        q = conn[0]
+        for c in conn[1:]:
+            if operator == '|':
+                q = q | c
+            else:
+                q = q & c
+        return q
+    return None
+
+class TranslationManager(Manager):
+
+    def get_query_set(self):
+        return QuerySet(self.model)
+
+    def find(self, data, user):
+        '''
+            query: {
+                conditions: [
+                    {
+                        value: "war"
+                    }
+                    {
+                        key: "year",
+                        value: "1970-1980,
+                        operator: "!="
+                    },
+                    {
+                        key: "country",
+                        value: "f",
+                        operator: "^"
+                    }
+                ],
+                operator: "&"
+            }
+        '''
+
+        #join query with operator
+        qs = self.get_query_set()
+
+        query = data.get('query', {})
+        conditions = parseConditions(query.get('conditions', []),
+                                     query.get('operator', '&'),
+                                     user)
+        if conditions:
+            qs = qs.filter(conditions)
+        qs = qs.distinct()
+        return qs

pandora/translation/migrations/0001_initial.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.13 on 2018-08-04 15:58
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Translation',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('modified', models.DateTimeField(default=django.utils.timezone.now, editable=False)),
+                ('lang', models.CharField(max_length=8, verbose_name='language')),
+                ('key', models.CharField(max_length=4096, verbose_name='key')),
+                ('value', models.CharField(blank=True, default=None, max_length=4096, null=True, verbose_name='translation')),
+            ],
+        ),
+        migrations.AlterUniqueTogether(
+            name='translation',
+            unique_together=set([('key', 'lang')]),
+        ),
+    ]

pandora/translation/migrations/0002_translation_type.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.13 on 2018-09-19 14:40
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('translation', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='translation',
+            name='type',
+            field=models.IntegerField(default=0, verbose_name='type'),
+        ),
+    ]

pandora/translation/models.py

@@ -0,0 +1,118 @@
+import hashlib
+
+from django.conf import settings
+from django.core.cache import cache
+from django.db import models
+import django.utils.translation
+from django.utils import timezone
+
+import ox
+
+from . import managers
+
+
+def get_cache_key(key, lang):
+    return '%s-%s' % (hashlib.sha1(key.encode()).hexdigest(), lang)
+
+
+def load_itemkey_translations():
+    from annotation.models import Annotation
+    from django.db.models import QuerySet
+    used_keys = []
+    for layer in settings.CONFIG['layers']:
+        if layer.get('translate'):
+            qs = Annotation.objects.filter(layer=layer['id'])
+            query = qs.query
+            query.group_by = ['value']
+            for value in QuerySet(query=query, model=Annotation).values_list('value', flat=True):
+                for lang in settings.CONFIG['languages']:
+                    if lang == settings.CONFIG['language']:
+                        continue
+                    used_keys.append(value)
+                    t, created = Translation.objects.get_or_create(lang=lang, key=value)
+                    if created:
+                        t.type = Translation.CONTENT
+                        t.save()
+
+    Translation.objects.filter(type=Translation.CONTENT).exclude(key__in=used_keys).delete()
+
+def load_translations():
+    import os
+    import json
+    from glob import glob
+    locale = {}
+    for file in glob('%s/json/locale.??.json' % settings.STATIC_ROOT):
+        lang = file.split('.')[-2]
+        if lang not in locale:
+            locale[lang] = {}
+        with open(os.path.join(file)) as fd:
+            locale[lang].update(json.load(fd))
+    for lang, locale in locale.items():
+        used_keys = []
+        if lang in settings.CONFIG['languages']:
+            for key, value in locale.items():
+                used_keys.append(key)
+                t, created = Translation.objects.get_or_create(lang=lang, key=key)
+                if created:
+                    t.type = Translation.UI
+                    t.value = value
+                    t.save()
+        Translation.objects.filter(type=Translation.UI, lang=lang).exclude(key__in=used_keys).delete()
+
+class Translation(models.Model):
+    CONTENT = 1
+    UI = 2
+
+    created = models.DateTimeField(auto_now_add=True, editable=False)
+    modified = models.DateTimeField(default=timezone.now, editable=False)
+
+    type = models.IntegerField('type', default=0)
+    lang = models.CharField('language', max_length=8)
+    key = models.CharField('key', max_length=4096)
+    value = models.CharField('translation', max_length=4096, null=True, blank=True, default=None)
+
+    objects = managers.TranslationManager()
+
+    class Meta:
+        unique_together = ('key', 'lang')
+
+    def __str__(self):
+        return '%s->%s [%s]' % (self.key, self.value, self.lang)
+
+    def json(self, keys=None, user=None):
+        data = {
+            'id': ox.toAZ(self.id)
+        }
+        for key in ('key', 'lang', 'value'):
+            data[key] = getattr(self, key)
+
+        return data
+
+    @classmethod
+    def get_translations(cls, key):
+        return list(cls.objects.filter(key=key).order_by('-lang').values_list('lang', flat=True))
+
+    @classmethod
+    def get_translation(cls, key, lang):
+        cache_key = get_cache_key(key, lang)
+        data = cache.get(cache_key)
+        if not data:
+            trans = None
+            for translation in cls.objects.filter(key=key, lang=lang):
+                trans = translation.get_value()
+                break
+            if trans is None:
+                cls.needs_translation(key)
+                trans = key
+            cache.set(cache_key, trans, 5*60)
+            return trans
+        return data
+
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        cache.delete(get_cache_key(self.key, self.lang))
+
+    def get_value(self):
+        if self.value:
+            return self.value
+        return self.key

pandora/translation/tasks.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+from __future__ import division, print_function, absolute_import
+
+from datetime import timedelta, datetime
+
+from celery.task import task, periodic_task
+from django.conf import settings
+
+from app.utils import limit_rate
+
+@periodic_task(run_every=timedelta(days=1), queue='encoding')
+def cronjob(**kwargs):
+    if limit_rate('translations.tasks.cronjob', 8 * 60 * 60):
+        load_translations()
+
+@task(ignore_results=True, queue='encoding')
+def load_translations():
+    from .models import load_itemkey_translations, load_translations
+    load_translations()
+    load_itemkey_translations()

pandora/translation/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

pandora/translation/views.py

@@ -0,0 +1,111 @@
+from django.shortcuts import render
+
+from oxdjango.shortcuts import render_to_json_response, json_response, get_object_or_404_json
+from oxdjango.api import actions
+import ox
+
+from item import utils
+from changelog.models import add_changelog
+from .models import Translation
+
+def locale_json(request, lang):
+    locale = {}
+    for t in Translation.objects.filter(lang=lang):
+        if t.value:
+            locale[t.key] = t.value
+    return render_to_json_response(locale)
+
+def editTranslation(request, data):
+    '''
+    Edits translation for a given key and language
+    takes {
+        key: string, // name key
+        lang: string // language i.e. en
+        value: string // translated value
+    }
+    returns {
+        id: string, // name id
+        key: string // key
+        ... // more properties
+    }
+    see: findTranslations
+    '''
+    response = json_response()
+    if not data['value']:
+        Translation.objects.filter(id=ox.fromAZ(data['id'])).delete()
+    else:
+        trans, created = Translation.objects.get_or_create(id=ox.fromAZ(data['id']))
+        trans.value = data['value']
+        trans.save()
+        response['data'] = trans.json()
+    add_changelog(request, data)
+    return render_to_json_response(response)
+actions.register(editTranslation)
+
+
+def parse_query(data, user):
+    query = {}
+    query['range'] = [0, 100]
+    query['sort'] = [{'key':'key', 'operator':'+'}]
+    for key in ('keys', 'range', 'sort', 'query'):
+        if key in data:
+            query[key] = data[key]
+    query['qs'] = Translation.objects.find(query, user)
+    return query
+
+def order_query(qs, sort):
+    order_by = []
+    for e in sort:
+        operator = e['operator']
+        if operator != '-':
+            operator = ''
+        key = {}.get(e['key'], e['key'])
+        order = '%s%s' % (operator, key)
+        order_by.append(order)
+    if order_by:
+        qs = qs.order_by(*order_by, nulls_last=True)
+    return qs
+
+
+def findTranslations(request, data):
+    '''
+    Finds translations for a given query
+    takes {
+        query: object, // query object, see `find`
+        sort: [object], // list of sort objects, see `find`
+        range: [int, int], // range of results to return
+        keys: [string] // list of properties to return
+    }
+    returns {
+        items: [object] // list of translation objects
+    }
+    see: editTranslation
+    '''
+    response = json_response()
+
+    query = parse_query(data, request.user)
+    qs = order_query(query['qs'], query['sort'])
+    qs = qs.distinct()
+
+    if 'keys' in data:
+        qs = qs.select_related()
+        qs = qs[query['range'][0]:query['range'][1]]
+        response['data']['items'] = [p.json(data['keys'], request.user) for p in qs]
+    elif 'position' in query:
+        ids = [i.get_id() for i in qs]
+        data['conditions'] = data['conditions'] + {
+            'value': data['position'],
+            'key': query['sort'][0]['key'],
+            'operator': '^'
+        }
+        query = parse_query(data, request.user)
+        qs = order_query(query['qs'], query['sort'])
+        if qs.count() > 0:
+            response['data']['position'] = utils.get_positions(ids, [qs[0].public_id])[0]
+    elif 'positions' in data:
+        ids = list(qs.values_list('id', flat=True))
+        response['data']['positions'] = utils.get_positions(ids, data['positions'], decode_id=True)
+    else:
+        response['data']['items'] = qs.count()
+    return render_to_json_response(response)
+actions.register(findTranslations)

pandora/urls.py

@@ -25,6 +25,7 @@ import edit.views
 import itemlist.views
 import item.views
 import item.urls
+import translation.views
 import urlalias.views
 
 def serve_static_file(path, location, content_type):
@@ -34,6 +35,7 @@ urlpatterns = [
     # Uncomment the admin/doc line below to enable admin documentation:
     # urlurl(r'^admin/doc/', include('django.contrib.admindocs.urls')),
     url(r'^admin/', include(admin.site.urls)),
+    url(r'^api/locale.(?P<lang>.*).json$', translation.views.locale_json),
     url(r'^api/upload/text/?$', text.views.upload),
     url(r'^api/upload/document/?$', document.views.upload),
     url(r'^api/upload/direct/?$', archive.views.direct_upload),

static/js/filter.js

@@ -15,6 +15,10 @@ pandora.ui.filter = function(id) {
                     align: 'left',
                     id: 'name',
                     format: function(value) {
+                        var layer = Ox.getObjectById(pandora.site.layers, filter.id);
+                        if (layer && layer.translate) {
+                            value = Ox._(value)
+                        }
                         return filter.flag
                             ? $('<div>')
                                 .append(

static/js/mainMenu.js

@@ -232,6 +232,7 @@ pandora.ui.mainMenu = function() {
                         { id: 'events', title: Ox._('Manage Events...'), disabled: !pandora.hasCapability('canManagePlacesAndEvents') },
                         {},
                         { id: 'users', title: Ox._('Manage Users...'), disabled: !pandora.hasCapability('canManageUsers') },
+                        { id: 'translations', title: Ox._('Manage Translations...'), disabled: !pandora.hasCapability('canManageTranslations') },
                         { id: 'statistics', title: Ox._('Statistics...'), disabled: !pandora.hasCapability('canManageUsers') },
                         {},
                         { id: 'changelog', title: Ox._('Changelog...'), disabled: !pandora.hasCapability('canManageUsers') }
@@ -652,6 +653,8 @@ pandora.ui.mainMenu = function() {
                     pandora.$ui.usersDialog = pandora.ui.usersDialog().open();
                 } else if (data.id == 'statistics') {
                     pandora.$ui.statisticsDialog = pandora.ui.statisticsDialog().open();
+                } else if (data.id == 'translations') {
+                    pandora.$ui.translationsDialog = pandora.ui.translationsDialog().open();
                 } else if (data.id == 'changelog') {
                     pandora.$ui.changelogDialog = pandora.ui.changelogDialog().open();
                 } else if (data.id == 'clearcache') {

static/js/translationsDialog.js

@@ -0,0 +1,234 @@
+'use strict';
+
+pandora.ui.translationsDialog = function() {
+
+    var height = Math.round((window.innerHeight - 48) * 0.9),
+        width = 576 + Ox.UI.SCROLLBAR_SIZE,
+
+        $languageSelect = Ox.Select({
+                id: 'selectlanguage',
+                items: [{
+                    id: '',
+                    title: Ox._('All')
+                }].concat(pandora.site.languages.filter(function(lang) {
+                    return lang != 'en'
+                }).map(function(lang) {
+                    return {
+                        id: lang,
+                        title: Ox.LOCALE_NAMES[lang]
+                    }
+                })),
+                value: pandora.site.language,
+                width: 96
+
+            })
+            .css({float: 'right', margin: '4px'})
+            .bindEvent({
+                change: function(data) {
+                    var value = $findInput.options('value')
+                    var query = prepareQuery(value, data.value)
+                    $list.options({
+                        query: query,
+                    });
+                }
+            }),
+
+        $findInput = Ox.Input({
+                changeOnKeypress: true,
+                clear: true,
+                placeholder: Ox._('Find'),
+                width: 192
+            })
+            .css({float: 'right', margin: '4px'})
+            .bindEvent({
+                change: function(data) {
+                    var lang = $languageSelect.options('value')
+                    var query = prepareQuery(data.value, lang)
+                    $list.options({
+                        query: query,
+                    });
+                }
+            }),
+
+        $list = Ox.TableList({
+                columns: [
+                    {
+                        id: 'id',
+                        title: Ox._('ID'),
+                        visible: false,
+                        width: 0
+                    },
+                    {
+                        id: 'key',
+                        operator: '+',
+                        removable: false,
+                        title: Ox._('Key'),
+                        format: function(data) {
+                            return Ox.encodeHTMLEntities(data)
+                        },
+                        visible: true,
+                        width: 256
+                    },
+                    {
+                        editable: true,
+                        id: 'value',
+                        operator: '+',
+                        title: Ox._('Value'),
+                        format: function(data) {
+                            return Ox.encodeHTMLEntities(data)
+                        },
+                        tooltip: Ox._('Edit Translation'),
+                        visible: true,
+                        width: 256
+                    },
+                    {
+                        id: 'lang',
+                        align: 'right',
+                        operator: '-',
+                        title: Ox._('Language'),
+                        format: function(lang) {
+                            return Ox.LOCALE_NAMES[lang]
+                        },
+                        visible: true,
+                        width: 64
+                    },
+                ],
+                columnsVisible: true,
+                items: pandora.api.findTranslations,
+                max: 1,
+                scrollbarVisible: true,
+                sort: [{key: 'key', operator: '+'}],
+                unique: 'id'
+            })
+            .bindEvent({
+                init: function(data) {
+                    $status.html(
+                        Ox.toTitleCase(Ox.formatCount(data.items, 'translation'))
+                    );
+                },
+                open: function(data) {
+                    $list.find('.OxItem.OxSelected > .OxCell.OxColumnSortname')
+                        .trigger('mousedown')
+                        .trigger('mouseup');
+                },
+                select: function(data) {
+                },
+                submit: function(data) {
+                    Ox.Request.clearCache('findTranslations');
+                    console.log(data)
+                    pandora.api.editTranslation({
+                        id: data.id,
+                        value: data.value
+                    });
+                }
+            }),
+
+
+        that = Ox.Dialog({
+                buttons: [
+                    {},
+                    Ox.Button({
+                        title: Ox._('Done'),
+                        width: 48
+                    }).bindEvent({
+                        click: function() {
+                            that.close();
+                        }
+                    })
+                ],
+                closeButton: true,
+                content: Ox.SplitPanel({
+                    elements: [
+                        {
+                            element: Ox.Bar({size: 24})
+                                .append($status)
+                                .append(
+                                    $findInput
+                                )
+                                .append(
+                                    $languageSelect
+                                ),
+                            size: 24
+                        },
+                        {
+                            element: $list
+                        }
+                    ],
+                    orientation: 'vertical'
+                }),
+                height: height,
+                maximizeButton: true,
+                minHeight: 256,
+                minWidth: 512,
+                padding: 0,
+                title: Ox._('Manage Translations'),
+                width: width
+            })
+            .bindEvent({
+                resizeend: function(data) {
+                    var width = (data.width - 64 - Ox.UI.SCROLLBAR_SIZE) / 2;
+                    [
+                        {id: 'name', width: Math.ceil(width)},
+                        {id: 'sortname', width: Math.floor(width)}
+                    ].forEach(function(column) {
+                        $list.resizeColumn(column.id, column.width);
+                    });
+                }
+            }),
+
+        $status = $('<div>')
+            .css({
+                position: 'absolute',
+                top: '4px',
+                left: '128px',
+                right: '32px',
+                bottom: '4px',
+                paddingTop: '2px',
+                fontSize: '9px',
+                textAlign: 'center'
+            })
+            .appendTo(that.find('.OxButtonsbar'));
+
+
+    function prepareQuery(value, lang) {
+        var query;
+        if (value) {
+            query = {
+                conditions: [
+                    {
+                        key: 'key',
+                        operator: '=',
+                        value: value
+                    },
+                    {
+                        key: 'value',
+                        operator: '=',
+                        value: value
+                    }
+                ],
+                operator: '|'
+            }
+        } else {
+            query = {
+                conditions: []
+            };
+        }
+        if (lang != '') {
+            query = {
+                conditions: [
+                    query,
+                    {
+                        key: 'lang',
+                        operator: '==',
+                        value: lang
+                    }
+                ],
+                operator: '&'
+            }
+        }
+        return query;
+    }
+
+    return that;
+
+};

static/js/utils.js

@@ -3062,9 +3062,13 @@ pandora.setLocale = function(locale, callback) {
             url = [
                 '/static/json/locale.pandora.' + locale + '.json',
                 '/static/json/locale.' + pandora.site.site.id + '.' + locale + '.json',
+                '/api/locale.' + locale + '.json'
             ];
         } else {
-            url = '/static/json/locale.' + locale + '.json';
+            url = [
+                '/static/json/locale.' + locale + '.json',
+                '/api/locale.' + locale + '.json'
+            ];
         }
     }
     Ox.setLocale(locale, url, callback);