add db based translations

load translations from files
and adds option to translate
string layers (i.e. keywords)
This commit is contained in:
j 2018-08-08 10:23:05 +01:00
parent 0a4c507346
commit f93ece1ab7
22 changed files with 682 additions and 1 deletions

View file

@ -187,6 +187,7 @@ class Annotation(models.Model):
if not delay_matches:
self.update_matches()
self.update_documents()
self.update_translations()
def update_matches(self):
from place.models import Place
@ -265,6 +266,15 @@ class Annotation(models.Model):
for document in Document.objects.filter(id__in=added):
self.documents.add(document)
def update_translations(self):
from translation.models import Translation
layer = self.get_layer()
if layer.get('translate'):
t, created = Translation.objects.get_or_create(lang=lang, key=self.value)
if created:
t.type = Translation.CONTENT
t.save()
def delete(self, *args, **kwargs):
with transaction.atomic():
super(Annotation, self).delete(*args, **kwargs)

View file

@ -60,6 +60,7 @@
"canManageHome": {},
"canManagePlacesAndEvents": {"staff": true, "admin": true},
"canManageTitlesAndNames": {"staff": true, "admin": true},
"canManageTranslations": {"admin": true},
"canManageUsers": {"staff": true, "admin": true},
"canPlayClips": {"guest": 2, "member": 2, "friend": 4, "staff": 4, "admin": 4},
"canPlayVideo": {"guest": 1, "member": 1, "friend": 4, "staff": 4, "admin": 4},

View file

@ -62,6 +62,7 @@
"canManageHome": {"staff": true, "admin": true},
"canManagePlacesAndEvents": {"member": true, "researcher": true, "staff": true, "admin": true},
"canManageTitlesAndNames": {"member": true, "researcher": true, "staff": true, "admin": true},
"canManageTranslations": {"admin": true},
"canManageUsers": {"staff": true, "admin": true},
"canPlayClips": {"guest": 3, "member": 3, "researcher": 3, "staff": 3, "admin": 3},
"canPlayVideo": {"guest": 1, "member": 1, "researcher": 3, "staff": 3, "admin": 3},

View file

@ -60,6 +60,7 @@
"canManageHome": {"staff": true, "admin": true},
"canManagePlacesAndEvents": {"member": true, "staff": true, "admin": true},
"canManageTitlesAndNames": {"member": true, "staff": true, "admin": true},
"canManageTranslations": {"admin": true},
"canManageUsers": {"staff": true, "admin": true},
"canPlayClips": {"guest": 1, "member": 1, "staff": 4, "admin": 4},
"canPlayVideo": {"guest": 1, "member": 1, "staff": 4, "admin": 4},

View file

@ -67,6 +67,7 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
"canManageHome": {"staff": true, "admin": true},
"canManagePlacesAndEvents": {"member": true, "staff": true, "admin": true},
"canManageTitlesAndNames": {"member": true, "staff": true, "admin": true},
"canManageTranslations": {"admin": true},
"canManageUsers": {"staff": true, "admin": true},
"canPlayClips": {"guest": 1, "member": 1, "staff": 4, "admin": 4},
"canPlayVideo": {"guest": 1, "member": 1, "staff": 4, "admin": 4},

View file

@ -141,6 +141,7 @@ INSTALLED_APPS = (
'news',
'user',
'urlalias',
'translation',
'tv',
'documentcollection',
'document',

View file

View file

@ -0,0 +1,3 @@
from django.contrib import admin
# Register your models here.

View file

@ -0,0 +1,5 @@
from django.apps import AppConfig
class TranslationConfig(AppConfig):
name = 'translation'

View file

@ -0,0 +1,107 @@
# -*- coding: utf-8 -*-
from django.db.models import Q, Manager
from django.conf import settings
from oxdjango.managers import get_operator
from oxdjango.query import QuerySet
keymap = {
'email': 'user__email',
'user': 'username',
'group': 'user__groups__name',
'groups': 'user__groups__name',
}
default_key = 'username'
def parseCondition(condition, user):
k = condition.get('key', default_key)
k = keymap.get(k, k)
v = condition['value']
op = condition.get('operator')
if not op:
op = '='
if op.startswith('!'):
op = op[1:]
exclude = True
else:
exclude = False
if k == 'level':
levels = ['robot'] + settings.CONFIG['userLevels']
if v in levels:
v = levels.index(v) - 1
else:
v = 0
key = k + get_operator(op, 'int')
else:
key = k + get_operator(op, 'istr')
key = str(key)
q = Q(**{key: v})
if exclude:
q = ~q
return q
def parseConditions(conditions, operator, user):
'''
conditions: [
],
operator: "&"
'''
conn = []
for condition in conditions:
if 'conditions' in condition:
q = parseConditions(condition['conditions'],
condition.get('operator', '&'), user)
if q:
conn.append(q)
pass
else:
conn.append(parseCondition(condition, user))
if conn:
q = conn[0]
for c in conn[1:]:
if operator == '|':
q = q | c
else:
q = q & c
return q
return None
class TranslationManager(Manager):
def get_query_set(self):
return QuerySet(self.model)
def find(self, data, user):
'''
query: {
conditions: [
{
value: "war"
}
{
key: "year",
value: "1970-1980,
operator: "!="
},
{
key: "country",
value: "f",
operator: "^"
}
],
operator: "&"
}
'''
#join query with operator
qs = self.get_query_set()
query = data.get('query', {})
conditions = parseConditions(query.get('conditions', []),
query.get('operator', '&'),
user)
if conditions:
qs = qs.filter(conditions)
qs = qs.distinct()
return qs

View file

@ -0,0 +1,32 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.13 on 2018-08-04 15:58
from __future__ import unicode_literals
from django.db import migrations, models
import django.utils.timezone
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='Translation',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created', models.DateTimeField(auto_now_add=True)),
('modified', models.DateTimeField(default=django.utils.timezone.now, editable=False)),
('lang', models.CharField(max_length=8, verbose_name='language')),
('key', models.CharField(max_length=4096, verbose_name='key')),
('value', models.CharField(blank=True, default=None, max_length=4096, null=True, verbose_name='translation')),
],
),
migrations.AlterUniqueTogether(
name='translation',
unique_together=set([('key', 'lang')]),
),
]

View file

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.13 on 2018-09-19 14:40
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('translation', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='translation',
name='type',
field=models.IntegerField(default=0, verbose_name='type'),
),
]

View file

@ -0,0 +1,118 @@
import hashlib
from django.conf import settings
from django.core.cache import cache
from django.db import models
import django.utils.translation
from django.utils import timezone
import ox
from . import managers
def get_cache_key(key, lang):
return '%s-%s' % (hashlib.sha1(key.encode()).hexdigest(), lang)
def load_itemkey_translations():
from annotation.models import Annotation
from django.db.models import QuerySet
used_keys = []
for layer in settings.CONFIG['layers']:
if layer.get('translate'):
qs = Annotation.objects.filter(layer=layer['id'])
query = qs.query
query.group_by = ['value']
for value in QuerySet(query=query, model=Annotation).values_list('value', flat=True):
for lang in settings.CONFIG['languages']:
if lang == settings.CONFIG['language']:
continue
used_keys.append(value)
t, created = Translation.objects.get_or_create(lang=lang, key=value)
if created:
t.type = Translation.CONTENT
t.save()
Translation.objects.filter(type=Translation.CONTENT).exclude(key__in=used_keys).delete()
def load_translations():
import os
import json
from glob import glob
locale = {}
for file in glob('%s/json/locale.??.json' % settings.STATIC_ROOT):
lang = file.split('.')[-2]
if lang not in locale:
locale[lang] = {}
with open(os.path.join(file)) as fd:
locale[lang].update(json.load(fd))
for lang, locale in locale.items():
used_keys = []
if lang in settings.CONFIG['languages']:
for key, value in locale.items():
used_keys.append(key)
t, created = Translation.objects.get_or_create(lang=lang, key=key)
if created:
t.type = Translation.UI
t.value = value
t.save()
Translation.objects.filter(type=Translation.UI, lang=lang).exclude(key__in=used_keys).delete()
class Translation(models.Model):
CONTENT = 1
UI = 2
created = models.DateTimeField(auto_now_add=True, editable=False)
modified = models.DateTimeField(default=timezone.now, editable=False)
type = models.IntegerField('type', default=0)
lang = models.CharField('language', max_length=8)
key = models.CharField('key', max_length=4096)
value = models.CharField('translation', max_length=4096, null=True, blank=True, default=None)
objects = managers.TranslationManager()
class Meta:
unique_together = ('key', 'lang')
def __str__(self):
return '%s->%s [%s]' % (self.key, self.value, self.lang)
def json(self, keys=None, user=None):
data = {
'id': ox.toAZ(self.id)
}
for key in ('key', 'lang', 'value'):
data[key] = getattr(self, key)
return data
@classmethod
def get_translations(cls, key):
return list(cls.objects.filter(key=key).order_by('-lang').values_list('lang', flat=True))
@classmethod
def get_translation(cls, key, lang):
cache_key = get_cache_key(key, lang)
data = cache.get(cache_key)
if not data:
trans = None
for translation in cls.objects.filter(key=key, lang=lang):
trans = translation.get_value()
break
if trans is None:
cls.needs_translation(key)
trans = key
cache.set(cache_key, trans, 5*60)
return trans
return data
def save(self, *args, **kwargs):
super().save(*args, **kwargs)
cache.delete(get_cache_key(self.key, self.lang))
def get_value(self):
if self.value:
return self.value
return self.key

View file

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import division, print_function, absolute_import
from datetime import timedelta, datetime
from celery.task import task, periodic_task
from django.conf import settings
from app.utils import limit_rate
@periodic_task(run_every=timedelta(days=1), queue='encoding')
def cronjob(**kwargs):
if limit_rate('translations.tasks.cronjob', 8 * 60 * 60):
load_translations()
@task(ignore_results=True, queue='encoding')
def load_translations():
from .models import load_itemkey_translations, load_translations
load_translations()
load_itemkey_translations()

View file

@ -0,0 +1,3 @@
from django.test import TestCase
# Create your tests here.

View file

@ -0,0 +1,111 @@
from django.shortcuts import render
from oxdjango.shortcuts import render_to_json_response, json_response, get_object_or_404_json
from oxdjango.api import actions
import ox
from item import utils
from changelog.models import add_changelog
from .models import Translation
def locale_json(request, lang):
locale = {}
for t in Translation.objects.filter(lang=lang):
if t.value:
locale[t.key] = t.value
return render_to_json_response(locale)
def editTranslation(request, data):
'''
Edits translation for a given key and language
takes {
key: string, // name key
lang: string // language i.e. en
value: string // translated value
}
returns {
id: string, // name id
key: string // key
... // more properties
}
see: findTranslations
'''
response = json_response()
if not data['value']:
Translation.objects.filter(id=ox.fromAZ(data['id'])).delete()
else:
trans, created = Translation.objects.get_or_create(id=ox.fromAZ(data['id']))
trans.value = data['value']
trans.save()
response['data'] = trans.json()
add_changelog(request, data)
return render_to_json_response(response)
actions.register(editTranslation)
def parse_query(data, user):
query = {}
query['range'] = [0, 100]
query['sort'] = [{'key':'key', 'operator':'+'}]
for key in ('keys', 'range', 'sort', 'query'):
if key in data:
query[key] = data[key]
query['qs'] = Translation.objects.find(query, user)
return query
def order_query(qs, sort):
order_by = []
for e in sort:
operator = e['operator']
if operator != '-':
operator = ''
key = {}.get(e['key'], e['key'])
order = '%s%s' % (operator, key)
order_by.append(order)
if order_by:
qs = qs.order_by(*order_by, nulls_last=True)
return qs
def findTranslations(request, data):
'''
Finds translations for a given query
takes {
query: object, // query object, see `find`
sort: [object], // list of sort objects, see `find`
range: [int, int], // range of results to return
keys: [string] // list of properties to return
}
returns {
items: [object] // list of translation objects
}
see: editTranslation
'''
response = json_response()
query = parse_query(data, request.user)
qs = order_query(query['qs'], query['sort'])
qs = qs.distinct()
if 'keys' in data:
qs = qs.select_related()
qs = qs[query['range'][0]:query['range'][1]]
response['data']['items'] = [p.json(data['keys'], request.user) for p in qs]
elif 'position' in query:
ids = [i.get_id() for i in qs]
data['conditions'] = data['conditions'] + {
'value': data['position'],
'key': query['sort'][0]['key'],
'operator': '^'
}
query = parse_query(data, request.user)
qs = order_query(query['qs'], query['sort'])
if qs.count() > 0:
response['data']['position'] = utils.get_positions(ids, [qs[0].public_id])[0]
elif 'positions' in data:
ids = list(qs.values_list('id', flat=True))
response['data']['positions'] = utils.get_positions(ids, data['positions'], decode_id=True)
else:
response['data']['items'] = qs.count()
return render_to_json_response(response)
actions.register(findTranslations)

View file

@ -25,6 +25,7 @@ import edit.views
import itemlist.views
import item.views
import item.urls
import translation.views
import urlalias.views
def serve_static_file(path, location, content_type):
@ -34,6 +35,7 @@ urlpatterns = [
# Uncomment the admin/doc line below to enable admin documentation:
# urlurl(r'^admin/doc/', include('django.contrib.admindocs.urls')),
url(r'^admin/', include(admin.site.urls)),
url(r'^api/locale.(?P<lang>.*).json$', translation.views.locale_json),
url(r'^api/upload/text/?$', text.views.upload),
url(r'^api/upload/document/?$', document.views.upload),
url(r'^api/upload/direct/?$', archive.views.direct_upload),

View file

@ -15,6 +15,10 @@ pandora.ui.filter = function(id) {
align: 'left',
id: 'name',
format: function(value) {
var layer = Ox.getObjectById(pandora.site.layers, filter.id);
if (layer && layer.translate) {
value = Ox._(value)
}
return filter.flag
? $('<div>')
.append(

View file

@ -232,6 +232,7 @@ pandora.ui.mainMenu = function() {
{ id: 'events', title: Ox._('Manage Events...'), disabled: !pandora.hasCapability('canManagePlacesAndEvents') },
{},
{ id: 'users', title: Ox._('Manage Users...'), disabled: !pandora.hasCapability('canManageUsers') },
{ id: 'translations', title: Ox._('Manage Translations...'), disabled: !pandora.hasCapability('canManageTranslations') },
{ id: 'statistics', title: Ox._('Statistics...'), disabled: !pandora.hasCapability('canManageUsers') },
{},
{ id: 'changelog', title: Ox._('Changelog...'), disabled: !pandora.hasCapability('canManageUsers') }
@ -652,6 +653,8 @@ pandora.ui.mainMenu = function() {
pandora.$ui.usersDialog = pandora.ui.usersDialog().open();
} else if (data.id == 'statistics') {
pandora.$ui.statisticsDialog = pandora.ui.statisticsDialog().open();
} else if (data.id == 'translations') {
pandora.$ui.translationsDialog = pandora.ui.translationsDialog().open();
} else if (data.id == 'changelog') {
pandora.$ui.changelogDialog = pandora.ui.changelogDialog().open();
} else if (data.id == 'clearcache') {

View file

@ -0,0 +1,234 @@
'use strict';
pandora.ui.translationsDialog = function() {
var height = Math.round((window.innerHeight - 48) * 0.9),
width = 576 + Ox.UI.SCROLLBAR_SIZE,
$languageSelect = Ox.Select({
id: 'selectlanguage',
items: [{
id: '',
title: Ox._('All')
}].concat(pandora.site.languages.filter(function(lang) {
return lang != 'en'
}).map(function(lang) {
return {
id: lang,
title: Ox.LOCALE_NAMES[lang]
}
})),
value: pandora.site.language,
width: 96
})
.css({float: 'right', margin: '4px'})
.bindEvent({
change: function(data) {
var value = $findInput.options('value')
var query = prepareQuery(value, data.value)
$list.options({
query: query,
});
}
}),
$findInput = Ox.Input({
changeOnKeypress: true,
clear: true,
placeholder: Ox._('Find'),
width: 192
})
.css({float: 'right', margin: '4px'})
.bindEvent({
change: function(data) {
var lang = $languageSelect.options('value')
var query = prepareQuery(data.value, lang)
$list.options({
query: query,
});
}
}),
$list = Ox.TableList({
columns: [
{
id: 'id',
title: Ox._('ID'),
visible: false,
width: 0
},
{
id: 'key',
operator: '+',
removable: false,
title: Ox._('Key'),
format: function(data) {
return Ox.encodeHTMLEntities(data)
},
visible: true,
width: 256
},
{
editable: true,
id: 'value',
operator: '+',
title: Ox._('Value'),
format: function(data) {
return Ox.encodeHTMLEntities(data)
},
tooltip: Ox._('Edit Translation'),
visible: true,
width: 256
},
{
id: 'lang',
align: 'right',
operator: '-',
title: Ox._('Language'),
format: function(lang) {
return Ox.LOCALE_NAMES[lang]
},
visible: true,
width: 64
},
],
columnsVisible: true,
items: pandora.api.findTranslations,
max: 1,
scrollbarVisible: true,
sort: [{key: 'key', operator: '+'}],
unique: 'id'
})
.bindEvent({
init: function(data) {
$status.html(
Ox.toTitleCase(Ox.formatCount(data.items, 'translation'))
);
},
open: function(data) {
$list.find('.OxItem.OxSelected > .OxCell.OxColumnSortname')
.trigger('mousedown')
.trigger('mouseup');
},
select: function(data) {
},
submit: function(data) {
Ox.Request.clearCache('findTranslations');
console.log(data)
pandora.api.editTranslation({
id: data.id,
value: data.value
});
}
}),
that = Ox.Dialog({
buttons: [
{},
Ox.Button({
title: Ox._('Done'),
width: 48
}).bindEvent({
click: function() {
that.close();
}
})
],
closeButton: true,
content: Ox.SplitPanel({
elements: [
{
element: Ox.Bar({size: 24})
.append($status)
.append(
$findInput
)
.append(
$languageSelect
),
size: 24
},
{
element: $list
}
],
orientation: 'vertical'
}),
height: height,
maximizeButton: true,
minHeight: 256,
minWidth: 512,
padding: 0,
title: Ox._('Manage Translations'),
width: width
})
.bindEvent({
resizeend: function(data) {
var width = (data.width - 64 - Ox.UI.SCROLLBAR_SIZE) / 2;
[
{id: 'name', width: Math.ceil(width)},
{id: 'sortname', width: Math.floor(width)}
].forEach(function(column) {
$list.resizeColumn(column.id, column.width);
});
}
}),
$status = $('<div>')
.css({
position: 'absolute',
top: '4px',
left: '128px',
right: '32px',
bottom: '4px',
paddingTop: '2px',
fontSize: '9px',
textAlign: 'center'
})
.appendTo(that.find('.OxButtonsbar'));
function prepareQuery(value, lang) {
var query;
if (value) {
query = {
conditions: [
{
key: 'key',
operator: '=',
value: value
},
{
key: 'value',
operator: '=',
value: value
}
],
operator: '|'
}
} else {
query = {
conditions: []
};
}
if (lang != '') {
query = {
conditions: [
query,
{
key: 'lang',
operator: '==',
value: lang
}
],
operator: '&'
}
}
return query;
}
return that;
};

View file

@ -3062,9 +3062,13 @@ pandora.setLocale = function(locale, callback) {
url = [
'/static/json/locale.pandora.' + locale + '.json',
'/static/json/locale.' + pandora.site.site.id + '.' + locale + '.json',
'/api/locale.' + locale + '.json'
];
} else {
url = '/static/json/locale.' + locale + '.json';
url = [
'/static/json/locale.' + locale + '.json',
'/api/locale.' + locale + '.json'
];
}
}
Ox.setLocale(locale, url, callback);