compute statistics on backend

This commit is contained in:
j 2013-10-22 15:58:47 +00:00
parent 198470e6d9
commit 559140ca76
4 changed files with 169 additions and 90 deletions

134
pandora/user/statistics.py Normal file
View file

@ -0,0 +1,134 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import ox.geo
colors = {
'system': {
'Android': [0, 255, 0],
'BlackBerry': [64, 64, 64],
'BSD': [255, 0, 0],
'iOS': [0, 128, 255],
'Java': [128, 128, 128],
'Linux': [255, 128, 0],
'Mac OS X': [0, 255, 255],
'Nokia': [255, 0, 255],
'PlayStation': [192, 192, 192],
'RIM Tablet OS': [64, 64, 64],
'Unix': [255, 255, 0],
'Wii': [192, 192, 192],
'Windows Phone': [0, 0, 128], #has to be before 'Windows'
'Windows': [0, 0, 255]
},
'browser': {
'Camino': [192, 192, 192],
'Chrome Frame': [255, 255, 0], #has to be before 'Chrome'
'Chrome': [0, 255, 0],
'Chromium': [128, 255, 0],
'Epiphany': [128, 128, 128],
'Firefox': [255, 128, 0],
'Internet Explorer': [0, 0, 255],
'Konqueror': [64, 64, 64],
'Nokia Browser': [255, 0, 255],
'Opera': [255, 0, 0],
'Safari': [0, 255, 255],
'WebKit': [0, 255, 128]
}
}
def get_name(key, version):
for name in colors[key]:
if version.startswith(name):
return name
return ''
class Statistics(dict):
def __init__(self):
for mode in ['all', 'registered']:
self[mode] = {
"year": {},
"month": {},
"day": {},
"weekday": {},
"hour": {},
"continent": {},
"region": {},
"country": {},
"city": {},
"system": {},
"browser": {},
"systemandbrowser": {},
"systemversion": {},
"browserversion": {},
"systemandbrowserversion": {}
}
def _increment(self, d, key, add=1, base=0):
if not key in d:
d[key] = base
d[key] += add
def add(self, item):
for mode in ['all', 'registered']:
if mode == 'all' or item['level'] != 'guest':
for key in ['firstseen', 'lastseen']:
year = '%s-%s' % (item[key].strftime('%Y'), key)
month = '%s-%s' % (item[key].strftime('%Y-%m'), key)
day = '%s-%s' % (item[key].strftime('%Y-%m-%d'), key)
weekday = item[key].strftime('%u')
hour = item[key].strftime('%H')
if not year in self[mode]['year']:
self[mode]['year'][year] = {}
self._increment(self[mode]['year'][year], month, 1)
self._increment(self[mode]['month'], month, 1)
if key == 'firstseen':
if not day in self[mode]['day']:
self[mode]['day'][day] = {}
self._increment(self[mode]['day'][day], hour, 1)
if not weekday in self[mode]['weekday']:
self[mode]['weekday'][weekday] = {}
self._increment(self[mode]['weekday'][weekday], hour, 1)
self._increment(self[mode]['hour'], hour)
if item['location']:
split = ox.geo.split_geoname(item['location'])
if len(split) == 1:
split.insert(0, None)
city, country = split
country_data = ox.geo.get_country(country)
continent = country_data.get('continent','')
region = ', '.join([continent, country_data.get('region', '')])
country = ', '.join([region, country])
city = ', '.join(country, city) if city else ''
self._increment(self[mode]['continent'], continent,)
self._increment(self[mode]['region'], region)
self._increment(self[mode]['country'], country)
if city:
self._increment(self[mode]['city'], city)
name = {}
for key in ['system', 'browser']:
version = item[key];
if version:
name[key] = get_name(key, version)
if name[key]:
self._increment(self[mode][key], name[key])
key = key + 'version';
self._increment(self[mode][key], version)
if name['system'] and name['browser']:
name = name['system'] + ' / ' + name['browser']
self._increment(self[mode]['systemandbrowser'], name)
name = item['system'] + ' / ' + item['browser']
self._increment(self[mode]['systemandbrowserversion'], name)

View file

@ -1,10 +1,28 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import json
from datetime import timedelta
from celery.task import task
from celery.task import task, periodic_task
import models
from app.models import Settings
from statistics import Statistics
@periodic_task(run_every=timedelta(hours=1), queue='encoding')
def cronjob(**kwargs):
update_statistics()
def update_statistics():
stats = Statistics()
ids = [i['session_key'] for i in models.SessionData.objects.all().values('session_key')]
for id in ids:
try:
u = models.SessionData.objects.get(pk=id)
stats.add(u.json())
except:
pass
Settings.set('statistics', stats)
@task(ignore_results=True, queue='default')
def parse_data(key):

View file

@ -795,3 +795,17 @@ def setUI(request):
return render_to_json_response(response)
actions.register(setUI, cache=False)
@capability_required_json('canManageUsers')
def statistics(request):
'''
'''
response = json_response()
from app.models import Settings
stats = Settings.get('statistics')
if not stats:
import tasks
tasks.update_stats()
stats = Settings.get('statistics')
response['data'] = stats
return render_to_json_response(response)
actions.register(statistics, cache=False)

View file

@ -86,100 +86,13 @@ pandora.ui.statisticsDialog = function() {
$tabPanel;
pandora.api.findUsers({
keys: ['browser', 'firstseen', 'lastseen', 'level', 'location', 'system'],
query: {
conditions: [{key: 'level', value: 'robot', operator: '!='}],
operator: '&'
},
range: [0, 1000000],
sort: [{key: 'username', operator: '+'}]
}, function(result) {
var data = {},
pandora.api.statistics({}, function(result) {
var data = result.data,
flagCountry = {},
$guestsCheckbox;
['all', 'registered'].forEach(function(mode) {
data[mode] = {
year: {},
month: {},
day: {},
weekday: {},
hour: {},
continent: {},
region: {},
country: {},
city: {},
system: {},
browser: {},
systemandbrowser: {},
systemversion: {},
browserversion: {},
systemandbrowserversion: {}
};
result.data.items.forEach(function(item) {
var city, continent, country, countryData, name = {}, region, split;
if (mode == 'all' || item.level != 'guest') {
['firstseen', 'lastseen'].forEach(function(key, i) {
var year = Ox.formatDate(item[key], '%Y') + '-' + key,
month = Ox.formatDate(item[key], '%Y-%m') + '-' + key,
day = Ox.formatDate(item[key], '%Y-%m-%d'),
weekday = Ox.formatDate(item[key], '%u'),
hour = Ox.formatDate(item[key], '%H');
data[mode].year[year] = data[mode].year[year] || {};
data[mode].year[year][month] = (data[mode].year[year][month] || 0) + 1;
data[mode].month[month] = (data[mode].month[month] || 0) + 1;
if (key == 'firstseen') {
data[mode].day[day] = data[mode].day[day] || {};
data[mode].day[day][hour] = (data[mode].day[day][hour] || 0) + 1;
data[mode].weekday[weekday] = data[mode].weekday[weekday] || {};
data[mode].weekday[weekday][hour] = (data[mode].weekday[weekday][hour] || 0) + 1;
data[mode].hour[hour] = (data[mode].hour[hour] || 0) + 1;
}
});
if (item.location) {
split = Ox.splitGeoname(item.location);
if (split.length == 1) {
country = split[0];
} else {
city = split[0];
country = split[1];
}
countryData = Ox.getCountryByName(country) || {continent: '', region: ''};
continent = countryData.continent;
region = [continent, countryData.region].join(', ');
country = [region, country].join(', ')
city = city ? [country, city].join(', ') : '';
data[mode].continent[continent] = (data[mode].continent[continent] || 0) + 1;
data[mode].region[region] = (data[mode].region[region] || 0) + 1;
data[mode].country[country] = (data[mode].country[country] || 0) + 1;
if (city) {
data[mode].city[city] = (data[mode].city[city] || 0) + 1;
}
}
['system', 'browser'].forEach(function(key) {
var version = item[key];
if (version) {
name[key] = getName(key, version);
if (name[key]) {
data[mode][key][name[key]] = (data[mode][key][name[key]] || 0) + 1;
key = key + 'version';
data[mode][key][version] = (data[mode][key][version] || 0) + 1;
}
}
});
if (name.system && name.browser) {
name = name.system + ' / ' + name.browser;
data[mode].systemandbrowser[name] = (data[mode].systemandbrowser[name] || 0) + 1;
name = item.system + ' / ' + item.browser;
data[mode].systemandbrowserversion[name] = (data[mode].systemandbrowserversion[name] || 0) + 1;
}
}
});
var keys, firstKey, lastKey;
keys = Object.keys(data[mode].month).map(function(key) {
return key.slice(0, 7);