forked from 0x2620/pandora
compute statistics on backend
This commit is contained in:
j
parent
198470e6d9
commit
559140ca76
4 changed files with 169 additions and 90 deletions
134
pandora/user/statistics.py
Normal file
134
pandora/user/statistics.py
Normal file
|
|
@ -0,0 +1,134 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# vi:si:et:sw=4:sts=4:ts=4
|
||||||
|
import ox.geo
|
||||||
|
|
||||||
|
colors = {
|
||||||
|
'system': {
|
||||||
|
'Android': [0, 255, 0],
|
||||||
|
'BlackBerry': [64, 64, 64],
|
||||||
|
'BSD': [255, 0, 0],
|
||||||
|
'iOS': [0, 128, 255],
|
||||||
|
'Java': [128, 128, 128],
|
||||||
|
'Linux': [255, 128, 0],
|
||||||
|
'Mac OS X': [0, 255, 255],
|
||||||
|
'Nokia': [255, 0, 255],
|
||||||
|
'PlayStation': [192, 192, 192],
|
||||||
|
'RIM Tablet OS': [64, 64, 64],
|
||||||
|
'Unix': [255, 255, 0],
|
||||||
|
'Wii': [192, 192, 192],
|
||||||
|
'Windows Phone': [0, 0, 128], #has to be before 'Windows'
|
||||||
|
'Windows': [0, 0, 255]
|
||||||
|
},
|
||||||
|
'browser': {
|
||||||
|
'Camino': [192, 192, 192],
|
||||||
|
'Chrome Frame': [255, 255, 0], #has to be before 'Chrome'
|
||||||
|
'Chrome': [0, 255, 0],
|
||||||
|
'Chromium': [128, 255, 0],
|
||||||
|
'Epiphany': [128, 128, 128],
|
||||||
|
'Firefox': [255, 128, 0],
|
||||||
|
'Internet Explorer': [0, 0, 255],
|
||||||
|
'Konqueror': [64, 64, 64],
|
||||||
|
'Nokia Browser': [255, 0, 255],
|
||||||
|
'Opera': [255, 0, 0],
|
||||||
|
'Safari': [0, 255, 255],
|
||||||
|
'WebKit': [0, 255, 128]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def get_name(key, version):
|
||||||
|
for name in colors[key]:
|
||||||
|
if version.startswith(name):
|
||||||
|
return name
|
||||||
|
return ''
|
||||||
|
|
||||||
|
class Statistics(dict):
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
|
||||||
|
for mode in ['all', 'registered']:
|
||||||
|
self[mode] = {
|
||||||
|
"year": {},
|
||||||
|
"month": {},
|
||||||
|
"day": {},
|
||||||
|
"weekday": {},
|
||||||
|
"hour": {},
|
||||||
|
"continent": {},
|
||||||
|
"region": {},
|
||||||
|
"country": {},
|
||||||
|
"city": {},
|
||||||
|
"system": {},
|
||||||
|
"browser": {},
|
||||||
|
"systemandbrowser": {},
|
||||||
|
"systemversion": {},
|
||||||
|
"browserversion": {},
|
||||||
|
"systemandbrowserversion": {}
|
||||||
|
}
|
||||||
|
|
||||||
|
def _increment(self, d, key, add=1, base=0):
|
||||||
|
if not key in d:
|
||||||
|
d[key] = base
|
||||||
|
d[key] += add
|
||||||
|
|
||||||
|
def add(self, item):
|
||||||
|
for mode in ['all', 'registered']:
|
||||||
|
if mode == 'all' or item['level'] != 'guest':
|
||||||
|
for key in ['firstseen', 'lastseen']:
|
||||||
|
year = '%s-%s' % (item[key].strftime('%Y'), key)
|
||||||
|
month = '%s-%s' % (item[key].strftime('%Y-%m'), key)
|
||||||
|
day = '%s-%s' % (item[key].strftime('%Y-%m-%d'), key)
|
||||||
|
weekday = item[key].strftime('%u')
|
||||||
|
hour = item[key].strftime('%H')
|
||||||
|
|
||||||
|
if not year in self[mode]['year']:
|
||||||
|
self[mode]['year'][year] = {}
|
||||||
|
self._increment(self[mode]['year'][year], month, 1)
|
||||||
|
|
||||||
|
self._increment(self[mode]['month'], month, 1)
|
||||||
|
|
||||||
|
if key == 'firstseen':
|
||||||
|
if not day in self[mode]['day']:
|
||||||
|
self[mode]['day'][day] = {}
|
||||||
|
self._increment(self[mode]['day'][day], hour, 1)
|
||||||
|
|
||||||
|
if not weekday in self[mode]['weekday']:
|
||||||
|
self[mode]['weekday'][weekday] = {}
|
||||||
|
self._increment(self[mode]['weekday'][weekday], hour, 1)
|
||||||
|
|
||||||
|
self._increment(self[mode]['hour'], hour)
|
||||||
|
|
||||||
|
if item['location']:
|
||||||
|
split = ox.geo.split_geoname(item['location'])
|
||||||
|
if len(split) == 1:
|
||||||
|
split.insert(0, None)
|
||||||
|
city, country = split
|
||||||
|
|
||||||
|
country_data = ox.geo.get_country(country)
|
||||||
|
continent = country_data.get('continent','')
|
||||||
|
region = ', '.join([continent, country_data.get('region', '')])
|
||||||
|
country = ', '.join([region, country])
|
||||||
|
city = ', '.join(country, city) if city else ''
|
||||||
|
|
||||||
|
self._increment(self[mode]['continent'], continent,)
|
||||||
|
self._increment(self[mode]['region'], region)
|
||||||
|
self._increment(self[mode]['country'], country)
|
||||||
|
if city:
|
||||||
|
self._increment(self[mode]['city'], city)
|
||||||
|
|
||||||
|
name = {}
|
||||||
|
for key in ['system', 'browser']:
|
||||||
|
version = item[key];
|
||||||
|
if version:
|
||||||
|
name[key] = get_name(key, version)
|
||||||
|
if name[key]:
|
||||||
|
self._increment(self[mode][key], name[key])
|
||||||
|
|
||||||
|
key = key + 'version';
|
||||||
|
self._increment(self[mode][key], version)
|
||||||
|
|
||||||
|
if name['system'] and name['browser']:
|
||||||
|
name = name['system'] + ' / ' + name['browser']
|
||||||
|
self._increment(self[mode]['systemandbrowser'], name)
|
||||||
|
|
||||||
|
name = item['system'] + ' / ' + item['browser']
|
||||||
|
self._increment(self[mode]['systemandbrowserversion'], name)
|
||||||
|
|
||||||
|
|
@ -1,10 +1,28 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
# vi:si:et:sw=4:sts=4:ts=4
|
# vi:si:et:sw=4:sts=4:ts=4
|
||||||
|
import json
|
||||||
|
from datetime import timedelta
|
||||||
|
|
||||||
from celery.task import task
|
from celery.task import task, periodic_task
|
||||||
|
|
||||||
import models
|
import models
|
||||||
|
from app.models import Settings
|
||||||
|
from statistics import Statistics
|
||||||
|
|
||||||
|
@periodic_task(run_every=timedelta(hours=1), queue='encoding')
|
||||||
|
def cronjob(**kwargs):
|
||||||
|
update_statistics()
|
||||||
|
|
||||||
|
def update_statistics():
|
||||||
|
stats = Statistics()
|
||||||
|
ids = [i['session_key'] for i in models.SessionData.objects.all().values('session_key')]
|
||||||
|
for id in ids:
|
||||||
|
try:
|
||||||
|
u = models.SessionData.objects.get(pk=id)
|
||||||
|
stats.add(u.json())
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
Settings.set('statistics', stats)
|
||||||
|
|
||||||
@task(ignore_results=True, queue='default')
|
@task(ignore_results=True, queue='default')
|
||||||
def parse_data(key):
|
def parse_data(key):
|
||||||
|
|
|
||||||
|
|
@ -795,3 +795,17 @@ def setUI(request):
|
||||||
return render_to_json_response(response)
|
return render_to_json_response(response)
|
||||||
actions.register(setUI, cache=False)
|
actions.register(setUI, cache=False)
|
||||||
|
|
||||||
|
@capability_required_json('canManageUsers')
|
||||||
|
def statistics(request):
|
||||||
|
'''
|
||||||
|
'''
|
||||||
|
response = json_response()
|
||||||
|
from app.models import Settings
|
||||||
|
stats = Settings.get('statistics')
|
||||||
|
if not stats:
|
||||||
|
import tasks
|
||||||
|
tasks.update_stats()
|
||||||
|
stats = Settings.get('statistics')
|
||||||
|
response['data'] = stats
|
||||||
|
return render_to_json_response(response)
|
||||||
|
actions.register(statistics, cache=False)
|
||||||
|
|
|
||||||
|
|
@ -86,100 +86,13 @@ pandora.ui.statisticsDialog = function() {
|
||||||
|
|
||||||
$tabPanel;
|
$tabPanel;
|
||||||
|
|
||||||
pandora.api.findUsers({
|
pandora.api.statistics({}, function(result) {
|
||||||
keys: ['browser', 'firstseen', 'lastseen', 'level', 'location', 'system'],
|
var data = result.data,
|
||||||
query: {
|
|
||||||
conditions: [{key: 'level', value: 'robot', operator: '!='}],
|
|
||||||
operator: '&'
|
|
||||||
},
|
|
||||||
range: [0, 1000000],
|
|
||||||
sort: [{key: 'username', operator: '+'}]
|
|
||||||
}, function(result) {
|
|
||||||
|
|
||||||
var data = {},
|
|
||||||
flagCountry = {},
|
flagCountry = {},
|
||||||
$guestsCheckbox;
|
$guestsCheckbox;
|
||||||
|
|
||||||
['all', 'registered'].forEach(function(mode) {
|
['all', 'registered'].forEach(function(mode) {
|
||||||
|
|
||||||
data[mode] = {
|
|
||||||
year: {},
|
|
||||||
month: {},
|
|
||||||
day: {},
|
|
||||||
weekday: {},
|
|
||||||
hour: {},
|
|
||||||
continent: {},
|
|
||||||
region: {},
|
|
||||||
country: {},
|
|
||||||
city: {},
|
|
||||||
system: {},
|
|
||||||
browser: {},
|
|
||||||
systemandbrowser: {},
|
|
||||||
systemversion: {},
|
|
||||||
browserversion: {},
|
|
||||||
systemandbrowserversion: {}
|
|
||||||
};
|
|
||||||
|
|
||||||
result.data.items.forEach(function(item) {
|
|
||||||
var city, continent, country, countryData, name = {}, region, split;
|
|
||||||
if (mode == 'all' || item.level != 'guest') {
|
|
||||||
['firstseen', 'lastseen'].forEach(function(key, i) {
|
|
||||||
var year = Ox.formatDate(item[key], '%Y') + '-' + key,
|
|
||||||
month = Ox.formatDate(item[key], '%Y-%m') + '-' + key,
|
|
||||||
day = Ox.formatDate(item[key], '%Y-%m-%d'),
|
|
||||||
weekday = Ox.formatDate(item[key], '%u'),
|
|
||||||
hour = Ox.formatDate(item[key], '%H');
|
|
||||||
data[mode].year[year] = data[mode].year[year] || {};
|
|
||||||
data[mode].year[year][month] = (data[mode].year[year][month] || 0) + 1;
|
|
||||||
data[mode].month[month] = (data[mode].month[month] || 0) + 1;
|
|
||||||
if (key == 'firstseen') {
|
|
||||||
data[mode].day[day] = data[mode].day[day] || {};
|
|
||||||
data[mode].day[day][hour] = (data[mode].day[day][hour] || 0) + 1;
|
|
||||||
data[mode].weekday[weekday] = data[mode].weekday[weekday] || {};
|
|
||||||
data[mode].weekday[weekday][hour] = (data[mode].weekday[weekday][hour] || 0) + 1;
|
|
||||||
data[mode].hour[hour] = (data[mode].hour[hour] || 0) + 1;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
if (item.location) {
|
|
||||||
split = Ox.splitGeoname(item.location);
|
|
||||||
if (split.length == 1) {
|
|
||||||
country = split[0];
|
|
||||||
} else {
|
|
||||||
city = split[0];
|
|
||||||
country = split[1];
|
|
||||||
}
|
|
||||||
countryData = Ox.getCountryByName(country) || {continent: '', region: ''};
|
|
||||||
continent = countryData.continent;
|
|
||||||
region = [continent, countryData.region].join(', ');
|
|
||||||
country = [region, country].join(', ')
|
|
||||||
city = city ? [country, city].join(', ') : '';
|
|
||||||
data[mode].continent[continent] = (data[mode].continent[continent] || 0) + 1;
|
|
||||||
data[mode].region[region] = (data[mode].region[region] || 0) + 1;
|
|
||||||
data[mode].country[country] = (data[mode].country[country] || 0) + 1;
|
|
||||||
if (city) {
|
|
||||||
data[mode].city[city] = (data[mode].city[city] || 0) + 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
['system', 'browser'].forEach(function(key) {
|
|
||||||
var version = item[key];
|
|
||||||
if (version) {
|
|
||||||
name[key] = getName(key, version);
|
|
||||||
if (name[key]) {
|
|
||||||
data[mode][key][name[key]] = (data[mode][key][name[key]] || 0) + 1;
|
|
||||||
key = key + 'version';
|
|
||||||
data[mode][key][version] = (data[mode][key][version] || 0) + 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
if (name.system && name.browser) {
|
|
||||||
name = name.system + ' / ' + name.browser;
|
|
||||||
data[mode].systemandbrowser[name] = (data[mode].systemandbrowser[name] || 0) + 1;
|
|
||||||
name = item.system + ' / ' + item.browser;
|
|
||||||
data[mode].systemandbrowserversion[name] = (data[mode].systemandbrowserversion[name] || 0) + 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
var keys, firstKey, lastKey;
|
var keys, firstKey, lastKey;
|
||||||
keys = Object.keys(data[mode].month).map(function(key) {
|
keys = Object.keys(data[mode].month).map(function(key) {
|
||||||
return key.slice(0, 7);
|
return key.slice(0, 7);
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue