forked from 0x2620/pandora
compute statistics on backend
This commit is contained in:
j
parent
198470e6d9
commit
559140ca76
4 changed files with 169 additions and 90 deletions
134
pandora/user/statistics.py
Normal file
134
pandora/user/statistics.py
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# vi:si:et:sw=4:sts=4:ts=4
|
||||
import ox.geo
|
||||
|
||||
colors = {
|
||||
'system': {
|
||||
'Android': [0, 255, 0],
|
||||
'BlackBerry': [64, 64, 64],
|
||||
'BSD': [255, 0, 0],
|
||||
'iOS': [0, 128, 255],
|
||||
'Java': [128, 128, 128],
|
||||
'Linux': [255, 128, 0],
|
||||
'Mac OS X': [0, 255, 255],
|
||||
'Nokia': [255, 0, 255],
|
||||
'PlayStation': [192, 192, 192],
|
||||
'RIM Tablet OS': [64, 64, 64],
|
||||
'Unix': [255, 255, 0],
|
||||
'Wii': [192, 192, 192],
|
||||
'Windows Phone': [0, 0, 128], #has to be before 'Windows'
|
||||
'Windows': [0, 0, 255]
|
||||
},
|
||||
'browser': {
|
||||
'Camino': [192, 192, 192],
|
||||
'Chrome Frame': [255, 255, 0], #has to be before 'Chrome'
|
||||
'Chrome': [0, 255, 0],
|
||||
'Chromium': [128, 255, 0],
|
||||
'Epiphany': [128, 128, 128],
|
||||
'Firefox': [255, 128, 0],
|
||||
'Internet Explorer': [0, 0, 255],
|
||||
'Konqueror': [64, 64, 64],
|
||||
'Nokia Browser': [255, 0, 255],
|
||||
'Opera': [255, 0, 0],
|
||||
'Safari': [0, 255, 255],
|
||||
'WebKit': [0, 255, 128]
|
||||
}
|
||||
}
|
||||
|
||||
def get_name(key, version):
|
||||
for name in colors[key]:
|
||||
if version.startswith(name):
|
||||
return name
|
||||
return ''
|
||||
|
||||
class Statistics(dict):
|
||||
|
||||
def __init__(self):
|
||||
|
||||
for mode in ['all', 'registered']:
|
||||
self[mode] = {
|
||||
"year": {},
|
||||
"month": {},
|
||||
"day": {},
|
||||
"weekday": {},
|
||||
"hour": {},
|
||||
"continent": {},
|
||||
"region": {},
|
||||
"country": {},
|
||||
"city": {},
|
||||
"system": {},
|
||||
"browser": {},
|
||||
"systemandbrowser": {},
|
||||
"systemversion": {},
|
||||
"browserversion": {},
|
||||
"systemandbrowserversion": {}
|
||||
}
|
||||
|
||||
def _increment(self, d, key, add=1, base=0):
|
||||
if not key in d:
|
||||
d[key] = base
|
||||
d[key] += add
|
||||
|
||||
def add(self, item):
|
||||
for mode in ['all', 'registered']:
|
||||
if mode == 'all' or item['level'] != 'guest':
|
||||
for key in ['firstseen', 'lastseen']:
|
||||
year = '%s-%s' % (item[key].strftime('%Y'), key)
|
||||
month = '%s-%s' % (item[key].strftime('%Y-%m'), key)
|
||||
day = '%s-%s' % (item[key].strftime('%Y-%m-%d'), key)
|
||||
weekday = item[key].strftime('%u')
|
||||
hour = item[key].strftime('%H')
|
||||
|
||||
if not year in self[mode]['year']:
|
||||
self[mode]['year'][year] = {}
|
||||
self._increment(self[mode]['year'][year], month, 1)
|
||||
|
||||
self._increment(self[mode]['month'], month, 1)
|
||||
|
||||
if key == 'firstseen':
|
||||
if not day in self[mode]['day']:
|
||||
self[mode]['day'][day] = {}
|
||||
self._increment(self[mode]['day'][day], hour, 1)
|
||||
|
||||
if not weekday in self[mode]['weekday']:
|
||||
self[mode]['weekday'][weekday] = {}
|
||||
self._increment(self[mode]['weekday'][weekday], hour, 1)
|
||||
|
||||
self._increment(self[mode]['hour'], hour)
|
||||
|
||||
if item['location']:
|
||||
split = ox.geo.split_geoname(item['location'])
|
||||
if len(split) == 1:
|
||||
split.insert(0, None)
|
||||
city, country = split
|
||||
|
||||
country_data = ox.geo.get_country(country)
|
||||
continent = country_data.get('continent','')
|
||||
region = ', '.join([continent, country_data.get('region', '')])
|
||||
country = ', '.join([region, country])
|
||||
city = ', '.join(country, city) if city else ''
|
||||
|
||||
self._increment(self[mode]['continent'], continent,)
|
||||
self._increment(self[mode]['region'], region)
|
||||
self._increment(self[mode]['country'], country)
|
||||
if city:
|
||||
self._increment(self[mode]['city'], city)
|
||||
|
||||
name = {}
|
||||
for key in ['system', 'browser']:
|
||||
version = item[key];
|
||||
if version:
|
||||
name[key] = get_name(key, version)
|
||||
if name[key]:
|
||||
self._increment(self[mode][key], name[key])
|
||||
|
||||
key = key + 'version';
|
||||
self._increment(self[mode][key], version)
|
||||
|
||||
if name['system'] and name['browser']:
|
||||
name = name['system'] + ' / ' + name['browser']
|
||||
self._increment(self[mode]['systemandbrowser'], name)
|
||||
|
||||
name = item['system'] + ' / ' + item['browser']
|
||||
self._increment(self[mode]['systemandbrowserversion'], name)
|
||||
|
||||
|
|
@ -1,10 +1,28 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# vi:si:et:sw=4:sts=4:ts=4
|
||||
import json
|
||||
from datetime import timedelta
|
||||
|
||||
from celery.task import task
|
||||
from celery.task import task, periodic_task
|
||||
|
||||
import models
|
||||
from app.models import Settings
|
||||
from statistics import Statistics
|
||||
|
||||
@periodic_task(run_every=timedelta(hours=1), queue='encoding')
|
||||
def cronjob(**kwargs):
|
||||
update_statistics()
|
||||
|
||||
def update_statistics():
|
||||
stats = Statistics()
|
||||
ids = [i['session_key'] for i in models.SessionData.objects.all().values('session_key')]
|
||||
for id in ids:
|
||||
try:
|
||||
u = models.SessionData.objects.get(pk=id)
|
||||
stats.add(u.json())
|
||||
except:
|
||||
pass
|
||||
Settings.set('statistics', stats)
|
||||
|
||||
@task(ignore_results=True, queue='default')
|
||||
def parse_data(key):
|
||||
|
|
|
|||
|
|
@ -795,3 +795,17 @@ def setUI(request):
|
|||
return render_to_json_response(response)
|
||||
actions.register(setUI, cache=False)
|
||||
|
||||
@capability_required_json('canManageUsers')
|
||||
def statistics(request):
|
||||
'''
|
||||
'''
|
||||
response = json_response()
|
||||
from app.models import Settings
|
||||
stats = Settings.get('statistics')
|
||||
if not stats:
|
||||
import tasks
|
||||
tasks.update_stats()
|
||||
stats = Settings.get('statistics')
|
||||
response['data'] = stats
|
||||
return render_to_json_response(response)
|
||||
actions.register(statistics, cache=False)
|
||||
|
|
|
|||
|
|
@ -86,100 +86,13 @@ pandora.ui.statisticsDialog = function() {
|
|||
|
||||
$tabPanel;
|
||||
|
||||
pandora.api.findUsers({
|
||||
keys: ['browser', 'firstseen', 'lastseen', 'level', 'location', 'system'],
|
||||
query: {
|
||||
conditions: [{key: 'level', value: 'robot', operator: '!='}],
|
||||
operator: '&'
|
||||
},
|
||||
range: [0, 1000000],
|
||||
sort: [{key: 'username', operator: '+'}]
|
||||
}, function(result) {
|
||||
|
||||
var data = {},
|
||||
pandora.api.statistics({}, function(result) {
|
||||
var data = result.data,
|
||||
flagCountry = {},
|
||||
$guestsCheckbox;
|
||||
|
||||
['all', 'registered'].forEach(function(mode) {
|
||||
|
||||
data[mode] = {
|
||||
year: {},
|
||||
month: {},
|
||||
day: {},
|
||||
weekday: {},
|
||||
hour: {},
|
||||
continent: {},
|
||||
region: {},
|
||||
country: {},
|
||||
city: {},
|
||||
system: {},
|
||||
browser: {},
|
||||
systemandbrowser: {},
|
||||
systemversion: {},
|
||||
browserversion: {},
|
||||
systemandbrowserversion: {}
|
||||
};
|
||||
|
||||
result.data.items.forEach(function(item) {
|
||||
var city, continent, country, countryData, name = {}, region, split;
|
||||
if (mode == 'all' || item.level != 'guest') {
|
||||
['firstseen', 'lastseen'].forEach(function(key, i) {
|
||||
var year = Ox.formatDate(item[key], '%Y') + '-' + key,
|
||||
month = Ox.formatDate(item[key], '%Y-%m') + '-' + key,
|
||||
day = Ox.formatDate(item[key], '%Y-%m-%d'),
|
||||
weekday = Ox.formatDate(item[key], '%u'),
|
||||
hour = Ox.formatDate(item[key], '%H');
|
||||
data[mode].year[year] = data[mode].year[year] || {};
|
||||
data[mode].year[year][month] = (data[mode].year[year][month] || 0) + 1;
|
||||
data[mode].month[month] = (data[mode].month[month] || 0) + 1;
|
||||
if (key == 'firstseen') {
|
||||
data[mode].day[day] = data[mode].day[day] || {};
|
||||
data[mode].day[day][hour] = (data[mode].day[day][hour] || 0) + 1;
|
||||
data[mode].weekday[weekday] = data[mode].weekday[weekday] || {};
|
||||
data[mode].weekday[weekday][hour] = (data[mode].weekday[weekday][hour] || 0) + 1;
|
||||
data[mode].hour[hour] = (data[mode].hour[hour] || 0) + 1;
|
||||
}
|
||||
});
|
||||
if (item.location) {
|
||||
split = Ox.splitGeoname(item.location);
|
||||
if (split.length == 1) {
|
||||
country = split[0];
|
||||
} else {
|
||||
city = split[0];
|
||||
country = split[1];
|
||||
}
|
||||
countryData = Ox.getCountryByName(country) || {continent: '', region: ''};
|
||||
continent = countryData.continent;
|
||||
region = [continent, countryData.region].join(', ');
|
||||
country = [region, country].join(', ')
|
||||
city = city ? [country, city].join(', ') : '';
|
||||
data[mode].continent[continent] = (data[mode].continent[continent] || 0) + 1;
|
||||
data[mode].region[region] = (data[mode].region[region] || 0) + 1;
|
||||
data[mode].country[country] = (data[mode].country[country] || 0) + 1;
|
||||
if (city) {
|
||||
data[mode].city[city] = (data[mode].city[city] || 0) + 1;
|
||||
}
|
||||
}
|
||||
['system', 'browser'].forEach(function(key) {
|
||||
var version = item[key];
|
||||
if (version) {
|
||||
name[key] = getName(key, version);
|
||||
if (name[key]) {
|
||||
data[mode][key][name[key]] = (data[mode][key][name[key]] || 0) + 1;
|
||||
key = key + 'version';
|
||||
data[mode][key][version] = (data[mode][key][version] || 0) + 1;
|
||||
}
|
||||
}
|
||||
});
|
||||
if (name.system && name.browser) {
|
||||
name = name.system + ' / ' + name.browser;
|
||||
data[mode].systemandbrowser[name] = (data[mode].systemandbrowser[name] || 0) + 1;
|
||||
name = item.system + ' / ' + item.browser;
|
||||
data[mode].systemandbrowserversion[name] = (data[mode].systemandbrowserversion[name] || 0) + 1;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
var keys, firstKey, lastKey;
|
||||
keys = Object.keys(data[mode].month).map(function(key) {
|
||||
return key.slice(0, 7);
|
||||
|
|
|
|||
Loading…
Reference in a new issue