From 559140ca7677192baa9aa52f43bf7283879ac8a0 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Tue, 22 Oct 2013 15:58:47 +0000 Subject: [PATCH] compute statistics on backend --- pandora/user/statistics.py | 134 ++++++++++++++++++++++++++++++++++ pandora/user/tasks.py | 20 ++++- pandora/user/views.py | 14 ++++ static/js/statisticsDialog.js | 91 +---------------------- 4 files changed, 169 insertions(+), 90 deletions(-) create mode 100644 pandora/user/statistics.py diff --git a/pandora/user/statistics.py b/pandora/user/statistics.py new file mode 100644 index 000000000..3e4620a27 --- /dev/null +++ b/pandora/user/statistics.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +import ox.geo + +colors = { + 'system': { + 'Android': [0, 255, 0], + 'BlackBerry': [64, 64, 64], + 'BSD': [255, 0, 0], + 'iOS': [0, 128, 255], + 'Java': [128, 128, 128], + 'Linux': [255, 128, 0], + 'Mac OS X': [0, 255, 255], + 'Nokia': [255, 0, 255], + 'PlayStation': [192, 192, 192], + 'RIM Tablet OS': [64, 64, 64], + 'Unix': [255, 255, 0], + 'Wii': [192, 192, 192], + 'Windows Phone': [0, 0, 128], #has to be before 'Windows' + 'Windows': [0, 0, 255] + }, + 'browser': { + 'Camino': [192, 192, 192], + 'Chrome Frame': [255, 255, 0], #has to be before 'Chrome' + 'Chrome': [0, 255, 0], + 'Chromium': [128, 255, 0], + 'Epiphany': [128, 128, 128], + 'Firefox': [255, 128, 0], + 'Internet Explorer': [0, 0, 255], + 'Konqueror': [64, 64, 64], + 'Nokia Browser': [255, 0, 255], + 'Opera': [255, 0, 0], + 'Safari': [0, 255, 255], + 'WebKit': [0, 255, 128] + } +} + +def get_name(key, version): + for name in colors[key]: + if version.startswith(name): + return name + return '' + +class Statistics(dict): + + def __init__(self): + + for mode in ['all', 'registered']: + self[mode] = { + "year": {}, + "month": {}, + "day": {}, + "weekday": {}, + "hour": {}, + "continent": {}, + "region": {}, + "country": {}, + "city": {}, + "system": {}, + "browser": {}, + "systemandbrowser": {}, + "systemversion": {}, + "browserversion": {}, + "systemandbrowserversion": {} + } + + def _increment(self, d, key, add=1, base=0): + if not key in d: + d[key] = base + d[key] += add + + def add(self, item): + for mode in ['all', 'registered']: + if mode == 'all' or item['level'] != 'guest': + for key in ['firstseen', 'lastseen']: + year = '%s-%s' % (item[key].strftime('%Y'), key) + month = '%s-%s' % (item[key].strftime('%Y-%m'), key) + day = '%s-%s' % (item[key].strftime('%Y-%m-%d'), key) + weekday = item[key].strftime('%u') + hour = item[key].strftime('%H') + + if not year in self[mode]['year']: + self[mode]['year'][year] = {} + self._increment(self[mode]['year'][year], month, 1) + + self._increment(self[mode]['month'], month, 1) + + if key == 'firstseen': + if not day in self[mode]['day']: + self[mode]['day'][day] = {} + self._increment(self[mode]['day'][day], hour, 1) + + if not weekday in self[mode]['weekday']: + self[mode]['weekday'][weekday] = {} + self._increment(self[mode]['weekday'][weekday], hour, 1) + + self._increment(self[mode]['hour'], hour) + + if item['location']: + split = ox.geo.split_geoname(item['location']) + if len(split) == 1: + split.insert(0, None) + city, country = split + + country_data = ox.geo.get_country(country) + continent = country_data.get('continent','') + region = ', '.join([continent, country_data.get('region', '')]) + country = ', '.join([region, country]) + city = ', '.join(country, city) if city else '' + + self._increment(self[mode]['continent'], continent,) + self._increment(self[mode]['region'], region) + self._increment(self[mode]['country'], country) + if city: + self._increment(self[mode]['city'], city) + + name = {} + for key in ['system', 'browser']: + version = item[key]; + if version: + name[key] = get_name(key, version) + if name[key]: + self._increment(self[mode][key], name[key]) + + key = key + 'version'; + self._increment(self[mode][key], version) + + if name['system'] and name['browser']: + name = name['system'] + ' / ' + name['browser'] + self._increment(self[mode]['systemandbrowser'], name) + + name = item['system'] + ' / ' + item['browser'] + self._increment(self[mode]['systemandbrowserversion'], name) + diff --git a/pandora/user/tasks.py b/pandora/user/tasks.py index b8825aaee..daec15366 100644 --- a/pandora/user/tasks.py +++ b/pandora/user/tasks.py @@ -1,10 +1,28 @@ # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 +import json +from datetime import timedelta -from celery.task import task +from celery.task import task, periodic_task import models +from app.models import Settings +from statistics import Statistics +@periodic_task(run_every=timedelta(hours=1), queue='encoding') +def cronjob(**kwargs): + update_statistics() + +def update_statistics(): + stats = Statistics() + ids = [i['session_key'] for i in models.SessionData.objects.all().values('session_key')] + for id in ids: + try: + u = models.SessionData.objects.get(pk=id) + stats.add(u.json()) + except: + pass + Settings.set('statistics', stats) @task(ignore_results=True, queue='default') def parse_data(key): diff --git a/pandora/user/views.py b/pandora/user/views.py index 92a736ef8..7fc004e69 100644 --- a/pandora/user/views.py +++ b/pandora/user/views.py @@ -795,3 +795,17 @@ def setUI(request): return render_to_json_response(response) actions.register(setUI, cache=False) +@capability_required_json('canManageUsers') +def statistics(request): + ''' + ''' + response = json_response() + from app.models import Settings + stats = Settings.get('statistics') + if not stats: + import tasks + tasks.update_stats() + stats = Settings.get('statistics') + response['data'] = stats + return render_to_json_response(response) +actions.register(statistics, cache=False) diff --git a/static/js/statisticsDialog.js b/static/js/statisticsDialog.js index 5b024baad..1518adb0c 100644 --- a/static/js/statisticsDialog.js +++ b/static/js/statisticsDialog.js @@ -86,100 +86,13 @@ pandora.ui.statisticsDialog = function() { $tabPanel; - pandora.api.findUsers({ - keys: ['browser', 'firstseen', 'lastseen', 'level', 'location', 'system'], - query: { - conditions: [{key: 'level', value: 'robot', operator: '!='}], - operator: '&' - }, - range: [0, 1000000], - sort: [{key: 'username', operator: '+'}] - }, function(result) { - - var data = {}, + pandora.api.statistics({}, function(result) { + var data = result.data, flagCountry = {}, $guestsCheckbox; ['all', 'registered'].forEach(function(mode) { - data[mode] = { - year: {}, - month: {}, - day: {}, - weekday: {}, - hour: {}, - continent: {}, - region: {}, - country: {}, - city: {}, - system: {}, - browser: {}, - systemandbrowser: {}, - systemversion: {}, - browserversion: {}, - systemandbrowserversion: {} - }; - - result.data.items.forEach(function(item) { - var city, continent, country, countryData, name = {}, region, split; - if (mode == 'all' || item.level != 'guest') { - ['firstseen', 'lastseen'].forEach(function(key, i) { - var year = Ox.formatDate(item[key], '%Y') + '-' + key, - month = Ox.formatDate(item[key], '%Y-%m') + '-' + key, - day = Ox.formatDate(item[key], '%Y-%m-%d'), - weekday = Ox.formatDate(item[key], '%u'), - hour = Ox.formatDate(item[key], '%H'); - data[mode].year[year] = data[mode].year[year] || {}; - data[mode].year[year][month] = (data[mode].year[year][month] || 0) + 1; - data[mode].month[month] = (data[mode].month[month] || 0) + 1; - if (key == 'firstseen') { - data[mode].day[day] = data[mode].day[day] || {}; - data[mode].day[day][hour] = (data[mode].day[day][hour] || 0) + 1; - data[mode].weekday[weekday] = data[mode].weekday[weekday] || {}; - data[mode].weekday[weekday][hour] = (data[mode].weekday[weekday][hour] || 0) + 1; - data[mode].hour[hour] = (data[mode].hour[hour] || 0) + 1; - } - }); - if (item.location) { - split = Ox.splitGeoname(item.location); - if (split.length == 1) { - country = split[0]; - } else { - city = split[0]; - country = split[1]; - } - countryData = Ox.getCountryByName(country) || {continent: '', region: ''}; - continent = countryData.continent; - region = [continent, countryData.region].join(', '); - country = [region, country].join(', ') - city = city ? [country, city].join(', ') : ''; - data[mode].continent[continent] = (data[mode].continent[continent] || 0) + 1; - data[mode].region[region] = (data[mode].region[region] || 0) + 1; - data[mode].country[country] = (data[mode].country[country] || 0) + 1; - if (city) { - data[mode].city[city] = (data[mode].city[city] || 0) + 1; - } - } - ['system', 'browser'].forEach(function(key) { - var version = item[key]; - if (version) { - name[key] = getName(key, version); - if (name[key]) { - data[mode][key][name[key]] = (data[mode][key][name[key]] || 0) + 1; - key = key + 'version'; - data[mode][key][version] = (data[mode][key][version] || 0) + 1; - } - } - }); - if (name.system && name.browser) { - name = name.system + ' / ' + name.browser; - data[mode].systemandbrowser[name] = (data[mode].systemandbrowser[name] || 0) + 1; - name = item.system + ' / ' + item.browser; - data[mode].systemandbrowserversion[name] = (data[mode].systemandbrowserversion[name] || 0) + 1; - } - } - }); - var keys, firstKey, lastKey; keys = Object.keys(data[mode].month).map(function(key) { return key.slice(0, 7);