compute statistics on backend

pandora/user/statistics.py

@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+import ox.geo
+
+colors = {
+    'system': {
+        'Android': [0, 255, 0],
+        'BlackBerry': [64, 64, 64],
+        'BSD': [255, 0, 0],
+        'iOS': [0, 128, 255],
+        'Java': [128, 128, 128],
+        'Linux': [255, 128, 0],
+        'Mac OS X': [0, 255, 255],
+        'Nokia': [255, 0, 255],
+        'PlayStation': [192, 192, 192],
+        'RIM Tablet OS': [64, 64, 64],
+        'Unix': [255, 255, 0],
+        'Wii': [192, 192, 192],
+        'Windows Phone': [0, 0, 128], #has to be before 'Windows'
+        'Windows': [0, 0, 255]
+    },
+    'browser': {
+        'Camino': [192, 192, 192],
+        'Chrome Frame': [255, 255, 0], #has to be before 'Chrome'
+        'Chrome': [0, 255, 0],
+        'Chromium': [128, 255, 0],
+        'Epiphany': [128, 128, 128],
+        'Firefox': [255, 128, 0],
+        'Internet Explorer': [0, 0, 255],
+        'Konqueror': [64, 64, 64],
+        'Nokia Browser': [255, 0, 255],
+        'Opera': [255, 0, 0],
+        'Safari': [0, 255, 255],
+        'WebKit': [0, 255, 128]
+    }
+}
+
+def get_name(key, version):
+    for name in colors[key]:
+        if version.startswith(name):
+            return name
+    return ''
+
+class Statistics(dict):
+
+    def __init__(self):
+        
+        for mode in ['all', 'registered']:
+            self[mode] = {
+                 "year": {},
+                 "month": {},
+                 "day": {},
+                 "weekday": {},
+                 "hour": {},
+                 "continent": {},
+                 "region": {},
+                 "country": {},
+                 "city": {},
+                 "system": {},
+                 "browser": {},
+                 "systemandbrowser": {},
+                 "systemversion": {},
+                 "browserversion": {},
+                 "systemandbrowserversion": {}
+            }
+
+    def _increment(self, d, key, add=1, base=0):
+        if not key in d:
+            d[key] = base
+        d[key] += add
+
+    def add(self, item):
+        for mode in ['all', 'registered']:
+            if mode == 'all' or item['level'] != 'guest':
+                for key in ['firstseen', 'lastseen']:
+                    year = '%s-%s' % (item[key].strftime('%Y'), key)
+                    month = '%s-%s' % (item[key].strftime('%Y-%m'), key)
+                    day = '%s-%s' % (item[key].strftime('%Y-%m-%d'), key)
+                    weekday = item[key].strftime('%u')
+                    hour = item[key].strftime('%H')
+
+                    if not year in self[mode]['year']:
+                        self[mode]['year'][year] = {}
+                    self._increment(self[mode]['year'][year], month, 1)
+
+                    self._increment(self[mode]['month'], month, 1)
+
+                    if key == 'firstseen':
+                        if not day in self[mode]['day']:
+                            self[mode]['day'][day] = {}
+                        self._increment(self[mode]['day'][day], hour, 1)
+
+                        if not weekday in self[mode]['weekday']:
+                            self[mode]['weekday'][weekday] = {}
+                        self._increment(self[mode]['weekday'][weekday], hour, 1)
+
+                        self._increment(self[mode]['hour'], hour)
+
+                if item['location']:
+                    split = ox.geo.split_geoname(item['location'])
+                    if len(split) == 1:
+                        split.insert(0, None)
+                    city, country = split
+
+                    country_data = ox.geo.get_country(country)
+                    continent = country_data.get('continent','')
+                    region = ', '.join([continent, country_data.get('region', '')])
+                    country = ', '.join([region, country])
+                    city = ', '.join(country, city) if city else ''
+
+                    self._increment(self[mode]['continent'], continent,)
+                    self._increment(self[mode]['region'], region)
+                    self._increment(self[mode]['country'], country)
+                    if city:
+                        self._increment(self[mode]['city'], city)
+
+                name = {}
+                for key in ['system', 'browser']:
+                    version = item[key];
+                    if version:
+                        name[key] = get_name(key, version)
+                        if name[key]:
+                            self._increment(self[mode][key], name[key])
+
+                            key = key + 'version';
+                            self._increment(self[mode][key], version)
+
+                if name['system'] and name['browser']:
+                    name = name['system'] + ' / ' + name['browser']
+                    self._increment(self[mode]['systemandbrowser'], name)
+                    
+                    name = item['system'] + ' / ' + item['browser']
+                    self._increment(self[mode]['systemandbrowserversion'], name)
+

pandora/user/tasks.py

@@ -1,10 +1,28 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
+import json
+from datetime import timedelta
 
-from celery.task import task
+from celery.task import task, periodic_task
 
 import models
+from app.models import Settings
+from statistics import Statistics
 
+@periodic_task(run_every=timedelta(hours=1), queue='encoding')
+def cronjob(**kwargs):
+    update_statistics()
+
+def update_statistics():
+    stats = Statistics()
+    ids = [i['session_key'] for i in models.SessionData.objects.all().values('session_key')]
+    for id in ids:
+        try:
+            u = models.SessionData.objects.get(pk=id)
+            stats.add(u.json())
+        except:
+            pass
+    Settings.set('statistics', stats)
 
 @task(ignore_results=True, queue='default')
 def parse_data(key):

pandora/user/views.py

@@ -795,3 +795,17 @@ def setUI(request):
     return render_to_json_response(response)
 actions.register(setUI, cache=False)
 
+@capability_required_json('canManageUsers')
+def statistics(request):
+    '''
+    '''
+    response = json_response()
+    from app.models import Settings
+    stats = Settings.get('statistics')
+    if not stats:
+        import tasks
+        tasks.update_stats()
+        stats = Settings.get('statistics')
+    response['data'] = stats
+    return render_to_json_response(response)
+actions.register(statistics, cache=False)