From d63f02081093c5f9729155be3cf71d6f6deb142f Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Mon, 5 Sep 2011 01:26:04 +0200 Subject: [PATCH] hack a view --- oxbrowser/api/__init__.py | 0 oxbrowser/api/actions.py | 117 +++++ oxbrowser/api/management/__init__.py | 0 oxbrowser/api/management/commands/__init__.py | 0 oxbrowser/api/models.py | 3 + oxbrowser/api/urls.py | 10 + oxbrowser/api/views.py | 82 ++++ oxbrowser/app/__init__.py | 0 oxbrowser/app/admin.py | 18 + oxbrowser/app/models.py | 40 ++ oxbrowser/app/tests.py | 24 + oxbrowser/app/views.py | 89 ++++ oxbrowser/cables.json | 87 ++++ oxbrowser/item/__init__.py | 0 oxbrowser/item/models.py | 48 ++ oxbrowser/item/tests.py | 16 + oxbrowser/item/urls.py | 9 + oxbrowser/item/utils.py | 61 +++ oxbrowser/item/views.py | 439 ++++++++++++++++++ oxbrowser/settings.py | 13 +- oxbrowser/templates/index.html | 18 + oxbrowser/urls.py | 38 +- static/css/site.css | 3 + static/js/site.js | 124 +++++ 24 files changed, 1225 insertions(+), 14 deletions(-) create mode 100644 oxbrowser/api/__init__.py create mode 100644 oxbrowser/api/actions.py create mode 100644 oxbrowser/api/management/__init__.py create mode 100644 oxbrowser/api/management/commands/__init__.py create mode 100644 oxbrowser/api/models.py create mode 100644 oxbrowser/api/urls.py create mode 100644 oxbrowser/api/views.py create mode 100644 oxbrowser/app/__init__.py create mode 100644 oxbrowser/app/admin.py create mode 100644 oxbrowser/app/models.py create mode 100644 oxbrowser/app/tests.py create mode 100644 oxbrowser/app/views.py create mode 100644 oxbrowser/cables.json create mode 100644 oxbrowser/item/__init__.py create mode 100644 oxbrowser/item/models.py create mode 100644 oxbrowser/item/tests.py create mode 100644 oxbrowser/item/urls.py create mode 100644 oxbrowser/item/utils.py create mode 100644 oxbrowser/item/views.py create mode 100644 oxbrowser/templates/index.html create mode 100644 static/css/site.css create mode 100644 static/js/site.js diff --git a/oxbrowser/api/__init__.py b/oxbrowser/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oxbrowser/api/actions.py b/oxbrowser/api/actions.py new file mode 100644 index 0000000..f80a29f --- /dev/null +++ b/oxbrowser/api/actions.py @@ -0,0 +1,117 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +import sys +import inspect + +from django.conf import settings + +from ox.django.shortcuts import render_to_json_response, json_response +from ox.utils import json + + +def autodiscover(): + #register api actions from all installed apps + from django.utils.importlib import import_module + from django.utils.module_loading import module_has_submodule + for app in settings.INSTALLED_APPS: + if app != 'api': + mod = import_module(app) + try: + import_module('%s.views'%app) + except: + if module_has_submodule(mod, 'views'): + raise + +def trim(docstring): + if not docstring: + return '' + # Convert tabs to spaces (following the normal Python rules) + # and split into a list of lines: + lines = docstring.expandtabs().splitlines() + # Determine minimum indentation (first line doesn't count): + indent = sys.maxint + for line in lines[1:]: + stripped = line.lstrip() + if stripped: + indent = min(indent, len(line) - len(stripped)) + # Remove indentation (first line is special): + trimmed = [lines[0].strip()] + if indent < sys.maxint: + for line in lines[1:]: + trimmed.append(line[indent:].rstrip()) + # Strip off trailing and leading blank lines: + while trimmed and not trimmed[-1]: + trimmed.pop() + while trimmed and not trimmed[0]: + trimmed.pop(0) + # Return a single string: + return '\n'.join(trimmed) + + +class ApiActions(dict): + properties = {} + def __init__(self): + + def api(request): + ''' + returns list of all known api actions + param data { + docs: bool + } + if docs is true, action properties contain docstrings + return { + status: {'code': int, 'text': string}, + data: { + actions: { + 'api': { + cache: true, + doc: 'recursion' + }, + 'hello': { + cache: true, + .. + } + ... + } + } + } + ''' + data = json.loads(request.POST.get('data', '{}')) + docs = data.get('docs', False) + code = data.get('code', False) + _actions = self.keys() + _actions.sort() + actions = {} + for a in _actions: + actions[a] = self.properties[a] + if docs: + actions[a]['doc'] = self.doc(a) + if code: + actions[a]['code'] = self.code(a) + response = json_response({'actions': actions}) + return render_to_json_response(response) + self.register(api) + + def doc(self, f): + return trim(self[f].__doc__) + + def code(self, name): + f = self[name] + if name != 'api' and hasattr(f, 'func_closure') and f.func_closure: + f = f.func_closure[0].cell_contents + info = f.func_code.co_filename[len(settings.PROJECT_ROOT)+1:] + info = u'%s:%s' % (info, f.func_code.co_firstlineno) + return info, trim(inspect.getsource(f)) + + def register(self, method, action=None, cache=True): + if not action: + action = method.func_name + self[action] = method + self.properties[action] = {'cache': cache} + + def unregister(self, action): + if action in self: + del self[action] + +actions = ApiActions() + diff --git a/oxbrowser/api/management/__init__.py b/oxbrowser/api/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oxbrowser/api/management/commands/__init__.py b/oxbrowser/api/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oxbrowser/api/models.py b/oxbrowser/api/models.py new file mode 100644 index 0000000..04d0a3c --- /dev/null +++ b/oxbrowser/api/models.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 + diff --git a/oxbrowser/api/urls.py b/oxbrowser/api/urls.py new file mode 100644 index 0000000..fb0b40c --- /dev/null +++ b/oxbrowser/api/urls.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 + +from django.conf.urls.defaults import * + + +urlpatterns = patterns("api.views", + (r'^$', 'api'), +) + diff --git a/oxbrowser/api/views.py b/oxbrowser/api/views.py new file mode 100644 index 0000000..ad1ad0d --- /dev/null +++ b/oxbrowser/api/views.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from __future__ import division, with_statement + +import os + +from django.shortcuts import render_to_response +from django.template import RequestContext +from django.conf import settings +from django.db.models import Max, Sum + +from ox.django.shortcuts import render_to_json_response, json_response +from ox.utils import json + +from app.models import site_config + +from actions import actions + + +def api(request): + if request.META['REQUEST_METHOD'] == "OPTIONS": + response = render_to_json_response({'status': {'code': 200, + 'text': 'use POST'}}) + response['Access-Control-Allow-Origin'] = '*' + return response + if not 'action' in request.POST: + methods = actions.keys() + api = [] + for f in sorted(methods): + api.append({'name': f, + 'doc': actions.doc(f).replace('\n', '
\n')}) + context = RequestContext(request, {'api': api, + 'sitename': settings.SITENAME}) + return render_to_response('api.html', context) + function = request.POST['action'] + #FIXME: possible to do this in f + #data = json.loads(request.POST['data']) + + f = actions.get(function) + if f: + response = f(request) + else: + response = render_to_json_response(json_response(status=400, + text='Unknown function %s' % function)) + response['Access-Control-Allow-Origin'] = '*' + return response + + +def init(request): + ''' + return {'status': {'code': int, 'text': string}, + 'data': {user: object}} + ''' + #data = json.loads(request.POST['data']) + response = json_response({}) + config = site_config() + del config['keys'] #is this needed? + + #populate max values for percent requests + for key in filter(lambda k: 'format' in k, config['itemKeys']): + ''' + if key['format']['type'] == 'percent' and key['format']['args'][0] == 'auto': + name = key['id'] + if name == 'popularity': + name = 'item__accessed__accessed' + value = ItemSort.objects.aggregate(Sum(name))['%s__sum'%name] + else: + value = ItemSort.objects.aggregate(Max(name))['%s__max'%name] + key['format']['args'][0] = value + ''' + response['data']['site'] = config + return render_to_json_response(response) +actions.register(init) + + +def error(request): + ''' + this action is used to test api error codes, it should return a 503 error + ''' + success = error_is_success + return render_to_json_response({}) +actions.register(error) diff --git a/oxbrowser/app/__init__.py b/oxbrowser/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oxbrowser/app/admin.py b/oxbrowser/app/admin.py new file mode 100644 index 0000000..90cbd79 --- /dev/null +++ b/oxbrowser/app/admin.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 + +from django.contrib import admin + +import models + + +class PageAdmin(admin.ModelAdmin): + search_fields = ['name', 'body'] + +admin.site.register(models.Page, PageAdmin) + + +class SiteSettingsAdmin(admin.ModelAdmin): + search_fields = ['key', 'value'] + +admin.site.register(models.SiteSettings, SiteSettingsAdmin) diff --git a/oxbrowser/app/models.py b/oxbrowser/app/models.py new file mode 100644 index 0000000..8cfe979 --- /dev/null +++ b/oxbrowser/app/models.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from __future__ import division, with_statement +import os + +from django.db import models +from django.conf import settings +from ox.utils import json + + +class Page(models.Model): + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + name = models.CharField(max_length=1024, unique=True) + body = models.TextField(blank=True) + + def __unicode__(self): + return self.name + + +class SiteSettings(models.Model): + key = models.CharField(max_length=1024, unique=True) + value = models.TextField(blank=True) + + def __unicode__(self): + return self.key + +def site_config(): + with open(settings.SITE_CONFIG) as f: + site_config = json.load(f) + + site_config['site']['id'] = settings.SITEID + site_config['site']['name'] = settings.SITENAME + site_config['site']['sectionName'] = settings.SITENAME + site_config['site']['url'] = settings.URL + + site_config['keys'] = {} + for key in site_config['itemKeys']: + site_config['keys'][key['id']] = key + return site_config diff --git a/oxbrowser/app/tests.py b/oxbrowser/app/tests.py new file mode 100644 index 0000000..927cadf --- /dev/null +++ b/oxbrowser/app/tests.py @@ -0,0 +1,24 @@ +""" +This file demonstrates two different styles of tests (one doctest and one +unittest). These will both pass when you run "manage.py test". + +Replace these with more appropriate tests for your application. +""" + +from django.test import TestCase + + +class SimpleTest(TestCase): + + def test_basic_addition(self): + """ + Tests that 1 + 1 always equals 2. + """ + self.failUnlessEqual(1 + 1, 2) + +__test__ = {"doctest": """ +Another way to test that 1 + 1 is equal to 2. + +>>> 1 + 1 == 2 +True +"""} diff --git a/oxbrowser/app/views.py b/oxbrowser/app/views.py new file mode 100644 index 0000000..9d541cf --- /dev/null +++ b/oxbrowser/app/views.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from django.shortcuts import render_to_response +from django.template import RequestContext +from django.conf import settings + +from ox.django.shortcuts import json_response, render_to_json_response, get_object_or_404_json +from ox.django.decorators import login_required_json + +from ox.utils import json + +import models + +from api.actions import actions + + +def intro(request): + context = RequestContext(request, {'settings': settings}) + return render_to_response('intro.html', context) + + +def index(request): + context = RequestContext(request, {'settings': settings}) + return render_to_response('index.html', context) + + +def embed(request): + context = RequestContext(request, {'settings': settings}) + return render_to_response('embed.html', context) + + +def timeline(request): + context = RequestContext(request, {'settings': settings}) + return render_to_response('timeline.html', context) + + +def getPage(request): + ''' + param data { + name: pagename + } + return { + status: ... + data: { + name: + body: + } + } + ''' + data = json.loads(request.POST['data']) + if isinstance(data, basestring): + name = data + else: + name = data['name'] + page, created = models.Page.objects.get_or_create(name=name) + if created: + page.body = 'Insert text here' + page.save() + response = json_response({'name': page.name, 'body': page.body}) + return render_to_json_response(response) +actions.register(getPage) + + +@login_required_json +def editPage(request): + ''' + param data { + name: pagename + body: text + } + return { + status: ... + data: { + name: + body: + } + } + ''' + if not request.user.is_staff: + response = json_response(status=403, text='permission denied') + return render_to_json_response(response) + data = json.loads(request.POST['data']) + page, created = models.Page.objects.get_or_create(name=data['name']) + page.body = data['body'] + page.save() + response = json_response({'name': page.name, 'page': page.body}) + return render_to_json_response(response) +actions.register(getPage) + diff --git a/oxbrowser/cables.json b/oxbrowser/cables.json new file mode 100644 index 0000000..6035224 --- /dev/null +++ b/oxbrowser/cables.json @@ -0,0 +1,87 @@ +{ + "groups": [ + {"id": "origin", "title": "Origin"}, + {"id": "created", "title": "Created"}, + {"id": "released", "title": "Released"} + ], + "itemKeys": [ + { + "id": "refid", + "title": "ID", + "type": "string", + "columnRequired": false, + "columnWidth": 180 + }, + { + "id": "subject", + "title": "Subject", + "type": "string", + "columnRequired": true, + "columnWidth": 480, + "find": true, + "sort": "string" + }, + { + "id": "origin", + "title": "Origin", + "type": "string", + "columnWidth": 140, + "find": true, + "group": true, + "sort": "string" + }, + { + "id": "created", + "title": "Created", + "type": "date", + "columnWidth": 120, + "group": true, + "format": {"type": "date", "args": ["%a, %b %e, %Y"]} + + }, + { + "id": "released", + "title": "Released", + "type": "date", + "columnWidth": 120, + "group": true, + "format": {"type": "date", "args": ["%a, %b %e, %Y"]} + + }, + { + "id": "classification", + "title": "Classification", + "type": "string", + "columnWidth": 180, + "find": true, + "group": true + }, + { + "id": "tags", + "title": "Tag", + "type": ["string"], + "columnWidth": 120, + "find": true, + "group": true + } + ], + "itemName": { + "singular": "Cable", + "plural": "Cables" + }, + "itemViews": [ + {"id": "info", "title": "Info"} + ], + "listViews": [ + {"id": "list", "title": "as List"}, + {"id": "map", "title": "on Map"} + ], + "site": { + "id": "{{settings.SITEID}}", + "name": "{{settings.SITENAME}}", + "url": "{{settings.URL}}" + }, + "totals": [ + {"id": "items"} + ] +} diff --git a/oxbrowser/item/__init__.py b/oxbrowser/item/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oxbrowser/item/models.py b/oxbrowser/item/models.py new file mode 100644 index 0000000..8356ef6 --- /dev/null +++ b/oxbrowser/item/models.py @@ -0,0 +1,48 @@ +from django.db import models +from django.conf import settings + +from pymongo import Connection, ASCENDING, DESCENDING +try: + from bson import ObjectId +except: + from pymongo.bson import ObjectId + +from app.models import site_config + +connection = Connection() +db = connection[settings.MONGO_DB] + +items = db.cables +facets = db.facets + +#FIXME: ./manage.py update_indexes +''' +for key in site_config()['itemKeys']: + #items.ensure_index(key['id'], ASCENDING) + #items.ensure_index(key['id'], ASCENDING) + items.ensure_index(key['id']) +''' + +def save(item): + #update facets + for f in self.facets: + new = filter(lambda i: i not in self.old_document.get(key, []), self.document.get(key, [])) + removed = filter(lambda i: i not in self.document.get(key, []), self.old_document.get(key, [])) + + for k in new: + facet = facets.find_one({'facet': f, 'value': k}) + if not facet: + facet = facets.insert({'facet': f, 'value': k, 'count': 0}) + facet['count'] += 1 + facets.save(facet) + + for k in removed: + facet = facets.find_one({'facet': f, 'value': k}) + if facet: + facet['count'] -= 1 + if facet['count'] <= 0: + facets.remove(facet) + else: + facets.save(facet) + items.save(self.document) + diff --git a/oxbrowser/item/tests.py b/oxbrowser/item/tests.py new file mode 100644 index 0000000..501deb7 --- /dev/null +++ b/oxbrowser/item/tests.py @@ -0,0 +1,16 @@ +""" +This file demonstrates writing tests using the unittest module. These will pass +when you run "manage.py test". + +Replace this with more appropriate tests for your application. +""" + +from django.test import TestCase + + +class SimpleTest(TestCase): + def test_basic_addition(self): + """ + Tests that 1 + 1 always equals 2. + """ + self.assertEqual(1 + 1, 2) diff --git a/oxbrowser/item/urls.py b/oxbrowser/item/urls.py new file mode 100644 index 0000000..c5e1fd6 --- /dev/null +++ b/oxbrowser/item/urls.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 + +from django.conf.urls.defaults import * + + +urlpatterns = patterns("item.views", + +) diff --git a/oxbrowser/item/utils.py b/oxbrowser/item/utils.py new file mode 100644 index 0000000..f6fe34d --- /dev/null +++ b/oxbrowser/item/utils.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +# +from decimal import Decimal +import os +import re +import hashlib +import unicodedata + +from django.conf import settings +import ox +import ox.iso +from ox.normalize import normalizeName, normalizeTitle +import ox.web.imdb + + +def parse_decimal(string): + string = string.replace(':', '/') + if '/' not in string: + string = '%s/1' % string + d = string.split('/') + return Decimal(d[0]) / Decimal(d[1]) + + +def plural_key(term): + return { + 'country': 'countries', + }.get(term, term + 's') + + +def sort_string(string): + string = string.replace(u'Þ', 'Th') + #pad numbered titles + string = re.sub('(\d+)', lambda x: '%010d' % int(x.group(0)), string) + return unicodedata.normalize('NFKD', string) + + +def sort_title(title): + #title + title = re.sub(u'[\'!¿¡,\.;\-"\:\*\[\]]', '', title) + + #title = title.replace(u'Æ', 'Ae') + if isinstance(title, str): + title = unicode(title) + title = sort_string(title) + + return title.strip() + +def get_positions(ids, pos): + ''' + >>> get_positions([1,2,3,4], [2,4]) + {2: 1, 4: 3} + ''' + positions = {} + for i in pos: + try: + positions[i] = ids.index(i) + except: + pass + return positions diff --git a/oxbrowser/item/views.py b/oxbrowser/item/views.py new file mode 100644 index 0000000..c10e885 --- /dev/null +++ b/oxbrowser/item/views.py @@ -0,0 +1,439 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from __future__ import division + +import os.path +from datetime import datetime, timedelta +import mimetypes +import re + +from django.db.models import Count, Sum, Max +from django.http import HttpResponse, Http404 +from django.shortcuts import get_object_or_404, redirect +from django.conf import settings + +from ox.utils import json + +from ox.django.decorators import login_required_json +from ox.django.shortcuts import render_to_json_response, get_object_or_404_json, json_response +from ox.django.http import HttpFileResponse +import ox + +import models +import utils + +from api.actions import actions + +from pymongo import ASCENDING, DESCENDING + + +def _order_query(qs, sort, prefix='sort__'): + order_by = [] + if len(sort) == 1: + if sort[0]['key'] == 'title': + sort.append({'operator': '-', 'key': 'year'}) + sort.append({'operator': '+', 'key': 'director'}) + elif sort[0]['key'] == 'director': + sort.append({'operator': '-', 'key': 'year'}) + sort.append({'operator': '+', 'key': 'title'}) + elif sort[0]['key'] == 'year': + sort.append({'operator': '+', 'key': 'director'}) + sort.append({'operator': '+', 'key': 'title'}) + elif not sort[0]['key'] in ('value', 'value_sort'): + sort.append({'operator': '+', 'key': 'director'}) + sort.append({'operator': '-', 'key': 'year'}) + sort.append({'operator': '+', 'key': 'title'}) + + for e in sort: + operator = e['operator'] + if operator != '-': + operator = '' + key = { + 'id': 'itemId', + 'accessed': 'accessed__access', + 'viewed': 'accessed__access', + }.get(e['key'], e['key']) + if key not in ('accessed__access', 'accessed__accessed'): + key = "%s%s" % (prefix, key) + order = '%s%s' % (operator, key) + order_by.append(order) + if order_by: + qs = qs.order_by(*order_by, nulls_last=True) + return qs + +def _order_by_group(query): + if 'sort' in query: + if len(query['sort']) == 1 and query['sort'][0]['key'] == 'items': + if query['group'] == "year": + order_by = query['sort'][0]['operator'] == '-' and 'items' or '-items' + else: + order_by = query['sort'][0]['operator'] == '-' and '-items' or 'items' + if query['group'] != "keyword": + order_by = (order_by, 'value_sort') + else: + order_by = (order_by,) + else: + order_by = query['sort'][0]['operator'] == '-' and '-value_sort' or 'value_sort' + order_by = (order_by, 'items') + else: + order_by = ('-value_sort', 'items') + return order_by + +def parseQuery(q): + ''' + query: { + conditions: [ + { + value: "war" + } + { + key: "year", + value: "1970-1980, + operator: "!=" + }, + { + key: "country", + value: "f", + operator: "^" + } + ], + operator: "&" + } + ''' + #FIXME: support or operator + #FIXME: support sub conditions with $or/$and + #http://www.mongodb.org/display/DOCS/Advanced+Queries#AdvancedQueries-ConditionalOperators + conditions = q.get('conditions', []) + operator = q.get('operator', '&') + + query = {} + for c in conditions: + key = c.get('key', 'all') + op = c.get('operator') + value = c['value'] + if op == '>': + mop = '$gt' + elif op == '>=': + mop = '$gte' + elif op == '<': + mop = '$lt' + elif op == '<=': + mop = '$lte' + elif op == '!=': + mop = '$ne' + elif op == '^': + mop = None + value = re.compile(u'^%s'%value, re.IGNORECASE) + elif op == '$': + mop = None + value = re.compile(u'%s$'%value, re.IGNORECASE) + elif op == '=': + mop = None + else: + mop = None + value = re.compile(u'%s'%value, re.IGNORECASE) + + if mop: + query[key]= {mop: value} + else: + query[key] = value + + return query + +def parse_query(data, user): + query = {} + query['range'] = [0, 100] + query['sort'] = [{'key':'title', 'operator':'+'}] + for key in ('sort', 'keys', 'group', 'range', 'position', 'positions'): + if key in data: + query[key] = data[key] + query['q'] = parseQuery(data.get('query', {})) + #query['qs'] = models.Item.objects.find(data, user) + + #group by only allows sorting by name or number of itmes + return query + +def find(request): + ''' + param data { + 'query': query, + 'sort': array, + 'range': array + } + + query: query object, more on query syntax at + https://wiki.0x2620.org/wiki/pandora/QuerySyntax + sort: array of key, operator dics + [ + { + key: "year", + operator: "-" + }, + { + key: "director", + operator: "" + } + ] + range: result range, array [from, to] + keys: array of keys to return + group: group elements by, country, genre, director... + + with keys, items is list of dicts with requested properties: + return {'status': {'code': int, 'text': string}, + 'data': {items: array}} + +Groups + param data { + 'query': query, + 'key': string, + 'group': string, + 'range': array + } + + query: query object, more on query syntax at + https://wiki.0x2620.org/wiki/pandora/QuerySyntax + range: result range, array [from, to] + keys: array of keys to return + group: group elements by, country, genre, director... + + possible values for keys: name, items + + with keys + items contains list of {'name': string, 'items': int}: + return {'status': {'code': int, 'text': string}, + 'data': {items: array}} + + without keys: return number of items in given query + return {'status': {'code': int, 'text': string}, + 'data': {items: int}} + +Positions + param data { + 'query': query, + 'positions': [], + 'sort': array + } + + query: query object, more on query syntax at + https://wiki.0x2620.org/wiki/pandora/QuerySyntax + positions: ids of items for which positions are required + return { + status: {...}, + data: { + positions: { + id: position + } + } + } + ''' + data = json.loads(request.POST['data']) + if settings.JSON_DEBUG: + print json.dumps(data, indent=2) + query = parse_query(data, request.user) + + response = json_response({}) + if 'group' in query: + ''' + response['data']['items'] = [] + items = 'items' + item_qs = query['qs'] + order_by = _order_by_group(query) + qs = models.Facet.objects.filter(key=query['group']).filter(item__id__in=item_qs) + qs = qs.values('value').annotate(items=Count('id')).order_by(*order_by) + + if 'positions' in query: + response['data']['positions'] = {} + ids = [j['value'] for j in qs] + response['data']['positions'] = utils.get_positions(ids, query['positions']) + elif 'range' in data: + qs = qs[query['range'][0]:query['range'][1]] + response['data']['items'] = [{'name': i['value'], 'items': i[items]} for i in qs] + else: + response['data']['items'] = qs.count() + ''' + if 'positions' in query: + response['data']['positions'] = {} + elif 'range' in data: + response['data']['items'] = [] + else: + response['data']['items'] = 0 + elif 'position' in query: + ''' + qs = _order_query(query['qs'], query['sort']) + ids = [j['itemId'] for j in qs.values('itemId')] + data['conditions'] = data['conditions'] + { + 'value': query['position'], + 'key': query['sort'][0]['key'], + 'operator': '^' + } + query = parse_query(data, request.user) + qs = _order_query(query['qs'], query['sort']) + if qs.count() > 0: + response['data']['position'] = utils.get_positions(ids, [qs[0].itemId])[0] + ''' + response['data']['position'] = -1 + elif 'positions' in query: + ''' + qs = _order_query(query['qs'], query['sort']) + ids = [j['itemId'] for j in qs.values('itemId')] + response['data']['positions'] = utils.get_positions(ids, query['positions']) + ''' + response['data']['position'] = {} + elif 'keys' in query: + response['data']['items'] = [] + ''' + qs = _order_query(query['qs'], query['sort']) + _p = query['keys'] + def only_p_sums(m): + r = {} + for p in _p: + if p == 'viewed' and request.user.is_authenticated(): + value = m.accessed.filter(user=request.user).annotate(v=Max('access')) + r[p] = value.exists() and value[0].v or None + elif p == 'accessed': + r[p] = m.a + elif p == 'popularity': + r[p] = m.sort.popularity + else: + r[p] = m.json.get(p, '') + if 'annotations' in query: + n = query['annotations'] + r['annotations'] = [a.json(layer=True) + for a in query['aqs'].filter(itemID=m.id)[:n]] + return r + def only_p(m): + r = {} + if m: + m = json.loads(m, object_hook=ox.django.fields.from_json) + for p in _p: + r[p] = m.get(p, '') + if 'annotations' in query: + n = query['annotations'] + r['annotations'] = [a.json(layer=True) + for a in query['aqs'].filter(item__itemId=m['id'])[:n]] + return r + + qs = qs[query['range'][0]:query['range'][1]] + #response['data']['items'] = [m.get_json(_p) for m in qs] + if 'popularity' in _p: + qs = qs.annotate(popularity=Sum('accessed__accessed')) + if 'accessed' in _p: + qs = qs.annotate(a=Max('accessed__access')) + if 'viewed' in _p or 'popularity' in _p or 'accessed' in _p: + response['data']['items'] = [only_p_sums(m) for m in qs] + else: + response['data']['items'] = [only_p(m['json']) for m in qs.values('json')] + ''' + def only_p(i): + r = {} + for key in query['keys']: + r[key] = i.get(key, '') + return r + print query + skey = query['sort'][0]['key'] + if query['sort'][0]['operator'] == '+': + sdir = ASCENDING + else: + sdir = DESCENDING + qs = models.items.find(query['q']).sort(skey, sdir) + if query['range'][0] < query['range'][1]: + qs = qs[query['range'][0]:query['range'][1]] + response['data']['items'] = [only_p(i) for i in qs] + + else: # otherwise stats + ''' + items = query['qs'] + files = File.objects.filter(item__in=items).filter(size__gt=0) + r = files.aggregate( + Sum('duration'), + Sum('pixels'), + Sum('size') + ) + response['data']['duration'] = r['duration__sum'] + response['data']['files'] = files.count() + response['data']['items'] = items.count() + response['data']['pixels'] = r['pixels__sum'] + response['data']['runtime'] = items.aggregate(Sum('sort__runtime'))['sort__runtime__sum'] + response['data']['size'] = r['size__sum'] + for key in ('runtime', 'duration', 'pixels', 'size'): + if response['data'][key] == None: + response['data'][key] = 0 + ''' + response['data']['items'] = models.items.find(query['q']).count() + + return render_to_json_response(response) +actions.register(find) + + +def autocomplete(request): + ''' + param data + key + value + operator '', '^', '$' + range + return + ''' + data = json.loads(request.POST['data']) + if not 'range' in data: + data['range'] = [0, 10] + op = data.get('operator', '') + + site_config = models.site_config() + key = site_config['keys'][data['key']] + order_by = key.get('autocompleteSortKey', False) + if order_by: + order_by = '-sort__%s' % order_by + else: + order_by = '-items' + sort_type = key.get('sort', key.get('type', 'string')) + if sort_type == 'title': + qs = parse_query({'query': data.get('query', {})}, request.user)['qs'] + if data['value']: + if op == '': + qs = qs.filter(find__key=data['key'], find__value__icontains=data['value']) + elif op == '^': + qs = qs.filter(find__key=data['key'], find__value__istartswith=data['value']) + elif op == '$': + qs = qs.filter(find__key=data['key'], find__value__iendswith=data['value']) + qs = qs.order_by(order_by, nulls_last=True) + qs = qs[data['range'][0]:data['range'][1]] + response = json_response({}) + response['data']['items'] = [i.get(data['key']) for i in qs] + else: + qs = models.Facet.objects.filter(key=data['key']) + if data['value']: + if op == '': + qs = qs.filter(value__icontains=data['value']) + elif op == '^': + qs = qs.filter(value__istartswith=data['value']) + elif op == '$': + qs = qs.filter(value__iendswith=data['value']) + qs = qs.values('value').annotate(items=Count('id')) + qs = qs.order_by(order_by) + qs = qs[data['range'][0]:data['range'][1]] + response = json_response({}) + response['data']['items'] = [i['value'] for i in qs] + return render_to_json_response(response) +actions.register(autocomplete) + + +def get(request): + ''' + param data { + id: string + keys: array + } + return item array + ''' + response = json_response({}) + data = json.loads(request.POST['data']) + item = models.items.find_one({'refid': data['id']}) + if item: + del item['_id'] + response['data'] = item + else: + response = json_response(status=404, text='not found') + return render_to_json_response(response) +actions.register(get) + diff --git a/oxbrowser/settings.py b/oxbrowser/settings.py index 5f6f351..754b8fa 100644 --- a/oxbrowser/settings.py +++ b/oxbrowser/settings.py @@ -4,6 +4,8 @@ import os from os.path import join +SITEID = 'oxbrowser' +URL = 'http://cablegates.org' SITENAME = 'oxbrowser' PROJECT_ROOT = os.path.normpath(os.path.dirname(__file__)) @@ -32,7 +34,7 @@ DATABASES = { 'PORT': '' } } - +MONGO_DB = 'cablegates' #CACHE_BACKEND = 'memcached://127.0.0.1:11211/' # Local time zone for this installation. Choices can be found here: @@ -56,8 +58,8 @@ APPEND_SLASH = False # Absolute path to the directory that holds media. # Example: "/home/media/media.lawrence.com/" -MEDIA_ROOT = join(PROJECT_ROOT, 'media') -STATIC_ROOT = join(PROJECT_ROOT, 'static') +MEDIA_ROOT = join(PROJECT_ROOT, '..', 'media') +STATIC_ROOT = join(PROJECT_ROOT, '..', 'static') # URL that handles the media served from MEDIA_ROOT. Make sure to use a @@ -101,9 +103,14 @@ INSTALLED_APPS = ( 'django.contrib.humanize', 'django_extensions', #'south', + 'app', + 'api', + 'item', ) +SITE_CONFIG = join(PROJECT_ROOT, 'cables.json') + #overwrite default settings with local settings try: diff --git a/oxbrowser/templates/index.html b/oxbrowser/templates/index.html new file mode 100644 index 0000000..2cdd5e2 --- /dev/null +++ b/oxbrowser/templates/index.html @@ -0,0 +1,18 @@ + + + + {{settings.SITENAME}} + + + + + + + + + + diff --git a/oxbrowser/urls.py b/oxbrowser/urls.py index 1f56b72..4c18f61 100644 --- a/oxbrowser/urls.py +++ b/oxbrowser/urls.py @@ -1,17 +1,33 @@ from django.conf.urls.defaults import patterns, include, url -# Uncomment the next two lines to enable the admin: -# from django.contrib import admin -# admin.autodiscover() +import os +from ox.django.http import HttpFileResponse + +from django.conf import settings + +from django.contrib import admin +admin.autodiscover() + +from api import actions +actions.autodiscover() + +def serve_static_file(path, location, content_type): + return HttpFileResponse(location, content_type=content_type) urlpatterns = patterns('', - # Examples: - # url(r'^$', 'oxbrowser.views.home', name='home'), - # url(r'^oxbrowser/', include('oxbrowser.foo.urls')), + (r'^admin/', include(admin.site.urls)), - # Uncomment the admin/doc line below to enable admin documentation: - # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), - - # Uncomment the next line to enable the admin: - # url(r'^admin/', include(admin.site.urls)), + (r'^api/$', include('api.urls')), + (r'^$', 'app.views.index'), + (r'', include('item.urls')), + (r'^robots.txt$', serve_static_file, {'location': os.path.join(settings.STATIC_ROOT, 'robots.txt'), 'content_type': 'text/plain'}), + (r'^favicon.ico$', serve_static_file, {'location': os.path.join(settings.STATIC_ROOT, 'png/icon.16.png'), 'content_type': 'image/x-icon'}), ) + +if settings.DEBUG: + urlpatterns += patterns('', + (r'^data/(?P.*)$', 'django.views.static.serve', + {'document_root': settings.MEDIA_ROOT}), + (r'^static/(?P.*)$', 'django.views.static.serve', + {'document_root': settings.STATIC_ROOT}), + ) diff --git a/static/css/site.css b/static/css/site.css new file mode 100644 index 0000000..0a07c19 --- /dev/null +++ b/static/css/site.css @@ -0,0 +1,3 @@ +.highlight { + background: rgb(255, 255, 0); +} diff --git a/static/js/site.js b/static/js/site.js new file mode 100644 index 0000000..098ab69 --- /dev/null +++ b/static/js/site.js @@ -0,0 +1,124 @@ +// vim: et:ts=4:sw=4:sts=4:ft=javascript + +Ox.load('UI', function() { + Ox.Theme('classic'); + window.app = new Ox.App({url: '/api/'}).bindEvent({ + load: function(event, data) { + app.site = { + sortKeys: $.map(data.site.itemKeys, function(key, i) { + return key.columnWidth ? key : null; + }) + }; + var position = 0; + app.main = Ox.SplitPanel({ + elements: [ + { + element: app.list = Ox.TextList({ + columns: $.map(app.site.sortKeys, function(key, i) { + var pos = -1; + if($.inArray(key.id, ['subject', 'origin', 'created'])>=0) + pos = position++; + return { + align: ['string', 'text'].indexOf( + Ox.isArray(key.type) ? key.type[0]: key.type + ) > -1 ? 'left' : 'right', + defaultWidth: key.columnWidth, + format: key.format, + id: key.id, + operator: '+', + position: pos, + removable: !key.columnRequired, + title: key.title, + type: key.type, + unique: key.id == 'refid', + visible: $.inArray(key.id, ['subject', 'origin', 'created'])>=0, + width: key.columnWidth + }; + }), + columnsMovable: true, + columnsRemovable: true, + columnsResizable: true, + columnsVisible: true, + draggable: true, + items: function(data, callback) { + app.api.find(data, callback); + }, + scrollbarVisible: true, + sort: [{key: 'created', operator: '-'}] + }).bindEvent({ + select: function(data) { + data.ids.length && select(data.ids[0]); + } + }) + }, + { + element: app.cable = Ox.Element().addClass('OxSelectable').css({overflow: 'auto'}), + collapsible: true, + size: 600, + resizable: true, + resize: [200, 400, 600] + } + ], + orientation: 'horizontal' + }); + app.find = Ox.Input({ + clear: true, + id: 'input', + placeholder: 'Find...', + value: '', + width: 192 + }).css({ + float: 'right', + margin: '4px' + }) + .bindEvent({ + submit: function(data) { + find({conditions: [{key: 'content', value: data.value, operator: ''}]}); + } + }); + app.frame = Ox.SplitPanel({ + elements: [ + { + element: app.bar = Ox.Bar().append(app.find), + size: 24 + }, + { + element: app.main + } + ], + orientation: 'vertical' + }).appendTo($('body')); + if(document.location.hash) { + select(document.location.hash.substring(1)); + } + } + }); +}); +function select(id) { + app.api.get({id: id}, function(result) { + var term = app.find.value(), + header = result.data.header, + content = result.data.content; + document.location.hash = '#' + result.data.refid; + if(term) { + header = Ox.highlight(header, term, 'highlight'); + content = Ox.highlight(content, term, 'highlight'); + } + app.cable.html('') + .append($('
').css('padding', '8px') + .addClass('OxSelectable') + .append($('
').html(header))
+                     .append($('
').html(content)
+                 ))
+                 .scrollTop(0);
+    }); 
+}
+function find(query) {
+    app.list.options({
+        items: function(data, callback) {
+            app.api.find(Ox.extend(data, {
+                query: query
+            }), callback);
+        }
+    });
+}