use tornado for api requests

ctl

@@ -94,6 +94,13 @@ if [ "$1" == "update" ]; then
     python2 oml update_static > /dev/null
     exit
 fi
+if [ "$1" == "python" ]; then
+    cd $BASE/$NAME
+    echo `pwd`
+    shift
+    python2 $@
+    exit $?
+fi
 
 cd $BASE/$NAME
 python2 oml $@

oml/api.py

@@ -7,20 +7,19 @@ import json
 import os
 
 import ox
-from oxflask.api import actions
+from oxtornado import actions
-from oxflask.shortcuts import returns_json
+
 
 import item.api
 import user.api
 
-@returns_json
+
-def selectFolder(request):
+def selectFolder(data):
     '''
         returns {
             path
         }
     '''
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     cmd = ['./ctl', 'ui', 'folder']
     p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
     stdout, stderr = p.communicate()
@@ -30,14 +29,13 @@ def selectFolder(request):
     }
 actions.register(selectFolder, cache=False)
 
-@returns_json
+
-def selectFile(request):
+def selectFile(data):
     '''
         returns {
             path
         }
     '''
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     cmd = ['./ctl', 'ui', 'file']
     p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
     stdout, stderr = p.communicate()
@@ -48,8 +46,8 @@ def selectFile(request):
 actions.register(selectFile, cache=False)
 
 
-@returns_json
+
-def autocompleteFolder(request):
+def autocompleteFolder(data):
     '''
         takes {
             path
@@ -58,7 +56,6 @@ def autocompleteFolder(request):
             items
         }
     '''
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     path = data['path']
     path = os.path.expanduser(path)
     if os.path.isdir(path):

oml/item/api.py

@@ -5,8 +5,8 @@ from __future__ import division
 import os
 import json
 
-from oxflask.api import actions
+from oxtornado import actions
-from oxflask.shortcuts import returns_json
+
 from sqlalchemy.orm import load_only
 
 import query
@@ -22,8 +22,8 @@ import utils
 import logging
 logger = logging.getLogger('oml.item.api')
 
-@returns_json
+
-def find(request):
+def find(data):
     '''
         takes {
             query {
@@ -37,7 +37,6 @@ def find(request):
         }
     '''
     response = {}
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     q = query.parse(data)
     if 'group' in q:
         names = {}
@@ -88,8 +87,8 @@ def find(request):
     return response
 actions.register(find)
 
-@returns_json
+
-def get(request):
+def get(data):
     '''
         takes {
             id
@@ -97,15 +96,14 @@ def get(request):
         }
     '''
     response = {}
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     item = models.Item.get(data['id'])
     if item:
         response = item.json(data['keys'] if 'keys' in data else None)
     return response
 actions.register(get)
 
-@returns_json
+
-def edit(request):
+def edit(data):
     '''
         takes {
             id
@@ -114,7 +112,6 @@ def edit(request):
         setting identifier or base metadata is possible not both at the same time
     '''
     response = {}
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     logger.debug('edit %s', data)
     item = models.Item.get(data['id'])
     keys = filter(lambda k: k in models.Item.id_keys, data.keys())
@@ -139,14 +136,13 @@ def edit(request):
     return response
 actions.register(edit, cache=False)
 
-@returns_json
+
-def remove(request):
+def remove(data):
     '''
         takes {
             id
         }
     '''
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     logger.debug('remove files %s', data)
     if 'ids' in data and data['ids']:
         for i in models.Item.query.filter(models.Item.id.in_(data['ids'])):
@@ -154,8 +150,8 @@ def remove(request):
     return {}
 actions.register(remove, cache=False)
 
-@returns_json
+
-def findMetadata(request):
+def findMetadata(data):
     '''
         takes {
             title: string,
@@ -171,21 +167,19 @@ def findMetadata(request):
         key is one of the supported identifiers: isbn10, isbn13...
     '''
     response = {}
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     logger.debug('findMetadata %s', data)
     response['items'] = meta.find(**data)
     return response
 actions.register(findMetadata)
 
-@returns_json
+
-def getMetadata(request):
+def getMetadata(data):
     '''
         takes {
             key: value
         }
         key can be one of the supported identifiers: isbn10, isbn13, oclc, olid,...
     '''
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     logger.debug('getMetadata %s', data)
     key, value = data.iteritems().next()
     if key in ('isbn10', 'isbn13'):
@@ -196,15 +190,14 @@ def getMetadata(request):
     return response
 actions.register(getMetadata)
 
-@returns_json
+
-def download(request):
+def download(data):
     '''
         takes {
             id
         }
     '''
     response = {}
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     item = models.Item.get(data['id'])
     if item:
         item.queue_download()
@@ -213,15 +206,14 @@ def download(request):
     return response
 actions.register(download, cache=False)
 
-@returns_json
+
-def cancelDownloads(request):
+def cancelDownloads(data):
     '''
         takes {
             ids
         }
     '''
     response = {}
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     ids = data['ids']
     if ids:
         for item in models.Item.query.filter(models.Item.id.in_(ids)):
@@ -237,14 +229,14 @@ def cancelDownloads(request):
     return response
 actions.register(cancelDownloads, cache=False)
 
-@returns_json
+
-def scan(request):
+def scan(data):
     state.main.add_callback(state.websockets[0].put, json.dumps(['scan', {}]))
     return {}
 actions.register(scan, cache=False)
 
-@returns_json
+
-def _import(request):
+def _import(data):
     '''
         takes {
             path absolute path to import
@@ -252,14 +244,13 @@ def _import(request):
             mode copy|move
         }
     '''
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     logger.debug('api.import %s', data)
     state.main.add_callback(state.websockets[0].put, json.dumps(['import', data]))
     return {}
 actions.register(_import, 'import', cache=False)
 
-@returns_json
+
-def cancelImport(request):
+def cancelImport(data):
     state.activity['cancel'] = True
     return {}
 actions.register(cancelImport, cache=False)

oml/oxtornado.py

@@ -0,0 +1,184 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+from __future__ import division, with_statement
+
+import inspect
+import sys
+import json
+import datetime
+
+import tornado.ioloop
+import tornado.web
+import tornado.gen
+import tornado.concurrent
+from threading import Thread
+from functools import wraps
+
+import logging
+logger = logging.getLogger('oxtornado')
+
+
+def json_response(data=None, status=200, text='ok'):
+    if not data:
+        data = {}
+    return {'status': {'code': status, 'text': text}, 'data': data}
+
+def _to_json(python_object):
+    if isinstance(python_object, datetime.datetime):
+        if python_object.year < 1900:
+            tt = python_object.timetuple()
+            return '%d-%02d-%02dT%02d:%02d%02dZ' % tuple(list(tt)[:6])
+        return python_object.strftime('%Y-%m-%dT%H:%M:%SZ')
+    raise TypeError(u'%s %s is not JSON serializable' % (repr(python_object), type(python_object)))
+
+def json_dumps(obj):
+    indent = 2
+    return json.dumps(obj, indent=indent, default=_to_json, ensure_ascii=False).encode('utf-8')
+
+def run_async(func):
+  @wraps(func)
+  def async_func(*args, **kwargs):
+    func_hl = Thread(target = func, args = args, kwargs = kwargs)
+    func_hl.start()
+    return func_hl
+
+  return async_func
+
+def trim(docstring):
+    if not docstring:
+        return ''
+    # Convert tabs to spaces (following the normal Python rules)
+    # and split into a list of lines:
+    lines = docstring.expandtabs().splitlines()
+    # Determine minimum indentation (first line doesn't count):
+    indent = sys.maxint
+    for line in lines[1:]:
+        stripped = line.lstrip()
+        if stripped:
+            indent = min(indent, len(line) - len(stripped))
+    # Remove indentation (first line is special):
+    trimmed = [lines[0].strip()]
+    if indent < sys.maxint:
+        for line in lines[1:]:
+            trimmed.append(line[indent:].rstrip())
+    # Strip off trailing and leading blank lines:
+    while trimmed and not trimmed[-1]:
+        trimmed.pop()
+    while trimmed and not trimmed[0]:
+        trimmed.pop(0)
+    # Return a single string:
+    return '\n'.join(trimmed)
+
+
+@run_async
+def api_task(app, request, callback):
+    action = request.arguments.get('action', [None])[0]
+    data = request.arguments.get('data', ['{}'])[0]
+    data = json.loads(data) if data else {}
+    if not action:
+        methods = actions.keys()
+        api = []
+        for f in sorted(methods):
+            api.append({'name': f,
+                        'doc': actions.doc(f).replace('\n', '<br>\n')})
+        response = json_response(api)
+    else:
+        logger.debug('API %s %s', action, data)
+        f = actions.get(action)
+        if f:
+            with app.app_context():
+                response = f(data)
+        else:
+            response = json_response(status=400, text='Unknown action %s' % action)
+    callback(response)
+
+class ApiHandler(tornado.web.RequestHandler):
+    def initialize(self, app):
+        self._app = app
+
+    @tornado.web.asynchronous
+    @tornado.gen.coroutine
+    def post(self):
+        if 'origin' in self.request.headers and self.request.host not in self.request.headers['origin']:
+            logger.debug('reject cross site attempt to access api %s', self.request)
+            self.set_status(403)
+            self.write('')
+            self.finish()
+            return
+
+        response = yield tornado.gen.Task(api_task, self._app, self.request)
+        response = json_dumps(json_response(response))
+        self.set_header('Content-Type', 'application/json')
+        self.write(response)
+        self.finish()
+
+class ApiActions(dict):
+    properties = {}
+    versions = {}
+    def __init__(self):
+
+        def api(data):
+            '''
+                returns list of all known api actions
+                takes {
+                    docs: bool
+                }
+                if docs is true, action properties contain docstrings
+                returns {
+                    actions: {
+                        'api': {
+                            cache: true,
+                            doc: 'recursion'
+                        },
+                        'hello': {
+                            cache: true,
+                            ..
+                        }
+                        ...
+                    }
+                }
+            '''
+            data = data or {}
+            docs = data.get('docs', False)
+            code = data.get('code', False)
+            _actions = self.keys()
+            _actions.sort()
+            actions = {}
+            for a in _actions:
+                actions[a] = self.properties[a]
+                if docs:
+                    actions[a]['doc'] = self.doc(a)
+                if code:
+                    actions[a]['code'] = self.code(a)
+            return {'actions': actions}
+        self.register(api)
+
+    def doc(self, name):
+        f = self[name]
+        return trim(f.__doc__)
+
+    def code(self, name, version=None):
+        f = self[name]
+        if name != 'api' and hasattr(f, 'func_closure') and f.func_closure:
+            fc = filter(lambda c: hasattr(c.cell_contents, '__call__'), f.func_closure)
+            f = fc[len(fc)-1].cell_contents 
+        info = f.func_code.co_filename
+        info = u'%s:%s' % (info, f.func_code.co_firstlineno)
+        return info, trim(inspect.getsource(f))
+
+    def register(self, method, action=None, cache=True, version=None):
+        if not action:
+            action = method.func_name
+        if version:
+            if not version in self.versions:
+                self.versions[version] = {}
+            self.versions[version][action] = method
+        else:
+            self[action] = method
+        self.properties[action] = {'cache': cache}
+
+    def unregister(self, action):
+        if action in self:
+            del self[action]
+
+actions = ApiActions()

oml/server.py

@@ -14,6 +14,7 @@ import websocket
 
 import state
 import node.server
+import oxtornado
 
 def run():
     root_dir = os.path.normpath(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..'))
@@ -30,6 +31,7 @@ def run():
     handlers = [
         (r'/(favicon.ico)', StaticFileHandler, {'path': static_path}),
         (r'/static/(.*)', StaticFileHandler, {'path': static_path}),
+        (r'/api/', oxtornado.ApiHandler, dict(app=app)),
         (r'/ws', websocket.Handler),
         (r".*", FallbackHandler, dict(fallback=tr)),
     ]

oml/user/api.py

@@ -6,8 +6,8 @@ import os
 from copy import deepcopy
 import json
 
-from oxflask.api import actions
+from oxtornado import actions
-from oxflask.shortcuts import returns_json
+
 
 import models
 
@@ -20,8 +20,8 @@ from changelog import Changelog
 import logging
 logger = logging.getLogger('oml.user.api')
 
-@returns_json
+
-def init(request):
+def init(data):
     '''
         takes {
         }
@@ -49,34 +49,32 @@ def init(request):
     return response
 actions.register(init)
 
-@returns_json
+
-def setPreferences(request):
+def setPreferences(data):
     '''
         takes {
             key: value,
             'sub.key': value
         }
     '''
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     update_dict(settings.preferences, data)
     return settings.preferences
 actions.register(setPreferences)
 
-@returns_json
+
-def setUI(request):
+def setUI(data):
     '''
         takes {
             key: value,
             'sub.key': value
         }
     '''
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     update_dict(settings.ui, data)
     return settings.ui
 actions.register(setUI)
 
-@returns_json
+
-def getUsers(request):
+def getUsers(data):
     '''
         returns {
             users: []
@@ -90,8 +88,8 @@ def getUsers(request):
     }
 actions.register(getUsers)
 
-@returns_json
+
-def getLists(request):
+def getLists(data):
     '''
         returns {
             lists: []
@@ -121,8 +119,8 @@ def validate_query(query):
         ):
             raise Exception('invalid query condition', condition)
 
-@returns_json
+
-def addList(request):
+def addList(data):
     '''
         takes {
             name
@@ -130,7 +128,6 @@ def addList(request):
             query
         }
     '''
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     logger.debug('addList %s', data)
     user_id = settings.USER_ID
     if 'query' in data:
@@ -145,8 +142,8 @@ def addList(request):
     return {}
 actions.register(addList, cache=False)
 
-@returns_json
+
-def editList(request):
+def editList(data):
     '''
         takes {
             id
@@ -154,7 +151,6 @@ def editList(request):
             query
         }
     '''
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     logger.debug('editList %s', data)
     l = models.List.get_or_create(data['id'])
     name = l.name
@@ -169,14 +165,13 @@ def editList(request):
     return l.json()
 actions.register(editList, cache=False)
 
-@returns_json
+
-def removeList(request):
+def removeList(data):
     '''
         takes {
             id
         }
     '''
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     l = models.List.get(data['id'])
     if l:
         l.remove()
@@ -184,15 +179,14 @@ def removeList(request):
 actions.register(removeList, cache=False)
 
 
-@returns_json
+
-def addListItems(request):
+def addListItems(data):
     '''
         takes {
             list
             items
         }
     '''
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     if data['list'] == ':':
         from item.models import Item
         user = state.user()
@@ -208,15 +202,14 @@ def addListItems(request):
     return {}
 actions.register(addListItems, cache=False)
 
-@returns_json
+
-def removeListItems(request):
+def removeListItems(data):
     '''
         takes {
             list
             items
         }
     '''
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     l = models.List.get(data['list'])
     if l:
         l.remove_items(data['items'])
@@ -224,14 +217,13 @@ def removeListItems(request):
     return {}
 actions.register(removeListItems, cache=False)
 
-@returns_json
+
-def sortLists(request):
+def sortLists(data):
     '''
         takes {
             ids
         }
     '''
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     n = 0
     logger.debug('sortLists %s', data)
     for id in data['ids']:
@@ -243,15 +235,14 @@ def sortLists(request):
     return {}
 actions.register(sortLists, cache=False)
 
-@returns_json
+
-def editUser(request):
+def editUser(data):
     '''
         takes {
             id
             nickname
         }
     '''
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     if 'nickname' in data:
         p = models.User.get_or_create(data['id'])
         p.set_nickname(data['nickname'])
@@ -259,15 +250,14 @@ def editUser(request):
     return {}
 actions.register(editUser, cache=False)
 
-@returns_json
+
-def requestPeering(request):
+def dataPeering(data):
     '''
         takes {
             id
             message
         }
     '''
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     if len(data.get('id', '')) != 43:
         logger.debug('invalid user id')
         return {}
@@ -277,19 +267,18 @@ def requestPeering(request):
     u.info['message'] = data.get('message', '')
     u.save()
     state.nodes.queue('add', u.id)
-    state.nodes.queue(u.id, 'peering', 'requestPeering')
+    state.nodes.queue(u.id, 'peering', 'dataPeering')
     return {}
-actions.register(requestPeering, cache=False)
+actions.register(dataPeering, cache=False)
 
-@returns_json
+
-def acceptPeering(request):
+def acceptPeering(data):
     '''
         takes {
             id
             message
         }
     '''
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     if len(data.get('id', '')) != 43:
         logger.debug('invalid user id')
         return {}
@@ -302,15 +291,14 @@ def acceptPeering(request):
     return {}
 actions.register(acceptPeering, cache=False)
 
-@returns_json
+
-def rejectPeering(request):
+def rejectPeering(data):
     '''
         takes {
             id
             message
         }
     '''
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     if len(data.get('id', '')) != 43:
         logger.debug('invalid user id')
         return {}
@@ -322,15 +310,14 @@ def rejectPeering(request):
     return {}
 actions.register(rejectPeering, cache=False)
 
-@returns_json
+
-def removePeering(request):
+def removePeering(data):
     '''
         takes {
             id
             message
         }
     '''
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     if len(data.get('id', '')) != 43:
         logger.debug('invalid user id')
         return {}
@@ -342,13 +329,12 @@ def removePeering(request):
     return {}
 actions.register(removePeering, cache=False)
 
-@returns_json
+
-def cancelPeering(request):
+def cancelPeering(data):
     '''
         takes {
         }
     '''
-    data = json.loads(request.form['data']) if 'data' in request.form else {}
     if len(data.get('id', '')) != 43:
         logger.debug('invalid user id')
         return {}
@@ -360,8 +346,8 @@ def cancelPeering(request):
     return {}
 actions.register(cancelPeering, cache=False)
 
-@returns_json
+
-def getActivity(request):
+def getActivity(data):
     '''
         return {
             activity