From ed7053c0cbde22c38dbaa428548c0cfd85348e3d Mon Sep 17 00:00:00 2001 From: j Date: Mon, 19 May 2014 22:14:24 +0200 Subject: [PATCH] use tornado for api requests --- ctl | 7 ++ oml/api.py | 19 +++-- oml/item/api.py | 57 +++++++-------- oml/oxtornado.py | 184 +++++++++++++++++++++++++++++++++++++++++++++++ oml/server.py | 2 + oml/user/api.py | 94 +++++++++++------------- 6 files changed, 265 insertions(+), 98 deletions(-) create mode 100644 oml/oxtornado.py diff --git a/ctl b/ctl index bfb04be..eb8d849 100755 --- a/ctl +++ b/ctl @@ -94,6 +94,13 @@ if [ "$1" == "update" ]; then python2 oml update_static > /dev/null exit fi +if [ "$1" == "python" ]; then + cd $BASE/$NAME + echo `pwd` + shift + python2 $@ + exit $? +fi cd $BASE/$NAME python2 oml $@ diff --git a/oml/api.py b/oml/api.py index 1e0d190..7810942 100644 --- a/oml/api.py +++ b/oml/api.py @@ -7,20 +7,19 @@ import json import os import ox -from oxflask.api import actions -from oxflask.shortcuts import returns_json +from oxtornado import actions + import item.api import user.api -@returns_json -def selectFolder(request): + +def selectFolder(data): ''' returns { path } ''' - data = json.loads(request.form['data']) if 'data' in request.form else {} cmd = ['./ctl', 'ui', 'folder'] p = subprocess.Popen(cmd, stdout=subprocess.PIPE) stdout, stderr = p.communicate() @@ -30,14 +29,13 @@ def selectFolder(request): } actions.register(selectFolder, cache=False) -@returns_json -def selectFile(request): + +def selectFile(data): ''' returns { path } ''' - data = json.loads(request.form['data']) if 'data' in request.form else {} cmd = ['./ctl', 'ui', 'file'] p = subprocess.Popen(cmd, stdout=subprocess.PIPE) stdout, stderr = p.communicate() @@ -48,8 +46,8 @@ def selectFile(request): actions.register(selectFile, cache=False) -@returns_json -def autocompleteFolder(request): + +def autocompleteFolder(data): ''' takes { path @@ -58,7 +56,6 @@ def autocompleteFolder(request): items } ''' - data = json.loads(request.form['data']) if 'data' in request.form else {} path = data['path'] path = os.path.expanduser(path) if os.path.isdir(path): diff --git a/oml/item/api.py b/oml/item/api.py index a04407a..f293efa 100644 --- a/oml/item/api.py +++ b/oml/item/api.py @@ -5,8 +5,8 @@ from __future__ import division import os import json -from oxflask.api import actions -from oxflask.shortcuts import returns_json +from oxtornado import actions + from sqlalchemy.orm import load_only import query @@ -22,8 +22,8 @@ import utils import logging logger = logging.getLogger('oml.item.api') -@returns_json -def find(request): + +def find(data): ''' takes { query { @@ -37,7 +37,6 @@ def find(request): } ''' response = {} - data = json.loads(request.form['data']) if 'data' in request.form else {} q = query.parse(data) if 'group' in q: names = {} @@ -88,8 +87,8 @@ def find(request): return response actions.register(find) -@returns_json -def get(request): + +def get(data): ''' takes { id @@ -97,15 +96,14 @@ def get(request): } ''' response = {} - data = json.loads(request.form['data']) if 'data' in request.form else {} item = models.Item.get(data['id']) if item: response = item.json(data['keys'] if 'keys' in data else None) return response actions.register(get) -@returns_json -def edit(request): + +def edit(data): ''' takes { id @@ -114,7 +112,6 @@ def edit(request): setting identifier or base metadata is possible not both at the same time ''' response = {} - data = json.loads(request.form['data']) if 'data' in request.form else {} logger.debug('edit %s', data) item = models.Item.get(data['id']) keys = filter(lambda k: k in models.Item.id_keys, data.keys()) @@ -139,14 +136,13 @@ def edit(request): return response actions.register(edit, cache=False) -@returns_json -def remove(request): + +def remove(data): ''' takes { id } ''' - data = json.loads(request.form['data']) if 'data' in request.form else {} logger.debug('remove files %s', data) if 'ids' in data and data['ids']: for i in models.Item.query.filter(models.Item.id.in_(data['ids'])): @@ -154,8 +150,8 @@ def remove(request): return {} actions.register(remove, cache=False) -@returns_json -def findMetadata(request): + +def findMetadata(data): ''' takes { title: string, @@ -171,21 +167,19 @@ def findMetadata(request): key is one of the supported identifiers: isbn10, isbn13... ''' response = {} - data = json.loads(request.form['data']) if 'data' in request.form else {} logger.debug('findMetadata %s', data) response['items'] = meta.find(**data) return response actions.register(findMetadata) -@returns_json -def getMetadata(request): + +def getMetadata(data): ''' takes { key: value } key can be one of the supported identifiers: isbn10, isbn13, oclc, olid,... ''' - data = json.loads(request.form['data']) if 'data' in request.form else {} logger.debug('getMetadata %s', data) key, value = data.iteritems().next() if key in ('isbn10', 'isbn13'): @@ -196,15 +190,14 @@ def getMetadata(request): return response actions.register(getMetadata) -@returns_json -def download(request): + +def download(data): ''' takes { id } ''' response = {} - data = json.loads(request.form['data']) if 'data' in request.form else {} item = models.Item.get(data['id']) if item: item.queue_download() @@ -213,15 +206,14 @@ def download(request): return response actions.register(download, cache=False) -@returns_json -def cancelDownloads(request): + +def cancelDownloads(data): ''' takes { ids } ''' response = {} - data = json.loads(request.form['data']) if 'data' in request.form else {} ids = data['ids'] if ids: for item in models.Item.query.filter(models.Item.id.in_(ids)): @@ -237,14 +229,14 @@ def cancelDownloads(request): return response actions.register(cancelDownloads, cache=False) -@returns_json -def scan(request): + +def scan(data): state.main.add_callback(state.websockets[0].put, json.dumps(['scan', {}])) return {} actions.register(scan, cache=False) -@returns_json -def _import(request): + +def _import(data): ''' takes { path absolute path to import @@ -252,14 +244,13 @@ def _import(request): mode copy|move } ''' - data = json.loads(request.form['data']) if 'data' in request.form else {} logger.debug('api.import %s', data) state.main.add_callback(state.websockets[0].put, json.dumps(['import', data])) return {} actions.register(_import, 'import', cache=False) -@returns_json -def cancelImport(request): + +def cancelImport(data): state.activity['cancel'] = True return {} actions.register(cancelImport, cache=False) diff --git a/oml/oxtornado.py b/oml/oxtornado.py new file mode 100644 index 0000000..2a12d8d --- /dev/null +++ b/oml/oxtornado.py @@ -0,0 +1,184 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from __future__ import division, with_statement + +import inspect +import sys +import json +import datetime + +import tornado.ioloop +import tornado.web +import tornado.gen +import tornado.concurrent +from threading import Thread +from functools import wraps + +import logging +logger = logging.getLogger('oxtornado') + + +def json_response(data=None, status=200, text='ok'): + if not data: + data = {} + return {'status': {'code': status, 'text': text}, 'data': data} + +def _to_json(python_object): + if isinstance(python_object, datetime.datetime): + if python_object.year < 1900: + tt = python_object.timetuple() + return '%d-%02d-%02dT%02d:%02d%02dZ' % tuple(list(tt)[:6]) + return python_object.strftime('%Y-%m-%dT%H:%M:%SZ') + raise TypeError(u'%s %s is not JSON serializable' % (repr(python_object), type(python_object))) + +def json_dumps(obj): + indent = 2 + return json.dumps(obj, indent=indent, default=_to_json, ensure_ascii=False).encode('utf-8') + +def run_async(func): + @wraps(func) + def async_func(*args, **kwargs): + func_hl = Thread(target = func, args = args, kwargs = kwargs) + func_hl.start() + return func_hl + + return async_func + +def trim(docstring): + if not docstring: + return '' + # Convert tabs to spaces (following the normal Python rules) + # and split into a list of lines: + lines = docstring.expandtabs().splitlines() + # Determine minimum indentation (first line doesn't count): + indent = sys.maxint + for line in lines[1:]: + stripped = line.lstrip() + if stripped: + indent = min(indent, len(line) - len(stripped)) + # Remove indentation (first line is special): + trimmed = [lines[0].strip()] + if indent < sys.maxint: + for line in lines[1:]: + trimmed.append(line[indent:].rstrip()) + # Strip off trailing and leading blank lines: + while trimmed and not trimmed[-1]: + trimmed.pop() + while trimmed and not trimmed[0]: + trimmed.pop(0) + # Return a single string: + return '\n'.join(trimmed) + + +@run_async +def api_task(app, request, callback): + action = request.arguments.get('action', [None])[0] + data = request.arguments.get('data', ['{}'])[0] + data = json.loads(data) if data else {} + if not action: + methods = actions.keys() + api = [] + for f in sorted(methods): + api.append({'name': f, + 'doc': actions.doc(f).replace('\n', '
\n')}) + response = json_response(api) + else: + logger.debug('API %s %s', action, data) + f = actions.get(action) + if f: + with app.app_context(): + response = f(data) + else: + response = json_response(status=400, text='Unknown action %s' % action) + callback(response) + +class ApiHandler(tornado.web.RequestHandler): + def initialize(self, app): + self._app = app + + @tornado.web.asynchronous + @tornado.gen.coroutine + def post(self): + if 'origin' in self.request.headers and self.request.host not in self.request.headers['origin']: + logger.debug('reject cross site attempt to access api %s', self.request) + self.set_status(403) + self.write('') + self.finish() + return + + response = yield tornado.gen.Task(api_task, self._app, self.request) + response = json_dumps(json_response(response)) + self.set_header('Content-Type', 'application/json') + self.write(response) + self.finish() + +class ApiActions(dict): + properties = {} + versions = {} + def __init__(self): + + def api(data): + ''' + returns list of all known api actions + takes { + docs: bool + } + if docs is true, action properties contain docstrings + returns { + actions: { + 'api': { + cache: true, + doc: 'recursion' + }, + 'hello': { + cache: true, + .. + } + ... + } + } + ''' + data = data or {} + docs = data.get('docs', False) + code = data.get('code', False) + _actions = self.keys() + _actions.sort() + actions = {} + for a in _actions: + actions[a] = self.properties[a] + if docs: + actions[a]['doc'] = self.doc(a) + if code: + actions[a]['code'] = self.code(a) + return {'actions': actions} + self.register(api) + + def doc(self, name): + f = self[name] + return trim(f.__doc__) + + def code(self, name, version=None): + f = self[name] + if name != 'api' and hasattr(f, 'func_closure') and f.func_closure: + fc = filter(lambda c: hasattr(c.cell_contents, '__call__'), f.func_closure) + f = fc[len(fc)-1].cell_contents + info = f.func_code.co_filename + info = u'%s:%s' % (info, f.func_code.co_firstlineno) + return info, trim(inspect.getsource(f)) + + def register(self, method, action=None, cache=True, version=None): + if not action: + action = method.func_name + if version: + if not version in self.versions: + self.versions[version] = {} + self.versions[version][action] = method + else: + self[action] = method + self.properties[action] = {'cache': cache} + + def unregister(self, action): + if action in self: + del self[action] + +actions = ApiActions() diff --git a/oml/server.py b/oml/server.py index c3f9d1a..a6d75fd 100644 --- a/oml/server.py +++ b/oml/server.py @@ -14,6 +14,7 @@ import websocket import state import node.server +import oxtornado def run(): root_dir = os.path.normpath(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..')) @@ -30,6 +31,7 @@ def run(): handlers = [ (r'/(favicon.ico)', StaticFileHandler, {'path': static_path}), (r'/static/(.*)', StaticFileHandler, {'path': static_path}), + (r'/api/', oxtornado.ApiHandler, dict(app=app)), (r'/ws', websocket.Handler), (r".*", FallbackHandler, dict(fallback=tr)), ] diff --git a/oml/user/api.py b/oml/user/api.py index b700c6c..7b3113c 100644 --- a/oml/user/api.py +++ b/oml/user/api.py @@ -6,8 +6,8 @@ import os from copy import deepcopy import json -from oxflask.api import actions -from oxflask.shortcuts import returns_json +from oxtornado import actions + import models @@ -20,8 +20,8 @@ from changelog import Changelog import logging logger = logging.getLogger('oml.user.api') -@returns_json -def init(request): + +def init(data): ''' takes { } @@ -49,34 +49,32 @@ def init(request): return response actions.register(init) -@returns_json -def setPreferences(request): + +def setPreferences(data): ''' takes { key: value, 'sub.key': value } ''' - data = json.loads(request.form['data']) if 'data' in request.form else {} update_dict(settings.preferences, data) return settings.preferences actions.register(setPreferences) -@returns_json -def setUI(request): + +def setUI(data): ''' takes { key: value, 'sub.key': value } ''' - data = json.loads(request.form['data']) if 'data' in request.form else {} update_dict(settings.ui, data) return settings.ui actions.register(setUI) -@returns_json -def getUsers(request): + +def getUsers(data): ''' returns { users: [] @@ -90,8 +88,8 @@ def getUsers(request): } actions.register(getUsers) -@returns_json -def getLists(request): + +def getLists(data): ''' returns { lists: [] @@ -121,8 +119,8 @@ def validate_query(query): ): raise Exception('invalid query condition', condition) -@returns_json -def addList(request): + +def addList(data): ''' takes { name @@ -130,7 +128,6 @@ def addList(request): query } ''' - data = json.loads(request.form['data']) if 'data' in request.form else {} logger.debug('addList %s', data) user_id = settings.USER_ID if 'query' in data: @@ -145,8 +142,8 @@ def addList(request): return {} actions.register(addList, cache=False) -@returns_json -def editList(request): + +def editList(data): ''' takes { id @@ -154,7 +151,6 @@ def editList(request): query } ''' - data = json.loads(request.form['data']) if 'data' in request.form else {} logger.debug('editList %s', data) l = models.List.get_or_create(data['id']) name = l.name @@ -169,14 +165,13 @@ def editList(request): return l.json() actions.register(editList, cache=False) -@returns_json -def removeList(request): + +def removeList(data): ''' takes { id } ''' - data = json.loads(request.form['data']) if 'data' in request.form else {} l = models.List.get(data['id']) if l: l.remove() @@ -184,15 +179,14 @@ def removeList(request): actions.register(removeList, cache=False) -@returns_json -def addListItems(request): + +def addListItems(data): ''' takes { list items } ''' - data = json.loads(request.form['data']) if 'data' in request.form else {} if data['list'] == ':': from item.models import Item user = state.user() @@ -208,15 +202,14 @@ def addListItems(request): return {} actions.register(addListItems, cache=False) -@returns_json -def removeListItems(request): + +def removeListItems(data): ''' takes { list items } ''' - data = json.loads(request.form['data']) if 'data' in request.form else {} l = models.List.get(data['list']) if l: l.remove_items(data['items']) @@ -224,14 +217,13 @@ def removeListItems(request): return {} actions.register(removeListItems, cache=False) -@returns_json -def sortLists(request): + +def sortLists(data): ''' takes { ids } ''' - data = json.loads(request.form['data']) if 'data' in request.form else {} n = 0 logger.debug('sortLists %s', data) for id in data['ids']: @@ -243,15 +235,14 @@ def sortLists(request): return {} actions.register(sortLists, cache=False) -@returns_json -def editUser(request): + +def editUser(data): ''' takes { id nickname } ''' - data = json.loads(request.form['data']) if 'data' in request.form else {} if 'nickname' in data: p = models.User.get_or_create(data['id']) p.set_nickname(data['nickname']) @@ -259,15 +250,14 @@ def editUser(request): return {} actions.register(editUser, cache=False) -@returns_json -def requestPeering(request): + +def dataPeering(data): ''' takes { id message } ''' - data = json.loads(request.form['data']) if 'data' in request.form else {} if len(data.get('id', '')) != 43: logger.debug('invalid user id') return {} @@ -277,19 +267,18 @@ def requestPeering(request): u.info['message'] = data.get('message', '') u.save() state.nodes.queue('add', u.id) - state.nodes.queue(u.id, 'peering', 'requestPeering') + state.nodes.queue(u.id, 'peering', 'dataPeering') return {} -actions.register(requestPeering, cache=False) +actions.register(dataPeering, cache=False) -@returns_json -def acceptPeering(request): + +def acceptPeering(data): ''' takes { id message } ''' - data = json.loads(request.form['data']) if 'data' in request.form else {} if len(data.get('id', '')) != 43: logger.debug('invalid user id') return {} @@ -302,15 +291,14 @@ def acceptPeering(request): return {} actions.register(acceptPeering, cache=False) -@returns_json -def rejectPeering(request): + +def rejectPeering(data): ''' takes { id message } ''' - data = json.loads(request.form['data']) if 'data' in request.form else {} if len(data.get('id', '')) != 43: logger.debug('invalid user id') return {} @@ -322,15 +310,14 @@ def rejectPeering(request): return {} actions.register(rejectPeering, cache=False) -@returns_json -def removePeering(request): + +def removePeering(data): ''' takes { id message } ''' - data = json.loads(request.form['data']) if 'data' in request.form else {} if len(data.get('id', '')) != 43: logger.debug('invalid user id') return {} @@ -342,13 +329,12 @@ def removePeering(request): return {} actions.register(removePeering, cache=False) -@returns_json -def cancelPeering(request): + +def cancelPeering(data): ''' takes { } ''' - data = json.loads(request.form['data']) if 'data' in request.form else {} if len(data.get('id', '')) != 43: logger.debug('invalid user id') return {} @@ -360,8 +346,8 @@ def cancelPeering(request): return {} actions.register(cancelPeering, cache=False) -@returns_json -def getActivity(request): + +def getActivity(data): ''' return { activity