diff --git a/oxdata/api/__init__.py b/oxdata/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oxdata/api/actions.py b/oxdata/api/actions.py new file mode 100644 index 0000000..f80a29f --- /dev/null +++ b/oxdata/api/actions.py @@ -0,0 +1,117 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +import sys +import inspect + +from django.conf import settings + +from ox.django.shortcuts import render_to_json_response, json_response +from ox.utils import json + + +def autodiscover(): + #register api actions from all installed apps + from django.utils.importlib import import_module + from django.utils.module_loading import module_has_submodule + for app in settings.INSTALLED_APPS: + if app != 'api': + mod = import_module(app) + try: + import_module('%s.views'%app) + except: + if module_has_submodule(mod, 'views'): + raise + +def trim(docstring): + if not docstring: + return '' + # Convert tabs to spaces (following the normal Python rules) + # and split into a list of lines: + lines = docstring.expandtabs().splitlines() + # Determine minimum indentation (first line doesn't count): + indent = sys.maxint + for line in lines[1:]: + stripped = line.lstrip() + if stripped: + indent = min(indent, len(line) - len(stripped)) + # Remove indentation (first line is special): + trimmed = [lines[0].strip()] + if indent < sys.maxint: + for line in lines[1:]: + trimmed.append(line[indent:].rstrip()) + # Strip off trailing and leading blank lines: + while trimmed and not trimmed[-1]: + trimmed.pop() + while trimmed and not trimmed[0]: + trimmed.pop(0) + # Return a single string: + return '\n'.join(trimmed) + + +class ApiActions(dict): + properties = {} + def __init__(self): + + def api(request): + ''' + returns list of all known api actions + param data { + docs: bool + } + if docs is true, action properties contain docstrings + return { + status: {'code': int, 'text': string}, + data: { + actions: { + 'api': { + cache: true, + doc: 'recursion' + }, + 'hello': { + cache: true, + .. + } + ... + } + } + } + ''' + data = json.loads(request.POST.get('data', '{}')) + docs = data.get('docs', False) + code = data.get('code', False) + _actions = self.keys() + _actions.sort() + actions = {} + for a in _actions: + actions[a] = self.properties[a] + if docs: + actions[a]['doc'] = self.doc(a) + if code: + actions[a]['code'] = self.code(a) + response = json_response({'actions': actions}) + return render_to_json_response(response) + self.register(api) + + def doc(self, f): + return trim(self[f].__doc__) + + def code(self, name): + f = self[name] + if name != 'api' and hasattr(f, 'func_closure') and f.func_closure: + f = f.func_closure[0].cell_contents + info = f.func_code.co_filename[len(settings.PROJECT_ROOT)+1:] + info = u'%s:%s' % (info, f.func_code.co_firstlineno) + return info, trim(inspect.getsource(f)) + + def register(self, method, action=None, cache=True): + if not action: + action = method.func_name + self[action] = method + self.properties[action] = {'cache': cache} + + def unregister(self, action): + if action in self: + del self[action] + +actions = ApiActions() + diff --git a/oxdata/api/management/__init__.py b/oxdata/api/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oxdata/api/management/commands/__init__.py b/oxdata/api/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oxdata/api/models.py b/oxdata/api/models.py new file mode 100644 index 0000000..04d0a3c --- /dev/null +++ b/oxdata/api/models.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 + diff --git a/oxdata/api/templates/api.html b/oxdata/api/templates/api.html new file mode 100644 index 0000000..cabd746 --- /dev/null +++ b/oxdata/api/templates/api.html @@ -0,0 +1,10 @@ + + + + {{sitename}} API + + + + + + diff --git a/oxdata/api/urls.py b/oxdata/api/urls.py new file mode 100644 index 0000000..fb0b40c --- /dev/null +++ b/oxdata/api/urls.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 + +from django.conf.urls.defaults import * + + +urlpatterns = patterns("api.views", + (r'^$', 'api'), +) + diff --git a/oxdata/api/views.py b/oxdata/api/views.py new file mode 100644 index 0000000..0ce7bdf --- /dev/null +++ b/oxdata/api/views.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from __future__ import division, with_statement + +import os +import copy + +from django.shortcuts import render_to_response +from django.template import RequestContext +from django.conf import settings +from django.db.models import Max, Sum + +from ox.django.shortcuts import render_to_json_response, json_response +from ox.utils import json + +from actions import actions + + +def api(request): + if request.META['REQUEST_METHOD'] == "OPTIONS": + response = render_to_json_response({'status': {'code': 200, + 'text': 'use POST'}}) + response['Access-Control-Allow-Origin'] = '*' + return response + if not 'action' in request.POST: + methods = actions.keys() + api = [] + for f in sorted(methods): + api.append({'name': f, + 'doc': actions.doc(f).replace('\n', '
\n')}) + context = RequestContext(request, {'api': api, + 'sitename': settings.SITENAME}) + return render_to_response('api.html', context) + function = request.POST['action'] + #FIXME: possible to do this in f + #data = json.loads(request.POST['data']) + + f = actions.get(function) + if f: + response = f(request) + else: + response = render_to_json_response(json_response(status=400, + text='Unknown function %s' % function)) + response['Access-Control-Allow-Origin'] = '*' + return response + +def init(request): + return render_to_json_response(json_response()) +actions.register(init) + +def error(request): + ''' + this action is used to test api error codes, it should return a 503 error + ''' + success = error_is_success + return render_to_json_response({}) +actions.register(error) diff --git a/oxdata/movie/__init__.py b/oxdata/movie/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/oxdata/movie/models.py b/oxdata/movie/models.py new file mode 100644 index 0000000..98f9c71 --- /dev/null +++ b/oxdata/movie/models.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 + +import re + +from django.db import models +import ox + + +def find(info): + q = Imdb.objects.all() + for key in Imdb.keys: + if key in info and info[key]: + fkey = '%s_iexact' + if isinstance(info[key], list): + q = q.filter(**{fkey: '\n'.join(info[key])}) + else: + q = q.filter(**{fkey:info[key]}) + if q.count() == 1: + return q[0] + return None + +class Imdb(models.Model): + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + + imdb = models.CharField(max_length=7, unique=True) + title = models.CharField(max_length=1000, blank=True, default='') + year = models.CharField(max_length=4, blank=True, default='') + director = models.CharField(max_length=1000, blank=True, default='') + + season = models.IntegerField(blank=True, null=True) + episode = models.IntegerField(blank=True, null=True) + episodeTitle = models.CharField(max_length=1000, blank=True, default='') + episodeYear = models.CharField(max_length=4, blank=True, default='') + episodeDirector = models.CharField(max_length=1000, blank=True, default='') + + def __unicode__(self): + return u"%s (%s)" % (self.title, self.imdb) + + keys = ('title', 'director', 'year', 'season', 'episode', + 'episodeTitle', 'episodeYear', 'episodeDirector') + + def update(self): + info = ox.web.imdb.ImdbCombined(self.imdb) + if info: + for key in self.keys: + ikey = { + 'director': 'directors', + 'episodeTitle': 'episode_title', + 'episodeYear': 'episode_year', + 'episodeDirector': 'episode_directors', + }.get(key, key) + if ikey in info: + if ikey in info: + value = info[ikey] + if ikey == 'title' and 'series_title' in info: + value = info['series_title'] + if isinstance(value, list): + value = '\n'.join(value) + '\n' + setattr(self, key, value) + self.save() + + def json(self): + j = {} + j['imdbId'] = self.imdb + for key in self.keys: + j[key] = getattr(self, key) + for key in ('director', 'episodeDirector'): + if j[key].srip(): + j[key] = j[key].strip().split('\n') + else: + del j[key] + for key in j.keys(): + if not j[key]: + del j[key] + return j + +def get_new_ids(timeout=-1): + robot = ox.cache.readUrl('http://www.imdb.com/robots.txt', timeout=timeout) + sitemap_url = re.compile('\nSitemap: (http.+)').findall(robot)[0] + sitemap = ox.cache.readUrl(sitemap_url, timeout=timeout) + urls = re.compile('(.+?)').findall(sitemap) + for url in sorted(urls, reverse=True): + print url + s = ox.cache.readUrl(url, timeout=timeout) + ids = re.compile('http://www.imdb.com/title/tt(\d{7})/combined').findall(s) + for i in ids: + m, created = Imdb.objects.get_or_create(imdb=i) + if created: + m.update() diff --git a/oxdata/movie/views.py b/oxdata/movie/views.py new file mode 100644 index 0000000..bf03a2e --- /dev/null +++ b/oxdata/movie/views.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from __future__ import division + +import re +from urllib import quote + +from django.conf import settings +from ox.django.shortcuts import render_to_json_response, json_response +import ox.web.imdb +from ox.utils import json + +from api.actions import actions + +import models + + +def getId(request): + data = json.loads(request.POST['data']) + response = json_response() + movie = models.find(data) + if movie: + response['data'] = movie.json() + else: + response['status'] = {'text':'not found', 'code': 404} + return render_to_json_response(response) +actions.register(getId) + +def getData(request): + response = json_response() + data = json.loads(request.POST['data']) + id = data['id'] + if len(id) == 7: + data = ox.web.imdb.Imdb(id) + #FIXME: all this should be in ox.web.imdb.Imdb + for key in ('directors', 'writers', 'editors', 'producers', + 'cinematographers', 'languages', 'genres', 'keywords', + 'episode_directors'): + if key in data: + data[key[:-1]] = data.pop(key) + if 'countries' in data: + data['country'] = data.pop('countries') + if 'release date' in data: + data['releasedate'] = data.pop('release date') + if isinstance(data['releasedate'], list): + data['releasedate'] = min(data['releasedate']) + if 'plot' in data: + data['summary'] = data.pop('plot') + if 'cast' in data: + if isinstance(data['cast'][0], basestring): + data['cast'] = [data['cast']] + data['actor'] = [c[0] for c in data['cast']] + data['cast'] = map(lambda x: {'actor': x[0], 'character': x[1]}, data['cast']) + if 'trivia' in data: + def fix_links(t): + def fix_names(m): + return '%s' % ( + quote(m.group(2).encode('utf-8')), m.group(2) + ) + t = re.sub('(.*?)', fix_names, t) + def fix_titles(m): + return '%s' % ( + quote(m.group(2).encode('utf-8')), m.group(2) + ) + t = re.sub('(.*?)', fix_titles, t) + return t + data['trivia'] = [fix_links(t) for t in data['trivia']] + if 'aspectratio' in data: + data['aspectRatio'] = data.pop('aspectratio') + + if 'reviews' in data: + reviews = [] + for r in data['reviews']: + for url in settings.REVIEW_WHITELIST: + if url in r[0]: + reviews.append({ + 'source': settings.REVIEW_WHITELIST[url], + 'url': r[0] + }) + data['reviews'] = reviews + if not data['reviews']: + del data['reviews'] + + response['data'] = data + else: + response['status'] = {'text':'not found', 'code': 404} + + return render_to_json_response(response) +actions.register(getData) diff --git a/oxdata/settings.py b/oxdata/settings.py index faee9c9..db2e557 100644 --- a/oxdata/settings.py +++ b/oxdata/settings.py @@ -4,6 +4,8 @@ import os from os.path import join +SITENAME = 'oxdata' + PROJECT_ROOT = os.path.normpath(os.path.dirname(__file__)) DEBUG = False @@ -72,7 +74,7 @@ ADMIN_MEDIA_PREFIX = '/admin/media/' TEMPLATE_LOADERS = ( 'django.template.loaders.filesystem.load_template_source', 'django.template.loaders.app_directories.load_template_source', -# 'django.template.loaders.eggs.load_template_source', + 'django.template.loaders.eggs.load_template_source', ) MIDDLEWARE_CLASSES = ( @@ -95,12 +97,14 @@ INSTALLED_APPS = ( 'django.contrib.admin', 'django.contrib.humanize', 'south', + 'django_extensions', 'djcelery', - 'oxdata.lookup', -# 'oxdata.movie', - 'oxdata.poster', - 'oxdata.cover', + 'api', + 'lookup', + 'movie', + 'poster', + 'cover', ) LOGIN_REDIRECT_URL='/' @@ -113,6 +117,17 @@ BROKER_USER = "oxdata" BROKER_PASSWORD = "ox" BROKER_VHOST = "/oxdata" +#Movie related settings +REVIEW_WHITELIST = { + u'.filmcritic.com': u'Filmcritic', + u'metacritic.com': u'Metacritic', + u'nytimes.com': u'New York Times', + u'rottentomatoes.com': u'Rotten Tomatoes', + u'salon.com': u'Salon.com', + u'sensesofcinema.com': u'Senses of Cinema', + u'villagevoice.com': u'Village Voice' +} + #overwrite default settings with local settings try: from local_settings import * diff --git a/oxdata/static/js/pandora.api.js b/oxdata/static/js/pandora.api.js new file mode 100755 index 0000000..31b774b --- /dev/null +++ b/oxdata/static/js/pandora.api.js @@ -0,0 +1,157 @@ +/*** + Pandora API +***/ +Ox.load('UI', { + hideScreen: false, + showScreen: true, + theme: 'classic' +}, function() { + +var app = new Ox.App({ + apiURL: '/api/', +}).bindEvent('load', function(data) { + app.default_info = '

Pan.do/ra API Overview

use this api in the browser with Ox.app or use pandora_client it in python. Further description of the api can be found on the wiki
'; + app.$body = $('body'); + app.$document = $(document); + app.$window = $(window); + //app.$body.html(''); + Ox.UI.hideLoadingScreen(); + + app.$ui = {}; + app.$ui.actionList = constructList(); + app.$ui.actionInfo = Ox.Container().css({padding: '16px'}).html(app.default_info); + + app.api.api({docs: true, code: true}, function(results) { + app.actions = results.data.actions; + + if(document.location.hash) { + app.$ui.actionList.triggerEvent('select', {ids: document.location.hash.substring(1).split(',')}); + } + }); + + var $left = new Ox.SplitPanel({ + elements: [ + { + element: new Ox.Element().append(new Ox.Element() + .html('API').css({ + 'padding': '4px', + })).css({ + 'background-color': '#ddd', + 'font-weight': 'bold', + }), + size: 24 + }, + { + element: app.$ui.actionList + } + ], + orientation: 'vertical' + }); + var $main = new Ox.SplitPanel({ + elements: [ + { + element: $left, + size: 160 + }, + { + element: app.$ui.actionInfo, + } + ], + orientation: 'horizontal' + }); + + $main.appendTo(app.$body); +}); + +function constructList() { + return new Ox.TextList({ + columns: [ + { + align: "left", + id: "name", + operator: "+", + title: "Name", + unique: true, + visible: true, + width: 140 + }, + ], + columnsMovable: false, + columnsRemovable: false, + id: 'actionList', + items: function(data, callback) { + function _sort(a, b) { + if(a.name > b.name) + return 1; + else if(a.name == b.name) + return 0; + return -1; + } + if (!data.keys) { + app.api.api(function(results) { + var items = []; + Ox.forEach(results.data.actions, function(v, k) { + items.push({'name': k}) + }); + items.sort(_sort); + var result = {'data': {'items': items.length}}; + callback(result); + }); + } else { + app.api.api(function(results) { + var items = []; + Ox.forEach(results.data.actions, function(v, k) { + items.push({'name': k}) + }); + items.sort(_sort); + var result = {'data': {'items': items}}; + callback(result); + }); + } + }, + scrollbarVisible: true, + sort: [ + { + key: "name", + operator: "+" + } + ] + }).bindEvent({ + select: function(data) { + var info = $('
').addClass('OxSelectable'), + hash = '#'; + if(data.ids.length) + data.ids.forEach(function(id) { + info.append($("

").html(id)); + var $doc =$('
')
+                           .html(app.actions[id].doc.replace('/\n/
\n/g')) + .appendTo(info); + var $code = $('') + .html(app.actions[id].code[1].replace('/\n/
\n/g')) + .hide(); + var $button = new Ox.Button({ + title: [ + {id: "one", title: "right"}, + {id: "two", title: "down"}, + ], + type: "image" + }) + .addClass("margin") + .click(function() { $code.toggle()}) + .appendTo(info) + var f = app.actions[id].code[0]; + $('').html(' View Source ('+f+')').appendTo(info) + $('
').append($code).appendTo(info) 
+
+                hash += id + ','
+              });
+            else
+              info.html(app.default_info);
+
+            document.location.hash = hash.substring(0, hash.length-1);
+            app.$ui.actionInfo.html(info);
+       }
+    });
+}
+});
+
diff --git a/oxdata/urls.py b/oxdata/urls.py
index 3b6dfbe..c826d8f 100644
--- a/oxdata/urls.py
+++ b/oxdata/urls.py
@@ -10,11 +10,16 @@ from django.conf import settings
 from django.contrib import admin
 admin.autodiscover()
 
+from api import actions
+actions.autodiscover()
+
+
 def serve_static_file(path, location, content_type):
     return HttpFileResponse(location, content_type=content_type)
 
 urlpatterns = patterns('',
     (r'^$', 'oxdata.views.index'),
+    (r'^api/$', include('api.urls')),
     (r'^poster/', include('oxdata.poster.urls')),
     (r'^still/$', 'oxdata.poster.views.still'),
     (r'^id/', include('oxdata.lookup.urls')),
diff --git a/requirements.txt b/requirements.txt
index e935c59..f8e6753 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,3 +4,4 @@ South
 chardet
 django-celery
 gunicorn
+-e git://github.com/bit/django-extensions.git#egg=django_extensions