This commit is contained in:
j 2014-05-14 20:46:31 +02:00
parent d385853186
commit 0e6b9533b4
12 changed files with 521 additions and 154 deletions

17
README
View file

@ -4,6 +4,23 @@ Open Media Library
soon
== Networking ==
At this time you need a working IPv6 connection to use Open Media Libary.
If you dont have native IPv6 you can use Teredo/Miredo (apt-get install miredo)
or get a tunnel Hurricane Electric (https://tunnelbroker.net/)
or SixSS (https://sixxs.net).
== Platform ==
If you install Open Media Library on a architecture thats not directly supported,
you need a working python 2.7.x installation and the following packages:
apt-get install \
python-pypdf python-stdnum python-html5lib python-chardet python-openssl \
python-simplejson python-lxml
pip install -r requirements.txt
== Development ==
mkdir client

View file

@ -26,6 +26,7 @@
"columnRequired": true,
"columnWidth": 192,
"filter": true,
"find": true,
"sort": true,
"sortType": "person"
},

3
ctl
View file

@ -25,6 +25,9 @@ export PATH
PYTHONPATH="$PLATFORM_ENV/lib/python2.7/site-packages:$SHARED_ENV/lib/python2.7/site-packages:$BASE/$NAME"
export PYTHONPATH
oxCACHE="$BASE/config/ox"
export oxCACHE
#must be called to update commands in $PATH
hash -r 2>/dev/null

View file

@ -14,6 +14,8 @@ from changelog import Changelog
import re
import state
import meta
import utils
@returns_json
@ -108,7 +110,7 @@ actions.register(edit, cache=False)
@returns_json
def identify(request):
def findMetadata(request):
'''
takes {
title: string,
@ -124,26 +126,22 @@ def identify(request):
'''
response = {}
data = json.loads(request.form['data']) if 'data' in request.form else {}
response = {
'items': [
{
u'title': u'Cinema',
u'author': [u'Gilles Deleuze'],
u'date': u'1986-10',
u'publisher': u'University of Minnesota Press',
u'isbn10': u'0816613990',
},
{
u'title': u'How to Change the World: Reflections on Marx and Marxism',
u'author': [u'Eric Hobsbawm'],
u'date': u'2011-09-06',
u'publisher': u'Yale University Press',
u'isbn13': u'9780300176162',
}
]
}
print 'findMetadata', data
response['items'] = meta.find(**data)
return response
actions.register(identify)
actions.register(findMetadata)
@returns_json
def getMetadata(request):
data = json.loads(request.form['data']) if 'data' in request.form else {}
print 'getMetadata', data
key, value = data.iteritems().next()
if key in ('isbn10', 'isbn13'):
value = utils.normalize_isbn(value)
response = meta.lookup(key, value)
response['mainid'] = key
return response
actions.register(getMetadata)
@returns_json
def download(request):

View file

@ -16,8 +16,11 @@ providers = [
('abebooks', 'isbn10')
]
def find(title, author=None, publisher=None, year=None):
return []
def find(title, author=None, publisher=None, date=None):
results = openlibrary.find(title=title, author=author, publisher=publisher, date=date)
for r in results:
r['mainid'] = 'olid'
return results
def lookup(key, value):
data = {key: value}
@ -32,16 +35,16 @@ def lookup(key, value):
if not kv in ids:
ids.append(kv)
done = False
print ids
print 'lookup %s=%s =>' % ids[0], ids
for k, v in ids:
for provider, id in providers:
if id == k:
if id == k and provider not in provider_data:
provider_data[provider] = globals()[provider].lookup(v)
for provider in sorted(
provider_data.keys(),
key=lambda x: -len(provider_data[x])
):
print provider, len(provider_data[provider])
print provider, len(provider_data[provider]), provider_data[provider].keys()
for k_, v_ in provider_data[provider].iteritems():
if not k_ in data:
data[k_] = v_

View file

@ -1,6 +1,7 @@
from ox.cache import read_url
from ox import find_re, strip_tags
from ox import find_re, strip_tags, decode_html
import re
import stdnum.isbn
base = 'http://www.lookupbyisbn.com'
@ -13,6 +14,9 @@ def get_ids(key, value):
if m:
asin = m[0].split('/')[-3]
ids.append(('asin', asin))
if key == 'asin':
if stdnum.isbn.is_valid(value):
ids.append(('isbn10', value))
if ids:
print 'lookupbyisbn.get_ids', key, value
print ids
@ -43,10 +47,13 @@ def lookup(id):
r[key] = int(r[key])
desc = find_re(data, '<h2>Description:<\/h2>(.*?)<div ')
desc = desc.replace('<br /><br />', ' ').replace('<br /> ', ' ').replace('<br />', ' ')
r['description'] = strip_tags(desc).strip()
r['description'] = desc
if r['description'] == u'Description of this item is not available at this time.':
r['description'] = ''
r['cover'] = find_re(data, '<img src="(.*?)" alt="Book cover').replace('._SL160_', '')
for key in r:
if isinstance(r[key], basestring):
r[key] = decode_html(strip_tags(r[key])).strip()
if 'author' in r and isinstance(r['author'], basestring):
r['author'] = [r['author']]
return r

View file

@ -2,12 +2,60 @@
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division
from urllib import urlencode
from ox.cache import read_url
import json
from marc_countries import COUNTRIES
from utils import normalize_isbn
KEYS = {
'authors': 'author',
'covers': 'cover',
'dewey_decimal_class': 'classification',
'isbn_10': 'isbn10',
'isbn_13': 'isbn13',
'languages': 'language',
'lccn': 'lccn',
'number_of_pages': 'pages',
'oclc_numbers': 'oclc',
'publish_country': 'country',
'publish_date': 'date',
'publishers': 'publisher',
'publish_places': 'place',
'series': 'series',
'title': 'title',
}
def find(*args, **kargs):
args = [a.replace(':', ' ') for a in args]
for k in ('date', 'publisher'):
if k in kargs:
print 'ignoring %s on openlibrary' % k, kargs[k]
del kargs[k]
for k, v in kargs.iteritems():
key = KEYS.keys()[KEYS.values().index(k)]
if v:
if not isinstance(v, list):
v = [v]
#v = ['%s:"%s"' % (key, value.replace(':', '\:')) for value in v]
v = ['"%s"' % value.replace(':', ' ') for value in v]
args += v
query = ' '.join(args)
query = query.strip()
print 'openlibrary.find', query
r = api.search(query)
results = []
ids = [b for b in r.get('result', []) if b.startswith('/books')]
books = api.get_many(ids).get('result', [])
for olid, value in books.iteritems():
olid = olid.split('/')[-1]
book = format(value)
book['olid'] = olid
results.append(book)
return results
def get_ids(key, value):
ids = []
if key == 'olid':
@ -17,15 +65,13 @@ def get_ids(key, value):
for v in data[id]:
if (id, v) not in ids:
ids.append((id, v))
elif key in ('isbn10', 'isbn13'):
elif key in ('isbn10', 'isbn13', 'oclc', 'lccn'):
print 'openlibraryid.get_ids', key, value
r = find('isbn:%s' % value)
for d in sorted(r.get('docs', []), key=lambda d: -d['last_modified_i']):
if 'edition_key' in d:
v = d['edition_key']
if isinstance(v, list):
v = v[0]
for kv in [('olid', v)] + get_ids('olid', v):
r = api.things({'type': '/type/edition', key.replace('isbn', 'isbn_'): value})
for b in r.get('result', []):
if b.startswith('/books'):
olid = b.split('/')[-1]
for kv in [('olid', olid)] + get_ids('olid', olid):
if kv not in ids:
ids.append(kv)
if ids:
@ -35,38 +81,29 @@ def get_ids(key, value):
def lookup(id, return_all=False):
#print 'openlibrary.lookup', id
data = {
'olid': id
}
url = 'https://openlibrary.org/books/%s.json' % id
info = json.loads(read_url(url))
keys = {
'title': 'title',
'authors': 'author',
'publishers': 'publisher',
'languages': 'language',
'publish_places': 'place',
'publish_country': 'country',
'covers': 'cover',
'isbn_10': 'isbn10',
'isbn_13': 'isbn13',
'lccn': 'lccn',
'oclc_numbers': 'oclc',
'dewey_decimal_class': 'classification',
'number_of_pages': 'pages',
}
for key in keys:
info = api.get('/books/' + id).get('result', {})
#url = 'https://openlibrary.org/books/%s.json' % id
#info = json.loads(read_url(url))
data = format(info, return_all)
data['olid'] = id
print 'openlibrary.lookup', id, data.keys()
return data
def format(info, return_all=False):
data = {}
for key in KEYS:
if key in info:
value = info[key]
if key == 'authors':
value = authors(value)
value = resolve_names(value)
elif key == 'publish_country':
value = value.strip()
value = COUNTRIES.get(value, value)
elif key == 'covers':
value = 'https://covers.openlibrary.org/b/id/%s.jpg' % value[0]
value = COUNTRIES.get(value, value)
elif key == 'languages':
value = languages(value)
value = resolve_names(value)
elif not return_all and isinstance(value, list) and key not in ('publish_places'):
value = value[0]
if key in ('isbn_10', 'isbn_13'):
@ -74,27 +111,52 @@ def lookup(id, return_all=False):
value = map(normalize_isbn, value)
else:
value = normalize_isbn(value)
data[keys[key]] = value
data[KEYS[key]] = value
return data
info = lookup
def find(query):
url = 'https://openlibrary.org/search.json?q=%s' % query
data = json.loads(read_url(url))
return data
def authors(authors):
return resolve_names(authors)
def resolve_names(objects, key='name'):
r = []
for o in objects:
url = 'https://openlibrary.org%s.json' % o['key']
data = json.loads(read_url(url))
r.append(data[key])
data = api.get_many([k['key'] for k in objects]).get('result', {})
for k, value in data.iteritems():
if 'location' in value and value.get('type', {}).get('key') == '/type/redirect':
value = api.get(value['location']).get('result', {})
r.append(value[key])
return r
def languages(languages):
return resolve_names(languages)
class API(object):
base = 'https://openlibrary.org/api'
def _request(self, action, data):
for key in data:
if not isinstance(data[key], basestring):
data[key] = json.dumps(data[key])
url = self.base + '/' + action + '?' + urlencode(data)
result = json.loads(read_url(url))
if 'status' in result and result['status'] == 'error' or 'error' in result:
print 'FAILED', action, data
print 'URL', url
return result
def get(self, key):
data = self._request('get', {'key': key})
return data
def get_many(self, keys):
data = self._request('get_many', {'keys': keys})
return data
def search(self, query):
if isinstance(query, basestring):
query = {
'query': query
}
data = self._request('search', {'q': query})
if 'status' in data and data['status'] == 'error':
print 'FAILED', query
return data
def things(self, query):
data = self._request('things', {'query': query})
return data
api = API()

7
requirements-shared.txt Normal file
View file

@ -0,0 +1,7 @@
tornado==3.1.1
requests==2.2.1
chardet
html5lib
ox
python-stdnum==0.9
pyPdf==1.13

9
requirements.txt Normal file
View file

@ -0,0 +1,9 @@
Twisted
simplejson
ed25519
Flask==0.10.1
SQLAlchemy==0.9.4
Flask-SQLAlchemy==1.0
Flask-Script==2.0.3
Flask-Migrate==1.2.0
pyopenssl>=0.13.1

View file

@ -9,108 +9,130 @@ oml.ui.identifyDialog = function(data) {
].map(function(id) {
return {
id: id,
title: Ox._(Ox.getObjectById(oml.config.itemKeys, id).title)
title: Ox.getObjectById(oml.config.itemKeys, id).title
};
}),
keys = [
'title', 'author', 'publisher', 'date'
].map(function(id) {
var key = Ox.getObjectById(oml.config.sortKeys, id);
return {
format: key.format,
id: id,
title: Ox._(Ox.getObjectById(oml.config.itemKeys, id).title)
operator: key.operator,
width: {
title: 288,
author: 224,
publisher: 160,
date: 96 - Ox.UI.SCROLLBAR_SIZE
}[id],
title: key.title,
visible: true
};
}),
$input = Ox.FormElementGroup({
elements: [
Ox.Select({
items: ids,
overlap: 'right',
value: 'isbn10',
width: 128
}),
Ox.Input({
value: data['isbn10'] || '',
width: 610
})
]
})
.css({margin: '16px'}),
originalData = Ox.clone(data, true),
$preview = Ox.Element(),
$idForm = renderIdForm(data),
$preview = data.mainid
? oml.ui.infoView(data)
: Ox.Element(),
$idPanel = Ox.SplitPanel({
elements: [
{element: Ox.Element().append($input), size: 48},
{element: Ox.Element().append($idForm), size: 96},
{element: $preview}
],
orientation: 'vertical'
}),
$form = Ox.Form({
items: keys.map(function(key) {
return Ox.Input({
id: key.id,
labelWidth: 128,
label: key.title,
value: key == 'author'
? (data[key.id] || []).join(', ')
: data[key.id],
width: 736
});
})
})
.css({padding: '16px'})
.bindEvent({
change: function(data) {
Ox.print('FORM CHANGE', data);
}
}),
$titleForm = Ox.Element(),
$list = Ox.TableList({
columns: [
{
id: 'index'
},
{
id: 'title',
visible: true,
width: 288,
},
{
id: 'author',
visible: true,
width: 224
},
{
id: 'publisher',
visible: true,
width: 160
},
{
id: 'date',
visible: true,
width: 96
$inputs = keys.map(function(key, index) {
return Ox.Input({
label: Ox._(key.title),
labelWidth: 64,
value: data[key.id],
width: 360
})
.css({
position: 'absolute',
left: index < 2 ? '16px' : '392px',
top: index % 2 == 0 ? '16px' : '40px'
})
.bindEvent({
submit: function(data) {
$findButton.triggerEvent('click');
}
],
items: [],
max: 1,
sort: [{key: 'index', operator: '+'}],
unique: 'index'
})
.appendTo($titleForm);
}),
$clearButton = Ox.Button({
title: Ox._('Clear'),
width: 64
})
.css({
position: 'absolute',
right: '160px',
top: '64px'
})
.bindEvent({
select: function(data) {
$that.options('buttons')[1].options({
disabled: data.ids.length == 0
click: function() {
keys.forEach(function(key) {
inputValue(key.id, '');
});
updateButtons();
}
}),
})
.appendTo($titleForm),
$resetButton = Ox.Button({
disabled: true,
title: Ox._('Reset'),
width: 64
})
.css({
position: 'absolute',
right: '88px',
top: '64px'
})
.bindEvent({
click: function() {
keys.forEach(function(key) {
inputValue(key.id, originalData[key.id]);
});
updateButtons();
}
})
.appendTo($titleForm),
$findButton = Ox.Button({
title: Ox._('Find'),
width: 64
})
.css({
position: 'absolute',
right: '16px',
top: '64px'
})
.bindEvent({
click: function() {
var data = {};
keys.forEach(function(key) {
data[key.id] = inputValue(key.id);
});
findMetadata(data);
}
})
.appendTo($titleForm),
$titlePanel = Ox.SplitPanel({
elements: [
{element: Ox.Element().append($form), size: 120},
{element: $list}
{element: $titleForm, size: 96},
{element: renderResults([Ox.extend({index: '0'}, data)])}
],
orientation: 'vertical'
}),
@ -186,6 +208,233 @@ oml.ui.identifyDialog = function(data) {
width: 768
});
function findMetadata(data) {
$titlePanel.replaceElement(1, Ox.LoadingScreen().start());
oml.api.findMetadata(data, function(result) {
Ox.print('GOT RESULTS', result.data);
var items = result.data.items.map(function(item, index) {
return Ox.extend({index: index.toString()}, item);
}).concat([
Ox.extend({index: result.data.items.length.toString()}, data)
]);
$titlePanel.replaceElement(1, renderResults(items));
});
}
function getMetadata(key, value) {
$idPanel.replaceElement(1, Ox.LoadingScreen().start());
oml.api.getMetadata(Ox.extend({}, key, value), function(result) {
Ox.print('GOT RESULT', result.data);
$idForm = renderIdForm(result.data);
$preview = oml.ui.infoView(result.data);
$idPanel
.replaceElement(0, $idForm)
.replaceElement(1, $preview);
});
}
function inputValue(key, value) {
// FIXME: UNELEGANT
Ox.print('INPUTVALUE', key, value)
var $input = $inputs[[
'title', 'author', 'publisher', 'date'
].indexOf(key)];
if (Ox.isUndefined(value)) {
value = $input.value();
if (key == 'author') {
value = value ? value.split(', ') : [];
}
} else {
$input.value(
key == 'author' ? (value || []).join(', ') : value
);
}
return value;
}
function isEmpty(data) {
return Ox.every(data, Ox.isEmpty);
}
function isOriginal(data) {
return Ox.every(data, function(value, key) {
return value == originalData[key];
});
}
function renderIdForm(data) {
var $element = Ox.Element(),
$elements = ids.map(function(id, index) {
return Ox.FormElementGroup({
elements: [
Ox.Checkbox({
overlap: 'right',
title: Ox._(id.title),
value: id.id == data.mainid,
width: 80
})
.bindEvent({
change: function(data) {
var value = $elements[index].options('elements')[1].value()
if (data.value) {
if (value) {
$elements.forEach(function($element, i) {
if (i != index) {
$elements[i].options('elements')[0].value(false);
}
});
getMetadata(id.id, value, function() {
// ...
});
} else {
this.value(false);
}
} else {
this.value(true);
}
}
}),
Ox.Input({
value: data[id.id] || '',
width: 160
})
.bindEvent({
submit: function(data) {
if (data.value) {
$elements.forEach(function($element, i) {
$element.options('elements')[0].options({
disabled: true,
value: i == index
});
$element.options('elements')[1].options({
disabled: true
});
});
getMetadata(id.id, data.value, function() {
// ...
});
}
}
})
],
float: 'left'
})
.css({
position: 'absolute',
left: 16 + Math.floor(index / 2) * 248 + 'px',
top: 16 + (index % 2) * 24 + 'px'
})
.appendTo($element);
}),
$resetButton = Ox.Button({
disabled: true,
title: Ox._('Reset'),
width: 64
})
.css({
position: 'absolute',
right: '16px',
top: '64px'
})
.bindEvent({
click: function() {
/*
keys.forEach(function(key) {
inputValue(key.id, originalData[key.id]);
});
updateButtons();
*/
}
})
.appendTo($element);
return $element;
Ox.print('???', data.mainid)
return Ox.Form({
items: Ox.flatten(ids.map(function(id) {
return [
Ox.Checkbox({
disabled: !data[id.id] || id.id == data.mainid,
id: id.id + 'Checkbox',
title: Ox._(id.title),
value: id.id == data.mainid,
width: 128
})
.bindEvent({
change: function() {
getMetadata(id.id, data[id.id]);
}
}),
Ox.Input({
id: id.id + 'Input',
value: data[id.id] || '',
width: 128
})
.css({marginBottom: '16px'})
.bindEvent({
change: function(data) {
if (data.value) {
getMetadata(id.id, data.value, function() {
//...
});
} else {
Ox.print('this', this)
}
}
})
];
}))
})
.css({margin: '16px'});
return $form;
}
function renderResults(items) {
Ox.print('LIST ITEMS::::', items);
var $list = Ox.TableList({
columns: Ox.clone(keys, true),
items: items,
min: 1,
max: 1,
scrollbarVisible: true,
selected: ['0'],
sort: [{key: 'index', operator: '+'}],
unique: 'index'
})
.bindEvent({
select: function(data) {
var index = data.ids[0];
data = Ox.getObject(items, 'index', index);
$results.replaceElement(1, Ox.LoadingScreen().start());
Ox.print('OLID', data.olid);
oml.api.getMetadata({olid: data.olid}, function(result) {
Ox.print('#### GOT DATA', result.data);
$results.replaceElement(1, oml.ui.infoView(result.data));
that.options('buttons')[1].options({disabled: false});
});
}
}),
$results = Ox.SplitPanel({
elements: [
{element: $list, size: 80},
{element: oml.ui.infoView(items[0])}
],
orientation: 'vertical'
});
return $results;
}
function updateButtons() {
var data = {}, empty, original;
keys.forEach(function(key) {
data[key.id] = inputValue(key.id);
});
empty = isEmpty(data);
original = isOriginal(data);
$clearButton.options({disabled: empty});
$resetButton.options({disabled: original});
$findButton.options({disabled: empty});
}
return that;
};

View file

@ -1,6 +1,6 @@
'use strict';
oml.ui.infoView = function() {
oml.ui.infoView = function(identifyData) {
var ui = oml.user.ui,
@ -34,11 +34,14 @@ oml.ui.infoView = function() {
.css({
position: 'absolute',
left: '288px',
right: '176px',
right: !identifyData ? '176px' : 16 + Ox.UI.SCROLLBAR_SIZE + 'px',
top: '16px'
})
.appendTo(that),
$data;
if (!identifyData) {
$data = Ox.Element()
.addClass('OxSelectable')
.css({
@ -48,6 +51,7 @@ oml.ui.infoView = function() {
width: '128px'
})
.appendTo(that);
}
function formatLight(str) {
return '<span class="OxLight">' + str + '</span>';
@ -218,7 +222,9 @@ oml.ui.infoView = function() {
width = Math.round(ratio >= 1 ? size : size * ratio),
height = Math.round(ratio <= 1 ? size : size / ratio),
left = Math.floor((size - width) / 2),
src = '/' + data.id + '/cover256.jpg',
src = !identifyData
? '/' + data.id + '/cover256.jpg'
: data.cover,
reflectionSize = Math.round(size / 2);
$elements.forEach(function($element) {
@ -334,6 +340,7 @@ oml.ui.infoView = function() {
.appendTo($info);
}
$('<div>').css({height: '16px'}).appendTo($info);
} else if ($element == $data) {
@ -424,7 +431,11 @@ oml.ui.infoView = function() {
};
ui.item && that.update(ui.item);
if (!identifyData) {
ui.item && that.update(ui.item);
} else {
that.update(identifyData, [$cover, $info]);
}
oml.bindEvent({
transfer: function(data) {

View file

@ -73,8 +73,8 @@
? Ox['format' + Ox.toTitleCase(key.format.type)].apply(
this, [value].concat(key.format.args || [])
)
: Ox.isArray(key.type) ? value.join(', ')
: value;
: Ox.isArray(key.type) ? (value || []).join(', ')
: (value || '');
}
});
})