diff --git a/oml/meta/google.py b/oml/meta/google.py index 713647d..029124a 100644 --- a/oml/meta/google.py +++ b/oml/meta/google.py @@ -17,7 +17,7 @@ import logging logger = logging.getLogger(__name__) NAMESPACES = { - 'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/', + 'openSearch': 'http://a9.com/-/spec/opensearchrss/1.0/', 'atom' : 'http://www.w3.org/2005/Atom', 'dc' : 'http://purl.org/dc/terms', 'gbs' : 'http://schemas.google.com/books/2008', @@ -25,29 +25,6 @@ NAMESPACES = { } XPath = partial(etree.XPath, namespaces=NAMESPACES) -def find_(query): - logger.debug('find %s', query) - query += ' isbn' - isbns = [] - for r in ox.web.google.find(query): - isbns += find_isbns(' '.join(r)) - logger.debug('isbns', isbns) - results = [] - done = set() - for isbn in isbns: - if isbn not in done: - r = { - 'isbn': isbn, - 'primaryid': ['isbn', isbn] - } - results.append(r) - done.add(isbn) - if len(isbn) == 10: - done.add(stdnum.isbn.to_isbn13(isbn)) - if len(isbn) == 13 and isbn.startswith('978'): - done.add(stdnum.isbn.to_isbn10(isbn)) - return results - def parse_entry(entry_): entry_id = XPath('descendant::atom:id') creator = XPath('descendant::dc:creator') @@ -90,7 +67,7 @@ def parse_entry(entry_): info = decode_html_data(info) return info -def find(title=None, author=None): +def find_feeds(title=None, author=None): ''' parts = [] if title: @@ -107,36 +84,48 @@ def find(title=None, author=None): url = 'http://books.google.com/books/feeds/volumes?' + urlencode({ 'q': q.strip(), 'max-results': 20, - 'start-index':1, - 'min-viewability':'none', + 'start-index': 1, + 'min-viewability': 'none', }) data = read_url(url) - feed = etree.fromstring(data, - parser=etree.XMLParser(recover=True, no_network=True)) + feed = etree.fromstring(data, parser=etree.XMLParser(recover=True, no_network=True)) results = [] isbns = set() for entry_ in XPath('//atom:entry')(feed): - info = parse_entry(entry_) - if 'isbn' in info and not 'isbn' in isbns: - results.append(info) + entry = parse_entry(entry_) + if 'isbn' in entry and 'isbn' not in isbns: + results.append(info(entry['isbn'])) isbns.add(info['isbn']) return results -def info_old(isbn): - url = 'http://books.google.com/books/feeds/volumes?' + urlencode({ - 'q': 'isnb:' + isbn, - 'max-results':1, - 'start-index':1, - 'min-viewability':'none', - }) - data = read_url(url) - feed = etree.fromstring(data, - parser=etree.XMLParser(recover=True, no_network=True)) - for entry_ in XPath('//atom:entry')(feed): - info = parse_entry(entry_) - info['isbn'] = isbn - return info - return {} +def find(title=None, author=None): + q = '' + if title: + q += title + ' ' + if author: + q += author + url = 'https://www.googleapis.com/books/v1/volumes?q=%s' % q + api_key = settings.server.get('google_api_key') + if api_key: + url += '&key=' + api_key + if api_limit.error: + raise IOError(url) + while not api_limit.consume(1): + logger.debug('hitting google api to fast, waiting 1 second') + sleep(1) + r = get_json(url, timeout=-1) + if 'error' in r: + logger.debug('got google api error, dont call for 10 minutes') + store.delete(url) + api_limit.error = True + raise IOError(url, r) + if 'items' not in r: + logger.debug('unknown %s: %s [%s]', key, value, r) + return [] + results = [] + for item in r['items']: + results.append(parse_item(item)) + return results def info(value): key = 'isbn' @@ -155,11 +144,14 @@ def info(value): store.delete(url) api_limit.error = True raise IOError(url, r) - if not 'items' in r: + if 'items' not in r: logger.debug('unknown %s: %s [%s]', key, value, r) return {} - _data = r['items'][0]['volumeInfo'] - _id = r['items'][0]['id'] + return parse_item(r['items'][0]) + +def parse_item(item): + _data = item['volumeInfo'] + _id = item['id'] data = {} for key in [ 'authors', @@ -169,17 +161,17 @@ def info(value): 'publishedDate', 'publisher', 'title', - ]: + ]: if key in _data: data[{ 'authors': 'author', 'pageCount': 'pages', 'publishedDate': 'date', - }.get(key,key)] = _data[key] + }.get(key, key)] = _data[key] if 'subtitle' in _data and _data['subtitle'].strip(): data['title'] = '{title}: {subtitle}'.format(**_data) - if r['items'][0]['accessInfo']['viewability'] != 'NO_PAGES': + if item['accessInfo']['viewability'] != 'NO_PAGES': #data['cover'] = 'https://books.google.com/books?id=%s&pg=PP1&img=1&zoom=0&hl=en' % _id data['cover'] = 'https://books.google.com/books/content/images/frontcover/%s?fife=w600-rw' % _id @@ -191,7 +183,7 @@ def info(value): if 'industryIdentifiers' in _data: for k in _data['industryIdentifiers']: if k['type'].startswith('ISBN'): - if not 'isbn' in data: + if 'isbn' not in data: data['isbn'] = [] data['isbn'].append(k['identifier']) else: