use new google api to find books by title/author

oml/meta/google.py

@@ -25,29 +25,6 @@ NAMESPACES = {
 }
 XPath = partial(etree.XPath, namespaces=NAMESPACES)
 
-def find_(query):
-    logger.debug('find %s', query)
-    query += ' isbn'
-    isbns = []
-    for r in ox.web.google.find(query):
-        isbns += find_isbns(' '.join(r))
-    logger.debug('isbns', isbns)
-    results = []
-    done = set()
-    for isbn in isbns:
-        if isbn not in done:
-            r = {
-                'isbn': isbn,
-                'primaryid': ['isbn', isbn]
-            }
-            results.append(r)
-            done.add(isbn)
-            if len(isbn) == 10:
-                done.add(stdnum.isbn.to_isbn13(isbn))
-            if len(isbn) == 13 and isbn.startswith('978'):
-                done.add(stdnum.isbn.to_isbn10(isbn))
-    return results
-
 def parse_entry(entry_):
     entry_id       = XPath('descendant::atom:id')
     creator        = XPath('descendant::dc:creator')
@@ -90,7 +67,7 @@ def parse_entry(entry_):
     info = decode_html_data(info)
     return info
 
-def find(title=None, author=None):
+def find_feeds(title=None, author=None):
     '''
     parts = []
     if title:
@@ -111,32 +88,44 @@ def find(title=None, author=None):
         'min-viewability': 'none',
     })
     data = read_url(url)
-    feed = etree.fromstring(data,
-        parser=etree.XMLParser(recover=True, no_network=True))
+    feed = etree.fromstring(data, parser=etree.XMLParser(recover=True, no_network=True))
     results = []
     isbns = set()
     for entry_ in XPath('//atom:entry')(feed):
-        info = parse_entry(entry_)
-        if 'isbn' in info and not 'isbn' in isbns:
-            results.append(info)
+        entry = parse_entry(entry_)
+        if 'isbn' in entry and 'isbn' not in isbns:
+            results.append(info(entry['isbn']))
             isbns.add(info['isbn'])
     return results
 
-def info_old(isbn):
-    url = 'http://books.google.com/books/feeds/volumes?' + urlencode({
-        'q': 'isnb:' + isbn,
-        'max-results':1,
-        'start-index':1,
-        'min-viewability':'none',
-    })
-    data = read_url(url)
-    feed = etree.fromstring(data,
-        parser=etree.XMLParser(recover=True, no_network=True))
-    for entry_ in XPath('//atom:entry')(feed):
-        info = parse_entry(entry_)
-        info['isbn'] = isbn
-        return info
-    return {}
+def find(title=None, author=None):
+    q = ''
+    if title:
+        q += title + ' '
+    if author:
+        q += author
+    url = 'https://www.googleapis.com/books/v1/volumes?q=%s' % q
+    api_key = settings.server.get('google_api_key')
+    if api_key:
+        url += '&key=' + api_key
+    if api_limit.error:
+        raise IOError(url)
+    while not api_limit.consume(1):
+        logger.debug('hitting google api to fast, waiting 1 second')
+        sleep(1)
+    r = get_json(url, timeout=-1)
+    if 'error' in r:
+        logger.debug('got google api error, dont call for 10 minutes')
+        store.delete(url)
+        api_limit.error = True
+        raise IOError(url, r)
+    if 'items' not in r:
+        logger.debug('unknown %s: %s [%s]', key, value, r)
+        return []
+    results = []
+    for item in r['items']:
+        results.append(parse_item(item))
+    return results
 
 def info(value):
     key = 'isbn'
@@ -155,11 +144,14 @@ def info(value):
         store.delete(url)
         api_limit.error = True
         raise IOError(url, r)
-    if not 'items' in r:
+    if 'items' not in r:
         logger.debug('unknown %s: %s [%s]', key, value, r)
         return {}
-    _data = r['items'][0]['volumeInfo']
-    _id = r['items'][0]['id']
+    return parse_item(r['items'][0])
+
+def parse_item(item):
+    _data = item['volumeInfo']
+    _id = item['id']
     data = {}
     for key in [
             'authors',
@@ -179,7 +171,7 @@ def info(value):
 
     if 'subtitle' in _data and _data['subtitle'].strip():
         data['title'] = '{title}: {subtitle}'.format(**_data)
-    if r['items'][0]['accessInfo']['viewability'] != 'NO_PAGES':
+    if item['accessInfo']['viewability'] != 'NO_PAGES':
         #data['cover'] = 'https://books.google.com/books?id=%s&pg=PP1&img=1&zoom=0&hl=en' % _id
         data['cover'] = 'https://books.google.com/books/content/images/frontcover/%s?fife=w600-rw' % _id
 
@@ -191,7 +183,7 @@ def info(value):
     if 'industryIdentifiers' in _data:
         for k in _data['industryIdentifiers']:
             if k['type'].startswith('ISBN'):
-                if not 'isbn' in data:
+                if 'isbn' not in data:
                     data['isbn'] = []
                 data['isbn'].append(k['identifier'])
             else: