use new google api to find books by title/author
This commit is contained in:
parent
ac84e82a52
commit
70beacd848
1 changed files with 46 additions and 54 deletions
|
|
@ -17,7 +17,7 @@ import logging
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
NAMESPACES = {
|
||||
'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/',
|
||||
'openSearch': 'http://a9.com/-/spec/opensearchrss/1.0/',
|
||||
'atom' : 'http://www.w3.org/2005/Atom',
|
||||
'dc' : 'http://purl.org/dc/terms',
|
||||
'gbs' : 'http://schemas.google.com/books/2008',
|
||||
|
|
@ -25,29 +25,6 @@ NAMESPACES = {
|
|||
}
|
||||
XPath = partial(etree.XPath, namespaces=NAMESPACES)
|
||||
|
||||
def find_(query):
|
||||
logger.debug('find %s', query)
|
||||
query += ' isbn'
|
||||
isbns = []
|
||||
for r in ox.web.google.find(query):
|
||||
isbns += find_isbns(' '.join(r))
|
||||
logger.debug('isbns', isbns)
|
||||
results = []
|
||||
done = set()
|
||||
for isbn in isbns:
|
||||
if isbn not in done:
|
||||
r = {
|
||||
'isbn': isbn,
|
||||
'primaryid': ['isbn', isbn]
|
||||
}
|
||||
results.append(r)
|
||||
done.add(isbn)
|
||||
if len(isbn) == 10:
|
||||
done.add(stdnum.isbn.to_isbn13(isbn))
|
||||
if len(isbn) == 13 and isbn.startswith('978'):
|
||||
done.add(stdnum.isbn.to_isbn10(isbn))
|
||||
return results
|
||||
|
||||
def parse_entry(entry_):
|
||||
entry_id = XPath('descendant::atom:id')
|
||||
creator = XPath('descendant::dc:creator')
|
||||
|
|
@ -90,7 +67,7 @@ def parse_entry(entry_):
|
|||
info = decode_html_data(info)
|
||||
return info
|
||||
|
||||
def find(title=None, author=None):
|
||||
def find_feeds(title=None, author=None):
|
||||
'''
|
||||
parts = []
|
||||
if title:
|
||||
|
|
@ -107,36 +84,48 @@ def find(title=None, author=None):
|
|||
url = 'http://books.google.com/books/feeds/volumes?' + urlencode({
|
||||
'q': q.strip(),
|
||||
'max-results': 20,
|
||||
'start-index':1,
|
||||
'min-viewability':'none',
|
||||
'start-index': 1,
|
||||
'min-viewability': 'none',
|
||||
})
|
||||
data = read_url(url)
|
||||
feed = etree.fromstring(data,
|
||||
parser=etree.XMLParser(recover=True, no_network=True))
|
||||
feed = etree.fromstring(data, parser=etree.XMLParser(recover=True, no_network=True))
|
||||
results = []
|
||||
isbns = set()
|
||||
for entry_ in XPath('//atom:entry')(feed):
|
||||
info = parse_entry(entry_)
|
||||
if 'isbn' in info and not 'isbn' in isbns:
|
||||
results.append(info)
|
||||
entry = parse_entry(entry_)
|
||||
if 'isbn' in entry and 'isbn' not in isbns:
|
||||
results.append(info(entry['isbn']))
|
||||
isbns.add(info['isbn'])
|
||||
return results
|
||||
|
||||
def info_old(isbn):
|
||||
url = 'http://books.google.com/books/feeds/volumes?' + urlencode({
|
||||
'q': 'isnb:' + isbn,
|
||||
'max-results':1,
|
||||
'start-index':1,
|
||||
'min-viewability':'none',
|
||||
})
|
||||
data = read_url(url)
|
||||
feed = etree.fromstring(data,
|
||||
parser=etree.XMLParser(recover=True, no_network=True))
|
||||
for entry_ in XPath('//atom:entry')(feed):
|
||||
info = parse_entry(entry_)
|
||||
info['isbn'] = isbn
|
||||
return info
|
||||
return {}
|
||||
def find(title=None, author=None):
|
||||
q = ''
|
||||
if title:
|
||||
q += title + ' '
|
||||
if author:
|
||||
q += author
|
||||
url = 'https://www.googleapis.com/books/v1/volumes?q=%s' % q
|
||||
api_key = settings.server.get('google_api_key')
|
||||
if api_key:
|
||||
url += '&key=' + api_key
|
||||
if api_limit.error:
|
||||
raise IOError(url)
|
||||
while not api_limit.consume(1):
|
||||
logger.debug('hitting google api to fast, waiting 1 second')
|
||||
sleep(1)
|
||||
r = get_json(url, timeout=-1)
|
||||
if 'error' in r:
|
||||
logger.debug('got google api error, dont call for 10 minutes')
|
||||
store.delete(url)
|
||||
api_limit.error = True
|
||||
raise IOError(url, r)
|
||||
if 'items' not in r:
|
||||
logger.debug('unknown %s: %s [%s]', key, value, r)
|
||||
return []
|
||||
results = []
|
||||
for item in r['items']:
|
||||
results.append(parse_item(item))
|
||||
return results
|
||||
|
||||
def info(value):
|
||||
key = 'isbn'
|
||||
|
|
@ -155,11 +144,14 @@ def info(value):
|
|||
store.delete(url)
|
||||
api_limit.error = True
|
||||
raise IOError(url, r)
|
||||
if not 'items' in r:
|
||||
if 'items' not in r:
|
||||
logger.debug('unknown %s: %s [%s]', key, value, r)
|
||||
return {}
|
||||
_data = r['items'][0]['volumeInfo']
|
||||
_id = r['items'][0]['id']
|
||||
return parse_item(r['items'][0])
|
||||
|
||||
def parse_item(item):
|
||||
_data = item['volumeInfo']
|
||||
_id = item['id']
|
||||
data = {}
|
||||
for key in [
|
||||
'authors',
|
||||
|
|
@ -169,17 +161,17 @@ def info(value):
|
|||
'publishedDate',
|
||||
'publisher',
|
||||
'title',
|
||||
]:
|
||||
]:
|
||||
if key in _data:
|
||||
data[{
|
||||
'authors': 'author',
|
||||
'pageCount': 'pages',
|
||||
'publishedDate': 'date',
|
||||
}.get(key,key)] = _data[key]
|
||||
}.get(key, key)] = _data[key]
|
||||
|
||||
if 'subtitle' in _data and _data['subtitle'].strip():
|
||||
data['title'] = '{title}: {subtitle}'.format(**_data)
|
||||
if r['items'][0]['accessInfo']['viewability'] != 'NO_PAGES':
|
||||
if item['accessInfo']['viewability'] != 'NO_PAGES':
|
||||
#data['cover'] = 'https://books.google.com/books?id=%s&pg=PP1&img=1&zoom=0&hl=en' % _id
|
||||
data['cover'] = 'https://books.google.com/books/content/images/frontcover/%s?fife=w600-rw' % _id
|
||||
|
||||
|
|
@ -191,7 +183,7 @@ def info(value):
|
|||
if 'industryIdentifiers' in _data:
|
||||
for k in _data['industryIdentifiers']:
|
||||
if k['type'].startswith('ISBN'):
|
||||
if not 'isbn' in data:
|
||||
if 'isbn' not in data:
|
||||
data['isbn'] = []
|
||||
data['isbn'].append(k['identifier'])
|
||||
else:
|
||||
|
|
|
|||
Loading…
Reference in a new issue