0x2620/openmedialibrary
1
0
Fork 0
Code Issues 131 Pull requests Releases Wiki Activity

futher google api tuning

Browse source
This commit is contained in:
j 2016-01-23 18:56:36 +05:30
parent f008f3d200
commit 5bb1d4c162
1 changed files with 15 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
oml/meta/google.py
View file

@ -40,12 +40,16 @@ def info(key, value):
if key not in ('isbn', 'lccn', 'oclc'): if key not in ('isbn', 'lccn', 'oclc'):
raise IOError('unknwon key %s' % key) raise IOError('unknwon key %s' % key)
url = 'https://www.googleapis.com/books/v1/volumes?q=%s:%s' % (key, value) url = 'https://www.googleapis.com/books/v1/volumes?q=%s:%s' % (key, value)
if api_limit.error:
raise IOError(url)
while not api_limit.consume(1): while not api_limit.consume(1):
logger.debug('hitting google api to fast, waiting 1 second') logger.debug('hitting google api to fast, waiting 1 second')
sleep(1) sleep(1)
r = get_json(url, timeout=-1) r = get_json(url, timeout=-1)
if 'error' in r: if 'error' in r:
logger.debug('got google api error, dont call for 10 minutes')
store.delete(url) store.delete(url)
api_limit.error = True
raise IOError(url, r) raise IOError(url, r)
if not 'items' in r: if not 'items' in r:
print('unknown %s: %s [%s]' % (key, value, r)) print('unknown %s: %s [%s]' % (key, value, r))
@ -99,6 +103,7 @@ def info(key, value):
return data return data
class Limit(object): class Limit(object):
_error = False
def __init__(self, fill_rate, capacity): def __init__(self, fill_rate, capacity):
self.timestamp = time() self.timestamp = time()
@ -122,4 +127,13 @@ class Limit(object):
return self._tokens return self._tokens
tokens = property(get_tokens) tokens = property(get_tokens)
api_limit = Limit(fill_rate=1, capacity=10) def get_error(self):
if self._error and self._error < (time() - 10*60):
self._error = False
return self._error != False
def set_error(self, value):
self._error = time()
error = property(get_error, set_error)
api_limit = Limit(fill_rate=0.5, capacity=25)