futher google api tuning
This commit is contained in:
parent
f008f3d200
commit
5bb1d4c162
1 changed files with 15 additions and 1 deletions
|
@ -40,12 +40,16 @@ def info(key, value):
|
|||
if key not in ('isbn', 'lccn', 'oclc'):
|
||||
raise IOError('unknwon key %s' % key)
|
||||
url = 'https://www.googleapis.com/books/v1/volumes?q=%s:%s' % (key, value)
|
||||
if api_limit.error:
|
||||
raise IOError(url)
|
||||
while not api_limit.consume(1):
|
||||
logger.debug('hitting google api to fast, waiting 1 second')
|
||||
sleep(1)
|
||||
r = get_json(url, timeout=-1)
|
||||
if 'error' in r:
|
||||
logger.debug('got google api error, dont call for 10 minutes')
|
||||
store.delete(url)
|
||||
api_limit.error = True
|
||||
raise IOError(url, r)
|
||||
if not 'items' in r:
|
||||
print('unknown %s: %s [%s]' % (key, value, r))
|
||||
|
@ -99,6 +103,7 @@ def info(key, value):
|
|||
return data
|
||||
|
||||
class Limit(object):
|
||||
_error = False
|
||||
|
||||
def __init__(self, fill_rate, capacity):
|
||||
self.timestamp = time()
|
||||
|
@ -122,4 +127,13 @@ class Limit(object):
|
|||
return self._tokens
|
||||
tokens = property(get_tokens)
|
||||
|
||||
api_limit = Limit(fill_rate=1, capacity=10)
|
||||
def get_error(self):
|
||||
if self._error and self._error < (time() - 10*60):
|
||||
self._error = False
|
||||
return self._error != False
|
||||
|
||||
def set_error(self, value):
|
||||
self._error = time()
|
||||
error = property(get_error, set_error)
|
||||
|
||||
api_limit = Limit(fill_rate=0.5, capacity=25)
|
||||
|
|
Loading…
Reference in a new issue