diff --git a/oml/meta/google.py b/oml/meta/google.py index d298618..6c9b477 100644 --- a/oml/meta/google.py +++ b/oml/meta/google.py @@ -40,12 +40,16 @@ def info(key, value): if key not in ('isbn', 'lccn', 'oclc'): raise IOError('unknwon key %s' % key) url = 'https://www.googleapis.com/books/v1/volumes?q=%s:%s' % (key, value) + if api_limit.error: + raise IOError(url) while not api_limit.consume(1): logger.debug('hitting google api to fast, waiting 1 second') sleep(1) r = get_json(url, timeout=-1) if 'error' in r: + logger.debug('got google api error, dont call for 10 minutes') store.delete(url) + api_limit.error = True raise IOError(url, r) if not 'items' in r: print('unknown %s: %s [%s]' % (key, value, r)) @@ -99,6 +103,7 @@ def info(key, value): return data class Limit(object): + _error = False def __init__(self, fill_rate, capacity): self.timestamp = time() @@ -122,4 +127,13 @@ class Limit(object): return self._tokens tokens = property(get_tokens) -api_limit = Limit(fill_rate=1, capacity=10) + def get_error(self): + if self._error and self._error < (time() - 10*60): + self._error = False + return self._error != False + + def set_error(self, value): + self._error = time() + error = property(get_error, set_error) + +api_limit = Limit(fill_rate=0.5, capacity=25)