futher google api tuning

oml/meta/google.py

@@ -40,12 +40,16 @@ def info(key, value):
     if key not in ('isbn', 'lccn', 'oclc'):
         raise IOError('unknwon key %s' % key)
     url = 'https://www.googleapis.com/books/v1/volumes?q=%s:%s' % (key, value)
+    if api_limit.error:
+        raise IOError(url)
     while not api_limit.consume(1):
         logger.debug('hitting google api to fast, waiting 1 second')
         sleep(1)
     r = get_json(url, timeout=-1)
     if 'error' in r:
+        logger.debug('got google api error, dont call for 10 minutes')
         store.delete(url)
+        api_limit.error = True
         raise IOError(url, r)
     if not 'items' in r:
         print('unknown %s: %s [%s]' % (key, value, r))
@@ -99,6 +103,7 @@ def info(key, value):
     return data
 
 class Limit(object):
+    _error = False
 
     def __init__(self, fill_rate, capacity):
         self.timestamp = time()
@@ -122,4 +127,13 @@ class Limit(object):
         return self._tokens
     tokens = property(get_tokens)
 
-api_limit = Limit(fill_rate=1, capacity=10)
+    def get_error(self):
+        if self._error and self._error < (time() - 10*60):
+            self._error = False
+        return self._error != False
+
+    def set_error(self, value):
+        self._error = time()
+    error = property(get_error, set_error)
+
+api_limit = Limit(fill_rate=0.5, capacity=25)