From 8675edf19f3b03c38bba6ba8029484a2292f8a98 Mon Sep 17 00:00:00 2001 From: j Date: Mon, 29 Apr 2019 12:22:27 +0200 Subject: [PATCH 1/5] fallback cache --- ox/cache.py | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/ox/cache.py b/ox/cache.py index c475322..904f31d 100644 --- a/ox/cache.py +++ b/ox/cache.py @@ -456,7 +456,47 @@ class RedisCache(KVCache): self.backend = redis.from_url(self.url) -if cache_path().startswith('fs:'): +class FallbackCache(KVCache): + caches = [] + + def __init__(self): + fallback = cache_path() + for path in fallback.split('|'): + os.environ['oxCACHE'] = path + if path.startswith('redis:'): + store = RedisCache() + elif path.startswith('memcache:'): + store = MemCache() + self.caches.append(store) + os.environ['oxCACHE'] = fallback + + def get(self, url, data, headers=None, timeout=-1, value="data"): + if timeout == 0: + return None + + info_key, data_key = self._keys(url, data, headers) + for cache in self.caches: + try: + info = cache.backend.get(info_key) + except: + info = None + if info: + return cache.get(url, data, headers, timeout, value) + return None + + def set(self, url, post_data, data, headers): + self.caches[0].set(url, post_data, data, headers) + for cache in self.caches[1:]: + cache.delete(url, post_data, headers) + + def delete(self, url, data=None, headers=None): + for cache in self.caches: + cache.delete(url, data, headers) + + +if '|' in cache_path(): + store = FallbackCache() +elif cache_path().startswith('fs:'): store = FileCache() elif cache_path().startswith('redis:'): store = RedisCache() From 75b12dfb8675143bd6879c503ba5ba85495680d5 Mon Sep 17 00:00:00 2001 From: j Date: Tue, 30 Apr 2019 18:44:33 +0200 Subject: [PATCH 2/5] normalize api to end with / --- ox/api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ox/api.py b/ox/api.py index 042d0be..51742c2 100644 --- a/ox/api.py +++ b/ox/api.py @@ -227,6 +227,8 @@ def signin(url): url = 'https://%s/api/' % url else: site = url.split('/')[2] + if not url.endswith('/'): + url += '/' api = API(url) update = False try: From 8ecb14795f5e154777d44b418d26adfe85a8cbe6 Mon Sep 17 00:00:00 2001 From: j Date: Wed, 26 Jun 2019 06:53:01 +0200 Subject: [PATCH 3/5] fix release date --- ox/web/imdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ox/web/imdb.py b/ox/web/imdb.py index 641fdd7..913604c 100644 --- a/ox/web/imdb.py +++ b/ox/web/imdb.py @@ -224,7 +224,7 @@ class Imdb(SiteParser): 'releasedate': { 'page': 'releaseinfo', 're': [ - '(.*?)', + '(.*?)', strip_tags, ], 'type': 'list' From 0728847ffaefc8f14902bdb9090eed490e7fd31b Mon Sep 17 00:00:00 2001 From: j Date: Fri, 28 Jun 2019 09:58:47 +0200 Subject: [PATCH 4/5] fix keyword parsing --- ox/web/imdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ox/web/imdb.py b/ox/web/imdb.py index 913604c..4821b0c 100644 --- a/ox/web/imdb.py +++ b/ox/web/imdb.py @@ -187,7 +187,7 @@ class Imdb(SiteParser): ], type='int'), 'keyword': { 'page': 'keywords', - 're': '(.*?)']), From 2026b64faf121047e763f22276d1665d31007024 Mon Sep 17 00:00:00 2001 From: j Date: Fri, 28 Jun 2019 12:21:32 +0200 Subject: [PATCH 5/5] always utf-8, fixes #3209 --- ox/web/piratecinema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ox/web/piratecinema.py b/ox/web/piratecinema.py index 9ae3c24..4ed946b 100644 --- a/ox/web/piratecinema.py +++ b/ox/web/piratecinema.py @@ -7,7 +7,7 @@ from ox.net import read_url def get_poster_url(id): url = 'http://piratecinema.org/posters/' - html = read_url(url, unicode=True) + html = read_url(url).decode('utf-8') results = re.compile('src="(.+)" title=".+\((\d{7})\)"').findall(html) for result in results: if result[1] == id: