From a131c784823b657b48899cf643896b00db391641 Mon Sep 17 00:00:00 2001 From: j Date: Mon, 5 Aug 2019 12:13:18 +0200 Subject: [PATCH] continue after --- oxdata/movie/imdbids.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/oxdata/movie/imdbids.py b/oxdata/movie/imdbids.py index 0cb606c..94b8453 100644 --- a/oxdata/movie/imdbids.py +++ b/oxdata/movie/imdbids.py @@ -22,6 +22,7 @@ TIMEOUT = 90 * DAY DATA_ROOT = os.path.join(settings.MEDIA_ROOT, 'imdb') DEBUG = False +base_url = 'https://www.imdb.com' def debug(*args, **kwargs): if DEBUG: @@ -37,7 +38,6 @@ def read_url(url, timeout): return data def get_range(from_, to): - base_url = 'https://www.imdb.com' url = '%s/search/title?adult=include&release_date=%s,%s&sort=release_date,asc&count=50' % (base_url, from_, to) return url @@ -113,8 +113,6 @@ def update_month(year, month, film_counts): if film_count > MAX_PER_RANGE and film_count < 2*MAX_PER_RANGE: r = update_ids(year, month, day, sort='alpha', expected=film_count) save_film_counts(film_counts) - elif film_count > MAX_PER_RANGE: - print(key, '!!!to many per day', film_count, key) else: r = update_ids(year, month, day, expected=film_count) save_film_counts(film_counts) @@ -168,9 +166,13 @@ def update_ids(year, month=None, day=None, sort=None, expexted=None): while n: start += step next_link = 'start=%s&ref_=adv_nxt"' % (start) + after_link = 'href="(.*?after=.*?&ref_=adv_nxt)"' has_next = re.compile(next_link).findall(data) + has_after = re.compile(after_link).findall(data) if has_next: n = '%s&start=%s' % (url, start) + elif sort != 'alpha' and start > MAX_PER_RANGE and has_after: + n = '%s%s' % (base_url, has_after[0]) else: n = False doc = lxml.html.fromstring(data)