0x2620/oxdata
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

one better

Browse source
This commit is contained in:
j 2018-05-03 12:11:51 +02:00
parent 02d415b5fa
commit fe06a8c664
1 changed files with 65 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
oxdata/movie/imdbids.py
View file

@ -76,7 +76,9 @@ def update_year(year, film_counts):
film_counts[key] = film_count
update_month(year, month, film_counts)
else:
update_ids(year)
r = update_ids(year)
if r != film_counts[key]:
print('%s: count %s, got ids %s' % (key, film_counts[key], r))
save_film_counts(film_counts)
def update_month(year, month, film_counts):
@ -92,19 +94,28 @@ def update_month(year, month, film_counts):
if film_count != film_counts.get(key):
print_info(key, film_count, film_counts)
film_counts[key] = film_count
if film_count > MAX_PER_RANGE:
if film_count > MAX_PER_RANGE and film_count < 2*MAX_PER_RANGE:
r = update_ids(year, month, day, sort='alpha')
if r != film_counts[key]:
print('%s: count %s, got ids %s' % (key, film_counts[key], r))
save_film_counts(film_counts)
elif film_count > MAX_PER_RANGE:
print(key, '!!!to many per day')
else:
update_ids(year, month, day)
r = update_ids(year, month, day)
if r != film_counts[key]:
print('%s: count %s, got ids %s' % (key, film_counts[key], r))
save_film_counts(film_counts)
if days_total != month_total:
print('!! month and days don\'t add up: %s month vs %s days total' % (month_total, days_total))
else:
update_ids(year, month)
r = update_ids(year, month)
if r != film_counts[key]:
print('%s: count %s, got ids %s' % (key, film_counts[key], r))
save_film_counts(film_counts)
def update_ids(year, month=None, day=None):
def update_ids(year, month=None, day=None, sort=None):
films = {}
if day is not None:
url = get_day(year, month, day)
@ -115,6 +126,14 @@ def update_ids(year, month=None, day=None):
else:
url = get_year(year)
key = '%04d' % year
if sort == 'alpha':
urls = [
url.replace('sort=release_date,asc', 'sort=alpha,asc'),
url.replace('sort=release_date,asc', 'sort=alpha,desc'),
]
else:
urls = [url]
for url in urls:
data = ox.web.imdb.read_url(url, unicode=True, timeout=TIMEOUT)
n = True
page = 2
@ -158,6 +177,7 @@ def update_ids(year, month=None, day=None):
path = get_path('ids/%s.json' % key)
with open(path, 'w') as fd:
json.dump(films, fd, indent=4, ensure_ascii=False, sort_keys=True)
return len(films)
def save_film_counts(film_counts):
with open(get_path('film_counts.json'), 'w') as fd: