0x2620/oxdata
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

py3

Browse source
This commit is contained in:
j 2016-09-19 10:55:14 +02:00
parent a26af3d2b1
commit fc1efbc7fe
1 changed files with 2 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
oxdata/movie/models.py
View file

@ -240,13 +240,13 @@ def get_new_ids(timeout=-1):
with open(new_ids_cache) as fd:
new_ids = set(json.load(fd))
else:
robot = ox.cache.read_url('http://www.imdb.com/robots.txt', timeout=timeout)
robot = ox.cache.read_url('http://www.imdb.com/robots.txt', timeout=timeout).decode('utf-8')
sitemap_url = re.compile('\nSitemap: (http.+)').findall(robot)[0]
sitemap = ox.cache.read_url(sitemap_url, timeout=timeout)
urls = re.compile('<loc>(.+?)</loc>').findall(sitemap)
ids = set()
for url in sorted(urls, reverse=False):
s = ox.cache.read_url(url, timeout=timeout)
s = ox.cache.read_url(url, timeout=timeout).decode('utf-8')
ids |= set(re.compile('<loc>http://www.imdb.com/title/tt(\d{7})/combined</loc>').findall(s))
#print url, len(ids)
known_ids = frozenset([i['imdb'] for i in Imdb.objects.all().values('imdb')])