faster check for known ids

oxdata/movie/models.py

@@ -94,15 +94,18 @@ class Imdb(models.Model):
         return j
 
 def get_new_ids(timeout=-1):
+    known_ids = frozenset([i['imdb'] for i in Imdb.objects.all().values('imdb')])
     robot = ox.cache.readUrl('http://www.imdb.com/robots.txt', timeout=timeout)
     sitemap_url = re.compile('\nSitemap: (http.+)').findall(robot)[0]
     sitemap = ox.cache.readUrl(sitemap_url, timeout=timeout)
     urls = re.compile('<loc>(.+?)</loc>').findall(sitemap)
     for url in sorted(urls, reverse=True):
-        print url
         s = ox.cache.readUrl(url, timeout=timeout)
         ids = re.compile('<loc>http://www.imdb.com/title/tt(\d{7})/combined</loc>').findall(s)
-        for i in ids:
-            m, created = Imdb.objects.get_or_create(imdb=i)
-            if created:
+        added = 0
+        for i in frozenset(ids) - known_ids:
+            m= Imdb(imdb=i)
             m.update()
+            added += 1
+        if added:
+            print url, added