scrape async

oml/changelog.py

@@ -156,11 +156,11 @@ class Changelog(db.Model):
             primary = [key, meta[key]]
             if not meta[key] and i.meta.get('primaryid', [''])[0] == key:
                 logger.debug('remove id mapping %s %s', i.id, primary)
-                i.update_primaryid(*primary)
+                i.update_primaryid(*primary, scrape=False)
                 i.modified = ts2datetime(timestamp)
             elif meta[key] and i.meta.get('primaryid') != primary:
                 logger.debug('edit mapping %s %s', i.id, primary)
-                i.update_primaryid(*primary)
+                i.update_primaryid(*primary, scrape=False)
                 i.modified = ts2datetime(timestamp)
         else:
             if 'primaryid' in i.meta:

oml/downloads.py

@@ -77,6 +77,7 @@ class ScrapeThread(Thread):
                 s.item.scrape()
                 for f in s.item.files:
                     f.move()
+                s.item.update_icons()
                 s.remove()
                 trigger_event('change', {})
                 scraped = True

oml/item/models.py

@@ -260,7 +260,7 @@ class Item(db.Model):
             if record and user in self.users:
                 Changelog.record(user, 'edititem', self.id, record)
 
-    def update_primaryid(self, key=None, id=None):
+    def update_primaryid(self, key=None, id=None, scrape=True):
         if key is None and id is None:
             if 'primaryid' not in self.meta:
                 return
@@ -285,10 +285,13 @@ class Item(db.Model):
         logger.debug('set primaryid %s %s', key, id)
 
         # get metadata from external resources
-        self.scrape()
+        if scrape:
+            self.scrape()
         self.update_icons()
         self.modified = datetime.utcnow()
         self.save()
+        if not scrape:
+            Scrape.get_or_create(self.id)
         for f in self.files.all():
             f.move()
         user = state.user()

oml/nodes.py

@@ -133,14 +133,14 @@ class Node(Thread):
         return None
 
     def request(self, action, *args):
-        logger.debug('request %s%s', action, args)
+        logger.debug('request[%s] %s%s', self.user_id, action, args)
         self.resolve()
         url = self.url
         if not url:
             logger.debug('unable to find host %s', self.user_id)
             self.online = False
             return None
-        logger.debug('url=%s', url)
+        #logger.debug('url=%s', url)
         content = json.dumps([action, args]).encode()
         #sig = settings.sk.sign(content, encoding=ENCODING).decode()
         headers = {
@@ -151,7 +151,7 @@ class Node(Thread):
             'Content-Type': 'application/json',
         }
         self._opener.addheaders = list(zip(headers.keys(), headers.values()))
-        logger.debug('headers: %s', self._opener.addheaders)
+        #logger.debug('headers: %s', self._opener.addheaders)
         try:
             self._opener.timeout = self.TIMEOUT
             r = self._opener.open(url, data=content)
@@ -196,7 +196,6 @@ class Node(Thread):
             response = None
         '''
         response = json.loads(data.decode('utf-8'))
-        logger.debug('response: %s', response)
         return response
 
     def _valid(self, data, sig):
@@ -284,7 +283,6 @@ class Node(Thread):
             self.trigger_status()
             logger.debug('%s went offline', self.user.name)
             return False
-        logger.debug('changes: %s', changes)
         if not changes:
             return False
         with db.session():