0x2620/openmedialibrary
1
0
Fork 0
Code Issues 131 Pull requests Releases Wiki Activity

scrape async

Browse source
This commit is contained in:
j 2015-12-02 16:30:37 +01:00
parent 19f552bd31
commit 3b166eccbb
4 changed files with 11 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
oml/changelog.py
View file

@ -156,11 +156,11 @@ class Changelog(db.Model):
primary = [key, meta[key]]
if not meta[key] and i.meta.get('primaryid', [''])[0] == key:
logger.debug('remove id mapping %s %s', i.id, primary)
i.update_primaryid(*primary)
i.update_primaryid(*primary, scrape=False)
i.modified = ts2datetime(timestamp)
elif meta[key] and i.meta.get('primaryid') != primary:
logger.debug('edit mapping %s %s', i.id, primary)
i.update_primaryid(*primary)
i.update_primaryid(*primary, scrape=False)
i.modified = ts2datetime(timestamp)
else:
if 'primaryid' in i.meta:

1
oml/downloads.py
View file

@ -77,6 +77,7 @@ class ScrapeThread(Thread):
s.item.scrape()
for f in s.item.files:
f.move()
s.item.update_icons()
s.remove()
trigger_event('change', {})
scraped = True

5
oml/item/models.py
View file

@ -260,7 +260,7 @@ class Item(db.Model):
if record and user in self.users:
Changelog.record(user, 'edititem', self.id, record)
def update_primaryid(self, key=None, id=None):
def update_primaryid(self, key=None, id=None, scrape=True):
if key is None and id is None:
if 'primaryid' not in self.meta:
return
@ -285,10 +285,13 @@ class Item(db.Model):
logger.debug('set primaryid %s %s', key, id)
# get metadata from external resources
if scrape:
self.scrape()
self.update_icons()
self.modified = datetime.utcnow()
self.save()
if not scrape:
Scrape.get_or_create(self.id)
for f in self.files.all():
f.move()
user = state.user()

8
oml/nodes.py
View file

@ -133,14 +133,14 @@ class Node(Thread):
return None
def request(self, action, *args):
logger.debug('request %s%s', action, args)
logger.debug('request[%s] %s%s', self.user_id, action, args)
self.resolve()
url = self.url
if not url:
logger.debug('unable to find host %s', self.user_id)
self.online = False
return None
logger.debug('url=%s', url)
#logger.debug('url=%s', url)
content = json.dumps([action, args]).encode()
#sig = settings.sk.sign(content, encoding=ENCODING).decode()
headers = {
@ -151,7 +151,7 @@ class Node(Thread):
'Content-Type': 'application/json',
}
self._opener.addheaders = list(zip(headers.keys(), headers.values()))
logger.debug('headers: %s', self._opener.addheaders)
#logger.debug('headers: %s', self._opener.addheaders)
try:
self._opener.timeout = self.TIMEOUT
r = self._opener.open(url, data=content)
@ -196,7 +196,6 @@ class Node(Thread):
response = None
'''
response = json.loads(data.decode('utf-8'))
logger.debug('response: %s', response)
return response
def _valid(self, data, sig):
@ -284,7 +283,6 @@ class Node(Thread):
self.trigger_status()
logger.debug('%s went offline', self.user.name)
return False
logger.debug('changes: %s', changes)
if not changes:
return False
with db.session():