0x2620/openmedialibrary
1
0
Fork 0
Code Issues 131 Pull requests Releases Wiki Activity

scrape async

Browse source
This commit is contained in:
j 2015-12-02 16:30:37 +01:00
parent 19f552bd31
commit 3b166eccbb
4 changed files with 11 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
oml/changelog.py
View file

@ -156,11 +156,11 @@ class Changelog(db.Model):
primary = [key, meta[key]] primary = [key, meta[key]]
if not meta[key] and i.meta.get('primaryid', [''])[0] == key: if not meta[key] and i.meta.get('primaryid', [''])[0] == key:
logger.debug('remove id mapping %s %s', i.id, primary) logger.debug('remove id mapping %s %s', i.id, primary)
i.update_primaryid(*primary) i.update_primaryid(*primary, scrape=False)
i.modified = ts2datetime(timestamp) i.modified = ts2datetime(timestamp)
elif meta[key] and i.meta.get('primaryid') != primary: elif meta[key] and i.meta.get('primaryid') != primary:
logger.debug('edit mapping %s %s', i.id, primary) logger.debug('edit mapping %s %s', i.id, primary)
i.update_primaryid(*primary) i.update_primaryid(*primary, scrape=False)
i.modified = ts2datetime(timestamp) i.modified = ts2datetime(timestamp)
else: else:
if 'primaryid' in i.meta: if 'primaryid' in i.meta:

1
oml/downloads.py
View file

@ -77,6 +77,7 @@ class ScrapeThread(Thread):
s.item.scrape() s.item.scrape()
for f in s.item.files: for f in s.item.files:
f.move() f.move()
s.item.update_icons()
s.remove() s.remove()
trigger_event('change', {}) trigger_event('change', {})
scraped = True scraped = True

5
oml/item/models.py
View file

@ -260,7 +260,7 @@ class Item(db.Model):
if record and user in self.users: if record and user in self.users:
Changelog.record(user, 'edititem', self.id, record) Changelog.record(user, 'edititem', self.id, record)
def update_primaryid(self, key=None, id=None): def update_primaryid(self, key=None, id=None, scrape=True):
if key is None and id is None: if key is None and id is None:
if 'primaryid' not in self.meta: if 'primaryid' not in self.meta:
return return
@ -285,10 +285,13 @@ class Item(db.Model):
logger.debug('set primaryid %s %s', key, id) logger.debug('set primaryid %s %s', key, id)
# get metadata from external resources # get metadata from external resources
if scrape:
self.scrape() self.scrape()
self.update_icons() self.update_icons()
self.modified = datetime.utcnow() self.modified = datetime.utcnow()
self.save() self.save()
if not scrape:
Scrape.get_or_create(self.id)
for f in self.files.all(): for f in self.files.all():
f.move() f.move()
user = state.user() user = state.user()

8
oml/nodes.py
View file

@ -133,14 +133,14 @@ class Node(Thread):
return None return None
def request(self, action, *args): def request(self, action, *args):
logger.debug('request %s%s', action, args) logger.debug('request[%s] %s%s', self.user_id, action, args)
self.resolve() self.resolve()
url = self.url url = self.url
if not url: if not url:
logger.debug('unable to find host %s', self.user_id) logger.debug('unable to find host %s', self.user_id)
self.online = False self.online = False
return None return None
logger.debug('url=%s', url) #logger.debug('url=%s', url)
content = json.dumps([action, args]).encode() content = json.dumps([action, args]).encode()
#sig = settings.sk.sign(content, encoding=ENCODING).decode() #sig = settings.sk.sign(content, encoding=ENCODING).decode()
headers = { headers = {
@ -151,7 +151,7 @@ class Node(Thread):
'Content-Type': 'application/json', 'Content-Type': 'application/json',
} }
self._opener.addheaders = list(zip(headers.keys(), headers.values())) self._opener.addheaders = list(zip(headers.keys(), headers.values()))
logger.debug('headers: %s', self._opener.addheaders) #logger.debug('headers: %s', self._opener.addheaders)
try: try:
self._opener.timeout = self.TIMEOUT self._opener.timeout = self.TIMEOUT
r = self._opener.open(url, data=content) r = self._opener.open(url, data=content)
@ -196,7 +196,6 @@ class Node(Thread):
response = None response = None
''' '''
response = json.loads(data.decode('utf-8')) response = json.loads(data.decode('utf-8'))
logger.debug('response: %s', response)
return response return response
def _valid(self, data, sig): def _valid(self, data, sig):
@ -284,7 +283,6 @@ class Node(Thread):
self.trigger_status() self.trigger_status()
logger.debug('%s went offline', self.user.name) logger.debug('%s went offline', self.user.name)
return False return False
logger.debug('changes: %s', changes)
if not changes: if not changes:
return False return False
with db.session(): with db.session():