dont end download thread if one download fails
This commit is contained in:
parent
1e9a1ef4ef
commit
2e09464c4d
2 changed files with 34 additions and 26 deletions
|
@ -4,13 +4,13 @@ from __future__ import division
|
|||
|
||||
from threading import Thread
|
||||
import time
|
||||
import logging
|
||||
|
||||
import db
|
||||
import state
|
||||
import settings
|
||||
import update
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger('oml.downloads')
|
||||
|
||||
class Downloads(Thread):
|
||||
|
|
|
@ -298,8 +298,13 @@ class Node(Thread):
|
|||
t1 = datetime.utcnow()
|
||||
logger.debug('download %s', url)
|
||||
self._opener.addheaders = zip(headers.keys(), headers.values())
|
||||
try:
|
||||
r = self._opener.open(url, timeout=self.TIMEOUT*2)
|
||||
except:
|
||||
logger.debug('openurl failed %s', url, exec_info=1)
|
||||
return False
|
||||
if r.getcode() == 200:
|
||||
try:
|
||||
if r.headers.get('content-encoding', None) == 'gzip':
|
||||
content = gzip.GzipFile(fileobj=r).read()
|
||||
else:
|
||||
|
@ -325,6 +330,9 @@ class Node(Thread):
|
|||
self.download_speed = len(content) / duration
|
||||
logger.debug('SPEED %s', ox.format_bits(self.download_speed))
|
||||
return item.save_file(content)
|
||||
except:
|
||||
logger.debug('download failed %s', url, exec_info=1)
|
||||
return False
|
||||
else:
|
||||
logger.debug('FAILED %s', url)
|
||||
return False
|
||||
|
|
Loading…
Reference in a new issue