dont end download thread if one download fails

This commit is contained in:
j 2014-09-05 19:10:47 +02:00
parent 1e9a1ef4ef
commit 2e09464c4d
2 changed files with 34 additions and 26 deletions

View file

@ -4,13 +4,13 @@ from __future__ import division
from threading import Thread
import time
import logging
import db
import state
import settings
import update
import logging
logger = logging.getLogger('oml.downloads')
class Downloads(Thread):

View file

@ -298,8 +298,13 @@ class Node(Thread):
t1 = datetime.utcnow()
logger.debug('download %s', url)
self._opener.addheaders = zip(headers.keys(), headers.values())
try:
r = self._opener.open(url, timeout=self.TIMEOUT*2)
except:
logger.debug('openurl failed %s', url, exec_info=1)
return False
if r.getcode() == 200:
try:
if r.headers.get('content-encoding', None) == 'gzip':
content = gzip.GzipFile(fileobj=r).read()
else:
@ -325,6 +330,9 @@ class Node(Thread):
self.download_speed = len(content) / duration
logger.debug('SPEED %s', ox.format_bits(self.download_speed))
return item.save_file(content)
except:
logger.debug('download failed %s', url, exec_info=1)
return False
else:
logger.debug('FAILED %s', url)
return False