0x2620/openmedialibrary
1
0
Fork 0
Code Issues 131 Pull requests Releases Wiki Activity

dont end download thread if one download fails

Browse source
This commit is contained in:
j 2014-09-05 19:10:47 +02:00
parent 1e9a1ef4ef
commit 2e09464c4d
2 changed files with 34 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
oml/downloads.py
View file

@ -4,13 +4,13 @@ from __future__ import division
from threading import Thread from threading import Thread
import time import time
import logging
import db import db
import state import state
import settings import settings
import update import update
import logging
logger = logging.getLogger('oml.downloads') logger = logging.getLogger('oml.downloads')
class Downloads(Thread): class Downloads(Thread):

8
oml/nodes.py
View file

@ -298,8 +298,13 @@ class Node(Thread):
t1 = datetime.utcnow() t1 = datetime.utcnow()
logger.debug('download %s', url) logger.debug('download %s', url)
self._opener.addheaders = zip(headers.keys(), headers.values()) self._opener.addheaders = zip(headers.keys(), headers.values())
try:
r = self._opener.open(url, timeout=self.TIMEOUT*2) r = self._opener.open(url, timeout=self.TIMEOUT*2)
except:
logger.debug('openurl failed %s', url, exec_info=1)
return False
if r.getcode() == 200: if r.getcode() == 200:
try:
if r.headers.get('content-encoding', None) == 'gzip': if r.headers.get('content-encoding', None) == 'gzip':
content = gzip.GzipFile(fileobj=r).read() content = gzip.GzipFile(fileobj=r).read()
else: else:
@ -325,6 +330,9 @@ class Node(Thread):
self.download_speed = len(content) / duration self.download_speed = len(content) / duration
logger.debug('SPEED %s', ox.format_bits(self.download_speed)) logger.debug('SPEED %s', ox.format_bits(self.download_speed))
return item.save_file(content) return item.save_file(content)
except:
logger.debug('download failed %s', url, exec_info=1)
return False
else: else:
logger.debug('FAILED %s', url) logger.debug('FAILED %s', url)
return False return False