diff --git a/README.md b/README.md index 7f0ac20..8e6dbeb 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,8 @@ Development Now checkout the source and prepare for use: - mkdir client - cd client + mkdir openmedialibrary + cd openmedialibrary git clone https://git.0x2620.org/openmedialibrary.git git clone https://git.0x2620.org/openmedialibrary_platform.git platform ln -s openmedialibrary/ctl ctl @@ -47,11 +47,10 @@ On Linux you need a working python2 installation with pillow, pyhon-lxml and pop Platform ---------- -If you install Open Media Library on a architecture that is currently not upported, -you need a working python 2.7 installation and the following packages: +If you install Open Media Library on a architecture/os that is currently +not upported, you need a working python 2.7 installation and the dependencies +listed in requirements.txt and requirements-shared.txt: - apt-get install \ - python-pypdf python-stdnum python-html5lib python-chardet python-openssl \ - python-simplejson python-lxml pip install -r requirements.txt + pip install -r requirements-shared.txt diff --git a/oml/commands.py b/oml/commands.py index 315aa54..befbc1c 100644 --- a/oml/commands.py +++ b/oml/commands.py @@ -149,6 +149,7 @@ def command_release(*args): EXCLUDE=[ '--exclude', '.git', '--exclude', '.bzr', + '--exclude', 'pip_cache', '--exclude', '.*.swp', '--exclude', '._*', '--exclude', '.DS_Store' ] diff --git a/oml/downloads.py b/oml/downloads.py index b6cd57d..943d68e 100644 --- a/oml/downloads.py +++ b/oml/downloads.py @@ -4,13 +4,13 @@ from threading import Thread import time -import logging import db import state import settings import update +import logging logger = logging.getLogger('oml.downloads') class Downloads(Thread): diff --git a/oml/localnodes.py b/oml/localnodes.py index 5da89c1..5a6c6a6 100644 --- a/oml/localnodes.py +++ b/oml/localnodes.py @@ -162,6 +162,7 @@ class LocalNodes4(LocalNodesBase): def send(self): packet = self.get_packet() if packet: + logger.debug('send4 %s', packet) sockaddr = (self._BROADCAST, self._PORT) s = socket.socket (socket.AF_INET, socket.SOCK_DGRAM) s.setsockopt (socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, self._TTL) @@ -174,6 +175,7 @@ class LocalNodes4(LocalNodesBase): def get_socket(self): s = socket.socket (socket.AF_INET, socket.SOCK_DGRAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) mreq = struct.pack("=4sl", socket.inet_aton(self._BROADCAST), socket.INADDR_ANY) s.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) self._socket = s @@ -206,6 +208,7 @@ class LocalNodes6(LocalNodesBase): def get_socket(self): s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) group_bin = socket.inet_pton(socket.AF_INET6, self._BROADCAST) + b'\0'*4 s.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, group_bin) self._socket = s diff --git a/oml/node/cert.py b/oml/node/cert.py index bda83f1..c01d4bd 100644 --- a/oml/node/cert.py +++ b/oml/node/cert.py @@ -13,7 +13,7 @@ def get_fingerprint(): with open(settings.ssl_cert_path) as fd: data = fd.read() cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, data) - return hashlib.sha1(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, cert)).hexdigest() + return hashlib.sha256(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, cert)).hexdigest() def generate_ssl(): key = OpenSSL.crypto.PKey() diff --git a/oml/nodes.py b/oml/nodes.py index c7f158b..6970117 100644 --- a/oml/nodes.py +++ b/oml/nodes.py @@ -305,37 +305,42 @@ class Node(Thread): } t1 = datetime.utcnow() logger.debug('download %s', url) - self._opener.addheaders = list(zip(list(headers.keys()), list(headers.values()))) - self._opener.timeout = self.TIMEOUT*2 - r = self._opener.open(url) + self._opener.addheaders = zip(headers.keys(), headers.values()) + try: + r = self._opener.open(url, timeout=self.TIMEOUT*2) + except: + logger.debug('openurl failed %s', url, exec_info=1) + return False if r.getcode() == 200: - if r.headers.get('content-encoding', None) == 'gzip': - content = gzip.GzipFile(fileobj=r).read() - else: - content = b'' - ct = datetime.utcnow() - ''' - for chunk in iter(lambda: r.read(16*1024), b''): - content += chunk - if (datetime.utcnow() - ct).total_seconds() > 1: - ct = datetime.utcnow() - t = Transfer.get(item.id) - t.progress = len(content) / item.info['size'] - t.save() - trigger_event('transfer', { - 'id': item.id, 'progress': t.progress - }) + try: + if r.headers.get('content-encoding', None) == 'gzip': + content = gzip.GzipFile(fileobj=r).read() + else: + content = b'' + ct = datetime.utcnow() + for chunk in iter(lambda: r.read(16*1024), b''): + content += chunk + if (datetime.utcnow() - ct).total_seconds() > 1: + ct = datetime.utcnow() + t = Transfer.get(item.id) + t.progress = len(content) / item.info['size'] + t.save() + trigger_event('transfer', { + 'id': item.id, 'progress': t.progress + }) + ''' + content = r.read() + ''' - ''' - content = r.read() - logger.debug('download done %s', item.id) - - t2 = datetime.utcnow() - duration = (t2-t1).total_seconds() - if duration: - self.download_speed = len(content) / duration - logger.debug('SPEED %s', ox.format_bits(self.download_speed)) - return item.save_file(content) + t2 = datetime.utcnow() + duration = (t2-t1).total_seconds() + if duration: + self.download_speed = len(content) / duration + logger.debug('SPEED %s', ox.format_bits(self.download_speed)) + return item.save_file(content) + except: + logger.debug('download failed %s', url, exec_info=1) + return False else: logger.debug('FAILED %s', url) return False diff --git a/oml/server.py b/oml/server.py index f29020d..551b555 100644 --- a/oml/server.py +++ b/oml/server.py @@ -5,6 +5,7 @@ import os import sys import signal +import time from tornado.httpserver import HTTPServer from tornado.ioloop import IOLoop @@ -12,7 +13,7 @@ from tornado.web import StaticFileHandler, Application from cache import Cache from item.handlers import EpubHandler, ReaderHandler, FileHandler -from item.handlers import OMLHandler, serve_static +from item.handlers import OMLHandler from item.icons import IconHandler import db import node.server @@ -29,7 +30,15 @@ class MainHandler(OMLHandler): def get(self, path): path = os.path.join(settings.static_path, 'html/oml.html') - serve_static(self, path, 'text/html') + with open(path) as fd: + content = fd.read() + version = settings.MINOR_VERSION + if version == 'git': + version = int(time.mktime(time.gmtime())) + content = content.replace('oml.js?1', 'oml.js?%s' % version) + self.set_header('Content-Type', 'text/html') + self.set_header('Content-Length', str(len(content))) + self.write(content) def run(): setup.create_db() diff --git a/oml/ssl_request.py b/oml/ssl_request.py index caa4bec..f783e04 100644 --- a/oml/ssl_request.py +++ b/oml/ssl_request.py @@ -30,7 +30,15 @@ class CertValidatingHTTPSConnection(http.client.HTTPConnection): self.cert_reqs = ssl.CERT_NONE def _ValidateCertificateFingerprint(self, cert): - fingerprint = hashlib.sha1(cert).hexdigest() + if len(self.fingerprint) == 40: + fingerprint = hashlib.sha1(cert).hexdigest() + elif len(self.fingerprint) == 64: + fingerprint = hashlib.sha256(cert).hexdigest() + elif len(self.fingerprint) == 128: + fingerprint = hashlib.sha512(cert).hexdigest() + else: + logging.error('unkown fingerprint length %s (%s)', self.fingerprint, len(self.fingerprint)) + return False return fingerprint == self.fingerprint def connect(self): diff --git a/oml/update.py b/oml/update.py index e11ab08..58694d4 100644 --- a/oml/update.py +++ b/oml/update.py @@ -65,9 +65,7 @@ def download(): return True release_data = get(RELEASE_URL) release = json.loads(release_data) - old = settings.release['modules']['openmedialibrary']['version'] - new = release['modules']['openmedialibrary']['version'] - if verify(release) and old < new: + if verify(release): ox.makedirs(settings.updates_path) os.chdir(os.path.dirname(settings.base_dir)) current_files = {'release.json'} @@ -117,6 +115,7 @@ def install(): shutil.rmtree('%s_old' % module) shutil.rmtree(new) else: + os.unlink(module_tar) return False shutil.copy(os.path.join(settings.updates_path, 'release.json'), os.path.join(settings.config_path, 'release.json')) for cmd in [ diff --git a/requirements.txt b/requirements.txt index 4b56c22..fad5ebc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +lxml simplejson ed25519 SQLAlchemy==0.9.7