From 6cb3e13fb75f8e6ffa47796d6d53be59c4aa8d6b Mon Sep 17 00:00:00 2001 From: j Date: Fri, 5 Sep 2014 17:38:24 +0200 Subject: [PATCH 1/6] update requirements, cleanup readme --- README.md | 13 ++++++------- requirements.txt | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 9855a92..8440921 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,8 @@ Development Now checkout the source and prepare for use: - mkdir client - cd client + mkdir openmedialibrary + cd openmedialibrary git clone https://git.0x2620.org/openmedialibrary.git git clone https://git.0x2620.org/openmedialibrary_platform.git platform ln -s openmedialibrary/ctl ctl @@ -47,11 +47,10 @@ On Linux you need a working python2 installation with PIL, pyhon-lxml and popple Platform ---------- -If you install Open Media Library on a architecture that is currently not upported, -you need a working python 2.7 installation and the following packages: +If you install Open Media Library on a architecture/os that is currently +not upported, you need a working python 2.7 installation and the dependencies +listed in requirements.txt and requirements-shared.txt: - apt-get install \ - python-pypdf python-stdnum python-html5lib python-chardet python-openssl \ - python-simplejson python-lxml pip install -r requirements.txt + pip install -r requirements-shared.txt diff --git a/requirements.txt b/requirements.txt index 134b16e..37db19c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Twisted +lxml simplejson ed25519 SQLAlchemy==0.9.4 From 0ef522f620c8363af91602711633b8eaa0d0ce36 Mon Sep 17 00:00:00 2001 From: j Date: Fri, 5 Sep 2014 18:36:03 +0200 Subject: [PATCH 2/6] reuse port --- oml/commands.py | 1 + oml/localnodes.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/oml/commands.py b/oml/commands.py index 34e17e8..bac5005 100644 --- a/oml/commands.py +++ b/oml/commands.py @@ -149,6 +149,7 @@ def command_release(*args): EXCLUDE=[ '--exclude', '.git', '--exclude', '.bzr', + '--exclude', 'pip_cache', '--exclude', '.*.swp', '--exclude', '._*', '--exclude', '.DS_Store' ] diff --git a/oml/localnodes.py b/oml/localnodes.py index 3140b14..c41ac44 100644 --- a/oml/localnodes.py +++ b/oml/localnodes.py @@ -162,6 +162,7 @@ class LocalNodes4(LocalNodesBase): def send(self): packet = self.get_packet() if packet: + logger.debug('send4 %s', packet) sockaddr = (self._BROADCAST, self._PORT) s = socket.socket (socket.AF_INET, socket.SOCK_DGRAM) s.setsockopt (socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, self._TTL) @@ -174,6 +175,7 @@ class LocalNodes4(LocalNodesBase): def get_socket(self): s = socket.socket (socket.AF_INET, socket.SOCK_DGRAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) mreq = struct.pack("=4sl", socket.inet_aton(self._BROADCAST), socket.INADDR_ANY) s.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) self._socket = s @@ -206,6 +208,7 @@ class LocalNodes6(LocalNodesBase): def get_socket(self): s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) group_bin = socket.inet_pton(socket.AF_INET6, self._BROADCAST) + '\0'*4 s.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, group_bin) self._socket = s From 1e9a1ef4ef10bf08e6a5f7ef782c7f96bc7eaa4b Mon Sep 17 00:00:00 2001 From: j Date: Fri, 5 Sep 2014 19:02:19 +0200 Subject: [PATCH 3/6] load new version --- oml/server.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/oml/server.py b/oml/server.py index 6c72380..d2af7f2 100644 --- a/oml/server.py +++ b/oml/server.py @@ -5,6 +5,7 @@ from __future__ import division, print_function import os import sys import signal +import time from tornado.httpserver import HTTPServer from tornado.ioloop import IOLoop @@ -12,7 +13,7 @@ from tornado.web import StaticFileHandler, Application from cache import Cache from item.handlers import EpubHandler, ReaderHandler, FileHandler -from item.handlers import OMLHandler, serve_static +from item.handlers import OMLHandler from item.icons import IconHandler import db import node.server @@ -29,7 +30,15 @@ class MainHandler(OMLHandler): def get(self, path): path = os.path.join(settings.static_path, 'html/oml.html') - serve_static(self, path, 'text/html') + with open(path) as fd: + content = fd.read() + version = settings.MINOR_VERSION + if version == 'git': + version = int(time.mktime(time.gmtime())) + content = content.replace('oml.js?1', 'oml.js?%s' % version) + self.set_header('Content-Type', 'text/html') + self.set_header('Content-Length', str(len(content))) + self.write(content) def run(): setup.create_db() From 2e09464c4d2b3c6701053239be89244c535114eb Mon Sep 17 00:00:00 2001 From: j Date: Fri, 5 Sep 2014 19:10:47 +0200 Subject: [PATCH 4/6] dont end download thread if one download fails --- oml/downloads.py | 2 +- oml/nodes.py | 58 +++++++++++++++++++++++++++--------------------- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/oml/downloads.py b/oml/downloads.py index e7457ee..a42a6b0 100644 --- a/oml/downloads.py +++ b/oml/downloads.py @@ -4,13 +4,13 @@ from __future__ import division from threading import Thread import time -import logging import db import state import settings import update +import logging logger = logging.getLogger('oml.downloads') class Downloads(Thread): diff --git a/oml/nodes.py b/oml/nodes.py index 7399caf..510f938 100644 --- a/oml/nodes.py +++ b/oml/nodes.py @@ -298,33 +298,41 @@ class Node(Thread): t1 = datetime.utcnow() logger.debug('download %s', url) self._opener.addheaders = zip(headers.keys(), headers.values()) - r = self._opener.open(url, timeout=self.TIMEOUT*2) + try: + r = self._opener.open(url, timeout=self.TIMEOUT*2) + except: + logger.debug('openurl failed %s', url, exec_info=1) + return False if r.getcode() == 200: - if r.headers.get('content-encoding', None) == 'gzip': - content = gzip.GzipFile(fileobj=r).read() - else: - content = '' - ct = datetime.utcnow() - for chunk in iter(lambda: r.read(16*1024), ''): - content += chunk - if (datetime.utcnow() - ct).total_seconds() > 1: - ct = datetime.utcnow() - t = Transfer.get(item.id) - t.progress = len(content) / item.info['size'] - t.save() - trigger_event('transfer', { - 'id': item.id, 'progress': t.progress - }) - ''' - content = r.read() - ''' + try: + if r.headers.get('content-encoding', None) == 'gzip': + content = gzip.GzipFile(fileobj=r).read() + else: + content = '' + ct = datetime.utcnow() + for chunk in iter(lambda: r.read(16*1024), ''): + content += chunk + if (datetime.utcnow() - ct).total_seconds() > 1: + ct = datetime.utcnow() + t = Transfer.get(item.id) + t.progress = len(content) / item.info['size'] + t.save() + trigger_event('transfer', { + 'id': item.id, 'progress': t.progress + }) + ''' + content = r.read() + ''' - t2 = datetime.utcnow() - duration = (t2-t1).total_seconds() - if duration: - self.download_speed = len(content) / duration - logger.debug('SPEED %s', ox.format_bits(self.download_speed)) - return item.save_file(content) + t2 = datetime.utcnow() + duration = (t2-t1).total_seconds() + if duration: + self.download_speed = len(content) / duration + logger.debug('SPEED %s', ox.format_bits(self.download_speed)) + return item.save_file(content) + except: + logger.debug('download failed %s', url, exec_info=1) + return False else: logger.debug('FAILED %s', url) return False From c3441c8a108eb3fd3addfc473de34eb034c117f5 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 6 Sep 2014 01:44:17 +0200 Subject: [PATCH 5/6] support longer tls fingerprints --- oml/node/cert.py | 2 +- oml/ssl_request.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/oml/node/cert.py b/oml/node/cert.py index a820b14..7fac562 100644 --- a/oml/node/cert.py +++ b/oml/node/cert.py @@ -13,7 +13,7 @@ def get_fingerprint(): with open(settings.ssl_cert_path) as fd: data = fd.read() cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, data) - return hashlib.sha1(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, cert)).hexdigest() + return hashlib.sha256(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, cert)).hexdigest() def generate_ssl(): key = OpenSSL.crypto.PKey() diff --git a/oml/ssl_request.py b/oml/ssl_request.py index 1ce8fcf..0cd4670 100644 --- a/oml/ssl_request.py +++ b/oml/ssl_request.py @@ -30,7 +30,15 @@ class CertValidatingHTTPSConnection(httplib.HTTPConnection): self.cert_reqs = ssl.CERT_NONE def _ValidateCertificateFingerprint(self, cert): - fingerprint = hashlib.sha1(cert).hexdigest() + if len(self.fingerprint) == 40: + fingerprint = hashlib.sha1(cert).hexdigest() + elif len(self.fingerprint) == 64: + fingerprint = hashlib.sha256(cert).hexdigest() + elif len(self.fingerprint) == 128: + fingerprint = hashlib.sha512(cert).hexdigest() + else: + logging.error('unkown fingerprint length %s (%s)', self.fingerprint, len(self.fingerprint)) + return False return fingerprint == self.fingerprint def connect(self): From 0c11681de33f90223745313d865b183c6f4b9238 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 6 Sep 2014 01:56:37 +0200 Subject: [PATCH 6/6] check if all modules are available --- oml/update.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/oml/update.py b/oml/update.py index 2fb8718..299b43f 100644 --- a/oml/update.py +++ b/oml/update.py @@ -65,9 +65,7 @@ def download(): return True release_data = get(RELEASE_URL) release = json.loads(release_data) - old = settings.release['modules']['openmedialibrary']['version'] - new = release['modules']['openmedialibrary']['version'] - if verify(release) and old < new: + if verify(release): ox.makedirs(settings.updates_path) os.chdir(os.path.dirname(settings.base_dir)) current_files = {'release.json'} @@ -117,6 +115,7 @@ def install(): shutil.rmtree('%s_old' % module) shutil.rmtree(new) else: + os.unlink(module_tar) return False shutil.copy(os.path.join(settings.updates_path, 'release.json'), os.path.join(settings.config_path, 'release.json')) for cmd in [