Merge branch 'master' into py3

README.md

@@ -25,8 +25,8 @@ Development
 
 Now checkout the source and prepare for use:
 
-    mkdir client
+    mkdir openmedialibrary
-    cd client
+    cd openmedialibrary
     git clone https://git.0x2620.org/openmedialibrary.git
     git clone https://git.0x2620.org/openmedialibrary_platform.git platform
     ln -s openmedialibrary/ctl ctl
@@ -47,11 +47,10 @@ On Linux you need a working python2 installation with pillow, pyhon-lxml and pop
 Platform
 ----------
 
-If you install Open Media Library on a architecture that is currently not upported,
+If you install Open Media Library on a architecture/os that is currently
-you need a working python 2.7 installation and the following packages:
+not upported, you need a working python 2.7 installation and the dependencies
+listed in requirements.txt and requirements-shared.txt:
 
-    apt-get install \
-        python-pypdf python-stdnum python-html5lib python-chardet python-openssl \
-        python-simplejson python-lxml
     pip install -r requirements.txt
+    pip install -r requirements-shared.txt
 

oml/commands.py

@@ -149,6 +149,7 @@ def command_release(*args):
 
     EXCLUDE=[
         '--exclude', '.git', '--exclude', '.bzr',
+        '--exclude', 'pip_cache',
         '--exclude', '.*.swp', '--exclude', '._*', '--exclude', '.DS_Store'
     ]
 

oml/downloads.py

@@ -4,13 +4,13 @@
 
 from threading import Thread
 import time
-import logging
 
 import db
 import state
 import settings
 import update
 
+import logging
 logger = logging.getLogger('oml.downloads')
 
 class Downloads(Thread):

oml/localnodes.py

@@ -162,6 +162,7 @@ class LocalNodes4(LocalNodesBase):
     def send(self):
         packet = self.get_packet()
         if packet:
+            logger.debug('send4 %s', packet)
             sockaddr = (self._BROADCAST, self._PORT)
             s = socket.socket (socket.AF_INET, socket.SOCK_DGRAM)
             s.setsockopt (socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, self._TTL)
@@ -174,6 +175,7 @@ class LocalNodes4(LocalNodesBase):
     def get_socket(self):
         s = socket.socket (socket.AF_INET, socket.SOCK_DGRAM)
         s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
         mreq = struct.pack("=4sl", socket.inet_aton(self._BROADCAST), socket.INADDR_ANY)
         s.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
         self._socket = s
@@ -206,6 +208,7 @@ class LocalNodes6(LocalNodesBase):
     def get_socket(self):
         s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
         s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
         group_bin = socket.inet_pton(socket.AF_INET6, self._BROADCAST) + b'\0'*4
         s.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, group_bin)
         self._socket = s

oml/node/cert.py

@@ -13,7 +13,7 @@ def get_fingerprint():
     with open(settings.ssl_cert_path) as fd:
         data = fd.read()
     cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, data)
-    return hashlib.sha1(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, cert)).hexdigest()
+    return hashlib.sha256(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, cert)).hexdigest()
 
 def generate_ssl():
     key = OpenSSL.crypto.PKey()

oml/nodes.py

@@ -305,37 +305,42 @@ class Node(Thread):
         }
         t1 = datetime.utcnow()
         logger.debug('download %s', url)
-        self._opener.addheaders = list(zip(list(headers.keys()), list(headers.values())))
+        self._opener.addheaders = zip(headers.keys(), headers.values())
-        self._opener.timeout = self.TIMEOUT*2
+        try:
-        r = self._opener.open(url)
+            r = self._opener.open(url, timeout=self.TIMEOUT*2)
+        except:
+            logger.debug('openurl failed %s', url, exec_info=1)
+            return False
         if r.getcode() == 200:
-            if r.headers.get('content-encoding', None) == 'gzip':
+            try:
-                content = gzip.GzipFile(fileobj=r).read()
+                if r.headers.get('content-encoding', None) == 'gzip':
-            else:
+                    content = gzip.GzipFile(fileobj=r).read()
-                content = b''
+                else:
-                ct = datetime.utcnow()
+                    content = b''
-                '''
+                    ct = datetime.utcnow()
-                for chunk in iter(lambda: r.read(16*1024), b''):
+                    for chunk in iter(lambda: r.read(16*1024), b''):
-                    content += chunk
+                        content += chunk
-                    if (datetime.utcnow() - ct).total_seconds() > 1:
+                        if (datetime.utcnow() - ct).total_seconds() > 1:
-                        ct = datetime.utcnow()
+                            ct = datetime.utcnow()
-                        t = Transfer.get(item.id)
+                            t = Transfer.get(item.id)
-                        t.progress = len(content) / item.info['size']
+                            t.progress = len(content) / item.info['size']
-                        t.save()
+                            t.save()
-                        trigger_event('transfer', {
+                            trigger_event('transfer', {
-                            'id': item.id, 'progress': t.progress
+                                'id': item.id, 'progress': t.progress
-                        })
+                            })
+                    '''
+                    content = r.read()
+                    '''
 
-                '''
+                t2 = datetime.utcnow()
-                content = r.read()
+                duration = (t2-t1).total_seconds()
-                logger.debug('download done %s', item.id)
+                if duration:
-
+                    self.download_speed = len(content) / duration
-            t2 = datetime.utcnow()
+                logger.debug('SPEED %s', ox.format_bits(self.download_speed))
-            duration = (t2-t1).total_seconds()
+                return item.save_file(content)
-            if duration:
+            except:
-                self.download_speed = len(content) / duration
+                logger.debug('download failed %s', url, exec_info=1)
-            logger.debug('SPEED %s', ox.format_bits(self.download_speed))
+                return False
-            return item.save_file(content)
         else:
             logger.debug('FAILED %s', url)
             return False

oml/server.py

@@ -5,6 +5,7 @@
 import os
 import sys
 import signal
+import time
 
 from tornado.httpserver import HTTPServer
 from tornado.ioloop import IOLoop
@@ -12,7 +13,7 @@ from tornado.web import StaticFileHandler, Application
 
 from cache import Cache
 from item.handlers import EpubHandler, ReaderHandler, FileHandler
-from item.handlers import OMLHandler, serve_static
+from item.handlers import OMLHandler
 from item.icons import IconHandler
 import db
 import node.server
@@ -29,7 +30,15 @@ class MainHandler(OMLHandler):
 
     def get(self, path):
         path = os.path.join(settings.static_path, 'html/oml.html')
-        serve_static(self, path, 'text/html')
+        with open(path) as fd:
+            content = fd.read()
+        version = settings.MINOR_VERSION
+        if version == 'git':
+            version = int(time.mktime(time.gmtime()))
+        content = content.replace('oml.js?1', 'oml.js?%s' % version)
+        self.set_header('Content-Type', 'text/html')
+        self.set_header('Content-Length', str(len(content)))
+        self.write(content)
 
 def run():
     setup.create_db()

oml/ssl_request.py

@@ -30,7 +30,15 @@ class CertValidatingHTTPSConnection(http.client.HTTPConnection):
         self.cert_reqs = ssl.CERT_NONE
 
     def _ValidateCertificateFingerprint(self, cert):
-        fingerprint = hashlib.sha1(cert).hexdigest()
+        if len(self.fingerprint) == 40:
+            fingerprint = hashlib.sha1(cert).hexdigest()
+        elif len(self.fingerprint) == 64:
+            fingerprint = hashlib.sha256(cert).hexdigest()
+        elif len(self.fingerprint) == 128:
+            fingerprint = hashlib.sha512(cert).hexdigest()
+        else:
+            logging.error('unkown fingerprint length %s (%s)', self.fingerprint, len(self.fingerprint))
+            return False
         return fingerprint == self.fingerprint
 
     def connect(self):

oml/update.py

@@ -65,9 +65,7 @@ def download():
         return True
     release_data = get(RELEASE_URL)
     release = json.loads(release_data)
-    old = settings.release['modules']['openmedialibrary']['version']
+    if verify(release):
-    new = release['modules']['openmedialibrary']['version']
-    if verify(release) and old < new:
         ox.makedirs(settings.updates_path)
         os.chdir(os.path.dirname(settings.base_dir))
         current_files = {'release.json'}
@@ -117,6 +115,7 @@ def install():
                     shutil.rmtree('%s_old' % module)
                     shutil.rmtree(new)
                 else:
+                    os.unlink(module_tar)
                     return False
         shutil.copy(os.path.join(settings.updates_path, 'release.json'), os.path.join(settings.config_path, 'release.json'))
         for cmd in [

requirements.txt

@@ -1,3 +1,4 @@
+lxml
 simplejson
 ed25519
 SQLAlchemy==0.9.7