diff --git a/oml/changelog.py b/oml/changelog.py index f641fd4..025611a 100644 --- a/oml/changelog.py +++ b/oml/changelog.py @@ -297,7 +297,7 @@ class Changelog(db.Model): return True def action_addpeer(self, user, timestamp, peerid, username): - if len(peerid) == 16: + if len(peerid) == settings.ID_LENGTH: from user.models import User if not 'users' in user.info: user.info['users'] = {} @@ -318,7 +318,7 @@ class Changelog(db.Model): return True def action_editpeer(self, user, timestamp, peerid, data): - if len(peerid) == 16: + if len(peerid) == settings.ID_LENGTH: from user.models import User peer = User.get_or_create(peerid) update = False @@ -466,7 +466,7 @@ class Changelog(db.Model): elif op == 'addpeer': peer_id = data[1] username = data[2] - if len(peer_id) == 16: + if len(peer_id) == settings.ID_LENGTH: peer = User.get(peer_id) if peer: username = peer.json().get('username', 'anonymous') diff --git a/oml/library.py b/oml/library.py index d840872..b490e16 100644 --- a/oml/library.py +++ b/oml/library.py @@ -153,7 +153,7 @@ class Peer(object): self.info['lists'][name] = list(set(self.info['lists'][name]) - set(ids)) elif action == 'addpeer': peerid, username = args - if len(peerid) == 16: + if len(peerid) == settings.ID_LENGTH: self.info['peers'][peerid] = {'username': username} # fixme, just trigger peer update here from user.models import User @@ -164,7 +164,7 @@ class Peer(object): peer.save() elif action == 'editpeer': peerid, data = args - if len(peerid) == 16: + if len(peerid) == settings.ID_LENGTH: if peerid not in self.info['peers']: self.info['peers'][peerid] = {} for key in ('username', 'contact'): diff --git a/oml/node/server.py b/oml/node/server.py index bc2e816..ae5e8df 100644 --- a/oml/node/server.py +++ b/oml/node/server.py @@ -12,18 +12,20 @@ import socket import socketserver import time -from Crypto.PublicKey import RSA -from Crypto.Util.asn1 import DerSequence -from OpenSSL.crypto import dump_privatekey, FILETYPE_ASN1 from OpenSSL.SSL import ( - Context, Connection, TLSv1_2_METHOD, - VERIFY_PEER, VERIFY_FAIL_IF_NO_PEER_CERT, VERIFY_CLIENT_ONCE + Connection, + Context, + TLSv1_2_METHOD, + VERIFY_CLIENT_ONCE, + VERIFY_FAIL_IF_NO_PEER_CERT, + VERIFY_PEER, ) import db import settings import state import user +import utils from changelog import changelog_size, changelog_path from websocket import trigger_event @@ -34,16 +36,15 @@ import logging logger = logging.getLogger(__name__) -def get_service_id(key): - ''' - service_id is the first half of the sha1 of the rsa public key encoded in base32 - ''' - # compute sha1 of public key and encode first half in base32 - pub_der = DerSequence() - pub_der.decode(dump_privatekey(FILETYPE_ASN1, key)) - public_key = RSA.construct((pub_der._seq[1], pub_der._seq[2])).exportKey('DER')[22:] - service_id = base64.b32encode(hashlib.sha1(public_key).digest()[:10]).lower().decode() - return service_id +def get_service_id(connection): + certs = connection.get_peer_cert_chain() + for cert in certs: + if cert.get_signature_algorithm().decode() == "ED25519": + pubkey = cert.get_pubkey() + public_key = pubkey.to_cryptography_key().public_bytes_raw() + service_id = utils.get_onion(public_key) + return service_id + raise Exception("connection with invalid certificate") class TLSTCPServer(socketserver.TCPServer): @@ -55,7 +56,7 @@ class TLSTCPServer(socketserver.TCPServer): socketserver.TCPServer.__init__(self, server_address, HandlerClass) ctx = Context(TLSv1_2_METHOD) ctx.use_privatekey_file(settings.ssl_key_path) - ctx.use_certificate_file(settings.ssl_cert_path) + ctx.use_certificate_chain_file(settings.ssl_cert_path) # only allow clients with cert: ctx.set_verify(VERIFY_PEER | VERIFY_CLIENT_ONCE | VERIFY_FAIL_IF_NO_PEER_CERT, self._accept) #ctx.set_verify(VERIFY_PEER | VERIFY_CLIENT_ONCE, self._accept) @@ -111,8 +112,7 @@ class Handler(http.server.SimpleHTTPRequestHandler): return self.do_GET() def do_GET(self): - #x509 = self.connection.get_peer_certificate() - #user_id = get_service_id(x509.get_pubkey()) if x509 else None + user_id = get_service_id(self.connection) import item.models parts = self.path.split('/') if len(parts) == 3 and parts[1] in ('get', 'preview'): @@ -185,8 +185,7 @@ class Handler(http.server.SimpleHTTPRequestHandler): self.end_headers() def _changelog(self): - x509 = self.connection.get_peer_certificate() - user_id = get_service_id(x509.get_pubkey()) if x509 else None + user_id = get_service_id(self.connection) with db.session(): u = user.models.User.get(user_id) if not u: @@ -257,8 +256,7 @@ class Handler(http.server.SimpleHTTPRequestHandler): ping responds public ip ''' - x509 = self.connection.get_peer_certificate() - user_id = get_service_id(x509.get_pubkey()) if x509 else None + user_id = get_service_id(self.connection) content = {} try: diff --git a/oml/settings.py b/oml/settings.py index 5c06f74..346af60 100644 --- a/oml/settings.py +++ b/oml/settings.py @@ -24,9 +24,11 @@ if not os.path.exists(data_path): db_path = os.path.join(data_path, 'data.db') log_path = os.path.join(data_path, 'debug.log') -ssl_cert_path = os.path.join(data_path, 'node.ssl.crt') -ssl_key_path = os.path.join(data_path, 'tor', 'private_key') +ca_key_path = os.path.join(data_path, 'node.ca.key') +ca_cert_path = os.path.join(data_path, 'node.ca.crt') +ssl_cert_path = os.path.join(data_path, 'node.tls.crt') +ssl_key_path = os.path.join(data_path, 'node.tls.key') if os.path.exists(oml_data_path): with open(oml_data_path) as fd: @@ -57,7 +59,7 @@ for key in server_defaults: release = pdict(os.path.join(data_path, 'release.json')) -USER_ID = get_user_id(ssl_key_path, ssl_cert_path) +USER_ID = get_user_id(ssl_key_path, ssl_cert_path, ca_key_path, ca_cert_path) OML_UPDATE_KEY = 'K55EZpPYbP3X+3mA66cztlw1sSaUMqGwfTDKQyP2qOU' OML_UPDATE_CERT = '''-----BEGIN CERTIFICATE----- @@ -96,3 +98,5 @@ if not FULLTEXT_SUPPORT: config['itemKeys'] = [k for k in config['itemKeys'] if k['id'] != 'fulltext'] DB_VERSION = 20 + +ID_LENGTH = 56 diff --git a/oml/tor.py b/oml/tor.py index 17056ca..a5f3933 100644 --- a/oml/tor.py +++ b/oml/tor.py @@ -22,6 +22,7 @@ import logging logging.getLogger('stem').setLevel(logging.ERROR) logger = logging.getLogger(__name__) + class TorDaemon(Thread): installing = False running = True @@ -201,18 +202,20 @@ class Tor(object): return False controller = self.controller if controller.get_version() >= stem.version.Requirement.ADD_ONION: - with open(settings.ssl_key_path, 'rb') as fd: - private_key = fd.read() - key_content = RSA.importKey(private_key).exportKey().decode() - key_content = ''.join(key_content.strip().split('\n')[1:-1]) + private_key, public_key = utils.load_pem_key(settings.ca_key_path) + key_type, key_content = utils.get_onion_key(private_key) ports = {9851: settings.server['node_port']} if settings.preferences.get('enableReadOnlyService'): ports[80] = settings.server['public_port'] controller.remove_ephemeral_hidden_service(settings.USER_ID) - response = controller.create_ephemeral_hidden_service(ports, - key_type='RSA1024', key_content=key_content, - detached=True) + response = controller.create_ephemeral_hidden_service( + ports, + key_type=key_type, key_content=key_content, + detached=True + ) if response.is_ok(): + if response.service_id != settings.USER_ID: + logger.error("Something is wrong with tor id %s vs %s", response.service_id, settings.USER_ID) logger.debug('published node as https://%s.onion:%s', settings.USER_ID, settings.server_defaults['node_port']) if settings.preferences.get('enableReadOnlyService'): diff --git a/oml/tor_request.py b/oml/tor_request.py index 0fd6455..bfeaa7e 100644 --- a/oml/tor_request.py +++ b/oml/tor_request.py @@ -66,27 +66,30 @@ class TorHTTPSConnection(http.client.HTTPSConnection): def __init__(self, host, port=None, service_id=None, check_hostname=None, context=None, **kwargs): self._service_id = service_id if self._service_id: - if hasattr(ssl, '_create_default_https_context'): - context = ssl._create_default_https_context() - elif hasattr(ssl, '_create_stdlib_context'): - context = ssl._create_stdlib_context() + context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) if context: context.check_hostname = False context.verify_mode = ssl.CERT_NONE - # tor keys are still 1024 bit, debian started to require 2048 by default, - # try to lower requirements to 1024 if needed - try: - context.load_cert_chain(settings.ssl_cert_path, settings.ssl_key_path) - except ssl.SSLError: - context.set_ciphers('DEFAULT@SECLEVEL=1') - context.load_cert_chain(settings.ssl_cert_path, settings.ssl_key_path) + context.load_cert_chain(settings.ssl_cert_path, settings.ssl_key_path) context.load_default_certs() + context.set_alpn_protocols(['http/1.1']) + context.post_handshake_auth = True http.client.HTTPSConnection.__init__(self, host, port, check_hostname=check_hostname, context=context, **kwargs) if not is_local(host): self._create_connection = create_tor_connection + def get_service_id_cert(self): + for cert in self.sock._sslobj.get_verified_chain(): + info = cert.get_info() + subject = info.get("subject") + if subject: + CN = subject[0][0][1] + if CN == self._service_id: + cert = cert.public_bytes() + return cert + def _check_service_id(self, cert): service_id = get_service_id(cert=cert) if service_id != self._service_id: @@ -96,11 +99,9 @@ class TorHTTPSConnection(http.client.HTTPSConnection): def connect(self): http.client.HTTPSConnection.connect(self) if self._service_id: - cert = self.sock.getpeercert(binary_form=True) + cert = self.get_service_id_cert() if not self._check_service_id(cert): - raise InvalidCertificateException(self._service_id, cert, - 'service_id mismatch') - #logger.debug('CIPHER %s VERSION %s', self.sock.cipher(), self.sock.ssl_version) + raise InvalidCertificateException(self._service_id, cert, 'service_id mismatch') class TorHTTPSHandler(urllib.request.HTTPSHandler): def __init__(self, debuglevel=0, context=None, check_hostname=None, service_id=None): diff --git a/oml/user/api.py b/oml/user/api.py index 53448b3..a1c9a8f 100644 --- a/oml/user/api.py +++ b/oml/user/api.py @@ -411,7 +411,7 @@ def requestPeering(data): nickname (optional) } ''' - if len(data.get('id', '')) != 16: + if len(data.get('id', '')) != settings.ID_LENGTH: logger.debug('invalid user id') return {} u = models.User.get_or_create(data['id']) @@ -434,7 +434,7 @@ def acceptPeering(data): message } ''' - if len(data.get('id', '')) != 16: + if len(data.get('id', '')) != settings.ID_LENGTH: logger.debug('invalid user id') return {} logger.debug('acceptPeering... %s', data) @@ -453,8 +453,8 @@ def rejectPeering(data): message } ''' - if len(data.get('id', '')) not in (16, 43): - logger.debug('invalid user id') + if len(data.get('id', '')) not in (16, 43, 56): + logger.debug('invalid user id: %s', data) return {} u = models.User.get_or_create(data['id']) u.info['message'] = data.get('message', '') @@ -471,8 +471,8 @@ def removePeering(data): message } ''' - if len(data.get('id', '')) not in (16, 43): - logger.debug('invalid user id') + if len(data.get('id', '')) not in (16, 43, 56): + logger.debug('invalid user id: %s', data) return {} u = models.User.get(data['id'], for_update=True) if u: @@ -488,8 +488,8 @@ def cancelPeering(data): takes { } ''' - if len(data.get('id', '')) != 16: - logger.debug('invalid user id') + if len(data.get('id', '')) != settings.ID_LENGTH: + logger.debug('invalid user id: %s', data) return {} u = models.User.get_or_create(data['id']) u.info['message'] = data.get('message', '') diff --git a/oml/user/models.py b/oml/user/models.py index ffe60f5..bb737a4 100644 --- a/oml/user/models.py +++ b/oml/user/models.py @@ -27,7 +27,7 @@ class User(db.Model): created = sa.Column(sa.DateTime()) modified = sa.Column(sa.DateTime()) - id = sa.Column(sa.String(43), primary_key=True) + id = sa.Column(sa.String(128), primary_key=True) info = sa.Column(MutableDict.as_mutable(sa.PickleType(pickler=json_pickler))) nickname = sa.Column(sa.String(256), index=True) @@ -256,7 +256,7 @@ class List(db.Model): type = sa.Column(sa.String(64)) _query = sa.Column('query', MutableDict.as_mutable(sa.PickleType(pickler=json_pickler))) - user_id = sa.Column(sa.String(43), sa.ForeignKey('user.id')) + user_id = sa.Column(sa.String(128), sa.ForeignKey('user.id')) user = sa.orm.relationship('User', backref=sa.orm.backref('lists', lazy='dynamic')) items = sa.orm.relationship('Item', secondary=list_items, @@ -456,7 +456,7 @@ class Metadata(db.Model): id = sa.Column(sa.Integer(), primary_key=True) item_id = sa.Column(sa.String(32)) - user_id = sa.Column(sa.String(43), sa.ForeignKey('user.id')) + user_id = sa.Column(sa.String(128), sa.ForeignKey('user.id')) data_hash = sa.Column(sa.String(40), index=True) data = sa.Column(MutableDict.as_mutable(sa.PickleType(pickler=json_pickler))) diff --git a/oml/utils.py b/oml/utils.py index ad2b65a..7d07ca6 100644 --- a/oml/utils.py +++ b/oml/utils.py @@ -17,19 +17,26 @@ import time import unicodedata import ox +import OpenSSL.crypto from OpenSSL.crypto import ( - load_privatekey, load_certificate, - dump_privatekey, dump_certificate, - FILETYPE_ASN1, FILETYPE_PEM, PKey, TYPE_RSA, - X509, X509Extension + dump_certificate, + dump_privatekey, + FILETYPE_PEM, + load_certificate, + load_privatekey, + PKey, + TYPE_RSA, + X509, + X509Extension ) -from Crypto.PublicKey import RSA -from Crypto.Util.asn1 import DerSequence +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ed25519 from meta.utils import normalize_isbn, find_isbns, get_language, to_isbn13 from win32utils import get_short_path_name + import logging logging.getLogger('PIL').setLevel(logging.ERROR) logger = logging.getLogger(__name__) @@ -92,7 +99,7 @@ def resize_image(data, width=None, size=None): height = max(height, 1) if width < source_width: - resize_method = Image.ANTIALIAS + resize_method = Image.LANCZOS else: resize_method = Image.BICUBIC output = source.resize((width, height), resize_method) @@ -119,78 +126,157 @@ def get_position_by_id(list, key): return i return -1 -def get_user_id(private_key, cert_path): - if os.path.exists(private_key): - with open(private_key) as fd: - key = load_privatekey(FILETYPE_PEM, fd.read()) - if key.bits() != 1024: - os.unlink(private_key) +def sign_cert(cert, key): + # pyOpenSSL sgin api does not allow NULL hash + # return cert.sign(key, None) + return OpenSSL.crypto._lib.X509_sign(cert._x509, key._pkey, OpenSSL.crypto._ffi.NULL) + +def load_pem_key(pem): + with open(pem) as fd: + ca_key_pem = fd.read() + key = load_privatekey(FILETYPE_PEM, ca_key_pem) + if key.bits() != 256: + raise Exception("Invalid key %s" % pem) + key = key.to_cryptography_key() + private_key = key.private_bytes_raw() + public_key = key.public_key().public_bytes_raw() + return private_key, public_key + + +def expand_private_key(secret_key) -> bytes: + hash = hashlib.sha512(secret_key).digest() + hash = bytearray(hash) + hash[0] &= 248 + hash[31] &= 127 + hash[31] |= 64 + return bytes(hash) + +def get_onion(pubkey): + version_byte = b"\x03" + checksum_str = ".onion checksum".encode() + checksum = hashlib.sha3_256(checksum_str + pubkey + version_byte).digest()[:2] + return base64.b32encode(pubkey + checksum + version_byte).decode().lower() + +def get_onion_key(private_key): + onion_key = expand_private_key(private_key) + key_type = 'ED25519-V3' + key_content = base64.encodebytes(onion_key).decode().strip().replace('\n', '') + return key_type, key_content + +def get_user_id(key_path, cert_path, ca_key_path, ca_cert_path): + if os.path.exists(ca_key_path): + try: + private_key, public_key = load_pem_key(ca_key_path) + except: + os.unlink(ca_key_path) else: - user_id = get_service_id(private_key) - if not os.path.exists(private_key): - if os.path.exists(cert_path): - os.unlink(cert_path) - folder = os.path.dirname(private_key) - if not os.path.exists(folder): - os.makedirs(folder) - os.chmod(folder, 0o700) - key = PKey() - key.generate_key(TYPE_RSA, 1024) - with open(private_key, 'wb') as fd: - os.chmod(private_key, 0o600) - fd.write(dump_privatekey(FILETYPE_PEM, key)) - os.chmod(private_key, 0o400) - user_id = get_service_id(private_key) - if not os.path.exists(cert_path) or \ - (datetime.now() - datetime.fromtimestamp(os.path.getmtime(cert_path))).days > 60: + user_id = get_onion(public_key) + + if not os.path.exists(ca_key_path): + private_key = ed25519.Ed25519PrivateKey.generate() + private_bytes = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption() + ) + with open(ca_key_path, 'wb') as fd: + fd.write(private_bytes) + + public_key = private_key.public_key().public_bytes_raw() + user_id = get_onion(public_key) + + if not os.path.exists(ca_cert_path) or \ + (datetime.now() - datetime.fromtimestamp(os.path.getmtime(ca_cert_path))).days > 5*365: + with open(ca_key_path, 'rb') as key_file: + key_data = key_file.read() + cakey = load_privatekey(FILETYPE_PEM, key_data) ca = X509() ca.set_version(2) ca.set_serial_number(1) ca.get_subject().CN = user_id ca.gmtime_adj_notBefore(0) - ca.gmtime_adj_notAfter(90 * 24 * 60 * 60) + ca.gmtime_adj_notAfter(10 * 356 * 24 * 60 * 60) ca.set_issuer(ca.get_subject()) - ca.set_pubkey(key) + ca.set_pubkey(cakey) ca.add_extensions([ - X509Extension(b"basicConstraints", True, b"CA:TRUE, pathlen:0"), - X509Extension(b"nsCertType", True, b"sslCA"), + X509Extension(b"basicConstraints", False, b"CA:TRUE"), + X509Extension(b"keyUsage", False, b"keyCertSign, cRLSign"), + X509Extension( + b"subjectKeyIdentifier", False, b"hash", subject=ca + ), + ]) + ca.add_extensions([ + X509Extension( + b"authorityKeyIdentifier", False, b"keyid:always", issuer=ca + ) + ]) + + sign_cert(ca, cakey) + + with open(ca_cert_path, 'wb') as fd: + fd.write(dump_certificate(FILETYPE_PEM, ca)) + + if os.path.exists(cert_path): + os.unlink(cert_path) + if os.path.exists(key_path): + os.unlink(key_path) + else: + with open(ca_cert_path) as fd: + ca = load_certificate(FILETYPE_PEM, fd.read()) + with open(ca_key_path) as fd: + cakey = load_privatekey(FILETYPE_PEM, fd.read()) + + + # create RSA intermediate certificate since clients don't quite like Ed25519 yet + if not os.path.exists(cert_path) or \ + (datetime.now() - datetime.fromtimestamp(os.path.getmtime(cert_path))).days > 60: + + key = PKey() + key.generate_key(TYPE_RSA, 2048) + + cert = X509() + cert.set_version(2) + cert.set_serial_number(2) + cert.get_subject().CN = user_id + ".onion" + cert.gmtime_adj_notBefore(0) + cert.gmtime_adj_notAfter(90 * 24 * 60 * 60) + cert.set_issuer(ca.get_subject()) + cert.set_pubkey(key) + subject_alt_names = b"DNS: %s.onion" % user_id.encode() + cert.add_extensions([ + X509Extension(b"basicConstraints", True, b"CA:FALSE"), X509Extension(b"extendedKeyUsage", True, b"serverAuth,clientAuth,emailProtection,timeStamping,msCodeInd,msCodeCom,msCTLSign,msSGC,msEFS,nsSGC"), X509Extension(b"keyUsage", False, b"keyCertSign, cRLSign"), X509Extension(b"subjectKeyIdentifier", False, b"hash", subject=ca), + X509Extension(b"subjectAltName", critical=True, value=subject_alt_names), ]) - ca.sign(key, "sha256") + sign_cert(cert, cakey) with open(cert_path, 'wb') as fd: + fd.write(dump_certificate(FILETYPE_PEM, cert)) fd.write(dump_certificate(FILETYPE_PEM, ca)) + with open(key_path, 'wb') as fd: + fd.write(dump_privatekey(FILETYPE_PEM, key)) return user_id + def get_service_id(private_key_file=None, cert=None): ''' service_id is the first half of the sha1 of the rsa public key encoded in base32 ''' if private_key_file: - with open(private_key_file, 'rb') as fd: - private_key = fd.read() - public_key = RSA.importKey(private_key).publickey().exportKey('DER')[22:] - # compute sha1 of public key and encode first half in base32 - service_id = base64.b32encode(hashlib.sha1(public_key).digest()[:10]).lower().decode() - ''' - # compute public key from priate key and export in DER format - # ignoring the SPKI header(22 bytes) - key = load_privatekey(FILETYPE_PEM, private_key) - cert = X509() - cert.set_pubkey(key) - public_key = dump_privatekey(FILETYPE_ASN1, cert.get_pubkey())[22:] - # compute sha1 of public key and encode first half in base32 - service_id = base64.b32encode(hashlib.sha1(public_key).digest()[:10]).lower().decode() - ''' + with open(private_key_file, 'rb') as key_file: + key_type, key_content = key_file.read().split(b':', 1) + private_key = base64.decodebytes(key_content) + public_key = Ed25519().public_key_from_hash(private_key) + service_id = get_onion(public_key) elif cert: - # compute sha1 of public key and encode first half in base32 - key = load_certificate(FILETYPE_ASN1, cert).get_pubkey() - pub_der = DerSequence() - pub_der.decode(dump_privatekey(FILETYPE_ASN1, key)) - public_key = RSA.construct((pub_der._seq[1], pub_der._seq[2])).exportKey('DER')[22:] - service_id = base64.b32encode(hashlib.sha1(public_key).digest()[:10]).lower().decode() + cert_ = load_certificate(FILETYPE_PEM, cert) + key = cert_.get_pubkey() + public_key = key.to_cryptography_key().public_bytes_raw() + service_id = get_onion(public_key) + else: + service_id = None return service_id def update_dict(root, data): diff --git a/static/js/utils.js b/static/js/utils.js index f4bb2b9..a78aac3 100644 --- a/static/js/utils.js +++ b/static/js/utils.js @@ -1028,7 +1028,7 @@ oml.updateFilterMenus = function() { }; oml.validatePublicKey = function(value) { - return /^[a-z0-9+\/]{16}$/.test(value); + return /^[a-z0-9+\/]{56}$/.test(value); }; oml.updateDebugMenu = function() {