0x2620/openmedialibrary
1
0
Fork 0
Code Issues 131 Pull requests Releases Wiki Activity

switch to onion v3 ids

Browse source
This commit is contained in:
j 2024-06-08 12:39:27 +01:00
parent e175c72a40
commit 71634c9ed1
10 changed files with 212 additions and 120 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
oml/changelog.py
View file

@ -297,7 +297,7 @@ class Changelog(db.Model):
return True
def action_addpeer(self, user, timestamp, peerid, username):
if len(peerid) == 16:
if len(peerid) == settings.ID_LENGTH:
from user.models import User
if not 'users' in user.info:
user.info['users'] = {}
@ -318,7 +318,7 @@ class Changelog(db.Model):
return True
def action_editpeer(self, user, timestamp, peerid, data):
if len(peerid) == 16:
if len(peerid) == settings.ID_LENGTH:
from user.models import User
peer = User.get_or_create(peerid)
update = False
@ -466,7 +466,7 @@ class Changelog(db.Model):
elif op == 'addpeer':
peer_id = data[1]
username = data[2]
if len(peer_id) == 16:
if len(peer_id) == settings.ID_LENGTH:
peer = User.get(peer_id)
if peer:
username = peer.json().get('username', 'anonymous')

4
oml/library.py
View file

@ -153,7 +153,7 @@ class Peer(object):
self.info['lists'][name] = list(set(self.info['lists'][name]) - set(ids))
elif action == 'addpeer':
peerid, username = args
if len(peerid) == 16:
if len(peerid) == settings.ID_LENGTH:
self.info['peers'][peerid] = {'username': username}
# fixme, just trigger peer update here
from user.models import User
@ -164,7 +164,7 @@ class Peer(object):
peer.save()
elif action == 'editpeer':
peerid, data = args
if len(peerid) == 16:
if len(peerid) == settings.ID_LENGTH:
if peerid not in self.info['peers']:
self.info['peers'][peerid] = {}
for key in ('username', 'contact'):

40
oml/node/server.py
View file

@ -12,18 +12,20 @@ import socket
import socketserver
import time
from Crypto.PublicKey import RSA
from Crypto.Util.asn1 import DerSequence
from OpenSSL.crypto import dump_privatekey, FILETYPE_ASN1
from OpenSSL.SSL import (
Context, Connection, TLSv1_2_METHOD,
VERIFY_PEER, VERIFY_FAIL_IF_NO_PEER_CERT, VERIFY_CLIENT_ONCE
Connection,
Context,
TLSv1_2_METHOD,
VERIFY_CLIENT_ONCE,
VERIFY_FAIL_IF_NO_PEER_CERT,
VERIFY_PEER,
)
import db
import settings
import state
import user
import utils
from changelog import changelog_size, changelog_path
from websocket import trigger_event
@ -34,16 +36,15 @@ import logging
logger = logging.getLogger(__name__)
def get_service_id(key):
'''
service_id is the first half of the sha1 of the rsa public key encoded in base32
'''
# compute sha1 of public key and encode first half in base32
pub_der = DerSequence()
pub_der.decode(dump_privatekey(FILETYPE_ASN1, key))
public_key = RSA.construct((pub_der._seq[1], pub_der._seq[2])).exportKey('DER')[22:]
service_id = base64.b32encode(hashlib.sha1(public_key).digest()[:10]).lower().decode()
def get_service_id(connection):
certs = connection.get_peer_cert_chain()
for cert in certs:
if cert.get_signature_algorithm().decode() == "ED25519":
pubkey = cert.get_pubkey()
public_key = pubkey.to_cryptography_key().public_bytes_raw()
service_id = utils.get_onion(public_key)
return service_id
raise Exception("connection with invalid certificate")
class TLSTCPServer(socketserver.TCPServer):
@ -55,7 +56,7 @@ class TLSTCPServer(socketserver.TCPServer):
socketserver.TCPServer.__init__(self, server_address, HandlerClass)
ctx = Context(TLSv1_2_METHOD)
ctx.use_privatekey_file(settings.ssl_key_path)
ctx.use_certificate_file(settings.ssl_cert_path)
ctx.use_certificate_chain_file(settings.ssl_cert_path)
# only allow clients with cert:
ctx.set_verify(VERIFY_PEER | VERIFY_CLIENT_ONCE | VERIFY_FAIL_IF_NO_PEER_CERT, self._accept)
#ctx.set_verify(VERIFY_PEER | VERIFY_CLIENT_ONCE, self._accept)
@ -111,8 +112,7 @@ class Handler(http.server.SimpleHTTPRequestHandler):
return self.do_GET()
def do_GET(self):
#x509 = self.connection.get_peer_certificate()
#user_id = get_service_id(x509.get_pubkey()) if x509 else None
user_id = get_service_id(self.connection)
import item.models
parts = self.path.split('/')
if len(parts) == 3 and parts[1] in ('get', 'preview'):
@ -185,8 +185,7 @@ class Handler(http.server.SimpleHTTPRequestHandler):
self.end_headers()
def _changelog(self):
x509 = self.connection.get_peer_certificate()
user_id = get_service_id(x509.get_pubkey()) if x509 else None
user_id = get_service_id(self.connection)
with db.session():
u = user.models.User.get(user_id)
if not u:
@ -257,8 +256,7 @@ class Handler(http.server.SimpleHTTPRequestHandler):
ping responds public ip
'''
x509 = self.connection.get_peer_certificate()
user_id = get_service_id(x509.get_pubkey()) if x509 else None
user_id = get_service_id(self.connection)
content = {}
try:

10
oml/settings.py
View file

@ -24,9 +24,11 @@ if not os.path.exists(data_path):
db_path = os.path.join(data_path, 'data.db')
log_path = os.path.join(data_path, 'debug.log')
ssl_cert_path = os.path.join(data_path, 'node.ssl.crt')
ssl_key_path = os.path.join(data_path, 'tor', 'private_key')
ca_key_path = os.path.join(data_path, 'node.ca.key')
ca_cert_path = os.path.join(data_path, 'node.ca.crt')
ssl_cert_path = os.path.join(data_path, 'node.tls.crt')
ssl_key_path = os.path.join(data_path, 'node.tls.key')
if os.path.exists(oml_data_path):
with open(oml_data_path) as fd:
@ -57,7 +59,7 @@ for key in server_defaults:
release = pdict(os.path.join(data_path, 'release.json'))
USER_ID = get_user_id(ssl_key_path, ssl_cert_path)
USER_ID = get_user_id(ssl_key_path, ssl_cert_path, ca_key_path, ca_cert_path)
OML_UPDATE_KEY = 'K55EZpPYbP3X+3mA66cztlw1sSaUMqGwfTDKQyP2qOU'
OML_UPDATE_CERT = '''-----BEGIN CERTIFICATE-----
@ -96,3 +98,5 @@ if not FULLTEXT_SUPPORT:
config['itemKeys'] = [k for k in config['itemKeys'] if k['id'] != 'fulltext']
DB_VERSION = 20
ID_LENGTH = 56

17
oml/tor.py
View file

@ -22,6 +22,7 @@ import logging
logging.getLogger('stem').setLevel(logging.ERROR)
logger = logging.getLogger(__name__)
class TorDaemon(Thread):
installing = False
running = True
@ -201,18 +202,20 @@ class Tor(object):
return False
controller = self.controller
if controller.get_version() >= stem.version.Requirement.ADD_ONION:
with open(settings.ssl_key_path, 'rb') as fd:
private_key = fd.read()
key_content = RSA.importKey(private_key).exportKey().decode()
key_content = ''.join(key_content.strip().split('\n')[1:-1])
private_key, public_key = utils.load_pem_key(settings.ca_key_path)
key_type, key_content = utils.get_onion_key(private_key)
ports = {9851: settings.server['node_port']}
if settings.preferences.get('enableReadOnlyService'):
ports[80] = settings.server['public_port']
controller.remove_ephemeral_hidden_service(settings.USER_ID)
response = controller.create_ephemeral_hidden_service(ports,
key_type='RSA1024', key_content=key_content,
detached=True)
response = controller.create_ephemeral_hidden_service(
ports,
key_type=key_type, key_content=key_content,
detached=True
)
if response.is_ok():
if response.service_id != settings.USER_ID:
logger.error("Something is wrong with tor id %s vs %s", response.service_id, settings.USER_ID)
logger.debug('published node as https://%s.onion:%s',
settings.USER_ID, settings.server_defaults['node_port'])
if settings.preferences.get('enableReadOnlyService'):

29
oml/tor_request.py
View file

@ -66,27 +66,30 @@ class TorHTTPSConnection(http.client.HTTPSConnection):
def __init__(self, host, port=None, service_id=None, check_hostname=None, context=None, **kwargs):
self._service_id = service_id
if self._service_id:
if hasattr(ssl, '_create_default_https_context'):
context = ssl._create_default_https_context()
elif hasattr(ssl, '_create_stdlib_context'):
context = ssl._create_stdlib_context()
context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
if context:
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
# tor keys are still 1024 bit, debian started to require 2048 by default,
# try to lower requirements to 1024 if needed
try:
context.load_cert_chain(settings.ssl_cert_path, settings.ssl_key_path)
except ssl.SSLError:
context.set_ciphers('DEFAULT@SECLEVEL=1')
context.load_cert_chain(settings.ssl_cert_path, settings.ssl_key_path)
context.load_default_certs()
context.set_alpn_protocols(['http/1.1'])
context.post_handshake_auth = True
http.client.HTTPSConnection.__init__(self, host, port,
check_hostname=check_hostname, context=context, **kwargs)
if not is_local(host):
self._create_connection = create_tor_connection
def get_service_id_cert(self):
for cert in self.sock._sslobj.get_verified_chain():
info = cert.get_info()
subject = info.get("subject")
if subject:
CN = subject[0][0][1]
if CN == self._service_id:
cert = cert.public_bytes()
return cert
def _check_service_id(self, cert):
service_id = get_service_id(cert=cert)
if service_id != self._service_id:
@ -96,11 +99,9 @@ class TorHTTPSConnection(http.client.HTTPSConnection):
def connect(self):
http.client.HTTPSConnection.connect(self)
if self._service_id:
cert = self.sock.getpeercert(binary_form=True)
cert = self.get_service_id_cert()
if not self._check_service_id(cert):
raise InvalidCertificateException(self._service_id, cert,
'service_id mismatch')
#logger.debug('CIPHER %s VERSION %s', self.sock.cipher(), self.sock.ssl_version)
raise InvalidCertificateException(self._service_id, cert, 'service_id mismatch')
class TorHTTPSHandler(urllib.request.HTTPSHandler):
def __init__(self, debuglevel=0, context=None, check_hostname=None, service_id=None):

16
oml/user/api.py
View file

@ -411,7 +411,7 @@ def requestPeering(data):
nickname (optional)
}
'''
if len(data.get('id', '')) != 16:
if len(data.get('id', '')) != settings.ID_LENGTH:
logger.debug('invalid user id')
return {}
u = models.User.get_or_create(data['id'])
@ -434,7 +434,7 @@ def acceptPeering(data):
message
}
'''
if len(data.get('id', '')) != 16:
if len(data.get('id', '')) != settings.ID_LENGTH:
logger.debug('invalid user id')
return {}
logger.debug('acceptPeering... %s', data)
@ -453,8 +453,8 @@ def rejectPeering(data):
message
}
'''
if len(data.get('id', '')) not in (16, 43):
logger.debug('invalid user id')
if len(data.get('id', '')) not in (16, 43, 56):
logger.debug('invalid user id: %s', data)
return {}
u = models.User.get_or_create(data['id'])
u.info['message'] = data.get('message', '')
@ -471,8 +471,8 @@ def removePeering(data):
message
}
'''
if len(data.get('id', '')) not in (16, 43):
logger.debug('invalid user id')
if len(data.get('id', '')) not in (16, 43, 56):
logger.debug('invalid user id: %s', data)
return {}
u = models.User.get(data['id'], for_update=True)
if u:
@ -488,8 +488,8 @@ def cancelPeering(data):
takes {
}
'''
if len(data.get('id', '')) != 16:
logger.debug('invalid user id')
if len(data.get('id', '')) != settings.ID_LENGTH:
logger.debug('invalid user id: %s', data)
return {}
u = models.User.get_or_create(data['id'])
u.info['message'] = data.get('message', '')

6
oml/user/models.py
View file

@ -27,7 +27,7 @@ class User(db.Model):
created = sa.Column(sa.DateTime())
modified = sa.Column(sa.DateTime())
id = sa.Column(sa.String(43), primary_key=True)
id = sa.Column(sa.String(128), primary_key=True)
info = sa.Column(MutableDict.as_mutable(sa.PickleType(pickler=json_pickler)))
nickname = sa.Column(sa.String(256), index=True)
@ -256,7 +256,7 @@ class List(db.Model):
type = sa.Column(sa.String(64))
_query = sa.Column('query', MutableDict.as_mutable(sa.PickleType(pickler=json_pickler)))
user_id = sa.Column(sa.String(43), sa.ForeignKey('user.id'))
user_id = sa.Column(sa.String(128), sa.ForeignKey('user.id'))
user = sa.orm.relationship('User', backref=sa.orm.backref('lists', lazy='dynamic'))
items = sa.orm.relationship('Item', secondary=list_items,
@ -456,7 +456,7 @@ class Metadata(db.Model):
id = sa.Column(sa.Integer(), primary_key=True)
item_id = sa.Column(sa.String(32))
user_id = sa.Column(sa.String(43), sa.ForeignKey('user.id'))
user_id = sa.Column(sa.String(128), sa.ForeignKey('user.id'))
data_hash = sa.Column(sa.String(40), index=True)
data = sa.Column(MutableDict.as_mutable(sa.PickleType(pickler=json_pickler)))

198
oml/utils.py
View file

@ -17,19 +17,26 @@ import time
import unicodedata
import ox
import OpenSSL.crypto
from OpenSSL.crypto import (
load_privatekey, load_certificate,
dump_privatekey, dump_certificate,
FILETYPE_ASN1, FILETYPE_PEM, PKey, TYPE_RSA,
X509, X509Extension
dump_certificate,
dump_privatekey,
FILETYPE_PEM,
load_certificate,
load_privatekey,
PKey,
TYPE_RSA,
X509,
X509Extension
)
from Crypto.PublicKey import RSA
from Crypto.Util.asn1 import DerSequence
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import ed25519
from meta.utils import normalize_isbn, find_isbns, get_language, to_isbn13
from win32utils import get_short_path_name
import logging
logging.getLogger('PIL').setLevel(logging.ERROR)
logger = logging.getLogger(__name__)
@ -92,7 +99,7 @@ def resize_image(data, width=None, size=None):
height = max(height, 1)
if width < source_width:
resize_method = Image.ANTIALIAS
resize_method = Image.LANCZOS
else:
resize_method = Image.BICUBIC
output = source.resize((width, height), resize_method)
@ -119,78 +126,157 @@ def get_position_by_id(list, key):
return i
return -1
def get_user_id(private_key, cert_path):
if os.path.exists(private_key):
with open(private_key) as fd:
key = load_privatekey(FILETYPE_PEM, fd.read())
if key.bits() != 1024:
os.unlink(private_key)
def sign_cert(cert, key):
# pyOpenSSL sgin api does not allow NULL hash
# return cert.sign(key, None)
return OpenSSL.crypto._lib.X509_sign(cert._x509, key._pkey, OpenSSL.crypto._ffi.NULL)
def load_pem_key(pem):
with open(pem) as fd:
ca_key_pem = fd.read()
key = load_privatekey(FILETYPE_PEM, ca_key_pem)
if key.bits() != 256:
raise Exception("Invalid key %s" % pem)
key = key.to_cryptography_key()
private_key = key.private_bytes_raw()
public_key = key.public_key().public_bytes_raw()
return private_key, public_key
def expand_private_key(secret_key) -> bytes:
hash = hashlib.sha512(secret_key).digest()
hash = bytearray(hash)
hash[0] &= 248
hash[31] &= 127
hash[31] |= 64
return bytes(hash)
def get_onion(pubkey):
version_byte = b"\x03"
checksum_str = ".onion checksum".encode()
checksum = hashlib.sha3_256(checksum_str + pubkey + version_byte).digest()[:2]
return base64.b32encode(pubkey + checksum + version_byte).decode().lower()
def get_onion_key(private_key):
onion_key = expand_private_key(private_key)
key_type = 'ED25519-V3'
key_content = base64.encodebytes(onion_key).decode().strip().replace('\n', '')
return key_type, key_content
def get_user_id(key_path, cert_path, ca_key_path, ca_cert_path):
if os.path.exists(ca_key_path):
try:
private_key, public_key = load_pem_key(ca_key_path)
except:
os.unlink(ca_key_path)
else:
user_id = get_service_id(private_key)
if not os.path.exists(private_key):
if os.path.exists(cert_path):
os.unlink(cert_path)
folder = os.path.dirname(private_key)
if not os.path.exists(folder):
os.makedirs(folder)
os.chmod(folder, 0o700)
key = PKey()
key.generate_key(TYPE_RSA, 1024)
with open(private_key, 'wb') as fd:
os.chmod(private_key, 0o600)
fd.write(dump_privatekey(FILETYPE_PEM, key))
os.chmod(private_key, 0o400)
user_id = get_service_id(private_key)
if not os.path.exists(cert_path) or \
(datetime.now() - datetime.fromtimestamp(os.path.getmtime(cert_path))).days > 60:
user_id = get_onion(public_key)
if not os.path.exists(ca_key_path):
private_key = ed25519.Ed25519PrivateKey.generate()
private_bytes = private_key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.PKCS8,
encryption_algorithm=serialization.NoEncryption()
)
with open(ca_key_path, 'wb') as fd:
fd.write(private_bytes)
public_key = private_key.public_key().public_bytes_raw()
user_id = get_onion(public_key)
if not os.path.exists(ca_cert_path) or \
(datetime.now() - datetime.fromtimestamp(os.path.getmtime(ca_cert_path))).days > 5*365:
with open(ca_key_path, 'rb') as key_file:
key_data = key_file.read()
cakey = load_privatekey(FILETYPE_PEM, key_data)
ca = X509()
ca.set_version(2)
ca.set_serial_number(1)
ca.get_subject().CN = user_id
ca.gmtime_adj_notBefore(0)
ca.gmtime_adj_notAfter(90 * 24 * 60 * 60)
ca.gmtime_adj_notAfter(10 * 356 * 24 * 60 * 60)
ca.set_issuer(ca.get_subject())
ca.set_pubkey(key)
ca.set_pubkey(cakey)
ca.add_extensions([
X509Extension(b"basicConstraints", True, b"CA:TRUE, pathlen:0"),
X509Extension(b"nsCertType", True, b"sslCA"),
X509Extension(b"basicConstraints", False, b"CA:TRUE"),
X509Extension(b"keyUsage", False, b"keyCertSign, cRLSign"),
X509Extension(
b"subjectKeyIdentifier", False, b"hash", subject=ca
),
])
ca.add_extensions([
X509Extension(
b"authorityKeyIdentifier", False, b"keyid:always", issuer=ca
)
])
sign_cert(ca, cakey)
with open(ca_cert_path, 'wb') as fd:
fd.write(dump_certificate(FILETYPE_PEM, ca))
if os.path.exists(cert_path):
os.unlink(cert_path)
if os.path.exists(key_path):
os.unlink(key_path)
else:
with open(ca_cert_path) as fd:
ca = load_certificate(FILETYPE_PEM, fd.read())
with open(ca_key_path) as fd:
cakey = load_privatekey(FILETYPE_PEM, fd.read())
# create RSA intermediate certificate since clients don't quite like Ed25519 yet
if not os.path.exists(cert_path) or \
(datetime.now() - datetime.fromtimestamp(os.path.getmtime(cert_path))).days > 60:
key = PKey()
key.generate_key(TYPE_RSA, 2048)
cert = X509()
cert.set_version(2)
cert.set_serial_number(2)
cert.get_subject().CN = user_id + ".onion"
cert.gmtime_adj_notBefore(0)
cert.gmtime_adj_notAfter(90 * 24 * 60 * 60)
cert.set_issuer(ca.get_subject())
cert.set_pubkey(key)
subject_alt_names = b"DNS: %s.onion" % user_id.encode()
cert.add_extensions([
X509Extension(b"basicConstraints", True, b"CA:FALSE"),
X509Extension(b"extendedKeyUsage", True,
b"serverAuth,clientAuth,emailProtection,timeStamping,msCodeInd,msCodeCom,msCTLSign,msSGC,msEFS,nsSGC"),
X509Extension(b"keyUsage", False, b"keyCertSign, cRLSign"),
X509Extension(b"subjectKeyIdentifier", False, b"hash", subject=ca),
X509Extension(b"subjectAltName", critical=True, value=subject_alt_names),
])
ca.sign(key, "sha256")
sign_cert(cert, cakey)
with open(cert_path, 'wb') as fd:
fd.write(dump_certificate(FILETYPE_PEM, cert))
fd.write(dump_certificate(FILETYPE_PEM, ca))
with open(key_path, 'wb') as fd:
fd.write(dump_privatekey(FILETYPE_PEM, key))
return user_id
def get_service_id(private_key_file=None, cert=None):
'''
service_id is the first half of the sha1 of the rsa public key encoded in base32
'''
if private_key_file:
with open(private_key_file, 'rb') as fd:
private_key = fd.read()
public_key = RSA.importKey(private_key).publickey().exportKey('DER')[22:]
# compute sha1 of public key and encode first half in base32
service_id = base64.b32encode(hashlib.sha1(public_key).digest()[:10]).lower().decode()
'''
# compute public key from priate key and export in DER format
# ignoring the SPKI header(22 bytes)
key = load_privatekey(FILETYPE_PEM, private_key)
cert = X509()
cert.set_pubkey(key)
public_key = dump_privatekey(FILETYPE_ASN1, cert.get_pubkey())[22:]
# compute sha1 of public key and encode first half in base32
service_id = base64.b32encode(hashlib.sha1(public_key).digest()[:10]).lower().decode()
'''
with open(private_key_file, 'rb') as key_file:
key_type, key_content = key_file.read().split(b':', 1)
private_key = base64.decodebytes(key_content)
public_key = Ed25519().public_key_from_hash(private_key)
service_id = get_onion(public_key)
elif cert:
# compute sha1 of public key and encode first half in base32
key = load_certificate(FILETYPE_ASN1, cert).get_pubkey()
pub_der = DerSequence()
pub_der.decode(dump_privatekey(FILETYPE_ASN1, key))
public_key = RSA.construct((pub_der._seq[1], pub_der._seq[2])).exportKey('DER')[22:]
service_id = base64.b32encode(hashlib.sha1(public_key).digest()[:10]).lower().decode()
cert_ = load_certificate(FILETYPE_PEM, cert)
key = cert_.get_pubkey()
public_key = key.to_cryptography_key().public_bytes_raw()
service_id = get_onion(public_key)
else:
service_id = None
return service_id
def update_dict(root, data):

2
static/js/utils.js
View file

@ -1028,7 +1028,7 @@ oml.updateFilterMenus = function() {
};
oml.validatePublicKey = function(value) {
return /^[a-z0-9+\/]{16}$/.test(value);
return /^[a-z0-9+\/]{56}$/.test(value);
};
oml.updateDebugMenu = function() {