don't fail if zeroconf interface was shut down

ctl

@@ -1,6 +1,5 @@
 #!/usr/bin/env bash
 NAME="openmedialibrary"
-PID="/tmp/$NAME.$USER.pid"
 
 cd "`dirname "$0"`"
 if [ -e oml ]; then
@@ -25,16 +24,7 @@ else
         mv "$BASE/config/release.json" "$BASE/data/release.json"
     fi
 fi
-if [ ! -e "$PID" ]; then
-if [ -e "$DATA/tor/hostname" ]; then
-    onion=$(cat "$DATA/tor/hostname")
-    id=${onion/.onion/}
-    PID="/tmp/$NAME.$USER.$id.pid"
-fi
-fi
-if [ ! -e "$PID" ]; then
-    PID="$DATA/$NAME.pid"
-fi
+PID="$DATA/$NAME.pid"
 
 PLATFORM_PYTHON=3.4
 SHARED_PYTHON=3.7
@@ -44,7 +34,7 @@ else
     if [ $SYSTEM == "Linux" ]; then
         if [ $PLATFORM == "x86_64" ]; then
             ARCH=64
-            PLATFORM_PYTHON=3.7
+            PLATFORM_PYTHON=3.11
         else
             ARCH=32
         fi
@@ -74,6 +64,10 @@ PATH="$PLATFORM_ENV/bin:$PATH"
 SHARED_ENV="$BASE/platform/Shared"
 export SHARED_ENV
 
+if [ -e "$SHARED_ENV/etc/openssl/openssl.cnf" ]; then
+    export OPENSSL_CONF="$SHARED_ENV/etc/openssl/openssl.cnf"
+fi
+
 PATH="$SHARED_ENV/bin:$PATH"
 export PATH
 

oml/changelog.py

@@ -297,7 +297,7 @@ class Changelog(db.Model):
         return True
 
     def action_addpeer(self, user, timestamp, peerid, username):
-        if len(peerid) == 16:
+        if len(peerid) == settings.ID_LENGTH:
             from user.models import User
             if not 'users' in user.info:
                 user.info['users'] = {}
@@ -318,7 +318,7 @@ class Changelog(db.Model):
         return True
 
     def action_editpeer(self, user, timestamp, peerid, data):
-        if len(peerid) == 16:
+        if len(peerid) == settings.ID_LENGTH:
             from user.models import User
             peer = User.get_or_create(peerid)
             update = False
@@ -466,7 +466,7 @@ class Changelog(db.Model):
             elif op == 'addpeer':
                 peer_id = data[1]
                 username = data[2]
-                if len(peer_id) == 16:
+                if len(peer_id) == settings.ID_LENGTH:
                     peer = User.get(peer_id)
                     if peer:
                         username = peer.json().get('username', 'anonymous')

oml/commands.py

@@ -154,7 +154,7 @@ def command_update_static(*args):
     import utils
     setup.create_db()
     old_oxjs = os.path.join(settings.static_path, 'oxjs')
-    oxjs = os.path.join(settings.base_dir, '..', 'oxjs')
+    oxjs = os.path.join(settings.top_dir, 'oxjs')
     if os.path.exists(old_oxjs) and not os.path.exists(oxjs):
         shutil.move(old_oxjs, oxjs)
     if not os.path.exists(oxjs):
@@ -163,7 +163,7 @@ def command_update_static(*args):
         os.system('cd "%s" && git pull' % oxjs)
     r('python3', os.path.join(oxjs, 'tools', 'build', 'build.py'), '-nogeo')
     utils.update_static()
-    reader = os.path.join(settings.base_dir, '..', 'reader')
+    reader = os.path.join(settings.top_dir, 'reader')
     if not os.path.exists(reader):
         r('git', 'clone', '--depth', '1', 'https://code.0x2620.org/0x2620/openmedialibrary_reader.git', reader)
     elif os.path.exists(os.path.join(reader, '.git')):

oml/item/api.py

@@ -5,7 +5,7 @@ import os
 import unicodedata
 
 from sqlalchemy.orm import load_only
-from sqlalchemy.sql.expression import text
+from sqlalchemy.sql.expression import text, column
 from sqlalchemy import func
 
 from oxtornado import actions
@@ -58,8 +58,13 @@ def find(data):
             qs = models.Find.query.filter_by(key=q['group'])
             if items is None or items.first():
                 if items is not None:
-                    qs = qs.filter(models.Find.item_id.in_(items))
-                values = list(qs.values('value', 'findvalue', 'sortvalue'))
+                    ids = [i[0] for i in items.with_entities(column('id'))]
+                    qs = qs.filter(models.Find.item_id.in_(ids))
+                values = list(qs.values(
+                    column('value'),
+                    column('findvalue'),
+                    column('sortvalue'),
+                ))
                 for f in values:
                     value = f[0]
                     findvalue = f[1]
@@ -167,7 +172,7 @@ actions.register(edit, cache=False)
 def remove(data):
     '''
         takes {
-            id
+            ids
         }
     '''
     if 'ids' in data and data['ids']:

oml/item/handlers.py

@@ -17,7 +17,8 @@ import db
 import settings
 import tornado.web
 import tornado.gen
-import tornado.concurrent
+from concurrent.futures import ThreadPoolExecutor
+from tornado.concurrent import run_on_executor
 
 from oxtornado import json_dumps, json_response
 
@@ -27,6 +28,8 @@ import state
 import logging
 logger = logging.getLogger(__name__)
 
+MAX_WORKERS = 4
+
 
 class OptionalBasicAuthMixin(object):
     class SendChallenge(Exception):
@@ -90,6 +93,23 @@ class EpubHandler(OMLHandler):
                 self.set_header('Content-Type', content_type)
                 self.write(z.read(filename))
 
+class CropHandler(OMLHandler):
+
+    def get(self, id, page, left, top, right, bottom):
+        from media.pdf import crop
+        with db.session():
+            item = Item.get(id)
+            path = item.get_path()
+            data = crop(path, page, left, top, right, bottom)
+            if path and data:
+                self.set_header('Content-Type', 'image/jpeg')
+                self.set_header('Content-Length', str(len(data)))
+                self.write(data)
+                return
+        self.set_status(404)
+        return
+
+
 def serve_static(handler, path, mimetype, include_body=True, disposition=None):
     handler.set_header('Content-Type', mimetype)
     size = os.stat(path).st_size
@@ -177,6 +197,7 @@ class ReaderHandler(OMLHandler):
         return serve_static(self, path, 'text/html')
 
 class UploadHandler(OMLHandler):
+    executor = ThreadPoolExecutor(max_workers=MAX_WORKERS)
 
     def initialize(self, context=None):
         self._context = context
@@ -184,7 +205,60 @@ class UploadHandler(OMLHandler):
     def get(self):
         self.write('use POST')
 
-    @tornado.web.asynchronous
+    @run_on_executor
+    def save_files(self, request):
+        listname = request.arguments.get('list', None)
+        if listname:
+            listname = listname[0]
+            if isinstance(listname, bytes):
+                listname = listname.decode('utf-8')
+        with self._context():
+            prefs = settings.preferences
+            ids = []
+            for upload in request.files.get('files', []):
+                filename = upload.filename
+                id = get_id(data=upload.body)
+                ids.append(id)
+                file = File.get(id)
+                if not file or not os.path.exists(file.fullpath()):
+                    logger.debug('add %s to library', id)
+                    prefix_books = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books' + os.sep)
+                    prefix_imported = os.path.join(prefix_books, '.import' + os.sep)
+                    ox.makedirs(prefix_imported)
+                    import_name = os.path.join(prefix_imported, filename)
+                    n = 1
+                    while os.path.exists(import_name):
+                        n += 1
+                        name, extension = filename.rsplit('.', 1)
+                        if extension == 'kepub':
+                            extension = 'epub'
+                        import_name = os.path.join(prefix_imported, '%s [%d].%s' % (name, n, extension))
+                    with open(import_name, 'wb') as fd:
+                        fd.write(upload.body)
+                    file = add_file(id, import_name, prefix_books)
+                    file.move()
+                else:
+                    user = state.user()
+                    if not file.item:
+                        item = Item.get_or_create(id=file.sha1, info=file.info)
+                        file.item_id = item.id
+                        state.db.session.add(file)
+                        state.db.session.commit()
+                    else:
+                        item = file.item
+                    if user not in item.users:
+                        logger.debug('add %s to local user', id)
+                        item.add_user(user)
+                        add_record('additem', item.id, file.info)
+                        add_record('edititem', item.id, item.meta)
+                        item.update()
+        if listname and ids:
+            list_ = List.get(settings.USER_ID, listname)
+            if list_:
+                list_.add_items(ids)
+        response = json_response({'ids': ids})
+        return response
+
     @tornado.gen.coroutine
     def post(self):
         if 'origin' in self.request.headers and self.request.host not in self.request.headers['origin']:
@@ -193,60 +267,7 @@ class UploadHandler(OMLHandler):
             self.write('')
             return
 
-        def save_files(context, request, callback):
-            listname = request.arguments.get('list', None)
-            if listname:
-                listname = listname[0]
-                if isinstance(listname, bytes):
-                    listname = listname.decode('utf-8')
-            with context():
-                prefs = settings.preferences
-                ids = []
-                for upload in request.files.get('files', []):
-                    filename = upload.filename
-                    id = get_id(data=upload.body)
-                    ids.append(id)
-                    file = File.get(id)
-                    if not file or not os.path.exists(file.fullpath()):
-                        logger.debug('add %s to library', id)
-                        prefix_books = os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books' + os.sep)
-                        prefix_imported = os.path.join(prefix_books, '.import' + os.sep)
-                        ox.makedirs(prefix_imported)
-                        import_name = os.path.join(prefix_imported, filename)
-                        n = 1
-                        while os.path.exists(import_name):
-                            n += 1
-                            name, extension = filename.rsplit('.', 1)
-                            if extension == 'kepub':
-                                extension = 'epub'
-                            import_name = os.path.join(prefix_imported, '%s [%d].%s' % (name, n, extension))
-                        with open(import_name, 'wb') as fd:
-                            fd.write(upload.body)
-                        file = add_file(id, import_name, prefix_books)
-                        file.move()
-                    else:
-                        user = state.user()
-                        if not file.item:
-                            item = Item.get_or_create(id=file.sha1, info=file.info)
-                            file.item_id = item.id
-                            state.db.session.add(file)
-                            state.db.session.commit()
-                        else:
-                            item = file.item
-                        if user not in item.users:
-                            logger.debug('add %s to local user', id)
-                            item.add_user(user)
-                            add_record('additem', item.id, file.info)
-                            add_record('edititem', item.id, item.meta)
-                            item.update()
-            if listname and ids:
-                l = List.get(settings.USER_ID, listname)
-                if l:
-                    l.add_items(ids)
-            response = json_response({'ids': ids})
-            callback(response)
-
-        response = yield tornado.gen.Task(save_files, self._context, self.request)
+        response = yield self.save_files(self.request)
         if 'status' not in response:
             response = json_response(response)
         response = json_dumps(response)

oml/item/models.py

@@ -322,7 +322,7 @@ class Item(db.Model):
 
     def remove_annotations(self):
         from annotation.models import Annotation
-        for a in Annotation.query.filter_by(item_id=self.id, user_id=state.user()):
+        for a in Annotation.query.filter_by(item_id=self.id, user_id=settings.USER_ID):
             a.add_record('removeannotation')
             a.delete()
 
@@ -733,7 +733,13 @@ class File(db.Model):
             return re.sub(r'^\.|\.$|:|/|\?|<|>|\\|\*|\||"', '_', string)
         prefs = settings.preferences
         prefix = os.sep.join(os.path.join(os.path.expanduser(prefs['libraryPath']), 'Books/').split('/'))
-        if not self.item:
+        not_item = False
+        try:
+            not_item = not self.item
+        except:
+            logger.debug('trying to move an item that was just deleted', exc_info=True)
+            not_item = True
+        if not_item:
             return
         j = self.item.json(keys=['title', 'author', 'publisher', 'date', 'extension'])
 

oml/item/query.py

@@ -23,7 +23,6 @@ def parse(data):
     if [r for r in query['range'] if not isinstance(r, int)]:
         logger.error('range must be 2 integers! got this: %s', query['range'])
         query['range'] = [0, 0]
-    #print data
     query['qs'] = models.Item.find(data)
     if 'group' not in query:
         query['qs'] = order(query['qs'], query['sort'])

oml/item/scan.py

@@ -225,6 +225,7 @@ def run_scan():
         missing = ids - library_items
         if missing:
             logger.debug('%s items in library without a record', len(missing))
+    settings.server['last_scan'] = time.mktime(time.gmtime())
 
 def change_path(old, new):
     old_icons = os.path.join(old, 'Metadata', 'icons.db')

oml/library.py

@@ -153,7 +153,7 @@ class Peer(object):
                 self.info['lists'][name] = list(set(self.info['lists'][name]) - set(ids))
         elif action == 'addpeer':
             peerid, username = args
-            if len(peerid) == 16:
+            if len(peerid) == settings.ID_LENGTH:
                 self.info['peers'][peerid] = {'username': username}
             # fixme, just trigger peer update here
             from user.models import User
@@ -164,7 +164,7 @@ class Peer(object):
                 peer.save()
         elif action == 'editpeer':
             peerid, data = args
-            if len(peerid) == 16:
+            if len(peerid) == settings.ID_LENGTH:
                 if peerid not in self.info['peers']:
                     self.info['peers'][peerid] = {}
                 for key in ('username', 'contact'):
@@ -377,6 +377,10 @@ def sync_db():
     from sqlalchemy.orm import load_only
     import item.models
     first = True
+    missing_previews = []
+    state.sync_db = True
+
+    #FIXME: why is this loop needed
     with db.session():
         sort_ids = {i.item_id for i in item.models.Sort.query.options(load_only('item_id'))}
         if sort_ids:
@@ -387,9 +391,11 @@ def sync_db():
                     if first:
                         first = False
                         logger.debug('sync items')
-                    i.update(commit=False)
-                    if i.info.get('mediastate') == 'unavailable' and state.tasks:
-                        state.tasks.queue('getpreview', i.id)
+                    #why?
+                    #i.update(commit=False)
+                    i.update_sort(commit=False)
+                    if i.info.get('mediastate') == 'unavailable':
+                        missing_previews.append(i.id)
                     commit = True
                     #logger.debug('sync:%s', i)
                     t0 = maybe_commit(t0)
@@ -397,6 +403,7 @@ def sync_db():
                     break
             if commit:
                 state.db.session.commit()
+
     if not first:
         logger.debug('synced items')
     if not state.shutdown:
@@ -408,6 +415,12 @@ def sync_db():
     item.models.Sort.query.filter_by(item_id=None).delete()
     item.models.Find.query.filter_by(item_id=None).delete()
 
+    if missing_previews and state.tasks:
+        logger.debug('queueing download of %s missing previews', len(missing_previews))
+        for id in missing_previews:
+            state.tasks.queue('getpreview', id)
+    state.sync_db = False
+
 def cleanup_lists():
     import item.models
     import user.models

oml/localnodes.py

@@ -1,22 +1,26 @@
 # -*- coding: utf-8 -*-
 
+import asyncio
 import socket
 
 import netifaces
 from zeroconf import (
-    ServiceBrowser, ServiceInfo, ServiceStateChange, Zeroconf
+    ServiceBrowser, ServiceInfo, ServiceStateChange
 )
+from zeroconf.asyncio import AsyncZeroconf
 from tornado.ioloop import PeriodicCallback
 
 import settings
 import state
 from tor_request import get_opener
+from utils import time_cache
 
 import logging
 logger = logging.getLogger(__name__)
 
 
-def can_connect(data):
+@time_cache(3)
+def can_connect(**data):
     try:
         opener = get_opener(data['id'])
         headers = {
@@ -60,100 +64,120 @@ class LocalNodes(dict):
             return
         self.setup()
         self._ip_changed = PeriodicCallback(self._update_if_ip_changed, 60000)
+        state.main.add_callback(self._ip_changed.start)
 
     def setup(self):
         self.local_ips = get_broadcast_interfaces()
-        self.zeroconf = {ip: Zeroconf(interfaces=[ip]) for ip in self.local_ips}
-        self.register_service()
+        self.zeroconf = {ip: AsyncZeroconf(interfaces=[ip]) for ip in self.local_ips}
+        asyncio.create_task(self.register_service())
         self.browse()
 
-    def _update_if_ip_changed(self):
+    async def _update_if_ip_changed(self):
         local_ips = get_broadcast_interfaces()
         username = settings.preferences.get('username', 'anonymous')
         if local_ips != self.local_ips or self.username != username:
-            self.close()
+            await self.close()
             self.setup()
 
     def browse(self):
         self.browser = {
-            ip: ServiceBrowser(self.zeroconf[ip], self.service_type, handlers=[self.on_service_state_change])
+            ip: ServiceBrowser(self.zeroconf[ip].zeroconf, self.service_type, handlers=[self.on_service_state_change])
             for ip in self.zeroconf
         }
 
-    def register_service(self):
+    async def register_service(self):
         if self.local_info:
             for local_ip, local_info in self.local_info:
-                self.zeroconf[local_ip].unregister_service(local_info)
+                self.zeroconf[local_ip].async_unregister_service(local_info)
             self.local_info = None
 
         local_name = socket.gethostname().partition('.')[0] + '.local.'
         port = settings.server['node_port']
+        self.local_info = []
         self.username = settings.preferences.get('username', 'anonymous')
         desc = {
-            'username': self.username
+            'username': self.username,
+            'id': settings.USER_ID,
         }
-        self.local_info = []
+        tasks = []
         for i, local_ip in enumerate(get_broadcast_interfaces()):
             if i:
-                name = '%s-%s [%s].%s' % (desc['username'], i+1, settings.USER_ID, self.service_type)
+                name = '%s [%s].%s' % (desc['username'], i, self.service_type)
             else:
-                name = '%s [%s].%s' % (desc['username'], settings.USER_ID, self.service_type)
-            local_info = ServiceInfo(self.service_type, name,
-                                     socket.inet_aton(local_ip), port, 0, 0, desc, local_name)
-            self.zeroconf[local_ip].register_service(local_info)
+                name = '%s.%s' % (desc['username'], self.service_type)
+
+            addresses = [socket.inet_aton(local_ip)]
+            local_info = ServiceInfo(self.service_type, name, port, 0, 0, desc, local_name, addresses=addresses)
+            task = self.zeroconf[local_ip].async_register_service(local_info)
+            tasks.append(task)
             self.local_info.append((local_ip, local_info))
+        await asyncio.gather(*tasks)
 
     def __del__(self):
         self.close()
 
-    def close(self):
+    async def close(self):
         if self.local_info:
+            tasks = []
             for local_ip, local_info in self.local_info:
                 try:
-                    self.zeroconf[local_ip].unregister_service(local_info)
+                    task = self.zeroconf[local_ip].async_unregister_service(local_info)
+                    tasks.append(task)
                 except:
                     logger.debug('exception closing zeroconf', exc_info=True)
             self.local_info = None
         if self.zeroconf:
             for local_ip in self.zeroconf:
                 try:
-                    self.zeroconf[local_ip].close()
+                    task = self.zeroconf[local_ip].async_close()
+                    tasks.append(task)
                 except:
                     logger.debug('exception closing zeroconf', exc_info=True)
             self.zeroconf = None
         for id in list(self):
             self.pop(id, None)
+        await asyncio.gather(*tasks)
 
     def on_service_state_change(self, zeroconf, service_type, name, state_change):
-        if '[' not in name:
-            id = name.split('.')[0]
-        else:
-            id = name.split('[')[1].split(']')[0]
-        if id == settings.USER_ID:
-            return
-        if state_change is ServiceStateChange.Added:
+        try:
             info = zeroconf.get_service_info(service_type, name)
-            if info:
+        except zeroconf._exceptions.NotRunningException:
+            return
+        if info and b'id' in info.properties:
+            id = info.properties[b'id'].decode()
+            if id == settings.USER_ID:
+                return
+            if len(id) != settings.ID_LENGTH:
+                return
+            if state_change is ServiceStateChange.Added:
+                new = id not in self
                 self[id] = {
                     'id': id,
-                    'host': socket.inet_ntoa(info.address),
+                    'host': socket.inet_ntoa(info.addresses[0]),
                     'port': info.port
                 }
                 if info.properties:
                     for key, value in info.properties.items():
                         key = key.decode()
                         self[id][key] = value.decode()
-                logger.debug('add: %s [%s] (%s:%s)', self[id].get('username', 'anon'), id, self[id]['host'], self[id]['port'])
+                logger.debug(
+                    '%s: %s [%s] (%s:%s)',
+                    'add' if new else 'update',
+                    self[id].get('username', 'anon'),
+                    id,
+                    self[id]['host'],
+                    self[id]['port']
+                )
                 if state.tasks and id in self:
                     state.tasks.queue('addlocalinfo', self[id])
-        elif state_change is ServiceStateChange.Removed:
-            logger.debug('remove: %s', id)
-            self.pop(id, None)
-            if state.tasks:
-                state.tasks.queue('removelocalinfo', id)
+            elif state_change is ServiceStateChange.Removed:
+                logger.debug('remove: %s', id)
+                self.pop(id, None)
+                if state.tasks:
+                    state.tasks.queue('removelocalinfo', id)
 
     def get_data(self, user_id):
         data = self.get(user_id)
-        if data and can_connect(data):
+        if data and can_connect(**data):
             return data
         return None

oml/media/epub.py

@@ -68,17 +68,17 @@ def cover(path):
         if manifest:
             manifest = manifest[0]
         if metadata and manifest:
-            for e in metadata.getchildren():
+            for e in list(metadata):
                 if e.tag == '{http://www.idpf.org/2007/opf}meta' and e.attrib.get('name') == 'cover':
                     cover_id = e.attrib['content']
-                    for e in manifest.getchildren():
+                    for e in list(manifest):
                         if e.attrib['id'] == cover_id:
                             filename = unquote(e.attrib['href'])
                             filename = normpath(os.path.join(os.path.dirname(opf[0]), filename))
                             if filename in files:
                                 return use(filename)
         if manifest:
-            images = [e for e in manifest.getchildren() if 'image' in e.attrib['media-type']]
+            images = [e for e in list(manifest) if 'image' in e.attrib['media-type']]
             if images:
                 image_data = []
                 for e in images:
@@ -89,7 +89,7 @@ def cover(path):
                 if image_data:
                     image_data.sort(key=lambda name: z.getinfo(name).file_size)
                     return use(image_data[-1])
-            for e in manifest.getchildren():
+            for e in list(manifest):
                 if 'html' in e.attrib['media-type']:
                     filename = unquote(e.attrib['href'])
                     filename = normpath(os.path.join(os.path.dirname(opf[0]), filename))
@@ -118,7 +118,7 @@ def info(epub):
         metadata = info.findall('{http://www.idpf.org/2007/opf}metadata')
         if metadata:
             metadata = metadata[0]
-            for e in metadata.getchildren():
+            for e in list(metadata):
                 if e.text and e.text.strip() and e.text not in ('unknown', 'none'):
                     key = e.tag.split('}')[-1]
                     key = {
@@ -148,7 +148,7 @@ def info(epub):
                 for point in nav_map.findall('{http://www.daisy.org/z3986/2005/ncx/}navPoint'):
                     label = point.find('{http://www.daisy.org/z3986/2005/ncx/}navLabel')
                     if label:
-                        txt = label.getchildren()[0].text
+                        txt = list(label)[0].text
                         if txt:
                             contents.append(txt)
                 if contents:

oml/media/pdf.py

@@ -10,6 +10,7 @@ from glob import glob
 from datetime import datetime
 
 from PyPDF2 import PdfFileReader
+from PIL import Image
 import ox
 
 import settings
@@ -24,13 +25,13 @@ def cover(pdf):
     else:
         return page(pdf, 1)
 
-def ql_cover(pdf):
+def ql_cover(pdf, size=1024):
     tmp = tempfile.mkdtemp()
     cmd = [
         'qlmanage',
         '-t',
         '-s',
-        '1024',
+        str(size),
         '-o',
         tmp,
         pdf
@@ -48,7 +49,7 @@ def ql_cover(pdf):
     shutil.rmtree(tmp)
     return data
 
-def page(pdf, page):
+def page(pdf, page, size=1024):
     tmp = tempfile.mkdtemp()
     if sys.platform == 'win32':
         pdf = get_short_path_name(pdf)
@@ -56,8 +57,8 @@ def page(pdf, page):
         'pdftocairo',
         pdf,
         '-jpeg',
-        '-f',  str(page), '-l', str(page),
-        '-scale-to', '1024', '-cropbox',
+        '-f', str(page), '-l', str(page),
+        '-scale-to', str(size), '-cropbox',
         os.path.join(tmp, 'page')
     ]
     if sys.platform == 'win32':
@@ -79,6 +80,46 @@ def page(pdf, page):
     shutil.rmtree(tmp)
     return data
 
+def crop(pdf, page, left, top, right, bottom):
+    size = 2048
+    tmp = tempfile.mkdtemp()
+    if sys.platform == 'win32':
+        pdf = get_short_path_name(pdf)
+    cmd = [
+        'pdftocairo',
+        pdf,
+        '-jpeg',
+        '-f', str(page), '-l', str(page),
+        '-scale-to', str(size), '-cropbox',
+        os.path.join(tmp, 'page')
+    ]
+    if sys.platform == 'win32':
+        startupinfo = subprocess.STARTUPINFO()
+        startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+        startupinfo.wShowWindow = subprocess.SW_HIDE
+        p = subprocess.Popen(cmd, close_fds=True, startupinfo=startupinfo)
+    else:
+        p = subprocess.Popen(cmd, close_fds=True)
+    p.wait()
+    image = glob('%s/*' % tmp)
+    if image:
+        image = image[0]
+        crop = [int(p) for p in (left, top, right, bottom)]
+        img = Image.open(image).crop(crop)
+        img.save(image)
+        with open(image, 'rb') as fd:
+            data = fd.read()
+    else:
+        logger.debug('pdftocairo %s %s', pdf, ' '.join(cmd))
+        data = None
+    shutil.rmtree(tmp)
+    return data
+    
+
+
+
+
+
 '''
 def page(pdf, page):
     image = tempfile.mkstemp('.jpg')[1]
@@ -281,3 +322,4 @@ def extract_isbn(text):
     isbns = find_isbns(text)
     if isbns:
         return isbns[0]
+

oml/media/txt.py

@@ -7,8 +7,9 @@ import tempfile
 import subprocess
 
 def cover(path):
+    import settings
     image = tempfile.mkstemp('.jpg')[1]
-    cmd = ['python3', '../reader/txt.js/txt.py', '-i', path, '-o', image]
+    cmd = ['python3', os.path.join(settings.top_dir, 'reader/txt.js/txt.py'), '-i', path, '-o', image]
     p = subprocess.Popen(cmd, close_fds=True)
     p.wait()
     with open(image, 'rb') as fd:

oml/node/server.py

@@ -12,18 +12,20 @@ import socket
 import socketserver
 import time
 
-from Crypto.PublicKey import RSA
-from Crypto.Util.asn1 import DerSequence
-from OpenSSL.crypto import dump_privatekey, FILETYPE_ASN1
 from OpenSSL.SSL import (
-    Context, Connection, TLSv1_2_METHOD,
-    VERIFY_PEER, VERIFY_FAIL_IF_NO_PEER_CERT, VERIFY_CLIENT_ONCE
+    Connection,
+    Context,
+    TLSv1_2_METHOD,
+    VERIFY_CLIENT_ONCE,
+    VERIFY_FAIL_IF_NO_PEER_CERT,
+    VERIFY_PEER,
 )
 
 import db
 import settings
 import state
 import user
+import utils
 from changelog import changelog_size, changelog_path
 from websocket import trigger_event
 
@@ -34,16 +36,15 @@ import logging
 logger = logging.getLogger(__name__)
 
 
-def get_service_id(key):
-    '''
-    service_id is the first half of the sha1 of the rsa public key encoded in base32
-    '''
-    # compute sha1 of public key and encode first half in base32
-    pub_der = DerSequence()
-    pub_der.decode(dump_privatekey(FILETYPE_ASN1, key))
-    public_key = RSA.construct((pub_der._seq[1], pub_der._seq[2])).exportKey('DER')[22:]
-    service_id = base64.b32encode(hashlib.sha1(public_key).digest()[:10]).lower().decode()
-    return service_id
+def get_service_id(connection):
+    certs = connection.get_peer_cert_chain()
+    for cert in certs:
+        if cert.get_signature_algorithm().decode() == "ED25519":
+            pubkey = cert.get_pubkey()
+            public_key = pubkey.to_cryptography_key().public_bytes_raw()
+            service_id = utils.get_onion(public_key)
+            return service_id
+    raise Exception("connection with invalid certificate")
 
 class TLSTCPServer(socketserver.TCPServer):
 
@@ -55,7 +56,7 @@ class TLSTCPServer(socketserver.TCPServer):
         socketserver.TCPServer.__init__(self, server_address, HandlerClass)
         ctx = Context(TLSv1_2_METHOD)
         ctx.use_privatekey_file(settings.ssl_key_path)
-        ctx.use_certificate_file(settings.ssl_cert_path)
+        ctx.use_certificate_chain_file(settings.ssl_cert_path)
         # only allow clients with cert:
         ctx.set_verify(VERIFY_PEER | VERIFY_CLIENT_ONCE | VERIFY_FAIL_IF_NO_PEER_CERT, self._accept)
         #ctx.set_verify(VERIFY_PEER | VERIFY_CLIENT_ONCE, self._accept)
@@ -111,8 +112,7 @@ class Handler(http.server.SimpleHTTPRequestHandler):
         return self.do_GET()
 
     def do_GET(self):
-        #x509 = self.connection.get_peer_certificate()
-        #user_id = get_service_id(x509.get_pubkey()) if x509 else None
+        user_id = get_service_id(self.connection)
         import item.models
         parts = self.path.split('/')
         if len(parts) == 3 and parts[1] in ('get', 'preview'):
@@ -185,8 +185,7 @@ class Handler(http.server.SimpleHTTPRequestHandler):
         self.end_headers()
 
     def _changelog(self):
-        x509 = self.connection.get_peer_certificate()
-        user_id = get_service_id(x509.get_pubkey()) if x509 else None
+        user_id = get_service_id(self.connection)
         with db.session():
             u = user.models.User.get(user_id)
             if not u:
@@ -257,8 +256,7 @@ class Handler(http.server.SimpleHTTPRequestHandler):
 
             ping            responds public ip
         '''
-        x509 = self.connection.get_peer_certificate()
-        user_id = get_service_id(x509.get_pubkey()) if x509 else None
+        user_id = get_service_id(self.connection)
 
         content = {}
         try:

oml/nodes.py

@@ -448,6 +448,9 @@ class Node(Thread):
         except socket.timeout:
             logger.debug('timeout %s', url)
             return False
+        except urllib.error.URLError as e:
+            logger.debug('urllib.error.URLError %s', e)
+            return False
         except socks.GeneralProxyError:
             logger.debug('download failed %s', url)
             return False
@@ -598,6 +601,8 @@ class Nodes(Thread):
     def _pull(self):
         if not state.sync_enabled or settings.preferences.get('downloadRate') == 0:
             return
+        if state.sync_db:
+            return
         if state.activity and state.activity.get('activity') == 'import':
             return
         self._pulling = True
@@ -617,12 +622,12 @@ class Nodes(Thread):
                 node.pullChanges()
         self._pulling = False
 
-    def join(self):
+    async def join(self):
         self._q.put(None)
         for node in list(self._nodes.values()):
             node.join()
         if self.local:
-            self.local.close()
+            await self.local.close()
         return super().join(1)
 
 def publish_node():

oml/oxtornado.py

@@ -83,9 +83,18 @@ class ApiHandler(tornado.web.RequestHandler):
         context = self._context
         if context is None:
             context = defaultcontext
-        action = request.arguments.get('action', [None])[0].decode('utf-8')
-        data = request.arguments.get('data', [b'{}'])[0]
-        data = json.loads(data.decode('utf-8')) if data else {}
+        action = None
+        if request.headers.get('Content-Type') == 'application/json':
+            try:
+                r = json.loads(request.body.decode())
+                action = r['action']
+                data = r['data']
+            except:
+                action = None
+        if not action:
+            action = request.arguments.get('action', [None])[0].decode('utf-8')
+            data = request.arguments.get('data', [b'{}'])[0]
+            data = json.loads(data.decode('utf-8')) if data else {}
         if not action:
             methods = list(actions.keys())
             api = []

oml/queryparser.py

@@ -2,9 +2,10 @@
 
 from datetime import datetime
 import unicodedata
+import sqlalchemy.orm.exc
 from sqlalchemy.sql import operators
 from sqlalchemy.orm import load_only
-from sqlalchemy.sql.expression import text
+from sqlalchemy.sql.expression import text, column
 
 import utils
 import settings
@@ -13,6 +14,7 @@ from fulltext import find_fulltext
 import logging
 logger = logging.getLogger(__name__)
 
+
 def get_operator(op, type='str'):
     return {
         'str': {
@@ -134,7 +136,8 @@ class Parser(object):
             q = get_operator(op)(self._find.findvalue, v)
             if k != '*':
                 q &= (self._find.key == k)
-            ids = self._model.query.join(self._find).filter(q).options(load_only('id'))
+            ids = self._find.query.filter(q).with_entities(column('item_id'))
+            ids = [i[0] for i in ids]
             return self.in_ids(ids, exclude)
         elif k == 'list':
             nickname, name = v.split(':', 1)
@@ -220,7 +223,7 @@ class Parser(object):
         for condition in conditions:
             if 'conditions' in condition:
                 q = self.parse_conditions(condition['conditions'],
-                                 condition.get('operator', '&'))
+                                          condition.get('operator', '&'))
             else:
                 q = self.parse_condition(condition)
             if isinstance(q, list):
@@ -265,8 +268,7 @@ class Parser(object):
         qs = self._model.query
         #only include items that have hard metadata
         conditions = self.parse_conditions(data.get('query', {}).get('conditions', []),
-                                     data.get('query', {}).get('operator', '&'))
+                                           data.get('query', {}).get('operator', '&'))
         for c in conditions:
             qs = qs.filter(c)
-        #print(qs)
         return qs

oml/server.py

@@ -1,16 +1,19 @@
 # -*- coding: utf-8 -*-
 
+import asyncio
 import os
 import sys
 import signal
 import time
 
 from tornado.ioloop import IOLoop
-from tornado.web import StaticFileHandler, Application
+import tornado.web
+from tornado.web import Application
 
 from cache import Cache
 from item.handlers import EpubHandler, ReaderHandler, FileHandler
 from item.handlers import OMLHandler, UploadHandler
+from item.handlers import CropHandler
 from item.icons import IconHandler
 import db
 import node.server
@@ -28,6 +31,12 @@ import logging
 
 logger = logging.getLogger(__name__)
 
+class StaticFileHandler(tornado.web.StaticFileHandler):
+    def get_content_type(self):
+        if self.request.path.split('?')[0].endswith('.mjs'):
+            return 'application/javascript'
+        return super().get_content_type()
+
 class MainHandler(OMLHandler):
 
     def get(self, path):
@@ -59,13 +68,13 @@ def log_request(handler):
         log_method("%d %s %.2fms", handler.get_status(),
                    handler._request_summary(), request_time)
 
-def shutdown():
+async def shutdown():
     state.shutdown = True
     if state.tor:
         state.tor._shutdown = True
     if state.nodes:
         logger.debug('shutdown nodes')
-        state.nodes.join()
+        await state.nodes.join()
     if state.downloads:
         logger.debug('shutdown downloads')
         state.downloads.join()
@@ -111,11 +120,11 @@ def run():
 
     common_handlers = [
         (r'/(favicon.ico)', StaticFileHandler, {'path': settings.static_path}),
-        (r'/static/oxjs/(.*)', StaticFileHandler, {'path': os.path.join(settings.base_dir, '..', 'oxjs')}),
-        (r'/static/cbr.js/(.*)', StaticFileHandler, {'path': os.path.join(settings.base_dir, '..', 'reader', 'cbr.js')}),
-        (r'/static/epub.js/(.*)', StaticFileHandler, {'path': os.path.join(settings.base_dir, '..', 'reader', 'epub.js')}),
-        (r'/static/pdf.js/(.*)', StaticFileHandler, {'path': os.path.join(settings.base_dir, '..', 'reader', 'pdf.js')}),
-        (r'/static/txt.js/(.*)', StaticFileHandler, {'path': os.path.join(settings.base_dir, '..', 'reader', 'txt.js')}),
+        (r'/static/oxjs/(.*)', StaticFileHandler, {'path': os.path.join(settings.top_dir, 'oxjs')}),
+        (r'/static/cbr.js/(.*)', StaticFileHandler, {'path': os.path.join(settings.top_dir, 'reader', 'cbr.js')}),
+        (r'/static/epub.js/(.*)', StaticFileHandler, {'path': os.path.join(settings.top_dir, 'reader', 'epub.js')}),
+        (r'/static/pdf.js/(.*)', StaticFileHandler, {'path': os.path.join(settings.top_dir, 'reader', 'pdf.js')}),
+        (r'/static/txt.js/(.*)', StaticFileHandler, {'path': os.path.join(settings.top_dir, 'reader', 'txt.js')}),
         (r'/static/(.*)', StaticFileHandler, {'path': settings.static_path}),
         (r'/(.*)/epub/(.*)', EpubHandler),
         (r'/(.*?)/reader/', ReaderHandler),
@@ -125,6 +134,7 @@ def run():
         (r'/(.*?)/get/', FileHandler, {
             'attachment': True
         }),
+        (r'/(.*)/2048p(\d*),(\d*),(\d*),(\d*),(\d*).jpg', CropHandler),
         (r'/(.*)/(cover|preview)(\d*).jpg', IconHandler),
     ]
     handlers = common_handlers + [
@@ -146,13 +156,12 @@ def run():
     http_server.listen(settings.server['port'], settings.server['address'], max_buffer_size=max_buffer_size)
 
     # public server
-    '''
-    public_port = settings.server.get('public_port')
-    public_address = settings.server['public_address']
-    if public_port:
-        public_server = Application(public_handlers, **options)
-        public_server.listen(public_port, public_address)
-    '''
+    if settings.preferences.get('enableReadOnlyService'):
+        public_port = settings.server.get('public_port')
+        public_address = settings.server['public_address']
+        if public_port:
+            public_server = Application(public_handlers, **options)
+            public_server.listen(public_port, public_address)
 
     if PID:
         with open(PID, 'w') as pid:
@@ -198,10 +207,10 @@ def run():
     print('open browser at %s' % url)
     logger.debug('Starting OML %s at %s', settings.VERSION, url)
 
-    signal.signal(signal.SIGTERM, shutdown)
+    signal.signal(signal.SIGTERM, lambda _, __: sys.exit(0))
 
     try:
         state.main.start()
     except:
         print('shutting down...')
-    shutdown()
+    asyncio.run(shutdown())

oml/settings.py

@@ -8,15 +8,17 @@ from oml.utils import get_user_id
 from oml import fulltext
 
 base_dir = os.path.normpath(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..'))
+top_dir = os.path.dirname(base_dir)
+
 static_path = os.path.join(base_dir, 'static')
-updates_path = os.path.normpath(os.path.join(base_dir, '..', 'updates'))
+updates_path = os.path.normpath(os.path.join(top_dir, 'updates'))
 
 oml_data_path = os.path.join(base_dir, 'config.json')
 
 
-data_path = os.path.normpath(os.path.join(base_dir, '..', 'data'))
+data_path = os.path.normpath(os.path.join(top_dir, 'data'))
 if not os.path.exists(data_path):
-    config_path = os.path.normpath(os.path.join(base_dir, '..', 'config'))
+    config_path = os.path.normpath(os.path.join(top_dir, 'config'))
     if os.path.exists(config_path):
         data_path = config_path
     else:
@@ -24,9 +26,11 @@ if not os.path.exists(data_path):
 
 db_path = os.path.join(data_path, 'data.db')
 log_path = os.path.join(data_path, 'debug.log')
-ssl_cert_path = os.path.join(data_path, 'node.ssl.crt')
-ssl_key_path = os.path.join(data_path, 'tor', 'private_key')
 
+ca_key_path = os.path.join(data_path, 'node.ca.key')
+ca_cert_path = os.path.join(data_path, 'node.ca.crt')
+ssl_cert_path = os.path.join(data_path, 'node.tls.crt')
+ssl_key_path = os.path.join(data_path, 'node.tls.key')
 
 if os.path.exists(oml_data_path):
     with open(oml_data_path) as fd:
@@ -57,7 +61,7 @@ for key in server_defaults:
 
 release = pdict(os.path.join(data_path, 'release.json'))
 
-USER_ID = get_user_id(ssl_key_path, ssl_cert_path)
+USER_ID = get_user_id(ssl_key_path, ssl_cert_path, ca_key_path, ca_cert_path)
 
 OML_UPDATE_KEY = 'K55EZpPYbP3X+3mA66cztlw1sSaUMqGwfTDKQyP2qOU'
 OML_UPDATE_CERT = '''-----BEGIN CERTIFICATE-----
@@ -96,3 +100,5 @@ if not FULLTEXT_SUPPORT:
     config['itemKeys'] = [k for k in config['itemKeys'] if k['id'] != 'fulltext']
 
 DB_VERSION = 20
+
+ID_LENGTH = 56

oml/setup.py

@@ -423,6 +423,27 @@ def upgrade_db(old, new=None):
                         )''')
             run_sql('CREATE UNIQUE INDEX IF NOT EXISTS user_metadata_index ON user_metadata(id, user_id)')
             run_sql('CREATE INDEX ix_user_metadata_data_hash ON user_metadata (data_hash)')
+    if old <= '20240608-1469-647a8b9':
+        old_hostname = os.path.join(settings.data_path, 'tor/hostname')
+        if os.path.exists(old_hostname):
+            with open(old_hostname) as fd:
+                OLD_USER_ID = fd.read().split('.')[0]
+                statements = [
+                    "UPDATE user SET id = '{nid}' WHERE id = '{oid}'",
+                    "UPDATE list SET user_id = '{nid}' WHERE user_id = '{oid}'",
+                    "UPDATE useritem SET user_id = '{nid}' WHERE user_id = '{oid}'",
+                    "UPDATE changelog SET user_id = '{nid}' WHERE user_id = '{oid}'",
+                ]
+                run_sql([
+                    sql.format(oid=OLD_USER_ID, nid=settings.USER_ID)
+                    for sql in statements
+                ])
+                for ext in ('log', 'db', 'json'):
+                    old_log = os.path.join(settings.data_path, 'peers/%s.%s' % (OLD_USER_ID, ext))
+                    new_log = os.path.join(settings.data_path, 'peers/%s.%s' % (USER_ID, ext))
+                    if os.path.exists(old_log) and not os.path.exists(new_log):
+                        os.rename(old_log, new_log)
+
 
 def create_default_lists(user_id=None):
     with db.session():

oml/state.py

@@ -16,6 +16,7 @@ websockets = []
 uisockets = []
 peers = {}
 changelog_size = None
+sync_db = False
 
 activity = {}
 removepeer = {}

oml/tasks.py

@@ -36,16 +36,15 @@ class Tasks(Thread):
 
     def run(self):
         self.load_tasks()
-        if time.mktime(time.gmtime()) - settings.server.get('last_scan', 0) > 24*60*60:
-            settings.server['last_scan'] = time.mktime(time.gmtime())
+        if (time.mktime(time.gmtime()) - settings.server.get('last_scan', 0)) > 24*60*60:
             self.queue('scan')
 
         import item.scan
         from item.models import sync_metadata, get_preview, get_cover
         from user.models import (
-                export_list, update_user_peering,
-                add_local_info, remove_local_info,
-                upload
+            export_list, update_user_peering,
+            add_local_info, remove_local_info,
+            upload
         )
         shutdown = False
         while not shutdown:
@@ -98,6 +97,7 @@ class Tasks(Thread):
 
     def load_tasks(self):
         if os.path.exists(self._taskspath):
+            logger.debug('loading tasks')
             try:
                 with open(self._taskspath) as f:
                     tasks = json.load(f)

oml/tor.py

@@ -22,9 +22,11 @@ import logging
 logging.getLogger('stem').setLevel(logging.ERROR)
 logger = logging.getLogger(__name__)
 
+
 class TorDaemon(Thread):
     installing = False
     running = True
+    ended = False
     p = None
 
     def __init__(self):
@@ -105,6 +107,8 @@ DirReqStatistics 0
                     logger.debug(line)
                 self.p.communicate()
                 time.sleep(0.5)
+                self.ended = True
+                self.running = False
             self.p = None
 
     def kill(self):
@@ -142,6 +146,10 @@ class Tor(object):
                 logger.debug("Start tor")
                 self.daemon = TorDaemon()
                 return self.connect()
+            elif self.daemon.ended:
+                logger.debug("Try starting tor again")
+                self.daemon = TorDaemon()
+                return self.connect()
             if not self.daemon.installing:
                 logger.debug("Failed to connect to tor")
             return False
@@ -201,18 +209,20 @@ class Tor(object):
             return False
         controller = self.controller
         if controller.get_version() >= stem.version.Requirement.ADD_ONION:
-            with open(settings.ssl_key_path, 'rb') as fd:
-                private_key = fd.read()
-            key_content = RSA.importKey(private_key).exportKey().decode()
-            key_content = ''.join(key_content.strip().split('\n')[1:-1])
+            private_key, public_key = utils.load_pem_key(settings.ca_key_path)
+            key_type, key_content = utils.get_onion_key(private_key)
             ports = {9851: settings.server['node_port']}
             if settings.preferences.get('enableReadOnlyService'):
                 ports[80] = settings.server['public_port']
             controller.remove_ephemeral_hidden_service(settings.USER_ID)
-            response = controller.create_ephemeral_hidden_service(ports,
-                    key_type='RSA1024', key_content=key_content,
-                    detached=True)
+            response = controller.create_ephemeral_hidden_service(
+                ports,
+                key_type=key_type, key_content=key_content,
+                detached=True
+            )
             if response.is_ok():
+                if response.service_id != settings.USER_ID:
+                    logger.error("Something is wrong with tor id %s vs %s", response.service_id, settings.USER_ID)
                 logger.debug('published node as https://%s.onion:%s',
                              settings.USER_ID, settings.server_defaults['node_port'])
                 if settings.preferences.get('enableReadOnlyService'):
@@ -259,65 +269,71 @@ def torbrowser_url(sys_platform=None):
     data = read_url(base_url, timeout=3*24*60*60).decode()
     versions = []
     for r in (
-        re.compile('href="(\d\.\d\.\d/)"'),
-        re.compile('href="(\d\.\d/)"'),
+        re.compile('href="(\d+\.\d+\.\d+/)"'),
+        re.compile('href="(\d+\.\d+/)"'),
     ):
         versions += r.findall(data)
+    if not versions:
+        return None
     current = sorted(versions)[-1]
     url = base_url + current
-    language = '.*?en'
     if sys_platform.startswith('linux'):
         if platform.architecture()[0] == '64bit':
-            osname = 'linux64'
+            osname = 'linux-x86_64'
         else:
-            osname = 'linux32'
+            osname = 'linux-x86_32'
         ext = 'xz'
     elif sys_platform == 'darwin':
-        osname = 'osx64'
+        osname = 'macos'
         ext = 'dmg'
     elif sys_platform == 'win32':
-        language = ''
-        osname = ''
-        ext = 'zip'
+        osname = 'windows-x86_64-portable'
+        ext = 'exe'
     else:
         logger.debug('no way to get torbrowser url for %s', sys.platform)
         return None
-    r = re.compile('href="(.*?{osname}{language}.*?{ext})"'.format(osname=osname,language=language,ext=ext))
-    torbrowser = sorted(r.findall(read_url(url).decode()))[-1]
+    data = read_url(url).decode()
+    r = re.compile('href="(.*?{osname}.*?{ext})"'.format(osname=osname, ext=ext)).findall(data)
+    if not r:
+        r = re.compile('href="(.*?{ext})"'.format(ext=ext)).findall(data)
+    torbrowser = sorted(r)[-1]
     url += torbrowser
     return url
 
 def get_tor():
     if sys.platform == 'darwin':
         for path in (
-            os.path.join(settings.base_dir, '..', 'platform_darwin64', 'tor', 'tor'),
+            os.path.join(settings.top_dir, 'platform_darwin64', 'tor', 'tor'),
             '/Applications/TorBrowser.app/TorBrowser/Tor/tor',
-            os.path.join(settings.base_dir, '..', 'tor', 'TorBrowser.app', 'TorBrowser', 'Tor', 'tor')
+            os.path.join(settings.top_dir, 'tor', 'TorBrowser.app', 'TorBrowser', 'Tor', 'tor')
         ):
             if os.path.isfile(path) and os.access(path, os.X_OK):
                 return path
     elif sys.platform == 'win32':
         paths = [
-            os.path.join(settings.base_dir, '..', 'platform_win32', 'tor', 'tor.exe')
+            os.path.join(settings.top_dir, 'platform_win32', 'tor', 'tor.exe')
         ]
-        exe = os.path.join('Tor Browser', 'Browser', 'TorBrowser', 'Tor', 'tor.exe')
-        for prefix in (
-            os.path.join(os.path.expanduser('~'), 'Desktop'),
-            os.path.join('C:', 'Program Files'),
-            os.path.join('C:', 'Program Files (x86)'),
+        for exe in (
+            os.path.join('Tor Browser', 'Browser', 'TorBrowser', 'Tor', 'tor.exe'),
+            os.path.join('Tor Browser', 'Browser', 'TorBrowser', 'Tor', 'Tor', 'tor.exe'),
         ):
-            path = os.path.join(prefix, exe)
-            paths.append(path)
-        paths.append(os.path.join(settings.base_dir, '..', 'tor', 'Tor', 'tor.exe'))
+            for prefix in (
+                os.path.join(os.path.expanduser('~'), 'Desktop'),
+                os.path.join('C:', 'Program Files'),
+                os.path.join('C:', 'Program Files (x86)'),
+            ):
+                path = os.path.join(prefix, exe)
+                paths.append(path)
+        paths.append(os.path.join(settings.top_dir, 'tor', 'Tor', 'tor.exe'))
         for path in paths:
             if os.path.isfile(path) and os.access(path, os.X_OK):
                 return os.path.normpath(path)
     elif sys.platform.startswith('linux'):
         for path in (
-            os.path.join(settings.base_dir, '..', 'platform_linux64', 'tor', 'tor'),
-            os.path.join(settings.base_dir, '..', 'platform_linux32', 'tor', 'tor'),
-            os.path.join(settings.base_dir, '..', 'platform_linux_armv7l', 'tor', 'tor'),
-            os.path.join(settings.base_dir, '..', 'platform_linux_aarch64', 'tor', 'tor'),
+            os.path.join(settings.top_dir, 'platform_linux64', 'tor', 'tor'),
+            os.path.join(settings.top_dir, 'platform_linux32', 'tor', 'tor'),
+            os.path.join(settings.top_dir, 'platform_linux_armv7l', 'tor', 'tor'),
+            os.path.join(settings.top_dir, 'platform_linux_aarch64', 'tor', 'tor'),
         ):
             if os.path.isfile(path) and os.access(path, os.X_OK):
                 return os.path.normpath(path)
@@ -331,9 +347,12 @@ def get_tor():
                     path = os.path.join(base, 'TorBrowser', 'Tor', 'tor')
                     if os.path.isfile(path) and os.access(path, os.X_OK):
                         return path
+                    path = os.path.join(base, 'TorBrowser', 'Tor', 'Tor', 'tor')
+                    if os.path.isfile(path) and os.access(path, os.X_OK):
+                        return path
                 except:
                     pass
-    local_tor = os.path.normpath(os.path.join(settings.base_dir, '..',
+    local_tor = os.path.normpath(os.path.join(settings.top_dir,
         'tor', 'tor-browser_en-US', 'Browser', 'TorBrowser', 'Tor', 'tor'))
     if os.path.exists(local_tor):
         return local_tor
@@ -342,7 +361,7 @@ def get_tor():
 def get_geoip(tor):
     geo = []
     for tordir in (
-        os.path.normpath(os.path.join(settings.base_dir, '..', 'platform', 'tor')),
+        os.path.normpath(os.path.join(settings.top_dir, 'platform', 'tor')),
         os.path.join(os.path.dirname(os.path.dirname(tor)), 'Data', 'Tor')
     ):
         gepipfile = os.path.join(tordir, 'geoip')
@@ -364,7 +383,7 @@ def install_tor():
         logger.debug('found existing tor installation')
         return
     url = torbrowser_url()
-    target = os.path.normpath(os.path.join(settings.base_dir, '..', 'tor'))
+    target = os.path.normpath(os.path.join(settings.top_dir, 'tor'))
     if url:
         logger.debug('downloading and installing tor')
         if sys.platform.startswith('linux'):

oml/tor_request.py

@@ -32,7 +32,7 @@ def getaddrinfo(*args):
     return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
 
 def create_tor_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
-                      source_address=None):
+                          source_address=None):
     host, port = address
     err = None
     af = socket.AF_INET
@@ -40,6 +40,9 @@ def create_tor_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
     proto = 6
     sa = address
     sock = None
+
+    logger.debug('make tor connection to: %s', address)
+
     try:
         sock = socks.socksocket(af, socktype, proto)
         if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
@@ -66,27 +69,30 @@ class TorHTTPSConnection(http.client.HTTPSConnection):
     def __init__(self, host, port=None, service_id=None, check_hostname=None, context=None, **kwargs):
         self._service_id = service_id
         if self._service_id:
-            if hasattr(ssl, '_create_default_https_context'):
-                context = ssl._create_default_https_context()
-            elif hasattr(ssl, '_create_stdlib_context'):
-                context = ssl._create_stdlib_context()
+            context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
             if context:
                 context.check_hostname = False
                 context.verify_mode = ssl.CERT_NONE
-                # tor keys are still 1024 bit, debian started to require 2048 by default,
-                # try to lower requirements to 1024 if needed
-                try:
-                    context.load_cert_chain(settings.ssl_cert_path, settings.ssl_key_path)
-                except ssl.SSLError:
-                    context.set_ciphers('DEFAULT@SECLEVEL=1')
-                    context.load_cert_chain(settings.ssl_cert_path, settings.ssl_key_path)
+                context.load_cert_chain(settings.ssl_cert_path, settings.ssl_key_path)
                 context.load_default_certs()
+                context.set_alpn_protocols(['http/1.1'])
+                context.post_handshake_auth = True
         http.client.HTTPSConnection.__init__(self, host, port,
             check_hostname=check_hostname, context=context, **kwargs)
 
         if not is_local(host):
             self._create_connection = create_tor_connection
 
+    def get_service_id_cert(self):
+        for cert in self.sock._sslobj.get_verified_chain():
+            info = cert.get_info()
+            subject = info.get("subject")
+            if subject:
+                CN = subject[0][0][1]
+                if CN == self._service_id:
+                    cert = cert.public_bytes()
+                    return cert
+
     def _check_service_id(self, cert):
         service_id = get_service_id(cert=cert)
         if service_id != self._service_id:
@@ -96,11 +102,9 @@ class TorHTTPSConnection(http.client.HTTPSConnection):
     def connect(self):
         http.client.HTTPSConnection.connect(self)
         if self._service_id:
-            cert = self.sock.getpeercert(binary_form=True)
+            cert = self.get_service_id_cert()
             if not self._check_service_id(cert):
-                raise InvalidCertificateException(self._service_id, cert,
-                                                  'service_id mismatch')
-        #logger.debug('CIPHER %s VERSION %s', self.sock.cipher(), self.sock.ssl_version)
+                raise InvalidCertificateException(self._service_id, cert, 'service_id mismatch')
 
 class TorHTTPSHandler(urllib.request.HTTPSHandler):
     def __init__(self, debuglevel=0, context=None, check_hostname=None, service_id=None):

oml/user/api.py

@@ -411,7 +411,7 @@ def requestPeering(data):
             nickname (optional)
         }
     '''
-    if len(data.get('id', '')) != 16:
+    if len(data.get('id', '')) != settings.ID_LENGTH:
         logger.debug('invalid user id')
         return {}
     u = models.User.get_or_create(data['id'])
@@ -434,7 +434,7 @@ def acceptPeering(data):
             message
         }
     '''
-    if len(data.get('id', '')) != 16:
+    if len(data.get('id', '')) != settings.ID_LENGTH:
         logger.debug('invalid user id')
         return {}
     logger.debug('acceptPeering... %s', data)
@@ -453,8 +453,8 @@ def rejectPeering(data):
             message
         }
     '''
-    if len(data.get('id', '')) not in (16, 43):
-        logger.debug('invalid user id')
+    if len(data.get('id', '')) not in (16, 43, 56):
+        logger.debug('invalid user id: %s', data)
         return {}
     u = models.User.get_or_create(data['id'])
     u.info['message'] = data.get('message', '')
@@ -471,8 +471,8 @@ def removePeering(data):
             message
         }
     '''
-    if len(data.get('id', '')) not in (16, 43):
-        logger.debug('invalid user id')
+    if len(data.get('id', '')) not in (16, 43, 56):
+        logger.debug('invalid user id: %s', data)
         return {}
     u = models.User.get(data['id'], for_update=True)
     if u:
@@ -488,8 +488,8 @@ def cancelPeering(data):
         takes {
         }
     '''
-    if len(data.get('id', '')) != 16:
-        logger.debug('invalid user id')
+    if len(data.get('id', '')) != settings.ID_LENGTH:
+        logger.debug('invalid user id: %s', data)
         return {}
     u = models.User.get_or_create(data['id'])
     u.info['message'] = data.get('message', '')

oml/user/models.py

@@ -27,7 +27,7 @@ class User(db.Model):
     created = sa.Column(sa.DateTime())
     modified = sa.Column(sa.DateTime())
 
-    id = sa.Column(sa.String(43), primary_key=True)
+    id = sa.Column(sa.String(128), primary_key=True)
     info = sa.Column(MutableDict.as_mutable(sa.PickleType(pickler=json_pickler)))
 
     nickname = sa.Column(sa.String(256), index=True)
@@ -256,7 +256,7 @@ class List(db.Model):
     type = sa.Column(sa.String(64))
     _query = sa.Column('query', MutableDict.as_mutable(sa.PickleType(pickler=json_pickler)))
 
-    user_id = sa.Column(sa.String(43), sa.ForeignKey('user.id'))
+    user_id = sa.Column(sa.String(128), sa.ForeignKey('user.id'))
     user = sa.orm.relationship('User', backref=sa.orm.backref('lists', lazy='dynamic'))
 
     items = sa.orm.relationship('Item', secondary=list_items,
@@ -456,7 +456,7 @@ class Metadata(db.Model):
 
     id = sa.Column(sa.Integer(), primary_key=True)
     item_id = sa.Column(sa.String(32))
-    user_id = sa.Column(sa.String(43), sa.ForeignKey('user.id'))
+    user_id = sa.Column(sa.String(128), sa.ForeignKey('user.id'))
     data_hash = sa.Column(sa.String(40), index=True)
     data = sa.Column(MutableDict.as_mutable(sa.PickleType(pickler=json_pickler)))
 

oml/utils.py

@@ -5,6 +5,7 @@ from datetime import datetime
 from io import StringIO, BytesIO
 from PIL import Image, ImageFile
 import base64
+import functools
 import hashlib
 import json
 import os
@@ -17,19 +18,26 @@ import time
 import unicodedata
 
 import ox
+import OpenSSL.crypto
 from OpenSSL.crypto import (
-    load_privatekey, load_certificate,
-    dump_privatekey, dump_certificate,
-    FILETYPE_ASN1, FILETYPE_PEM, PKey, TYPE_RSA,
-    X509, X509Extension
+    dump_certificate,
+    dump_privatekey,
+    FILETYPE_PEM,
+    load_certificate,
+    load_privatekey,
+    PKey,
+    TYPE_RSA,
+    X509,
+    X509Extension
 )
-from Crypto.PublicKey import RSA
-from Crypto.Util.asn1 import DerSequence
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import ed25519
 
 
 from meta.utils import normalize_isbn, find_isbns, get_language, to_isbn13
 from win32utils import get_short_path_name
 
+
 import logging
 logging.getLogger('PIL').setLevel(logging.ERROR)
 logger = logging.getLogger(__name__)
@@ -92,7 +100,7 @@ def resize_image(data, width=None, size=None):
     height = max(height, 1)
 
     if width < source_width:
-        resize_method = Image.ANTIALIAS
+        resize_method = Image.LANCZOS
     else:
         resize_method = Image.BICUBIC
     output = source.resize((width, height), resize_method)
@@ -119,78 +127,157 @@ def get_position_by_id(list, key):
             return i
     return -1
 
-def get_user_id(private_key, cert_path):
-    if os.path.exists(private_key):
-        with open(private_key) as fd:
-            key = load_privatekey(FILETYPE_PEM, fd.read())
-        if key.bits() != 1024:
-            os.unlink(private_key)
+def sign_cert(cert, key):
+    # pyOpenSSL sgin api does not allow NULL hash
+    # return cert.sign(key, None)
+    return OpenSSL.crypto._lib.X509_sign(cert._x509, key._pkey, OpenSSL.crypto._ffi.NULL)
+
+def load_pem_key(pem):
+    with open(pem) as fd:
+        ca_key_pem = fd.read()
+    key = load_privatekey(FILETYPE_PEM, ca_key_pem)
+    if key.bits() != 256:
+        raise Exception("Invalid key %s" % pem)
+    key = key.to_cryptography_key()
+    private_key = key.private_bytes_raw()
+    public_key = key.public_key().public_bytes_raw()
+    return private_key, public_key
+
+
+def expand_private_key(secret_key) -> bytes:
+    hash = hashlib.sha512(secret_key).digest()
+    hash = bytearray(hash)
+    hash[0] &= 248
+    hash[31] &= 127
+    hash[31] |= 64
+    return bytes(hash)
+
+def get_onion(pubkey):
+    version_byte = b"\x03"
+    checksum_str = ".onion checksum".encode()
+    checksum = hashlib.sha3_256(checksum_str + pubkey + version_byte).digest()[:2]
+    return base64.b32encode(pubkey + checksum + version_byte).decode().lower()
+
+def get_onion_key(private_key):
+    onion_key = expand_private_key(private_key)
+    key_type = 'ED25519-V3'
+    key_content = base64.encodebytes(onion_key).decode().strip().replace('\n', '')
+    return key_type, key_content
+
+def get_user_id(key_path, cert_path, ca_key_path, ca_cert_path):
+    if os.path.exists(ca_key_path):
+        try:
+            private_key, public_key = load_pem_key(ca_key_path)
+        except:
+            os.unlink(ca_key_path)
         else:
-            user_id = get_service_id(private_key)
-    if not os.path.exists(private_key):
-        if os.path.exists(cert_path):
-            os.unlink(cert_path)
-        folder = os.path.dirname(private_key)
-        if not os.path.exists(folder):
-            os.makedirs(folder)
-            os.chmod(folder, 0o700)
-        key = PKey()
-        key.generate_key(TYPE_RSA, 1024)
-        with open(private_key, 'wb') as fd:
-            os.chmod(private_key, 0o600)
-            fd.write(dump_privatekey(FILETYPE_PEM, key))
-            os.chmod(private_key, 0o400)
-        user_id = get_service_id(private_key)
-    if not os.path.exists(cert_path) or \
-            (datetime.now() - datetime.fromtimestamp(os.path.getmtime(cert_path))).days > 60:
+            user_id = get_onion(public_key)
+
+    if not os.path.exists(ca_key_path):
+        private_key = ed25519.Ed25519PrivateKey.generate()
+        private_bytes = private_key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.PKCS8,
+            encryption_algorithm=serialization.NoEncryption()
+        )
+        with open(ca_key_path, 'wb') as fd:
+            fd.write(private_bytes)
+
+        public_key = private_key.public_key().public_bytes_raw()
+        user_id = get_onion(public_key)
+
+    if not os.path.exists(ca_cert_path) or \
+            (datetime.now() - datetime.fromtimestamp(os.path.getmtime(ca_cert_path))).days > 5*365:
+        with open(ca_key_path, 'rb') as key_file:
+            key_data = key_file.read()
+        cakey = load_privatekey(FILETYPE_PEM, key_data)
         ca = X509()
         ca.set_version(2)
         ca.set_serial_number(1)
         ca.get_subject().CN = user_id
         ca.gmtime_adj_notBefore(0)
-        ca.gmtime_adj_notAfter(90 * 24 * 60 * 60)
+        ca.gmtime_adj_notAfter(10 * 356 * 24 * 60 * 60)
         ca.set_issuer(ca.get_subject())
-        ca.set_pubkey(key)
+        ca.set_pubkey(cakey)
         ca.add_extensions([
-          X509Extension(b"basicConstraints", True, b"CA:TRUE, pathlen:0"),
-          X509Extension(b"nsCertType", True, b"sslCA"),
+            X509Extension(b"basicConstraints", False, b"CA:TRUE"),
+            X509Extension(b"keyUsage", False, b"keyCertSign, cRLSign"),
+            X509Extension(
+                b"subjectKeyIdentifier", False, b"hash", subject=ca
+            ),
+        ])
+        ca.add_extensions([
+                X509Extension(
+                    b"authorityKeyIdentifier", False, b"keyid:always", issuer=ca
+                )
+        ])
+
+        sign_cert(ca, cakey)
+
+        with open(ca_cert_path, 'wb') as fd:
+            fd.write(dump_certificate(FILETYPE_PEM, ca))
+
+        if os.path.exists(cert_path):
+            os.unlink(cert_path)
+        if os.path.exists(key_path):
+            os.unlink(key_path)
+    else:
+        with open(ca_cert_path) as fd:
+            ca = load_certificate(FILETYPE_PEM, fd.read())
+        with open(ca_key_path) as fd:
+            cakey = load_privatekey(FILETYPE_PEM, fd.read())
+
+
+    # create RSA intermediate certificate since clients don't quite like Ed25519 yet
+    if not os.path.exists(cert_path) or \
+            (datetime.now() - datetime.fromtimestamp(os.path.getmtime(cert_path))).days > 60:
+
+        key = PKey()
+        key.generate_key(TYPE_RSA, 2048)
+
+        cert = X509()
+        cert.set_version(2)
+        cert.set_serial_number(2)
+        cert.get_subject().CN = user_id + ".onion"
+        cert.gmtime_adj_notBefore(0)
+        cert.gmtime_adj_notAfter(90 * 24 * 60 * 60)
+        cert.set_issuer(ca.get_subject())
+        cert.set_pubkey(key)
+        subject_alt_names = b"DNS: %s.onion" % user_id.encode()
+        cert.add_extensions([
+          X509Extension(b"basicConstraints", True, b"CA:FALSE"),
           X509Extension(b"extendedKeyUsage", True,
             b"serverAuth,clientAuth,emailProtection,timeStamping,msCodeInd,msCodeCom,msCTLSign,msSGC,msEFS,nsSGC"),
           X509Extension(b"keyUsage", False, b"keyCertSign, cRLSign"),
           X509Extension(b"subjectKeyIdentifier", False, b"hash", subject=ca),
+          X509Extension(b"subjectAltName", critical=True, value=subject_alt_names),
         ])
-        ca.sign(key, "sha256")
+        sign_cert(cert, cakey)
         with open(cert_path, 'wb') as fd:
+            fd.write(dump_certificate(FILETYPE_PEM, cert))
             fd.write(dump_certificate(FILETYPE_PEM, ca))
+        with open(key_path, 'wb') as fd:
+            fd.write(dump_privatekey(FILETYPE_PEM, key))
     return user_id
 
+
 def get_service_id(private_key_file=None, cert=None):
     '''
     service_id is the first half of the sha1 of the rsa public key encoded in base32
     '''
     if private_key_file:
-        with open(private_key_file, 'rb') as fd:
-            private_key = fd.read()
-        public_key = RSA.importKey(private_key).publickey().exportKey('DER')[22:]
-        # compute sha1 of public key and encode first half in base32
-        service_id = base64.b32encode(hashlib.sha1(public_key).digest()[:10]).lower().decode()
-        '''
-        # compute public key from priate key and export in DER format
-        # ignoring the SPKI header(22 bytes)
-        key = load_privatekey(FILETYPE_PEM, private_key)
-        cert = X509()
-        cert.set_pubkey(key)
-        public_key = dump_privatekey(FILETYPE_ASN1, cert.get_pubkey())[22:]
-        # compute sha1 of public key and encode first half in base32
-        service_id = base64.b32encode(hashlib.sha1(public_key).digest()[:10]).lower().decode()
-        '''
+        with open(private_key_file, 'rb') as key_file:
+            key_type, key_content = key_file.read().split(b':', 1)
+        private_key = base64.decodebytes(key_content)
+        public_key = Ed25519().public_key_from_hash(private_key)
+        service_id = get_onion(public_key)
     elif cert:
-        # compute sha1 of public key and encode first half in base32
-        key = load_certificate(FILETYPE_ASN1, cert).get_pubkey()
-        pub_der = DerSequence()
-        pub_der.decode(dump_privatekey(FILETYPE_ASN1, key))
-        public_key = RSA.construct((pub_der._seq[1], pub_der._seq[2])).exportKey('DER')[22:]
-        service_id = base64.b32encode(hashlib.sha1(public_key).digest()[:10]).lower().decode()
+        cert_ = load_certificate(FILETYPE_PEM, cert)
+        key = cert_.get_pubkey()
+        public_key = key.to_cryptography_key().public_bytes_raw()
+        service_id = get_onion(public_key)
+    else:
+        service_id = None
     return service_id
 
 def update_dict(root, data):
@@ -398,7 +485,7 @@ def check_pidfile(pid):
 def ctl(*args):
     import settings
     if sys.platform == 'win32':
-        platform_win32 = os.path.normpath(os.path.join(settings.base_dir, '..', 'platform_win32'))
+        platform_win32 = os.path.normpath(os.path.join(settings.top_dir, 'platform_win32'))
         python = os.path.join(platform_win32, 'pythonw.exe')
         cmd = [python, 'oml'] + list(args)
         startupinfo = subprocess.STARTUPINFO()
@@ -495,3 +582,36 @@ def iexists(path):
 
 def same_path(f1, f2):
     return unicodedata.normalize('NFC', f1) == unicodedata.normalize('NFC', f2)
+
+def time_cache(max_age, maxsize=128, typed=False):
+    def _decorator(fn):
+        @functools.lru_cache(maxsize=maxsize, typed=typed)
+        def _new(*args, __time_salt, **kwargs):
+            return fn(*args, **kwargs)
+
+        @functools.wraps(fn)
+        def _wrapped(*args, **kwargs):
+            return _new(*args, **kwargs, __time_salt=int(time.time() / max_age))
+
+        return _wrapped
+
+    return _decorator
+
+def migrate_userid(old_id, new_id):
+    from db import run_sql
+    import settings
+    statements = [
+        "UPDATE user SET id = '{nid}' WHERE id = '{oid}'",
+        "UPDATE list SET user_id = '{nid}' WHERE user_id = '{oid}'",
+        "UPDATE useritem SET user_id = '{nid}' WHERE user_id = '{oid}'",
+        "UPDATE changelog SET user_id = '{nid}' WHERE user_id = '{oid}'",
+    ]
+    run_sql([
+        sql.format(oid=old_id, nid=new_id)
+        for sql in statements
+    ])
+    for ext in ('log', 'db', 'json'):
+        old_file = os.path.join(settings.data_path, 'peers/%s.%s' % (old_id, ext))
+        new_file = os.path.join(settings.data_path, 'peers/%s.%s' % (new_id, ext))
+        if os.path.exists(old_file) and not os.path.exists(new_file):
+            os.rename(old_file, new_file)

requirements-shared.txt

@@ -2,7 +2,7 @@ requests==2.21.0
 chardet
 html5lib
 #ox>=2.0.666
-git+http://git.0x2620.org/python-ox.git#egg=python-ox
+git+https://code.0x2620.org/0x2620/python-ox.git#egg=ox
 python-stdnum==1.2
 PyPDF2==1.25.1
 pysocks

requirements.txt

@@ -1,9 +1,9 @@
 lxml
 simplejson
 ed25519>=1.4
-SQLAlchemy==1.0.12
+SQLAlchemy==1.4.46
 pyopenssl>=0.15
-pyCrypto>=2.6.1
+pycryptodome
 pillow
 netifaces
-tornado==5.1.1
+tornado==6.0.3

static/css/oml.css

@@ -10,6 +10,10 @@
     font-size: 14px;
     line-height: 21px;
 }
+.OMLQuote img {
+    max-width: 100%;
+    margin: auto;
+}
 
 .OMLAnnotation .OMLQuoteBackground {
     position: absolute;

static/html/pdf.html

@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+<!DOCTYPE html>
 <!--
 Copyright 2012 Mozilla Foundation
 
@@ -25,47 +25,53 @@ See https://github.com/adobe-type-tools/cmap-resources
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
     <meta name="google" content="notranslate">
-    <meta http-equiv="X-UA-Compatible" content="IE=edge">
-    <title></title>
+    <title>PDF.js viewer</title>
 
+<style>
+  #download, #openFile, #print, #viewBookmark {
+    display:none;
+  }
+</style>
+<link rel="resource" type="application/l10n" href="/static/pdf.js/locale/locale.json">
+<script src="/static/pdf.js/pdf.mjs" type="module"></script>
+<script src="/static/oxjs/min/Ox.js"></script>
 
-    <link rel="stylesheet" href="/static/pdf.js/viewer.css?3"/>
-    <script>
-        var DEFAULT_URL=document.location.pathname.replace(/\/reader\//, '/pdf/');
-    </script>
-    <style>
-        #download, #openFile, #print, #viewBookmark {
-            display:none;
-        }
-    </style>
+    <link rel="stylesheet" href="/static/pdf.js/viewer.css">
+    <link rel="stylesheet" href="/static/reader/pdf.css">
 
-    <script src="/static/oxjs/min/Ox.js?3"></script>
-    <script src="/static/pdf.js/compatibility.js?3"></script>
-
-    <!-- This snippet is used in production (included from viewer.html) -->
-    <link rel="resource" type="application/l10n" href="/static/pdf.js/locale/en-US/viewer.properties"/>
-    <script src="/static/pdf.js/l10n.js?3"></script>
-    <script src="/static/pdf.js/pdf.js?3"></script>
-
-    <script src="/static/pdf.js/viewer.js?3"></script>
-    <script src="/static/reader/pdf.js?3"></script>
+  <script src="/static/pdf.js/viewer.mjs" type="module"></script>
   </head>
 
-  <body tabindex="1" class="loadingInProgress">
+  <body tabindex="1">
     <div id="outerContainer">
 
       <div id="sidebarContainer">
         <div id="toolbarSidebar">
-          <div class="splitToolbarButton toggled">
-            <button id="viewThumbnail" class="toolbarButton toggled" title="Show Thumbnails" tabindex="2" data-l10n-id="thumbs">
-               <span data-l10n-id="thumbs_label">Thumbnails</span>
-            </button>
-            <button id="viewOutline" class="toolbarButton" title="Show Document Outline (double-click to expand/collapse all items)" tabindex="3" data-l10n-id="document_outline">
-               <span data-l10n-id="document_outline_label">Document Outline</span>
-            </button>
-            <button id="viewAttachments" class="toolbarButton" title="Show Attachments" tabindex="4" data-l10n-id="attachments">
-               <span data-l10n-id="attachments_label">Attachments</span>
-            </button>
+          <div id="toolbarSidebarLeft">
+            <div id="sidebarViewButtons" class="splitToolbarButton toggled" role="radiogroup">
+              <button id="viewThumbnail" class="toolbarButton toggled" title="Show Thumbnails" tabindex="2" data-l10n-id="pdfjs-thumbs-button" role="radio" aria-checked="true" aria-controls="thumbnailView">
+                 <span data-l10n-id="pdfjs-thumbs-button-label">Thumbnails</span>
+              </button>
+              <button id="viewOutline" class="toolbarButton" title="Show Document Outline (double-click to expand/collapse all items)" tabindex="3" data-l10n-id="pdfjs-document-outline-button" role="radio" aria-checked="false" aria-controls="outlineView">
+                 <span data-l10n-id="pdfjs-document-outline-button-label">Document Outline</span>
+              </button>
+              <button id="viewAttachments" class="toolbarButton" title="Show Attachments" tabindex="4" data-l10n-id="pdfjs-attachments-button" role="radio" aria-checked="false" aria-controls="attachmentsView">
+                 <span data-l10n-id="pdfjs-attachments-button-label">Attachments</span>
+              </button>
+              <button id="viewLayers" class="toolbarButton" title="Show Layers (double-click to reset all layers to the default state)" tabindex="5" data-l10n-id="pdfjs-layers-button" role="radio" aria-checked="false" aria-controls="layersView">
+                 <span data-l10n-id="pdfjs-layers-button-label">Layers</span>
+              </button>
+            </div>
+          </div>
+
+          <div id="toolbarSidebarRight">
+            <div id="outlineOptionsContainer">
+              <div class="verticalToolbarSeparator"></div>
+
+              <button id="currentOutlineItem" class="toolbarButton" disabled="disabled" title="Find Current Outline Item" tabindex="6" data-l10n-id="pdfjs-current-outline-item-button">
+                <span data-l10n-id="pdfjs-current-outline-item-button-label">Current Outline Item</span>
+              </button>
+            </div>
           </div>
         </div>
         <div id="sidebarContent">
@@ -75,119 +81,195 @@ See https://github.com/adobe-type-tools/cmap-resources
           </div>
           <div id="attachmentsView" class="hidden">
           </div>
+          <div id="layersView" class="hidden">
+          </div>
         </div>
-        <div id="sidebarResizer" class="hidden"></div>
+        <div id="sidebarResizer"></div>
       </div>  <!-- sidebarContainer -->
 
       <div id="mainContainer">
         <div class="findbar hidden doorHanger" id="findbar">
           <div id="findbarInputContainer">
-            <input id="findInput" class="toolbarField" title="Find" placeholder="Find in document…" tabindex="91" data-l10n-id="find_input">
+            <span class="loadingInput end">
+              <input id="findInput" class="toolbarField" title="Find" placeholder="Find in document…" tabindex="91" data-l10n-id="pdfjs-find-input" aria-invalid="false">
+            </span>
             <div class="splitToolbarButton">
-              <button id="findPrevious" class="toolbarButton findPrevious" title="Find the previous occurrence of the phrase" tabindex="92" data-l10n-id="find_previous">
-                <span data-l10n-id="find_previous_label">Previous</span>
+              <button id="findPrevious" class="toolbarButton" title="Find the previous occurrence of the phrase" tabindex="92" data-l10n-id="pdfjs-find-previous-button">
+                <span data-l10n-id="pdfjs-find-previous-button-label">Previous</span>
               </button>
               <div class="splitToolbarButtonSeparator"></div>
-              <button id="findNext" class="toolbarButton findNext" title="Find the next occurrence of the phrase" tabindex="93" data-l10n-id="find_next">
-                <span data-l10n-id="find_next_label">Next</span>
+              <button id="findNext" class="toolbarButton" title="Find the next occurrence of the phrase" tabindex="93" data-l10n-id="pdfjs-find-next-button">
+                <span data-l10n-id="pdfjs-find-next-button-label">Next</span>
               </button>
             </div>
           </div>
 
           <div id="findbarOptionsOneContainer">
             <input type="checkbox" id="findHighlightAll" class="toolbarField" tabindex="94">
-            <label for="findHighlightAll" class="toolbarLabel" data-l10n-id="find_highlight">Highlight all</label>
+            <label for="findHighlightAll" class="toolbarLabel" data-l10n-id="pdfjs-find-highlight-checkbox">Highlight All</label>
             <input type="checkbox" id="findMatchCase" class="toolbarField" tabindex="95">
-            <label for="findMatchCase" class="toolbarLabel" data-l10n-id="find_match_case_label">Match case</label>
+            <label for="findMatchCase" class="toolbarLabel" data-l10n-id="pdfjs-find-match-case-checkbox-label">Match Case</label>
           </div>
           <div id="findbarOptionsTwoContainer">
-            <input type="checkbox" id="findEntireWord" class="toolbarField" tabindex="96">
-            <label for="findEntireWord" class="toolbarLabel" data-l10n-id="find_entire_word_label">Whole words</label>
-            <span id="findResultsCount" class="toolbarLabel hidden"></span>
+            <input type="checkbox" id="findMatchDiacritics" class="toolbarField" tabindex="96">
+            <label for="findMatchDiacritics" class="toolbarLabel" data-l10n-id="pdfjs-find-match-diacritics-checkbox-label">Match Diacritics</label>
+            <input type="checkbox" id="findEntireWord" class="toolbarField" tabindex="97">
+            <label for="findEntireWord" class="toolbarLabel" data-l10n-id="pdfjs-find-entire-word-checkbox-label">Whole Words</label>
           </div>
 
-          <div id="findbarMessageContainer">
+          <div id="findbarMessageContainer" aria-live="polite">
+            <span id="findResultsCount" class="toolbarLabel"></span>
             <span id="findMsg" class="toolbarLabel"></span>
           </div>
         </div>  <!-- findbar -->
 
+        <div class="editorParamsToolbar hidden doorHangerRight" id="editorHighlightParamsToolbar">
+          <div id="highlightParamsToolbarContainer" class="editorParamsToolbarContainer">
+            <div id="editorHighlightColorPicker" class="colorPicker">
+              <span id="highlightColorPickerLabel" class="editorParamsLabel" data-l10n-id="pdfjs-editor-highlight-colorpicker-label">Highlight color</span>
+            </div>
+            <div id="editorHighlightThickness">
+              <label for="editorFreeHighlightThickness" class="editorParamsLabel" data-l10n-id="pdfjs-editor-free-highlight-thickness-input">Thickness</label>
+              <div class="thicknessPicker">
+                <input type="range" id="editorFreeHighlightThickness" class="editorParamsSlider" data-l10n-id="pdfjs-editor-free-highlight-thickness-title" value="12" min="8" max="24" step="1" tabindex="101">
+              </div>
+            </div>
+            <div id="editorHighlightVisibility">
+              <div class="divider"></div>
+              <div class="toggler">
+                <label for="editorHighlightShowAll" class="editorParamsLabel" data-l10n-id="pdfjs-editor-highlight-show-all-button-label">Show all</label>
+                <button id="editorHighlightShowAll" class="toggle-button" data-l10n-id="pdfjs-editor-highlight-show-all-button" aria-pressed="true" tabindex="102"></button>
+              </div>
+            </div>
+          </div>
+        </div>
+
+        <div class="editorParamsToolbar hidden doorHangerRight" id="editorFreeTextParamsToolbar">
+          <div class="editorParamsToolbarContainer">
+            <div class="editorParamsSetter">
+              <label for="editorFreeTextColor" class="editorParamsLabel" data-l10n-id="pdfjs-editor-free-text-color-input">Color</label>
+              <input type="color" id="editorFreeTextColor" class="editorParamsColor" tabindex="103">
+            </div>
+            <div class="editorParamsSetter">
+              <label for="editorFreeTextFontSize" class="editorParamsLabel" data-l10n-id="pdfjs-editor-free-text-size-input">Size</label>
+              <input type="range" id="editorFreeTextFontSize" class="editorParamsSlider" value="10" min="5" max="100" step="1" tabindex="104">
+            </div>
+          </div>
+        </div>
+
+        <div class="editorParamsToolbar hidden doorHangerRight" id="editorInkParamsToolbar">
+          <div class="editorParamsToolbarContainer">
+            <div class="editorParamsSetter">
+              <label for="editorInkColor" class="editorParamsLabel" data-l10n-id="pdfjs-editor-ink-color-input">Color</label>
+              <input type="color" id="editorInkColor" class="editorParamsColor" tabindex="105">
+            </div>
+            <div class="editorParamsSetter">
+              <label for="editorInkThickness" class="editorParamsLabel" data-l10n-id="pdfjs-editor-ink-thickness-input">Thickness</label>
+              <input type="range" id="editorInkThickness" class="editorParamsSlider" value="1" min="1" max="20" step="1" tabindex="106">
+            </div>
+            <div class="editorParamsSetter">
+              <label for="editorInkOpacity" class="editorParamsLabel" data-l10n-id="pdfjs-editor-ink-opacity-input">Opacity</label>
+              <input type="range" id="editorInkOpacity" class="editorParamsSlider" value="100" min="1" max="100" step="1" tabindex="107">
+            </div>
+          </div>
+        </div>
+
+        <div class="editorParamsToolbar hidden doorHangerRight" id="editorStampParamsToolbar">
+          <div class="editorParamsToolbarContainer">
+            <button id="editorStampAddImage" class="secondaryToolbarButton" title="Add image" tabindex="108" data-l10n-id="pdfjs-editor-stamp-add-image-button">
+              <span class="editorParamsLabel" data-l10n-id="pdfjs-editor-stamp-add-image-button-label">Add image</span>
+            </button>
+          </div>
+        </div>
+
         <div id="secondaryToolbar" class="secondaryToolbar hidden doorHangerRight">
           <div id="secondaryToolbarButtonContainer">
-            <button id="secondaryPresentationMode" class="secondaryToolbarButton presentationMode visibleLargeView" title="Switch to Presentation Mode" tabindex="51" data-l10n-id="presentation_mode">
-              <span data-l10n-id="presentation_mode_label">Presentation Mode</span>
+            <button id="secondaryOpenFile" class="secondaryToolbarButton" title="Open File" tabindex="51" data-l10n-id="pdfjs-open-file-button">
+              <span data-l10n-id="pdfjs-open-file-button-label">Open</span>
             </button>
 
-            <button id="secondaryOpenFile" class="secondaryToolbarButton openFile visibleLargeView" title="Open File" tabindex="52" data-l10n-id="open_file">
-              <span data-l10n-id="open_file_label">Open</span>
+            <button id="secondaryPrint" class="secondaryToolbarButton visibleMediumView" title="Print" tabindex="52" data-l10n-id="pdfjs-print-button">
+              <span data-l10n-id="pdfjs-print-button-label">Print</span>
             </button>
 
-            <button id="secondaryPrint" class="secondaryToolbarButton print visibleMediumView" title="Print" tabindex="53" data-l10n-id="print">
-              <span data-l10n-id="print_label">Print</span>
+            <button id="secondaryDownload" class="secondaryToolbarButton visibleMediumView" title="Save" tabindex="53" data-l10n-id="pdfjs-save-button">
+              <span data-l10n-id="pdfjs-save-button-label">Save</span>
             </button>
 
-            <button id="secondaryDownload" class="secondaryToolbarButton download visibleMediumView" title="Download" tabindex="54" data-l10n-id="download">
-              <span data-l10n-id="download_label">Download</span>
+            <div class="horizontalToolbarSeparator"></div>
+
+            <button id="presentationMode" class="secondaryToolbarButton" title="Switch to Presentation Mode" tabindex="54" data-l10n-id="pdfjs-presentation-mode-button">
+              <span data-l10n-id="pdfjs-presentation-mode-button-label">Presentation Mode</span>
             </button>
 
-            <a href="#" id="secondaryViewBookmark" class="secondaryToolbarButton bookmark visibleSmallView" title="Current view (copy or open in new window)" tabindex="55" data-l10n-id="bookmark">
-              <span data-l10n-id="bookmark_label">Current View</span>
+            <a href="#" id="viewBookmark" class="secondaryToolbarButton" title="Current Page (View URL from Current Page)" tabindex="55" data-l10n-id="pdfjs-bookmark-button">
+              <span data-l10n-id="pdfjs-bookmark-button-label">Current Page</span>
             </a>
 
-            <div class="horizontalToolbarSeparator visibleLargeView"></div>
+            <div id="viewBookmarkSeparator" class="horizontalToolbarSeparator"></div>
 
-            <button id="firstPage" class="secondaryToolbarButton firstPage" title="Go to First Page" tabindex="56" data-l10n-id="first_page">
-              <span data-l10n-id="first_page_label">Go to First Page</span>
+            <button id="firstPage" class="secondaryToolbarButton" title="Go to First Page" tabindex="56" data-l10n-id="pdfjs-first-page-button">
+              <span data-l10n-id="pdfjs-first-page-button-label">Go to First Page</span>
             </button>
-            <button id="lastPage" class="secondaryToolbarButton lastPage" title="Go to Last Page" tabindex="57" data-l10n-id="last_page">
-              <span data-l10n-id="last_page_label">Go to Last Page</span>
+            <button id="lastPage" class="secondaryToolbarButton" title="Go to Last Page" tabindex="57" data-l10n-id="pdfjs-last-page-button">
+              <span data-l10n-id="pdfjs-last-page-button-label">Go to Last Page</span>
             </button>
 
             <div class="horizontalToolbarSeparator"></div>
 
-            <button id="pageRotateCw" class="secondaryToolbarButton rotateCw" title="Rotate Clockwise" tabindex="58" data-l10n-id="page_rotate_cw">
-              <span data-l10n-id="page_rotate_cw_label">Rotate Clockwise</span>
+            <button id="pageRotateCw" class="secondaryToolbarButton" title="Rotate Clockwise" tabindex="58" data-l10n-id="pdfjs-page-rotate-cw-button">
+              <span data-l10n-id="pdfjs-page-rotate-cw-button-label">Rotate Clockwise</span>
             </button>
-            <button id="pageRotateCcw" class="secondaryToolbarButton rotateCcw" title="Rotate Counterclockwise" tabindex="59" data-l10n-id="page_rotate_ccw">
-              <span data-l10n-id="page_rotate_ccw_label">Rotate Counterclockwise</span>
+            <button id="pageRotateCcw" class="secondaryToolbarButton" title="Rotate Counterclockwise" tabindex="59" data-l10n-id="pdfjs-page-rotate-ccw-button">
+              <span data-l10n-id="pdfjs-page-rotate-ccw-button-label">Rotate Counterclockwise</span>
             </button>
 
             <div class="horizontalToolbarSeparator"></div>
 
-            <button id="cursorSelectTool" class="secondaryToolbarButton selectTool toggled" title="Enable Text Selection Tool" tabindex="60" data-l10n-id="cursor_text_select_tool">
-              <span data-l10n-id="cursor_text_select_tool_label">Text Selection Tool</span>
-            </button>
-            <button id="cursorHandTool" class="secondaryToolbarButton handTool" title="Enable Hand Tool" tabindex="61" data-l10n-id="cursor_hand_tool">
-              <span data-l10n-id="cursor_hand_tool_label">Hand Tool</span>
-            </button>
+            <div id="cursorToolButtons" role="radiogroup">
+              <button id="cursorSelectTool" class="secondaryToolbarButton toggled" title="Enable Text Selection Tool" tabindex="60" data-l10n-id="pdfjs-cursor-text-select-tool-button" role="radio" aria-checked="true">
+                <span data-l10n-id="pdfjs-cursor-text-select-tool-button-label">Text Selection Tool</span>
+              </button>
+              <button id="cursorHandTool" class="secondaryToolbarButton" title="Enable Hand Tool" tabindex="61" data-l10n-id="pdfjs-cursor-hand-tool-button" role="radio" aria-checked="false">
+                <span data-l10n-id="pdfjs-cursor-hand-tool-button-label">Hand Tool</span>
+              </button>
+            </div>
 
             <div class="horizontalToolbarSeparator"></div>
 
-            <button id="scrollVertical" class="secondaryToolbarButton scrollModeButtons scrollVertical toggled" title="Use Vertical Scrolling" tabindex="62" data-l10n-id="scroll_vertical">
-              <span data-l10n-id="scroll_vertical_label">Vertical Scrolling</span>
-            </button>
-            <button id="scrollHorizontal" class="secondaryToolbarButton scrollModeButtons scrollHorizontal" title="Use Horizontal Scrolling" tabindex="63" data-l10n-id="scroll_horizontal">
-              <span data-l10n-id="scroll_horizontal_label">Horizontal Scrolling</span>
-            </button>
-            <button id="scrollWrapped" class="secondaryToolbarButton scrollModeButtons scrollWrapped" title="Use Wrapped Scrolling" tabindex="64" data-l10n-id="scroll_wrapped">
-              <span data-l10n-id="scroll_wrapped_label">Wrapped Scrolling</span>
-            </button>
+            <div id="scrollModeButtons" role="radiogroup">
+              <button id="scrollPage" class="secondaryToolbarButton" title="Use Page Scrolling" tabindex="62" data-l10n-id="pdfjs-scroll-page-button" role="radio" aria-checked="false">
+                <span data-l10n-id="pdfjs-scroll-page-button-label">Page Scrolling</span>
+              </button>
+              <button id="scrollVertical" class="secondaryToolbarButton toggled" title="Use Vertical Scrolling" tabindex="63" data-l10n-id="pdfjs-scroll-vertical-button" role="radio" aria-checked="true">
+                <span data-l10n-id="pdfjs-scroll-vertical-button-label" >Vertical Scrolling</span>
+              </button>
+              <button id="scrollHorizontal" class="secondaryToolbarButton" title="Use Horizontal Scrolling" tabindex="64" data-l10n-id="pdfjs-scroll-horizontal-button" role="radio" aria-checked="false">
+                <span data-l10n-id="pdfjs-scroll-horizontal-button-label">Horizontal Scrolling</span>
+              </button>
+              <button id="scrollWrapped" class="secondaryToolbarButton" title="Use Wrapped Scrolling" tabindex="65" data-l10n-id="pdfjs-scroll-wrapped-button" role="radio" aria-checked="false">
+                <span data-l10n-id="pdfjs-scroll-wrapped-button-label">Wrapped Scrolling</span>
+              </button>
+            </div>
 
-            <div class="horizontalToolbarSeparator scrollModeButtons"></div>
+            <div class="horizontalToolbarSeparator"></div>
 
-            <button id="spreadNone" class="secondaryToolbarButton spreadModeButtons spreadNone toggled" title="Do not join page spreads" tabindex="65" data-l10n-id="spread_none">
-              <span data-l10n-id="spread_none_label">No Spreads</span>
-            </button>
-            <button id="spreadOdd" class="secondaryToolbarButton spreadModeButtons spreadOdd" title="Join page spreads starting with odd-numbered pages" tabindex="66" data-l10n-id="spread_odd">
-              <span data-l10n-id="spread_odd_label">Odd Spreads</span>
-            </button>
-            <button id="spreadEven" class="secondaryToolbarButton spreadModeButtons spreadEven" title="Join page spreads starting with even-numbered pages" tabindex="67" data-l10n-id="spread_even">
-              <span data-l10n-id="spread_even_label">Even Spreads</span>
-            </button>
+            <div id="spreadModeButtons" role="radiogroup">
+              <button id="spreadNone" class="secondaryToolbarButton toggled" title="Do not join page spreads" tabindex="66" data-l10n-id="pdfjs-spread-none-button" role="radio" aria-checked="true">
+                <span data-l10n-id="pdfjs-spread-none-button-label">No Spreads</span>
+              </button>
+              <button id="spreadOdd" class="secondaryToolbarButton" title="Join page spreads starting with odd-numbered pages" tabindex="67" data-l10n-id="pdfjs-spread-odd-button" role="radio" aria-checked="false">
+                <span data-l10n-id="pdfjs-spread-odd-button-label">Odd Spreads</span>
+              </button>
+              <button id="spreadEven" class="secondaryToolbarButton" title="Join page spreads starting with even-numbered pages" tabindex="68" data-l10n-id="pdfjs-spread-even-button" role="radio" aria-checked="false">
+                <span data-l10n-id="pdfjs-spread-even-button-label">Even Spreads</span>
+              </button>
+            </div>
 
-            <div class="horizontalToolbarSeparator spreadModeButtons"></div>
+            <div class="horizontalToolbarSeparator"></div>
 
-            <button id="documentProperties" class="secondaryToolbarButton documentProperties" title="Document Properties…" tabindex="68" data-l10n-id="document_properties">
-              <span data-l10n-id="document_properties_label">Document Properties…</span>
+            <button id="documentProperties" class="secondaryToolbarButton" title="Document Properties…" tabindex="69" data-l10n-id="pdfjs-document-properties-button" aria-controls="documentPropertiesDialog">
+              <span data-l10n-id="pdfjs-document-properties-button-label">Document Properties…</span>
             </button>
           </div>
         </div>  <!-- secondaryToolbar -->
@@ -196,76 +278,84 @@ See https://github.com/adobe-type-tools/cmap-resources
           <div id="toolbarContainer">
             <div id="toolbarViewer">
               <div id="toolbarViewerLeft">
-                <button id="sidebarToggle" class="toolbarButton" title="Toggle Sidebar" tabindex="11" data-l10n-id="toggle_sidebar">
-                  <span data-l10n-id="toggle_sidebar_label">Toggle Sidebar</span>
+                <button id="sidebarToggle" class="toolbarButton" title="Toggle Sidebar" tabindex="11" data-l10n-id="pdfjs-toggle-sidebar-button" aria-expanded="false" aria-controls="sidebarContainer">
+                  <span data-l10n-id="pdfjs-toggle-sidebar-button-label">Toggle Sidebar</span>
                 </button>
                 <div class="toolbarButtonSpacer"></div>
-                <button id="viewFind" class="toolbarButton" title="Find in Document" tabindex="12" data-l10n-id="findbar">
-                  <span data-l10n-id="findbar_label">Find</span>
+                <button id="viewFind" class="toolbarButton" title="Find in Document" tabindex="12" data-l10n-id="pdfjs-findbar-button" aria-expanded="false" aria-controls="findbar">
+                  <span data-l10n-id="pdfjs-findbar-button-label">Find</span>
                 </button>
                 <div class="splitToolbarButton hiddenSmallView">
-                  <button class="toolbarButton pageUp" title="Previous Page" id="previous" tabindex="13" data-l10n-id="previous">
-                    <span data-l10n-id="previous_label">Previous</span>
+                  <button class="toolbarButton" title="Previous Page" id="previous" tabindex="13" data-l10n-id="pdfjs-previous-button">
+                    <span data-l10n-id="pdfjs-previous-button-label">Previous</span>
                   </button>
                   <div class="splitToolbarButtonSeparator"></div>
-                  <button class="toolbarButton pageDown" title="Next Page" id="next" tabindex="14" data-l10n-id="next">
-                    <span data-l10n-id="next_label">Next</span>
+                  <button class="toolbarButton" title="Next Page" id="next" tabindex="14" data-l10n-id="pdfjs-next-button">
+                    <span data-l10n-id="pdfjs-next-button-label">Next</span>
                   </button>
                 </div>
-                <input type="number" id="pageNumber" class="toolbarField pageNumber" title="Page" value="1" size="4" min="1" tabindex="15" data-l10n-id="page">
+                <span class="loadingInput start">
+                  <input type="number" id="pageNumber" class="toolbarField" title="Page" value="1" min="1" tabindex="15" data-l10n-id="pdfjs-page-input" autocomplete="off">
+                </span>
                 <span id="numPages" class="toolbarLabel"></span>
               </div>
               <div id="toolbarViewerRight">
-                <button id="presentationMode" class="toolbarButton presentationMode hiddenLargeView" title="Switch to Presentation Mode" tabindex="31" data-l10n-id="presentation_mode">
-                  <span data-l10n-id="presentation_mode_label">Presentation Mode</span>
+                <div id="editorModeButtons" class="splitToolbarButton toggled" role="radiogroup">
+                  <button id="editorHighlight" class="toolbarButton" hidden="true" disabled="disabled" title="Highlight" role="radio" aria-checked="false" aria-controls="editorHighlightParamsToolbar" tabindex="31" data-l10n-id="pdfjs-editor-highlight-button">
+                    <span data-l10n-id="pdfjs-editor-highlight-button-label">Highlight</span>
+                  </button>
+                  <button id="editorFreeText" class="toolbarButton" disabled="disabled" title="Text" role="radio" aria-checked="false" aria-controls="editorFreeTextParamsToolbar" tabindex="32" data-l10n-id="pdfjs-editor-free-text-button">
+                    <span data-l10n-id="pdfjs-editor-free-text-button-label">Text</span>
+                  </button>
+                  <button id="editorInk" class="toolbarButton" disabled="disabled" title="Draw" role="radio" aria-checked="false" aria-controls="editorInkParamsToolbar" tabindex="33" data-l10n-id="pdfjs-editor-ink-button">
+                    <span data-l10n-id="pdfjs-editor-ink-button-label">Draw</span>
+                  </button>
+                  <button id="editorStamp" class="toolbarButton hidden" disabled="disabled" title="Add or edit images" role="radio" aria-checked="false" aria-controls="editorStampParamsToolbar" tabindex="34" data-l10n-id="pdfjs-editor-stamp-button">
+                    <span data-l10n-id="pdfjs-editor-stamp-button-label">Add or edit images</span>
+                  </button>
+                </div>
+
+                <div id="editorModeSeparator" class="verticalToolbarSeparator"></div>
+
+                <button id="print" class="toolbarButton hiddenMediumView" title="Print" tabindex="41" data-l10n-id="pdfjs-print-button">
+                  <span data-l10n-id="pdfjs-print-button-label">Print</span>
                 </button>
 
-                <button id="openFile" class="toolbarButton openFile hiddenLargeView" title="Open File" tabindex="32" data-l10n-id="open_file">
-                  <span data-l10n-id="open_file_label">Open</span>
+                <button id="download" class="toolbarButton hiddenMediumView" title="Save" tabindex="42" data-l10n-id="pdfjs-save-button">
+                  <span data-l10n-id="pdfjs-save-button-label">Save</span>
                 </button>
 
-                <button id="print" class="toolbarButton print hiddenMediumView" title="Print" tabindex="33" data-l10n-id="print">
-                  <span data-l10n-id="print_label">Print</span>
-                </button>
+                <div class="verticalToolbarSeparator hiddenMediumView"></div>
 
-                <button id="download" class="toolbarButton download hiddenMediumView" title="Download" tabindex="34" data-l10n-id="download">
-                  <span data-l10n-id="download_label">Download</span>
-                </button>
-                <a href="#" id="viewBookmark" class="toolbarButton bookmark hiddenSmallView" title="Current view (copy or open in new window)" tabindex="35" data-l10n-id="bookmark">
-                  <span data-l10n-id="bookmark_label">Current View</span>
-                </a>
-
-                <div class="verticalToolbarSeparator hiddenSmallView"></div>
-
-                <button id="secondaryToolbarToggle" class="toolbarButton" title="Tools" tabindex="36" data-l10n-id="tools">
-                  <span data-l10n-id="tools_label">Tools</span>
+                <button id="secondaryToolbarToggle" class="toolbarButton" title="Tools" tabindex="43" data-l10n-id="pdfjs-tools-button" aria-expanded="false" aria-controls="secondaryToolbar">
+                  <span data-l10n-id="pdfjs-tools-button-label">Tools</span>
                 </button>
               </div>
               <div id="toolbarViewerMiddle">
                 <div class="splitToolbarButton">
-                  <button id="zoomOut" class="toolbarButton zoomOut" title="Zoom Out" tabindex="21" data-l10n-id="zoom_out">
-                    <span data-l10n-id="zoom_out_label">Zoom Out</span>
+                  <button id="zoomOut" class="toolbarButton" title="Zoom Out" tabindex="21" data-l10n-id="pdfjs-zoom-out-button">
+                    <span data-l10n-id="pdfjs-zoom-out-button-label">Zoom Out</span>
                   </button>
                   <div class="splitToolbarButtonSeparator"></div>
-                  <button id="zoomIn" class="toolbarButton zoomIn" title="Zoom In" tabindex="22" data-l10n-id="zoom_in">
-                    <span data-l10n-id="zoom_in_label">Zoom In</span>
+                  <button id="zoomIn" class="toolbarButton" title="Zoom In" tabindex="22" data-l10n-id="pdfjs-zoom-in-button">
+                    <span data-l10n-id="pdfjs-zoom-in-button-label">Zoom In</span>
                    </button>
                 </div>
                 <span id="scaleSelectContainer" class="dropdownToolbarButton">
-                  <select id="scaleSelect" title="Zoom" tabindex="23" data-l10n-id="zoom">
-                    <option id="pageAutoOption" title="" value="auto" selected="selected" data-l10n-id="page_scale_auto">Automatic Zoom</option>
-                    <option id="pageActualOption" title="" value="page-actual" data-l10n-id="page_scale_actual">Actual Size</option>
-                    <option id="pageFitOption" title="" value="page-fit" data-l10n-id="page_scale_fit">Page Fit</option>
-                    <option id="pageWidthOption" title="" value="page-width" data-l10n-id="page_scale_width">Page Width</option>
-                    <option id="customScaleOption" title="" value="custom" disabled="disabled" hidden="true"></option>
-                    <option title="" value="0.5" data-l10n-id="page_scale_percent" data-l10n-args='{ "scale": 50 }'>50%</option>
-                    <option title="" value="0.75" data-l10n-id="page_scale_percent" data-l10n-args='{ "scale": 75 }'>75%</option>
-                    <option title="" value="1" data-l10n-id="page_scale_percent" data-l10n-args='{ "scale": 100 }'>100%</option>
-                    <option title="" value="1.25" data-l10n-id="page_scale_percent" data-l10n-args='{ "scale": 125 }'>125%</option>
-                    <option title="" value="1.5" data-l10n-id="page_scale_percent" data-l10n-args='{ "scale": 150 }'>150%</option>
-                    <option title="" value="2" data-l10n-id="page_scale_percent" data-l10n-args='{ "scale": 200 }'>200%</option>
-                    <option title="" value="3" data-l10n-id="page_scale_percent" data-l10n-args='{ "scale": 300 }'>300%</option>
-                    <option title="" value="4" data-l10n-id="page_scale_percent" data-l10n-args='{ "scale": 400 }'>400%</option>
+                  <select id="scaleSelect" title="Zoom" tabindex="23" data-l10n-id="pdfjs-zoom-select">
+                    <option id="pageAutoOption" title="" value="auto" selected="selected" data-l10n-id="pdfjs-page-scale-auto">Automatic Zoom</option>
+                    <option id="pageActualOption" title="" value="page-actual" data-l10n-id="pdfjs-page-scale-actual">Actual Size</option>
+                    <option id="pageFitOption" title="" value="page-fit" data-l10n-id="pdfjs-page-scale-fit">Page Fit</option>
+                    <option id="pageWidthOption" title="" value="page-width" data-l10n-id="pdfjs-page-scale-width">Page Width</option>
+                    <option id="customScaleOption" title="" value="custom" disabled="disabled" hidden="true" data-l10n-id="pdfjs-page-scale-percent" data-l10n-args='{ "scale": 0 }'>0%</option>
+                    <option title="" value="0.5" data-l10n-id="pdfjs-page-scale-percent" data-l10n-args='{ "scale": 50 }'>50%</option>
+                    <option title="" value="0.75" data-l10n-id="pdfjs-page-scale-percent" data-l10n-args='{ "scale": 75 }'>75%</option>
+                    <option title="" value="1" data-l10n-id="pdfjs-page-scale-percent" data-l10n-args='{ "scale": 100 }'>100%</option>
+                    <option title="" value="1.25" data-l10n-id="pdfjs-page-scale-percent" data-l10n-args='{ "scale": 125 }'>125%</option>
+                    <option title="" value="1.5" data-l10n-id="pdfjs-page-scale-percent" data-l10n-args='{ "scale": 150 }'>150%</option>
+                    <option title="" value="2" data-l10n-id="pdfjs-page-scale-percent" data-l10n-args='{ "scale": 200 }'>200%</option>
+                    <option title="" value="3" data-l10n-id="pdfjs-page-scale-percent" data-l10n-args='{ "scale": 300 }'>300%</option>
+                    <option title="" value="4" data-l10n-id="pdfjs-page-scale-percent" data-l10n-args='{ "scale": 400 }'>400%</option>
                   </select>
                 </span>
               </div>
@@ -279,125 +369,147 @@ See https://github.com/adobe-type-tools/cmap-resources
           </div>
         </div>
 
-        <menu type="context" id="viewerContextMenu">
-          <menuitem id="contextFirstPage" label="First Page"
-                    data-l10n-id="first_page"></menuitem>
-          <menuitem id="contextLastPage" label="Last Page"
-                    data-l10n-id="last_page"></menuitem>
-          <menuitem id="contextPageRotateCw" label="Rotate Clockwise"
-                    data-l10n-id="page_rotate_cw"></menuitem>
-          <menuitem id="contextPageRotateCcw" label="Rotate Counter-Clockwise"
-                    data-l10n-id="page_rotate_ccw"></menuitem>
-        </menu>
-
         <div id="viewerContainer" tabindex="0">
           <div id="viewer" class="pdfViewer"></div>
         </div>
-
-        <div id="errorWrapper" hidden='true'>
-          <div id="errorMessageLeft">
-            <span id="errorMessage"></span>
-            <button id="errorShowMore" data-l10n-id="error_more_info">
-              More Information
-            </button>
-            <button id="errorShowLess" data-l10n-id="error_less_info" hidden='true'>
-              Less Information
-            </button>
-          </div>
-          <div id="errorMessageRight">
-            <button id="errorClose" data-l10n-id="error_close">
-              Close
-            </button>
-          </div>
-          <div class="clearBoth"></div>
-          <textarea id="errorMoreInfo" hidden='true' readonly="readonly"></textarea>
-        </div>
       </div> <!-- mainContainer -->
 
-      <div id="overlayContainer" class="hidden">
-        <div id="passwordOverlay" class="container hidden">
-          <div class="dialog">
-            <div class="row">
-              <p id="passwordText" data-l10n-id="password_label">Enter the password to open this PDF file:</p>
+      <div id="dialogContainer">
+        <dialog id="passwordDialog">
+          <div class="row">
+            <label for="password" id="passwordText" data-l10n-id="pdfjs-password-label">Enter the password to open this PDF file:</label>
+          </div>
+          <div class="row">
+            <input type="password" id="password" class="toolbarField">
+          </div>
+          <div class="buttonRow">
+            <button id="passwordCancel" class="dialogButton"><span data-l10n-id="pdfjs-password-cancel-button">Cancel</span></button>
+            <button id="passwordSubmit" class="dialogButton"><span data-l10n-id="pdfjs-password-ok-button">OK</span></button>
+          </div>
+        </dialog>
+        <dialog id="documentPropertiesDialog">
+          <div class="row">
+            <span id="fileNameLabel" data-l10n-id="pdfjs-document-properties-file-name">File name:</span>
+            <p id="fileNameField" aria-labelledby="fileNameLabel">-</p>
+          </div>
+          <div class="row">
+            <span id="fileSizeLabel" data-l10n-id="pdfjs-document-properties-file-size">File size:</span>
+            <p id="fileSizeField" aria-labelledby="fileSizeLabel">-</p>
+          </div>
+          <div class="separator"></div>
+          <div class="row">
+            <span id="titleLabel" data-l10n-id="pdfjs-document-properties-title">Title:</span>
+            <p id="titleField" aria-labelledby="titleLabel">-</p>
+          </div>
+          <div class="row">
+            <span id="authorLabel" data-l10n-id="pdfjs-document-properties-author">Author:</span>
+            <p id="authorField" aria-labelledby="authorLabel">-</p>
+          </div>
+          <div class="row">
+            <span id="subjectLabel" data-l10n-id="pdfjs-document-properties-subject">Subject:</span>
+            <p id="subjectField" aria-labelledby="subjectLabel">-</p>
+          </div>
+          <div class="row">
+            <span id="keywordsLabel" data-l10n-id="pdfjs-document-properties-keywords">Keywords:</span>
+            <p id="keywordsField" aria-labelledby="keywordsLabel">-</p>
+          </div>
+          <div class="row">
+            <span id="creationDateLabel" data-l10n-id="pdfjs-document-properties-creation-date">Creation Date:</span>
+            <p id="creationDateField" aria-labelledby="creationDateLabel">-</p>
+          </div>
+          <div class="row">
+            <span id="modificationDateLabel" data-l10n-id="pdfjs-document-properties-modification-date">Modification Date:</span>
+            <p id="modificationDateField" aria-labelledby="modificationDateLabel">-</p>
+          </div>
+          <div class="row">
+            <span id="creatorLabel" data-l10n-id="pdfjs-document-properties-creator">Creator:</span>
+            <p id="creatorField" aria-labelledby="creatorLabel">-</p>
+          </div>
+          <div class="separator"></div>
+          <div class="row">
+            <span id="producerLabel" data-l10n-id="pdfjs-document-properties-producer">PDF Producer:</span>
+            <p id="producerField" aria-labelledby="producerLabel">-</p>
+          </div>
+          <div class="row">
+            <span id="versionLabel" data-l10n-id="pdfjs-document-properties-version">PDF Version:</span>
+            <p id="versionField" aria-labelledby="versionLabel">-</p>
+          </div>
+          <div class="row">
+            <span id="pageCountLabel" data-l10n-id="pdfjs-document-properties-page-count">Page Count:</span>
+            <p id="pageCountField" aria-labelledby="pageCountLabel">-</p>
+          </div>
+          <div class="row">
+            <span id="pageSizeLabel" data-l10n-id="pdfjs-document-properties-page-size">Page Size:</span>
+            <p id="pageSizeField" aria-labelledby="pageSizeLabel">-</p>
+          </div>
+          <div class="separator"></div>
+          <div class="row">
+            <span id="linearizedLabel" data-l10n-id="pdfjs-document-properties-linearized">Fast Web View:</span>
+            <p id="linearizedField" aria-labelledby="linearizedLabel">-</p>
+          </div>
+          <div class="buttonRow">
+            <button id="documentPropertiesClose" class="dialogButton"><span data-l10n-id="pdfjs-document-properties-close-button">Close</span></button>
+          </div>
+        </dialog>
+        <dialog class="dialog altText" id="altTextDialog" aria-labelledby="dialogLabel" aria-describedby="dialogDescription">
+          <div id="altTextContainer" class="mainContainer">
+            <div id="overallDescription">
+              <span id="dialogLabel" data-l10n-id="pdfjs-editor-alt-text-dialog-label" class="title">Choose an option</span>
+              <span id="dialogDescription" data-l10n-id="pdfjs-editor-alt-text-dialog-description">
+                Alt text (alternative text) helps when people can’t see the image or when it doesn’t load.
+              </span>
             </div>
-            <div class="row">
-              <input type="password" id="password" class="toolbarField">
+            <div id="addDescription">
+              <div class="radio">
+                <div class="radioButton">
+                  <input type="radio" id="descriptionButton" name="altTextOption" tabindex="0" aria-describedby="descriptionAreaLabel" checked>
+                  <label for="descriptionButton" data-l10n-id="pdfjs-editor-alt-text-add-description-label">Add a description</label>
+                </div>
+                <div class="radioLabel">
+                  <span id="descriptionAreaLabel" data-l10n-id="pdfjs-editor-alt-text-add-description-description">
+                    Aim for 1-2 sentences that describe the subject, setting, or actions.
+                  </span>
+                </div>
+              </div>
+              <div class="descriptionArea">
+                <textarea id="descriptionTextarea" placeholder="For example, “A young man sits down at a table to eat a meal”" aria-labelledby="descriptionAreaLabel" data-l10n-id="pdfjs-editor-alt-text-textarea" tabindex="0"></textarea>
+              </div>
             </div>
-            <div class="buttonRow">
-              <button id="passwordCancel" class="overlayButton"><span data-l10n-id="password_cancel">Cancel</span></button>
-              <button id="passwordSubmit" class="overlayButton"><span data-l10n-id="password_ok">OK</span></button>
+            <div id="markAsDecorative">
+              <div class="radio">
+                <div class="radioButton">
+                  <input type="radio" id="decorativeButton" name="altTextOption" aria-describedby="decorativeLabel">
+                  <label for="decorativeButton" data-l10n-id="pdfjs-editor-alt-text-mark-decorative-label">Mark as decorative</label>
+                </div>
+                <div class="radioLabel">
+                  <span id="decorativeLabel" data-l10n-id="pdfjs-editor-alt-text-mark-decorative-description">
+                    This is used for ornamental images, like borders or watermarks.
+                  </span>
+                </div>
+              </div>
+            </div>
+            <div id="buttons">
+              <button id="altTextCancel" class="secondaryButton" tabindex="0"><span data-l10n-id="pdfjs-editor-alt-text-cancel-button">Cancel</span></button>
+              <button id="altTextSave" class="primaryButton" tabindex="0"><span data-l10n-id="pdfjs-editor-alt-text-save-button">Save</span></button>
             </div>
           </div>
-        </div>
-        <div id="documentPropertiesOverlay" class="container hidden">
-          <div class="dialog">
-            <div class="row">
-              <span data-l10n-id="document_properties_file_name">File name:</span> <p id="fileNameField">-</p>
-            </div>
-            <div class="row">
-              <span data-l10n-id="document_properties_file_size">File size:</span> <p id="fileSizeField">-</p>
-            </div>
-            <div class="separator"></div>
-            <div class="row">
-              <span data-l10n-id="document_properties_title">Title:</span> <p id="titleField">-</p>
-            </div>
-            <div class="row">
-              <span data-l10n-id="document_properties_author">Author:</span> <p id="authorField">-</p>
-            </div>
-            <div class="row">
-              <span data-l10n-id="document_properties_subject">Subject:</span> <p id="subjectField">-</p>
-            </div>
-            <div class="row">
-              <span data-l10n-id="document_properties_keywords">Keywords:</span> <p id="keywordsField">-</p>
-            </div>
-            <div class="row">
-              <span data-l10n-id="document_properties_creation_date">Creation Date:</span> <p id="creationDateField">-</p>
-            </div>
-            <div class="row">
-              <span data-l10n-id="document_properties_modification_date">Modification Date:</span> <p id="modificationDateField">-</p>
-            </div>
-            <div class="row">
-              <span data-l10n-id="document_properties_creator">Creator:</span> <p id="creatorField">-</p>
-            </div>
-            <div class="separator"></div>
-            <div class="row">
-              <span data-l10n-id="document_properties_producer">PDF Producer:</span> <p id="producerField">-</p>
-            </div>
-            <div class="row">
-              <span data-l10n-id="document_properties_version">PDF Version:</span> <p id="versionField">-</p>
-            </div>
-            <div class="row">
-              <span data-l10n-id="document_properties_page_count">Page Count:</span> <p id="pageCountField">-</p>
-            </div>
-            <div class="row">
-              <span data-l10n-id="document_properties_page_size">Page Size:</span> <p id="pageSizeField">-</p>
-            </div>
-            <div class="separator"></div>
-            <div class="row">
-              <span data-l10n-id="document_properties_linearized">Fast Web View:</span> <p id="linearizedField">-</p>
-            </div>
-            <div class="buttonRow">
-              <button id="documentPropertiesClose" class="overlayButton"><span data-l10n-id="document_properties_close">Close</span></button>
-            </div>
+        </dialog>
+        <dialog id="printServiceDialog" style="min-width: 200px;">
+          <div class="row">
+            <span data-l10n-id="pdfjs-print-progress-message">Preparing document for printing…</span>
           </div>
-        </div>
-        <div id="printServiceOverlay" class="container hidden">
-          <div class="dialog">
-            <div class="row">
-              <span data-l10n-id="print_progress_message">Preparing document for printing…</span>
-            </div>
-            <div class="row">
-              <progress value="0" max="100"></progress>
-              <span data-l10n-id="print_progress_percent" data-l10n-args='{ "progress": 0 }' class="relative-progress">0%</span>
-            </div>
-            <div class="buttonRow">
-              <button id="printCancel" class="overlayButton"><span data-l10n-id="print_progress_close">Cancel</span></button>
-            </div>
+          <div class="row">
+            <progress value="0" max="100"></progress>
+            <span data-l10n-id="pdfjs-print-progress-percent" data-l10n-args='{ "progress": 0 }' class="relative-progress">0%</span>
           </div>
-        </div>
-      </div>  <!-- overlayContainer -->
+          <div class="buttonRow">
+            <button id="printCancel" class="dialogButton"><span data-l10n-id="pdfjs-print-progress-close-button">Cancel</span></button>
+          </div>
+        </dialog>
+      </div>  <!-- dialogContainer -->
 
     </div> <!-- outerContainer -->
     <div id="printContainer"></div>
+    <script src="/static/reader/pdf.js"></script>
   </body>
 </html>

static/js/annotation.js

@@ -1,9 +1,18 @@
 'use strict';
 
+oml.SELECTION = 0
+oml.HIGHLIGHT = 1
+
 oml.ui.annotation = function(annotation, $iframe) {
+    var value = Ox.encodeHTMLEntities(annotation.text).replace(/\n/g, '<br/>')
+    if (annotation.type == oml.HIGHLIGHT) {
+        let coord = annotation.coords[0].map(p => parseInt(p)).join(',')
+        let image = `/${oml.user.ui.item}/2048p${parseInt(annotation.page)},${coord}.jpg`
+        value = `<img src="${image}">`
+    }
     var $quoteText = Ox.Element()
         .addClass('OxSelectable OMLQuote')
-        .html(Ox.encodeHTMLEntities(annotation.text).replace(/\n/g, '<br/>'))
+        .html(value)
         .on({
             click: function(event) {
                 var id

static/js/folders.js

@@ -128,6 +128,10 @@ oml.ui.folders = function() {
                     .appendTo(that)
             );
             oml.$ui.librariesList.$body.css({height: '16px'}); // FIXME!
+            oml.$ui.librariesList.find('.OxColumnItems').css({
+                'text-overflow': 'reset',
+                'direction': 'rtl'
+            })
 
             users.forEach(function(user, index) {
 

static/js/oml.js

@@ -219,7 +219,7 @@
     function loadOML(browserSupported) {
         window.oml = Ox.App({
             name: 'oml',
-            socket: 'ws://' + document.location.host + '/ws',
+            socket: document.location.protocol.replace('http', 'ws') + '//' + document.location.host + '/ws',
             url: '/api/'
         }).bindEvent({
             load: function(data) {

static/js/utils.js

@@ -1028,7 +1028,7 @@ oml.updateFilterMenus = function() {
 };
 
 oml.validatePublicKey = function(value) {
-    return /^[a-z0-9+\/]{16}$/.test(value);
+    return /^[a-z0-9+\/]{56}$/.test(value);
 };
 
 oml.updateDebugMenu = function() {

static/js/viewer.js

@@ -151,7 +151,7 @@ oml.ui.viewer = function() {
                         height: '100%',
                         border: 0
                     }).onMessage(function(data, event) {
-                        console.log('got', event, data)
+                        // console.log('got', event, data, data.page)
                         if (event == 'addAnnotation') {
                             addAnnotation(data);
                             var $annotation = oml.ui.annotation(data, $iframe).bindEvent(annotationEvents)
@@ -215,7 +215,11 @@ oml.ui.viewer = function() {
             })
             return
         }
-        annotations = Ox.sortBy(annotations, sortKey)
+        var map = {}
+        map[sortKey] = function(value) {
+            return value ? value.toString() : '';
+        }
+        annotations = Ox.sortBy(annotations, sortKey, map)
         oml.$ui.annotationFolder.empty();
         var visibleAnnotations = [];
         var hasAnnotations = false;

static/reader/pdf.css

@@ -0,0 +1,62 @@
+
+.toolbarButton.cropFile::before,
+    .secondaryToolbarButton.cropFile::before {
+    mask-image: url(pdf/toolbarButton-crop.png);
+}
+.toolbarButton.embedPage::before,
+    .secondaryToolbarButton.embedPage::before {
+    mask-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNTYiIGhlaWdodD0iMjU2IiB2aWV3Qm94PSIwIDAgMjU2IDI1NiI+PGxpbmUgeDE9Ijg4IiB5MT0iNTYiIHgyPSIyNCIgeTI9IjEyOCIgc3Ryb2tlPSIjMDAwMDAwIiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS13aWR0aD0iNDgiLz48bGluZSB4MT0iMjQiIHkxPSIxMjgiIHgyPSI4OCIgeTI9IjIwMCIgc3Ryb2tlPSIjMDAwMDAwIiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS13aWR0aD0iNDgiLz48bGluZSB4MT0iMTY4IiB5MT0iNTYiIHgyPSIyMzIiIHkyPSIxMjgiIHN0cm9rZT0iIzAwMDAwMCIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2Utd2lkdGg9IjQ4Ii8+PGxpbmUgeDE9IjIzMiIgeTE9IjEyOCIgeDI9IjE2OCIgeTI9IjIwMCIgc3Ryb2tlPSIjMDAwMDAwIiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS13aWR0aD0iNDgiLz48L3N2Zz48IS0teyJjb2xvciI6ImRlZmF1bHQiLCJuYW1lIjoic3ltYm9sRW1iZWQiLCJ0aGVtZSI6Im94bWVkaXVtIn0tLT4=);
+}
+@media screen and (min-resolution: 2dppx) {
+    .toolbarButton.cropFile::before,
+    .secondaryToolbarButton.cropFile::before {
+        mask-image: url(pdf/toolbarButton-crop@2x.png);
+    }
+    .toolbarButton.embedPage::before,
+    .secondaryToolbarButton.embedPage::before {
+        mask-image: url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNTYiIGhlaWdodD0iMjU2IiB2aWV3Qm94PSIwIDAgMjU2IDI1NiI+PGxpbmUgeDE9Ijg4IiB5MT0iNTYiIHgyPSIyNCIgeTI9IjEyOCIgc3Ryb2tlPSIjMDAwMDAwIiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS13aWR0aD0iNDgiLz48bGluZSB4MT0iMjQiIHkxPSIxMjgiIHgyPSI4OCIgeTI9IjIwMCIgc3Ryb2tlPSIjMDAwMDAwIiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS13aWR0aD0iNDgiLz48bGluZSB4MT0iMTY4IiB5MT0iNTYiIHgyPSIyMzIiIHkyPSIxMjgiIHN0cm9rZT0iIzAwMDAwMCIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2Utd2lkdGg9IjQ4Ii8+PGxpbmUgeDE9IjIzMiIgeTE9IjEyOCIgeDI9IjE2OCIgeTI9IjIwMCIgc3Ryb2tlPSIjMDAwMDAwIiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS13aWR0aD0iNDgiLz48L3N2Zz48IS0teyJjb2xvciI6ImRlZmF1bHQiLCJuYW1lIjoic3ltYm9sRW1iZWQiLCJ0aGVtZSI6Im94bWVkaXVtIn0tLT4=);
+    }
+}
+
+.verticalToolbarSeparator.hiddenMediumView,
+#print,
+#secondaryPrint,
+#openFile,
+#secondaryOpenFile,
+#editorModeButtons {
+    display: none !important;
+}
+
+.page .crop-overlay {
+    position: absolute;
+    top: 0;
+    bottom: 0;
+    left: 0;
+    right: 0;
+    //background: rgba(0,0,0,0.5);
+    cursor: crosshair;
+    z-index: 100;
+}
+.page .crop-overlay.inactive {
+  pointer-events: none;
+  cursor: default;
+}
+
+.page .crop-overlay canvas {
+  width: 100%;
+  height: 100%;
+}
+.page .highlights {
+    position: absolute;
+    top: 0;
+    bottom: 0;
+    left: 0;
+    right: 0;
+    //background: rgba(0,0,0,0.5);
+    z-index: 101;
+    pointer-events: none;
+}
+.page .highlights canvas {
+  width: 100%;
+  height: 100%;
+}

static/reader/pdf.js

@@ -1,6 +1,97 @@
 var id = document.location.pathname.split('/')[1];
 var annotations = [];
 var currentPage = 1, rendered = false
+var highlightInactive = true
+var selectedAnnotation
+
+const SELECTION = 0
+const HIGHLIGHT = 1
+
+
+var div = document.createElement("div")
+div.innerHTML = `
+<button id="cropFile" class="toolbarButton cropFile hiddenLargeView" title="Highlight" tabindex="30" data-l10n-id="crop_file">
+    <span data-l10n-id="crop_file_label">Highlight</span>
+</button>
+`
+var cropFile = div.querySelector("#cropFile")
+
+document.querySelector('#toolbarViewerRight').insertBefore(cropFile, document.querySelector('#toolbarViewerRight').firstChild)
+
+// secondary menu
+div.innerHTML = `
+<button id="secondaryCropFile" class="secondaryToolbarButton visibleMediumView cropFile" title="Highlight" tabindex="50" data-l10n-id="crop">
+  <span data-l10n-id="crop_label">Highlight</span>
+</button>
+`
+var secondaryCropFile = div.querySelector("#secondaryCropFile")
+document.querySelector('#secondaryToolbarButtonContainer').insertBefore(
+    secondaryCropFile,
+    document.querySelector('#secondaryToolbarButtonContainer').firstChild
+)
+
+function initOverlay() {
+    document.querySelectorAll('#cropFile,.secondaryToolbarButton.cropFile').forEach(btn => {
+        btn.addEventListener('click', event=> {
+            if (highlightInactive) {
+                event.target.style.background = 'red'
+                highlightInactive = false
+                document.querySelectorAll('.crop-overlay.inactive').forEach(element => {
+                    element.classList.remove('inactive')
+                })
+            } else {
+                event.target.style.background = ''
+                highlightInactive = true
+                document.querySelectorAll('.crop-overlay').forEach(element => {
+                    element.classList.add('inactive')
+                })
+            }
+        })
+    })
+    PDFViewerApplication.initializedPromise.then(function() {
+        PDFViewerApplication.pdfViewer.eventBus.on("pagesinit", function(event) {
+            /*
+            document.querySelector('#viewerContainer').addEventListener('scroll', event => {
+                if (window.parent && window.parent.postMessage) {
+                    if (first) {
+                        first = false
+                    } else {
+                        window.parent.postMessage({event: 'scrolled', top: event.target.scrollTop})
+                    }
+                }
+            })
+            */
+        })
+        PDFViewerApplication.pdfViewer.eventBus.on("pagerender", function(event) {
+            var page = event.pageNumber.toString()
+            var div = event.source.div
+            var overlay = document.createElement('div')
+            overlay.classList.add('crop-overlay')
+            overlay.id = 'overlay' + page
+            if (highlightInactive) {
+                overlay.classList.add('inactive')
+            }
+            div.appendChild(overlay)
+
+            renderHighlightSelectionOverlay(overlay, id, page, event.source)
+            var highlights = document.createElement('div')
+            highlights.classList.add('highlights')
+            highlights.id = 'highlights' + page
+            var canvas = document.createElement('canvas')
+            highlights.appendChild(canvas)
+            div.appendChild(highlights)
+            renderHighlights(page)
+        })
+        PDFViewerApplication.eventBus.on('pagerendered', function(event) {
+            loadAnnotations(event.pageNumber)
+        })
+    })
+}
+
+document.addEventListener('DOMContentLoaded', function() {
+    window.PDFViewerApplication ? initOverlay() : document.addEventListener("webviewerloaded", initOverlay)
+})
+
 
 Ox.load({
     'UI': {
@@ -25,7 +116,15 @@ Ox.load({
                     document.querySelector('#viewerContainer').scrollTop = el.offsetTop + el.parentElement.offsetTop - 64;
                 }
             }, delay)
+            var oldSelection = selectedAnnotation
+            selectedAnnotation = data.id
             selectAnnotation(data.id)
+            if (oldSelection) {
+                var old = annotations.filter(function(a) { return a.id == oldSelection })[0]
+                if (old && old.type == HIGHLIGHT) {
+                    renderHighlights(old.page)
+                }
+            }
         } else if (event == 'addAnnotation') {
             createAnnotation()
         } else if (event == 'addAnnotations') {
@@ -37,7 +136,11 @@ Ox.load({
             }
             data.annotations.forEach(function(annotation) {
                 annotations.push(annotation)
-                renderAnnotation(annotation)
+                if (annotation.type == HIGHLIGHT) {
+                    renderHighlights(annotation.page)
+                } else {
+                    renderAnnotation(annotation)
+                }
             })
         } else if (event == 'removeAnnotation') {
             removeAnnotation(data.id)
@@ -75,18 +178,6 @@ window.addEventListener('mouseup', function(event) {
     }
 })
 
-function bindEvents() {
-    if (!window.PDFViewerApplication || !window.PDFViewerApplication.eventBus) {
-        setTimeout(bindEvents, 10)
-        return
-    }
-    PDFViewerApplication.eventBus.on('pagerendered', function(event) {
-        loadAnnotations(event.pageNumber)
-    })
-}
-
-bindEvents()
-
 function getHighlight() {
     var pageNumber = PDFViewerApplication.pdfViewer.currentPageNumber;
     var pageIndex = pageNumber - 1;
@@ -102,6 +193,7 @@ function getHighlight() {
     var text = selection.toString();
     var position = [pageNumber].concat(Ox.sort(selected.map(function(c) { return [c[1], c[0]]}))[0]);
     return {
+        type: SELECTION,
         page: pageNumber,
         pageLabel: PDFViewerApplication.pdfViewer.currentPageLabel,
         position: position,
@@ -134,25 +226,32 @@ function renderAnnotation(annotation) {
     }
     var pageElement = page.canvas.parentElement.parentElement;
     var viewport = page.viewport;
-    annotation.coords.forEach(function (rect) {
-        var bounds = viewport.convertToViewportRectangle(rect);
-        var el = document.createElement('div');
-        el.classList.add('oml-annotation')
-        el.classList.add('a' + annotation.id)
-        el.classList.add('page' + annotation.page)
-        el.dataset.id = annotation.id
-        el.setAttribute('style', 'position: absolute; background-color: yellow;opacity:0.3;' + 
-            'left:' + Math.min(bounds[0], bounds[2]) + 'px; top:' + Math.min(bounds[1], bounds[3]) + 'px;' +
-            'width:' + Math.abs(bounds[0] - bounds[2]) + 'px; height:' + Math.abs(bounds[1] - bounds[3]) + 'px;');
+    if (annotation.type == HIGHLIGHT) {
+        renderHighlights(annotation.page)
+    } else if (annotation.coords) {
+        pageElement.querySelectorAll('.oml-annotation').forEach(el => el.remove())
+        annotation.coords.forEach(function (rect) {
+            var bounds = viewport.convertToViewportRectangle(rect);
+            var el = document.createElement('div');
+            el.classList.add('oml-annotation')
+            el.classList.add('a' + annotation.id)
+            el.classList.add('page' + annotation.page)
+            el.dataset.id = annotation.id
+            el.setAttribute('style', 'position: absolute; background-color: yellow;opacity:0.3;' + 
+                'left:' + Math.min(bounds[0], bounds[2]) + 'px; top:' + Math.min(bounds[1], bounds[3]) + 'px;' +
+                'width:' + Math.abs(bounds[0] - bounds[2]) + 'px; height:' + Math.abs(bounds[1] - bounds[3]) + 'px;');
 
-        el.addEventListener('click', function() {
-            if (!el.classList.contains('selected')) {
-                selectAnnotation(annotation.id)
-                Ox.$parent.postMessage('selectAnnotation', {id: annotation.id})
-            }
+            el.addEventListener('click', function() {
+                if (!el.classList.contains('selected')) {
+                    selectAnnotation(annotation.id)
+                    Ox.$parent.postMessage('selectAnnotation', {id: annotation.id})
+                }
+            });
+            pageElement.appendChild(el);
         });
-        pageElement.appendChild(el);
-    });
+    } else {
+       // console.log("annotation without position", annotation)
+    }
 }
 
 function addAnnotation(annotation) {
@@ -169,6 +268,11 @@ function selectAnnotation(id) {
         g.classList.add('selected')
         g.style.backgroundColor = 'blue'
     })
+    annotations.forEach(a => {
+        if (a.id == id && a.type == HIGHLIGHT) {
+            renderHighlights(a.page)
+        }
+    })
 }
 
 function deselectAnnotation(id) {
@@ -207,7 +311,7 @@ function loadAnnotations(page) {
         e.remove()
     })
     annotations.filter(function(a) {
-        return a.page == page
+        return a.page == page && !a.type == HIGHLIGHT
     }).forEach(function(annot) {
         renderAnnotation(annot)
     })
@@ -220,3 +324,145 @@ function isInView(element) {
     var elementBottom = elementTop + $(element).height();
     return elementTop < docViewBottom && elementBottom > docViewTop;
 }
+
+function renderHighlightSelectionOverlay(root, documentId, page, source) {
+    var canvas = document.createElement('canvas')
+    root.appendChild(canvas)
+    canvas.width = canvas.clientWidth
+    canvas.height = canvas.clientHeight
+    var ctx = canvas.getContext('2d');
+    var viewerContainer = document.querySelector('#viewerContainer')
+    var bounds = root.getBoundingClientRect();
+    var base = 2048
+    var scale = Math.max(bounds.height, bounds.width) / base
+    var last_mousex = last_mousey = 0;
+    var mousex = mousey = 0;
+    var mousedown = false;
+    var p = {
+        top: 0,
+        left: 0,
+        bottom: 0,
+        right: 0
+    }
+    var inside = false
+
+    canvas.addEventListener('mousedown', function(e) {
+        if (inside) {
+            const coords = [
+                [p.left, p.top, p.right, p.bottom]
+            ]
+            addAnnotation({
+                type: HIGHLIGHT,
+                id: Ox.SHA1(pageNumber.toString() + JSON.stringify(p)),
+                text: "",
+                page: parseInt(page),
+                pageLabel: source.pageLabel,
+                coords: coords,
+            })
+            return
+        }
+        let bounds = root.getBoundingClientRect();
+        last_mousex = e.clientX - bounds.left;
+        last_mousey = e.clientY - bounds.top;
+        p.top = parseInt(last_mousey / scale)
+        p.left = parseInt(last_mousex / scale)
+        mousedown = true;
+    });
+
+    document.addEventListener('mouseup', function(e) {
+        if (mousedown) {
+            mousedown = false;
+            p.bottom = parseInt(mousey / scale)
+            p.right = parseInt(mousex / scale)
+
+            if (p.top > p.bottom) {
+                var t = p.top
+                p.top = p.bottom
+                p.bottom = t
+            }
+            if (p.left > p.right) {
+                var t = p.left
+                p.left = p.right
+                p.right = t
+            }
+            /*
+            var url = `${baseUrl}/documents/${documentId}/2048p${page},${p.left},${p.top},${p.right},${p.bottom}.jpg`
+            info.url = `${baseUrl}/document/${documentId}/${page}`
+            info.page = page
+            if (p.left != p.right && p.top != p.bottom) {
+                var context = formatOutput(info, url)
+                copyToClipboard(context)
+                addToRecent({
+                    document: documentId,
+                    page: parseInt(page),
+                    title: info.title,
+                    type: 'fragment',
+                    link: `${baseUrl}/documents/${documentId}/${page}`,
+                    src: url
+                })
+            }
+            */
+        }
+    });
+
+    canvas.addEventListener('mousemove', function(e) {
+        let bounds = root.getBoundingClientRect();
+        mousex = e.clientX - bounds.left;
+        mousey = e.clientY - bounds.top;
+
+        if(mousedown) {
+            ctx.clearRect(0, 0, canvas.clientWidth, canvas.clientHeight)
+            ctx.beginPath()
+            var width = mousex - last_mousex
+            var height = mousey - last_mousey
+            ctx.rect(last_mousex, last_mousey, width, height)
+            ctx.strokeStyle = 'black';
+            ctx.lineWidth = 2;
+            ctx.stroke();
+        } else {
+            let py = parseInt(mousey / scale)
+            let px = parseInt(mousex / scale)
+            if (py > p.top && py < p.bottom && px > p.left && px < p.right) {
+                inside = true
+                canvas.style.cursor = 'pointer'
+                canvas.title = 'Click to add highlight'
+            } else {
+                inside = false
+                canvas.style.cursor = ''
+                canvas.title = ''
+            }
+        }
+    });
+}
+function renderHighlights(page) {
+    var pageAnnotations = annotations.filter(annotation => {
+        return annotation.type == HIGHLIGHT && (!page || (annotation.page == page))
+    })
+    pageAnnotations.forEach(annotation => {
+        let page = annotation.page
+        var canvas = document.querySelector(`#highlights${page} canvas`)
+        if (canvas) {
+            canvas.width = canvas.clientWidth
+            canvas.height = canvas.clientHeight
+            var ctx = canvas.getContext('2d');
+            var viewerContainer = document.querySelector('#viewerContainer')
+            var bounds = canvas.parentElement.getBoundingClientRect();
+            var base = 2048
+            var scale = Math.max(bounds.height, bounds.width) / base
+            ctx.clearRect(0, 0, canvas.clientWidth, canvas.clientHeight)
+            pageAnnotations.forEach(annotation => {
+                    ctx.beginPath()
+                    ctx.strokeStyle = annotation.id == selectedAnnotation ? 'blue' : 'yellow';
+                    ctx.lineWidth = 2;
+                    annotation.coords.forEach(coord => {
+                        const width = coord[2] - coord[0],
+                              height = coord[3] - coord[1];
+                        ctx.rect(coord[0] * scale, coord[1] * scale, width * scale, height * scale)
+                    })
+                    ctx.stroke();
+            })
+
+        }
+    })
+
+}

static/reader/pdf/toolbarButton-crop.svg

@@ -0,0 +1,4 @@
+<?xml version="1.0"?>
+<svg xmlns="http://www.w3.org/2000/svg" height="54" width="54">
+<path stroke-linejoin="round" stroke="#333" stroke-linecap="round" stroke-width="5" fill="none" d="m13.2 39 35-34.6m-45 8.1h36.6v38m-27-47v36.6h38"/>
+</svg>