faster filters

oml/item/api.py

@@ -44,43 +44,42 @@ def find(data):
     if 'group' in q:
         names = {}
         groups = {}
+        sortvalues = {}
         _keydata = data.copy()
-        for key in ('range', 'position'):
+        for key in ('range', 'position', 'positions'):
             if key in _keydata:
                 del _keydata[key]
-        key = 'group:' + hashlib.sha1(json.dumps(_keydata).encode('utf-8')).hexdigest()
+        key = 'group:' + hashlib.sha1(json.dumps(_keydata, sort_keys=True).encode('utf-8')).hexdigest()
         g = state.cache.get(key)
         if g is None:
             items = q['qs'].options(load_only('id'))
             qs = models.Find.query.filter_by(key=q['group'])
             if items.first():
                 qs = qs.filter(models.Find.item_id.in_(items))
-                for f in qs.values('value', 'findvalue'):
+                for f in qs.values('value', 'findvalue', 'sortvalue'):
                     value = f[0]
                     findvalue = f[1]
+                    sortvalue = f[2]
                     if findvalue not in groups:
                         groups[findvalue] = 0
                     groups[findvalue] += 1
                     names[findvalue] = value
+                    sortvalues[value] = sortvalue
                 g = [{'name': names[k], 'items': groups[k]} for k in groups]
             else:
                 g = []
             if 'sort' in q:
-                sort_type = utils.get_by_id(settings.config['itemKeys'], q['group']).get('sortType')
+                reverse = q['sort'][0]['operator'] == '-'
                 def _sort_key(k):
-                    if sort_type == 'person':
+                    name = sortvalues[k['name']]
-                        name = get_sort_name(k['name'])
+                    if not name:
-                    else:
+                        name = '\uffff' if not reverse else ''
-                        name = k['name']
-                    if isinstance(name, str):
-                        name = unicodedata.normalize('NFKD', name).lower()
                     items = k['items']
                     if q['sort'][0]['key'] == 'name':
                         v = (name, items)
                     else:
                         v = (-items, name)
                     return v
-                reverse = q['sort'][0]['operator'] == '-'
                 if q['sort'][0]['key'] == 'items':
                     reverse = not reverse
                 g.sort(key=_sort_key, reverse=reverse)

oml/item/models.py

@@ -171,12 +171,8 @@ class Item(db.Model):
                         value = list(map(get_sort_name, value))
                         value = ox.sort_string('\n'.join(value)).lower()
                     elif sort_type == 'title':
-                        if isinstance(value, dict):
+                        value = self.get_sorttitle().lower()
-                            value = list(value.values())
+                        value = utils.sort_title(value)
-                        if isinstance(value, list):
-                            value = ''.join(value)
-                        value = ox.get_sort_title(value)
-                        value = utils.sort_title(value).lower()
                     elif sort_type == 'boolean':
                         pass
                     else:
@@ -204,6 +200,16 @@ class Item(db.Model):
             if f.value != v:
                 f.findvalue = unicodedata.normalize('NFKD', v).lower()
                 f.value = v
+                if k in self.filter_keys:
+                    sort_type = utils.get_by_id(settings.config['itemKeys'], k).get('sortType')
+                    if sort_type == 'person':
+                        f.sortvalue = get_sort_name(f.value)
+                    else:
+                        f.sortvalue = f.value
+                    if f.sortvalue:
+                        f.sortvalue = unicodedata.normalize('NFKD', f.sortvalue).lower()
+                else:
+                    f.sortvalue = None
                 state.db.session.add(f)
 
         keys = []
@@ -285,7 +291,8 @@ class Item(db.Model):
         'series',
         'sharemetadata',
         'tableofcontents',
-        'title'
+        'title',
+        'sorttitle'
     )
 
     def update_metadata(self, data, modified=None):
@@ -320,6 +327,13 @@ class Item(db.Model):
     def get_hash(self):
         return utils.get_meta_hash(self.meta.copy())
 
+    def get_sorttitle(self):
+        title = self.meta.get('sorttitle')
+        if title is None:
+            title = self.meta.get('title', 'Untitled')
+            title = ox.get_sort_title(title)
+        return title
+
     def sync_metadata(self):
         from user.models import Metadata
         if self.meta.get('sharemetadata'):
@@ -583,6 +597,7 @@ class Find(db.Model):
     key = sa.Column(sa.String(200), index=True)
     value = sa.Column(sa.Text())
     findvalue = sa.Column(sa.Text(), index=True)
+    sortvalue = sa.Column(sa.Text())
 
     def __repr__(self):
         return '%s=%s' % (self.key, self.findvalue)
@@ -782,6 +797,11 @@ def update_sort_table():
     sql = []
     for col in set(Item.sort_keys)-set(sort_indexes):
         sql.append('CREATE INDEX ix_sort_{col} ON sort ({col})'.format(col=col))
+    if not 'sortvalue' in db.get_table_columns('find'):
+        create_table = str(CreateTable(Find.__table__).compile(db.engine)).split('\n')
+        col = 'sortvalue'
+        add = [r for r in create_table if '\t%s ' % col in r][0].strip()[:-1]
+        sql.append('ALTER TABLE find ADD '+add)
     if sql:
         with db.session() as s:
             for q in sql:

oml/settings.py

@@ -99,4 +99,4 @@ USER_AGENT = 'OpenMediaLibrary/%s' % VERSION
 
 DEBUG_HTTP = server.get('debug_http', False)
 
-DB_VERSION = 9
+DB_VERSION = 10

oml/setup.py

@@ -77,6 +77,7 @@ CREATE TABLE find (
     "key" VARCHAR(200), 
     value TEXT, 
     findvalue TEXT, 
+    sortvalue TEXT, 
     PRIMARY KEY (id), 
     FOREIGN KEY(item_id) REFERENCES item (id)
 );

oml/update.py

@@ -307,6 +307,8 @@ class Update(Thread):
                 db_version = migrate_7()
             if db_version < 9:
                 db_version = migrate_8()
+            if db_version < 10:
+                db_version = migrate_10()
             settings.server['db_version'] = settings.DB_VERSION
 
     def run(self):
@@ -450,3 +452,30 @@ def migrate_8():
                     session.delete(peer)
         session.commit()
     return 8
+
+def migrate_10():
+    with db.session() as session:
+        from item.models import Item, Find
+        from utils import get_by_id
+        from item.person import get_sort_name
+        import unicodedata
+        sort_names = {}
+        for f in Find.query.filter(Find.key.in_(Item.sort_keys)):
+            if f.key in Item.filter_keys:
+                sort_type = get_by_id(settings.config['itemKeys'], f.key).get('sortType')
+                if sort_type == 'person':
+                    if f.value in sort_names:
+                        f.sortvalue = sort_names[f.value]
+                    else:
+                        f.sortvalue = sort_names[f.value] = get_sort_name(f.value)
+                else:
+                    f.sortvalue = f.value
+                if f.sortvalue:
+                    f.sortvalue = unicodedata.normalize('NFKD', f.sortvalue).lower()
+                else:
+                    f.sortvalue = None
+            else:
+                f.sortvalue = None
+            session.add(f)
+        session.commit()
+    return 10