From 2b3edd40dc2323b4061bc3b6e04f78b1a7b565e4 Mon Sep 17 00:00:00 2001 From: j Date: Mon, 25 Jan 2016 23:02:04 +0530 Subject: [PATCH] faster filters --- oml/item/api.py | 21 ++++++++++----------- oml/item/models.py | 34 +++++++++++++++++++++++++++------- oml/settings.py | 2 +- oml/setup.py | 1 + oml/update.py | 29 +++++++++++++++++++++++++++++ 5 files changed, 68 insertions(+), 19 deletions(-) diff --git a/oml/item/api.py b/oml/item/api.py index db816b1..20c2d28 100644 --- a/oml/item/api.py +++ b/oml/item/api.py @@ -44,43 +44,42 @@ def find(data): if 'group' in q: names = {} groups = {} + sortvalues = {} _keydata = data.copy() - for key in ('range', 'position'): + for key in ('range', 'position', 'positions'): if key in _keydata: del _keydata[key] - key = 'group:' + hashlib.sha1(json.dumps(_keydata).encode('utf-8')).hexdigest() + key = 'group:' + hashlib.sha1(json.dumps(_keydata, sort_keys=True).encode('utf-8')).hexdigest() g = state.cache.get(key) if g is None: items = q['qs'].options(load_only('id')) qs = models.Find.query.filter_by(key=q['group']) if items.first(): qs = qs.filter(models.Find.item_id.in_(items)) - for f in qs.values('value', 'findvalue'): + for f in qs.values('value', 'findvalue', 'sortvalue'): value = f[0] findvalue = f[1] + sortvalue = f[2] if findvalue not in groups: groups[findvalue] = 0 groups[findvalue] += 1 names[findvalue] = value + sortvalues[value] = sortvalue g = [{'name': names[k], 'items': groups[k]} for k in groups] else: g = [] if 'sort' in q: - sort_type = utils.get_by_id(settings.config['itemKeys'], q['group']).get('sortType') + reverse = q['sort'][0]['operator'] == '-' def _sort_key(k): - if sort_type == 'person': - name = get_sort_name(k['name']) - else: - name = k['name'] - if isinstance(name, str): - name = unicodedata.normalize('NFKD', name).lower() + name = sortvalues[k['name']] + if not name: + name = '\uffff' if not reverse else '' items = k['items'] if q['sort'][0]['key'] == 'name': v = (name, items) else: v = (-items, name) return v - reverse = q['sort'][0]['operator'] == '-' if q['sort'][0]['key'] == 'items': reverse = not reverse g.sort(key=_sort_key, reverse=reverse) diff --git a/oml/item/models.py b/oml/item/models.py index 729839b..6dbe863 100644 --- a/oml/item/models.py +++ b/oml/item/models.py @@ -171,12 +171,8 @@ class Item(db.Model): value = list(map(get_sort_name, value)) value = ox.sort_string('\n'.join(value)).lower() elif sort_type == 'title': - if isinstance(value, dict): - value = list(value.values()) - if isinstance(value, list): - value = ''.join(value) - value = ox.get_sort_title(value) - value = utils.sort_title(value).lower() + value = self.get_sorttitle().lower() + value = utils.sort_title(value) elif sort_type == 'boolean': pass else: @@ -204,6 +200,16 @@ class Item(db.Model): if f.value != v: f.findvalue = unicodedata.normalize('NFKD', v).lower() f.value = v + if k in self.filter_keys: + sort_type = utils.get_by_id(settings.config['itemKeys'], k).get('sortType') + if sort_type == 'person': + f.sortvalue = get_sort_name(f.value) + else: + f.sortvalue = f.value + if f.sortvalue: + f.sortvalue = unicodedata.normalize('NFKD', f.sortvalue).lower() + else: + f.sortvalue = None state.db.session.add(f) keys = [] @@ -285,7 +291,8 @@ class Item(db.Model): 'series', 'sharemetadata', 'tableofcontents', - 'title' + 'title', + 'sorttitle' ) def update_metadata(self, data, modified=None): @@ -320,6 +327,13 @@ class Item(db.Model): def get_hash(self): return utils.get_meta_hash(self.meta.copy()) + def get_sorttitle(self): + title = self.meta.get('sorttitle') + if title is None: + title = self.meta.get('title', 'Untitled') + title = ox.get_sort_title(title) + return title + def sync_metadata(self): from user.models import Metadata if self.meta.get('sharemetadata'): @@ -583,6 +597,7 @@ class Find(db.Model): key = sa.Column(sa.String(200), index=True) value = sa.Column(sa.Text()) findvalue = sa.Column(sa.Text(), index=True) + sortvalue = sa.Column(sa.Text()) def __repr__(self): return '%s=%s' % (self.key, self.findvalue) @@ -782,6 +797,11 @@ def update_sort_table(): sql = [] for col in set(Item.sort_keys)-set(sort_indexes): sql.append('CREATE INDEX ix_sort_{col} ON sort ({col})'.format(col=col)) + if not 'sortvalue' in db.get_table_columns('find'): + create_table = str(CreateTable(Find.__table__).compile(db.engine)).split('\n') + col = 'sortvalue' + add = [r for r in create_table if '\t%s ' % col in r][0].strip()[:-1] + sql.append('ALTER TABLE find ADD '+add) if sql: with db.session() as s: for q in sql: diff --git a/oml/settings.py b/oml/settings.py index e84a833..80a2e16 100644 --- a/oml/settings.py +++ b/oml/settings.py @@ -99,4 +99,4 @@ USER_AGENT = 'OpenMediaLibrary/%s' % VERSION DEBUG_HTTP = server.get('debug_http', False) -DB_VERSION = 9 +DB_VERSION = 10 diff --git a/oml/setup.py b/oml/setup.py index b2cb0a5..c476452 100644 --- a/oml/setup.py +++ b/oml/setup.py @@ -77,6 +77,7 @@ CREATE TABLE find ( "key" VARCHAR(200), value TEXT, findvalue TEXT, + sortvalue TEXT, PRIMARY KEY (id), FOREIGN KEY(item_id) REFERENCES item (id) ); diff --git a/oml/update.py b/oml/update.py index c52a64f..8a5c55c 100644 --- a/oml/update.py +++ b/oml/update.py @@ -307,6 +307,8 @@ class Update(Thread): db_version = migrate_7() if db_version < 9: db_version = migrate_8() + if db_version < 10: + db_version = migrate_10() settings.server['db_version'] = settings.DB_VERSION def run(self): @@ -450,3 +452,30 @@ def migrate_8(): session.delete(peer) session.commit() return 8 + +def migrate_10(): + with db.session() as session: + from item.models import Item, Find + from utils import get_by_id + from item.person import get_sort_name + import unicodedata + sort_names = {} + for f in Find.query.filter(Find.key.in_(Item.sort_keys)): + if f.key in Item.filter_keys: + sort_type = get_by_id(settings.config['itemKeys'], f.key).get('sortType') + if sort_type == 'person': + if f.value in sort_names: + f.sortvalue = sort_names[f.value] + else: + f.sortvalue = sort_names[f.value] = get_sort_name(f.value) + else: + f.sortvalue = f.value + if f.sortvalue: + f.sortvalue = unicodedata.normalize('NFKD', f.sortvalue).lower() + else: + f.sortvalue = None + else: + f.sortvalue = None + session.add(f) + session.commit() + return 10