0x2620
/
openmedialibrary
1
0
Fork 0
Code Issues 131 Pull Requests Releases Wiki Activity

faster filters

This commit is contained in:
j 2016-01-25 23:02:04 +05:30
parent 27908619ef
commit 2b3edd40dc
5 changed files with 68 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
oml/item/api.py
View File

@ -44,43 +44,42 @@ def find(data):
if 'group' in q:
names = {}
groups = {}
sortvalues = {}
_keydata = data.copy()
for key in ('range', 'position'):
for key in ('range', 'position', 'positions'):
if key in _keydata:
del _keydata[key]
key = 'group:' + hashlib.sha1(json.dumps(_keydata).encode('utf-8')).hexdigest()
key = 'group:' + hashlib.sha1(json.dumps(_keydata, sort_keys=True).encode('utf-8')).hexdigest()
g = state.cache.get(key)
if g is None:
items = q['qs'].options(load_only('id'))
qs = models.Find.query.filter_by(key=q['group'])
if items.first():
qs = qs.filter(models.Find.item_id.in_(items))
for f in qs.values('value', 'findvalue'):
for f in qs.values('value', 'findvalue', 'sortvalue'):
value = f[0]
findvalue = f[1]
sortvalue = f[2]
if findvalue not in groups:
groups[findvalue] = 0
groups[findvalue] += 1
names[findvalue] = value
sortvalues[value] = sortvalue
g = [{'name': names[k], 'items': groups[k]} for k in groups]
else:
g = []
if 'sort' in q:
sort_type = utils.get_by_id(settings.config['itemKeys'], q['group']).get('sortType')
reverse = q['sort'][0]['operator'] == '-'
def _sort_key(k):
if sort_type == 'person':
name = get_sort_name(k['name'])
else:
name = k['name']
if isinstance(name, str):
name = unicodedata.normalize('NFKD', name).lower()
name = sortvalues[k['name']]
if not name:
name = '\uffff' if not reverse else ''
items = k['items']
if q['sort'][0]['key'] == 'name':
v = (name, items)
else:
v = (-items, name)
return v
reverse = q['sort'][0]['operator'] == '-'
if q['sort'][0]['key'] == 'items':
reverse = not reverse
g.sort(key=_sort_key, reverse=reverse)

34
oml/item/models.py
View File

@ -171,12 +171,8 @@ class Item(db.Model):
value = list(map(get_sort_name, value))
value = ox.sort_string('\n'.join(value)).lower()
elif sort_type == 'title':
if isinstance(value, dict):
value = list(value.values())
if isinstance(value, list):
value = ''.join(value)
value = ox.get_sort_title(value)
value = utils.sort_title(value).lower()
value = self.get_sorttitle().lower()
value = utils.sort_title(value)
elif sort_type == 'boolean':
pass
else:
@ -204,6 +200,16 @@ class Item(db.Model):
if f.value != v:
f.findvalue = unicodedata.normalize('NFKD', v).lower()
f.value = v
if k in self.filter_keys:
sort_type = utils.get_by_id(settings.config['itemKeys'], k).get('sortType')
if sort_type == 'person':
f.sortvalue = get_sort_name(f.value)
else:
f.sortvalue = f.value
if f.sortvalue:
f.sortvalue = unicodedata.normalize('NFKD', f.sortvalue).lower()
else:
f.sortvalue = None
state.db.session.add(f)
keys = []
@ -285,7 +291,8 @@ class Item(db.Model):
'series',
'sharemetadata',
'tableofcontents',
'title'
'title',
'sorttitle'
)
def update_metadata(self, data, modified=None):
@ -320,6 +327,13 @@ class Item(db.Model):
def get_hash(self):
return utils.get_meta_hash(self.meta.copy())
def get_sorttitle(self):
title = self.meta.get('sorttitle')
if title is None:
title = self.meta.get('title', 'Untitled')
title = ox.get_sort_title(title)
return title
def sync_metadata(self):
from user.models import Metadata
if self.meta.get('sharemetadata'):
@ -583,6 +597,7 @@ class Find(db.Model):
key = sa.Column(sa.String(200), index=True)
value = sa.Column(sa.Text())
findvalue = sa.Column(sa.Text(), index=True)
sortvalue = sa.Column(sa.Text())
def __repr__(self):
return '%s=%s' % (self.key, self.findvalue)
@ -782,6 +797,11 @@ def update_sort_table():
sql = []
for col in set(Item.sort_keys)-set(sort_indexes):
sql.append('CREATE INDEX ix_sort_{col} ON sort ({col})'.format(col=col))
if not 'sortvalue' in db.get_table_columns('find'):
create_table = str(CreateTable(Find.__table__).compile(db.engine)).split('\n')
col = 'sortvalue'
add = [r for r in create_table if '\t%s ' % col in r][0].strip()[:-1]
sql.append('ALTER TABLE find ADD '+add)
if sql:
with db.session() as s:
for q in sql:

2
oml/settings.py
View File

@ -99,4 +99,4 @@ USER_AGENT = 'OpenMediaLibrary/%s' % VERSION
DEBUG_HTTP = server.get('debug_http', False)
DB_VERSION = 9
DB_VERSION = 10

1
oml/setup.py
View File

@ -77,6 +77,7 @@ CREATE TABLE find (
"key" VARCHAR(200),
value TEXT,
findvalue TEXT,
sortvalue TEXT,
PRIMARY KEY (id),
FOREIGN KEY(item_id) REFERENCES item (id)
);

29
oml/update.py
View File

@ -307,6 +307,8 @@ class Update(Thread):
db_version = migrate_7()
if db_version < 9:
db_version = migrate_8()
if db_version < 10:
db_version = migrate_10()
settings.server['db_version'] = settings.DB_VERSION
def run(self):
@ -450,3 +452,30 @@ def migrate_8():
session.delete(peer)
session.commit()
return 8
def migrate_10():
with db.session() as session:
from item.models import Item, Find
from utils import get_by_id
from item.person import get_sort_name
import unicodedata
sort_names = {}
for f in Find.query.filter(Find.key.in_(Item.sort_keys)):
if f.key in Item.filter_keys:
sort_type = get_by_id(settings.config['itemKeys'], f.key).get('sortType')
if sort_type == 'person':
if f.value in sort_names:
f.sortvalue = sort_names[f.value]
else:
f.sortvalue = sort_names[f.value] = get_sort_name(f.value)
else:
f.sortvalue = f.value
if f.sortvalue:
f.sortvalue = unicodedata.normalize('NFKD', f.sortvalue).lower()
else:
f.sortvalue = None
else:
f.sortvalue = None
session.add(f)
session.commit()
return 10