0x2620
/
openmedialibrary
1
0
Fork 0
Code Issues 131 Pull Requests Releases Wiki Activity

performance

This commit is contained in:
j 2014-05-27 01:45:29 +02:00
parent d451eb3063
commit 21d6324eb6
6 changed files with 47 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
oml/commands.py
View File

@ -98,11 +98,9 @@ class PostUpdate(Command):
changed = True
if changed:
f.save()
if old <= '20140526-117-39ea043' and new > '20140526-117-39ea043':
if old <= '20140526-118-d451eb3' and new > '20140526-118-d451eb3':
import item.models
for i in item.models.Item.query:
i.update_lists()
item.models.db.session.commit()
item.models.Find.query.filter_by(key='list').delete()
class Setup(Command):
"""

19
oml/item/api.py
View File

@ -87,15 +87,9 @@ def find(data):
j = i.json()
response['items'].append({k:j[k] for k in j if not data['keys'] or k in data['keys']})
else:
key = 'stats:' + hashlib.sha1(json.dumps(data)).hexdigest()
stats = state.cache.get(key)
if stats is None:
stats = {}
size = [i.info.get('size', 0) for i in q['qs'].join(models.Sort).options(load_only('id', 'info'))]
stats['items'] = len(size)
stats['size'] = sum(size)
state.cache.set(key, stats, ttl=60)
response = stats
size = [i.info.get('size', 0) for i in q['qs'].join(models.Sort).options(load_only('id', 'info'))]
response['items'] = len(size)
response['size'] = sum(size)
return response
actions.register(find)
@ -154,9 +148,14 @@ def remove(data):
if 'ids' in data and data['ids']:
for i in models.Item.query.filter(models.Item.id.in_(data['ids'])):
i.remove_file()
return {}
return {
'items': []
}
actions.register(remove, cache=False)
def autocomplete(data):
return {}
actions.register(remove, cache=False)
def findMetadata(data):
'''

27
oml/item/models.py
View File

@ -148,6 +148,10 @@ class Item(db.Model):
value = map(get_sort_name, value)
value = ox.sort_string(u'\n'.join(value))
elif sort_type == 'title':
if isinstance(value, dict):
value = value.values()
if isinstance(value, list):
value = u''.join(value)
value = utils.sort_title(value).lower()
else:
if isinstance(value, list):
@ -176,6 +180,8 @@ class Item(db.Model):
if value: value = value[0]
if value:
Find.query.filter_by(item_id=self.id, key=key['id']).delete()
if isinstance(value, dict):
value = ' '.join(value.values())
if not isinstance(value, list):
value = [value]
for v in value:
@ -185,24 +191,6 @@ class Item(db.Model):
if f:
db.session.delete(f)
def update_lists(self):
Find.query.filter_by(item_id=self.id, key='list').delete()
for p in self.users:
f = Find()
f.item_id = self.id
f.key = 'list'
if p.id == settings.USER_ID:
f.findvalue = f.value = ':'
else:
f.findvalue = f.value = '%s:' % p.id
db.session.add(f)
for l in self.lists:
f = Find()
f.item_id = self.id
f.key = 'list'
f.findvalue = f.value = l.find_id
db.session.add(f)
def update(self):
for key in ('mediastate', 'coverRatio', 'previewRatio'):
if key in self.meta:
@ -220,7 +208,6 @@ class Item(db.Model):
self.meta.update(Metadata.load(*self.meta['primaryid']))
self.update_sort()
self.update_find()
self.update_lists()
#self.modified = datetime.utcnow()
self.save()
@ -457,7 +444,7 @@ class Find(db.Model):
item = db.relationship('Item', backref=db.backref('find', lazy='dynamic'))
key = db.Column(db.String(200), index=True)
value = db.Column(db.Text())
findvalue = db.Column(db.Text())
findvalue = db.Column(db.Text(), index=True)
def __repr__(self):
return (u'%s=%s' % (self.key, self.findvalue)).encode('utf-8')

15
oml/media/pdf.py
View File

@ -121,11 +121,6 @@ def info(pdf):
except:
logger.debug('FAILED TO PARSE %s', pdf, exc_info=1)
if 'identifier' in data:
value = normalize_isbn(data['identifier'])
if stdnum.isbn.is_valid(value):
data['isbn'] = [value]
del data['identifier']
'''
cmd = ['pdfinfo', pdf]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@ -139,6 +134,16 @@ def info(pdf):
if not data[key]:
del data[key]
'''
if 'identifier' in data:
value = normalize_isbn(data['identifier'])
if stdnum.isbn.is_valid(value):
data['isbn'] = [value]
del data['identifier']
for key, value in data:
if isinstance(value, dict):
value = ' '.join(value.values())
data[key] = value
text = extract_text(pdf)
data['textsize'] = len(text)
if settings.server['extract_text']:

40
oml/queryparser.py
View File

@ -108,40 +108,35 @@ class Parser(object):
q = get_operator(op)(self._find.findvalue, v)
if k != '*':
q &= (self._find.key == k)
self._joins.append(self._find)
if exclude:
q = ~q
return q
elif k == 'list':
nickname, name = v.split(':', 1)
if nickname:
p = self._user.query.filter_by(nickname=nickname).first()
if p:
v = '%s:%s' % (p.id, name)
u = self._user.query.filter_by(nickname=nickname).first()
else:
p = self._user.query.filter_by(id=settings.USER_ID).first()
v = ':%s' % name
u = self._user.query.filter_by(id=settings.USER_ID).first()
if name:
l = self._list.query.filter_by(user_id=p.id, name=name).first()
l = self._list.query.filter_by(user_id=u.id, name=name).first()
else:
l = None
if l:
v = l.find_id
if l and l.type == 'smart':
if not l:
if not u:
q = (self._user.id == 0)
else:
q = (self._user.id == u.id)
self._joins.append(self._model.users)
elif l.type == 'smart':
data = l._query
q = self.parse_conditions(data.get('conditions', []),
data.get('operator', '&'))
else:
if exclude:
q = (self._find.key == 'list') & (self._find.findvalue == v)
ids = [i.id
for i in self._model.query.join(self._find).filter(q).group_by(self._model.id).options(load_only('id'))]
if ids:
q = ~self._model.id.in_(ids)
else:
q = (self._model.id != 0)
else:
q = (self._find.findvalue == v) & (self._find.key == 'list')
q = (self._list.id == l.id)
self._joins.append(self._list.items)
if exclude:
q = ~q
return q
elif key_type == 'date':
def parse_date(d):
@ -228,10 +223,13 @@ class Parser(object):
#join query with operator
qs = self._model.query
#only include items that have hard metadata
self._joins = []
conditions = self.parse_conditions(data.get('query', {}).get('conditions', []),
data.get('query', {}).get('operator', '&'))
for c in conditions:
qs = qs.join(self._find).filter(c)
if self._joins:
qs = qs.join(self._joins.pop(0))
qs = qs.filter(c)
qs = qs.group_by(self._model.id)
return qs

14
oml/user/models.py
View File

@ -108,8 +108,6 @@ class User(db.Model):
i.users.remove(self)
if not i.users:
i.delete()
else:
i.update_lists()
Changelog.query.filter_by(user_id=self.id).delete()
self.save()
if was_peering:
@ -219,12 +217,6 @@ class List(db.Model):
i.update()
db.session.add(self)
db.session.commit()
for item_id in items:
i = Item.get(item_id)
if i:
i.update_lists()
db.session.add(i)
db.session.commit()
if self.user_id == settings.USER_ID:
Changelog.record(self.user, 'addlistitems', self.name, items)
@ -237,12 +229,6 @@ class List(db.Model):
i.update()
db.session.add(self)
db.session.commit()
for item_id in items:
i = Item.get(item_id)
i.update_lists()
db.session.add(i)
db.session.commit()
db.session.commit()
if self.user_id == settings.USER_ID:
Changelog.record(self.user, 'removelistitems', self.name, items)