performance

oml/commands.py

@@ -98,11 +98,9 @@ class PostUpdate(Command):
                             changed = True
                     if changed:
                         f.save()
-            if old <= '20140526-117-39ea043' and new > '20140526-117-39ea043':
+            if old <= '20140526-118-d451eb3' and new > '20140526-118-d451eb3':
                 import item.models
-                for i in item.models.Item.query:
+                item.models.Find.query.filter_by(key='list').delete()
-                    i.update_lists()
-                item.models.db.session.commit()
 
 class Setup(Command):
         """

oml/item/api.py

@@ -87,15 +87,9 @@ def find(data):
             j = i.json()
             response['items'].append({k:j[k] for k in j if not data['keys'] or k in data['keys']})
     else:
-        key = 'stats:' + hashlib.sha1(json.dumps(data)).hexdigest()
+        size = [i.info.get('size', 0) for i in q['qs'].join(models.Sort).options(load_only('id', 'info'))]
-        stats = state.cache.get(key)
+        response['items'] = len(size)
-        if stats is None:
+        response['size'] = sum(size)
-            stats = {}
-            size = [i.info.get('size', 0) for i in q['qs'].join(models.Sort).options(load_only('id', 'info'))]
-            stats['items'] = len(size)
-            stats['size'] = sum(size)
-            state.cache.set(key, stats, ttl=60)
-        response = stats
     return response
 actions.register(find)
 
@@ -154,9 +148,14 @@ def remove(data):
     if 'ids' in data and data['ids']:
         for i in models.Item.query.filter(models.Item.id.in_(data['ids'])):
             i.remove_file()
-    return {}
+    return {
+        'items': []
+    }
 actions.register(remove, cache=False)
 
+def autocomplete(data):
+    return {}
+actions.register(remove, cache=False)
 
 def findMetadata(data):
     '''

oml/item/models.py

@@ -148,6 +148,10 @@ class Item(db.Model):
                         value = map(get_sort_name, value)
                         value = ox.sort_string(u'\n'.join(value))
                     elif sort_type == 'title':
+                        if isinstance(value, dict):
+                            value = value.values()
+                        if isinstance(value, list):
+                            value = u''.join(value)
                         value = utils.sort_title(value).lower()
                     else:
                         if isinstance(value, list):
@@ -176,6 +180,8 @@ class Item(db.Model):
                     if value: value = value[0]
                 if value:
                     Find.query.filter_by(item_id=self.id, key=key['id']).delete()
+                    if isinstance(value, dict):
+                        value = ' '.join(value.values())
                     if not isinstance(value, list):
                         value = [value]
                     for v in value:
@@ -185,24 +191,6 @@ class Item(db.Model):
                     if f:
                         db.session.delete(f)
 
-    def update_lists(self):
-        Find.query.filter_by(item_id=self.id, key='list').delete()
-        for p in self.users:
-            f = Find()
-            f.item_id = self.id
-            f.key = 'list'
-            if p.id == settings.USER_ID:
-                f.findvalue = f.value = ':'
-            else:
-                f.findvalue = f.value = '%s:' % p.id
-            db.session.add(f)
-        for l in self.lists:
-            f = Find()
-            f.item_id = self.id
-            f.key = 'list'
-            f.findvalue = f.value = l.find_id
-            db.session.add(f)
-
     def update(self):
         for key in ('mediastate', 'coverRatio', 'previewRatio'):
             if key in self.meta:
@@ -220,7 +208,6 @@ class Item(db.Model):
             self.meta.update(Metadata.load(*self.meta['primaryid']))
         self.update_sort()
         self.update_find()
-        self.update_lists()
         #self.modified = datetime.utcnow()
         self.save()
 
@@ -457,7 +444,7 @@ class Find(db.Model):
     item = db.relationship('Item', backref=db.backref('find', lazy='dynamic'))
     key = db.Column(db.String(200), index=True)
     value = db.Column(db.Text())
-    findvalue = db.Column(db.Text())
+    findvalue = db.Column(db.Text(), index=True)
 
     def __repr__(self):
         return (u'%s=%s' % (self.key, self.findvalue)).encode('utf-8')

oml/media/pdf.py

@@ -121,11 +121,6 @@ def info(pdf):
         except:
             logger.debug('FAILED TO PARSE %s', pdf, exc_info=1)
 
-        if 'identifier' in data:
-            value = normalize_isbn(data['identifier'])
-            if stdnum.isbn.is_valid(value):
-                data['isbn'] = [value]
-                del data['identifier']
     '''
     cmd = ['pdfinfo', pdf]
     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -139,6 +134,16 @@ def info(pdf):
         if not data[key]:
             del data[key]
     '''
+    if 'identifier' in data:
+        value = normalize_isbn(data['identifier'])
+        if stdnum.isbn.is_valid(value):
+            data['isbn'] = [value]
+            del data['identifier']
+
+    for key, value in data:
+        if isinstance(value, dict):
+            value = ' '.join(value.values())
+            data[key] = value
     text = extract_text(pdf)
     data['textsize'] = len(text)
     if settings.server['extract_text']:

oml/queryparser.py

@@ -108,40 +108,35 @@ class Parser(object):
             q = get_operator(op)(self._find.findvalue, v)
             if k != '*':
                 q &= (self._find.key == k)
+            self._joins.append(self._find)
             if exclude:
                 q = ~q
             return q
         elif k == 'list':
             nickname, name = v.split(':', 1)
             if nickname:
-                p = self._user.query.filter_by(nickname=nickname).first()
+                u = self._user.query.filter_by(nickname=nickname).first()
-                if p:
-                    v = '%s:%s' % (p.id, name)
             else:
-                p = self._user.query.filter_by(id=settings.USER_ID).first()
+                u = self._user.query.filter_by(id=settings.USER_ID).first()
-                v = ':%s' % name
             if name:
-                l = self._list.query.filter_by(user_id=p.id, name=name).first()
+                l = self._list.query.filter_by(user_id=u.id, name=name).first()
             else:
                 l = None
-            if l:
+            if not l:
-                v = l.find_id
+                if not u:
-            if l and l.type == 'smart':
+                    q = (self._user.id == 0)
+                else:
+                    q = (self._user.id == u.id)
+                self._joins.append(self._model.users)
+            elif l.type == 'smart':
                 data = l._query
                 q = self.parse_conditions(data.get('conditions', []),
                                     data.get('operator', '&'))
             else:
-                if exclude:
+                q = (self._list.id == l.id)
-                    q = (self._find.key == 'list') & (self._find.findvalue == v)
+                self._joins.append(self._list.items)
-                    ids = [i.id
+            if exclude:
-                        for i in self._model.query.join(self._find).filter(q).group_by(self._model.id).options(load_only('id'))]
+                q = ~q
-                    if ids:
-                        q = ~self._model.id.in_(ids)
-                    else:
-                        q = (self._model.id != 0)
-
-                else:
-                    q = (self._find.findvalue == v) & (self._find.key == 'list')
             return q
         elif key_type == 'date':
             def parse_date(d):
@@ -228,10 +223,13 @@ class Parser(object):
         #join query with operator
         qs = self._model.query
         #only include items that have hard metadata
+        self._joins = []
         conditions = self.parse_conditions(data.get('query', {}).get('conditions', []),
                                      data.get('query', {}).get('operator', '&'))
         for c in conditions:
-            qs = qs.join(self._find).filter(c)
+            if self._joins:
+                qs = qs.join(self._joins.pop(0))
+            qs = qs.filter(c)
         qs = qs.group_by(self._model.id)
         return qs
 

oml/user/models.py

@@ -108,8 +108,6 @@ class User(db.Model):
                 i.users.remove(self)
                 if not i.users:
                     i.delete()
-                else:
-                    i.update_lists()
             Changelog.query.filter_by(user_id=self.id).delete()
             self.save()
             if was_peering:
@@ -219,12 +217,6 @@ class List(db.Model):
                 i.update()
         db.session.add(self)
         db.session.commit()
-        for item_id in items:
-            i = Item.get(item_id)
-            if i:
-                i.update_lists()
-                db.session.add(i)
-        db.session.commit()
         if self.user_id == settings.USER_ID:
             Changelog.record(self.user, 'addlistitems', self.name, items)
 
@@ -237,12 +229,6 @@ class List(db.Model):
             i.update()
         db.session.add(self)
         db.session.commit()
-        for item_id in items:
-            i = Item.get(item_id)
-            i.update_lists()
-            db.session.add(i)
-        db.session.commit()
-        db.session.commit()
         if self.user_id == settings.USER_ID:
             Changelog.record(self.user, 'removelistitems', self.name, items)