diff --git a/oml/commands.py b/oml/commands.py index 8795776..795e480 100644 --- a/oml/commands.py +++ b/oml/commands.py @@ -98,11 +98,9 @@ class PostUpdate(Command): changed = True if changed: f.save() - if old <= '20140526-117-39ea043' and new > '20140526-117-39ea043': + if old <= '20140526-118-d451eb3' and new > '20140526-118-d451eb3': import item.models - for i in item.models.Item.query: - i.update_lists() - item.models.db.session.commit() + item.models.Find.query.filter_by(key='list').delete() class Setup(Command): """ diff --git a/oml/item/api.py b/oml/item/api.py index 2596ad7..8637d74 100644 --- a/oml/item/api.py +++ b/oml/item/api.py @@ -87,15 +87,9 @@ def find(data): j = i.json() response['items'].append({k:j[k] for k in j if not data['keys'] or k in data['keys']}) else: - key = 'stats:' + hashlib.sha1(json.dumps(data)).hexdigest() - stats = state.cache.get(key) - if stats is None: - stats = {} - size = [i.info.get('size', 0) for i in q['qs'].join(models.Sort).options(load_only('id', 'info'))] - stats['items'] = len(size) - stats['size'] = sum(size) - state.cache.set(key, stats, ttl=60) - response = stats + size = [i.info.get('size', 0) for i in q['qs'].join(models.Sort).options(load_only('id', 'info'))] + response['items'] = len(size) + response['size'] = sum(size) return response actions.register(find) @@ -154,9 +148,14 @@ def remove(data): if 'ids' in data and data['ids']: for i in models.Item.query.filter(models.Item.id.in_(data['ids'])): i.remove_file() - return {} + return { + 'items': [] + } actions.register(remove, cache=False) +def autocomplete(data): + return {} +actions.register(remove, cache=False) def findMetadata(data): ''' diff --git a/oml/item/models.py b/oml/item/models.py index 7e74255..dd91a44 100644 --- a/oml/item/models.py +++ b/oml/item/models.py @@ -148,6 +148,10 @@ class Item(db.Model): value = map(get_sort_name, value) value = ox.sort_string(u'\n'.join(value)) elif sort_type == 'title': + if isinstance(value, dict): + value = value.values() + if isinstance(value, list): + value = u''.join(value) value = utils.sort_title(value).lower() else: if isinstance(value, list): @@ -176,6 +180,8 @@ class Item(db.Model): if value: value = value[0] if value: Find.query.filter_by(item_id=self.id, key=key['id']).delete() + if isinstance(value, dict): + value = ' '.join(value.values()) if not isinstance(value, list): value = [value] for v in value: @@ -185,24 +191,6 @@ class Item(db.Model): if f: db.session.delete(f) - def update_lists(self): - Find.query.filter_by(item_id=self.id, key='list').delete() - for p in self.users: - f = Find() - f.item_id = self.id - f.key = 'list' - if p.id == settings.USER_ID: - f.findvalue = f.value = ':' - else: - f.findvalue = f.value = '%s:' % p.id - db.session.add(f) - for l in self.lists: - f = Find() - f.item_id = self.id - f.key = 'list' - f.findvalue = f.value = l.find_id - db.session.add(f) - def update(self): for key in ('mediastate', 'coverRatio', 'previewRatio'): if key in self.meta: @@ -220,7 +208,6 @@ class Item(db.Model): self.meta.update(Metadata.load(*self.meta['primaryid'])) self.update_sort() self.update_find() - self.update_lists() #self.modified = datetime.utcnow() self.save() @@ -457,7 +444,7 @@ class Find(db.Model): item = db.relationship('Item', backref=db.backref('find', lazy='dynamic')) key = db.Column(db.String(200), index=True) value = db.Column(db.Text()) - findvalue = db.Column(db.Text()) + findvalue = db.Column(db.Text(), index=True) def __repr__(self): return (u'%s=%s' % (self.key, self.findvalue)).encode('utf-8') diff --git a/oml/media/pdf.py b/oml/media/pdf.py index 9bb493e..00f953c 100644 --- a/oml/media/pdf.py +++ b/oml/media/pdf.py @@ -121,11 +121,6 @@ def info(pdf): except: logger.debug('FAILED TO PARSE %s', pdf, exc_info=1) - if 'identifier' in data: - value = normalize_isbn(data['identifier']) - if stdnum.isbn.is_valid(value): - data['isbn'] = [value] - del data['identifier'] ''' cmd = ['pdfinfo', pdf] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -139,6 +134,16 @@ def info(pdf): if not data[key]: del data[key] ''' + if 'identifier' in data: + value = normalize_isbn(data['identifier']) + if stdnum.isbn.is_valid(value): + data['isbn'] = [value] + del data['identifier'] + + for key, value in data: + if isinstance(value, dict): + value = ' '.join(value.values()) + data[key] = value text = extract_text(pdf) data['textsize'] = len(text) if settings.server['extract_text']: diff --git a/oml/queryparser.py b/oml/queryparser.py index 6bc5381..05aa828 100644 --- a/oml/queryparser.py +++ b/oml/queryparser.py @@ -108,40 +108,35 @@ class Parser(object): q = get_operator(op)(self._find.findvalue, v) if k != '*': q &= (self._find.key == k) + self._joins.append(self._find) if exclude: q = ~q return q elif k == 'list': nickname, name = v.split(':', 1) if nickname: - p = self._user.query.filter_by(nickname=nickname).first() - if p: - v = '%s:%s' % (p.id, name) + u = self._user.query.filter_by(nickname=nickname).first() else: - p = self._user.query.filter_by(id=settings.USER_ID).first() - v = ':%s' % name + u = self._user.query.filter_by(id=settings.USER_ID).first() if name: - l = self._list.query.filter_by(user_id=p.id, name=name).first() + l = self._list.query.filter_by(user_id=u.id, name=name).first() else: l = None - if l: - v = l.find_id - if l and l.type == 'smart': + if not l: + if not u: + q = (self._user.id == 0) + else: + q = (self._user.id == u.id) + self._joins.append(self._model.users) + elif l.type == 'smart': data = l._query q = self.parse_conditions(data.get('conditions', []), data.get('operator', '&')) else: - if exclude: - q = (self._find.key == 'list') & (self._find.findvalue == v) - ids = [i.id - for i in self._model.query.join(self._find).filter(q).group_by(self._model.id).options(load_only('id'))] - if ids: - q = ~self._model.id.in_(ids) - else: - q = (self._model.id != 0) - - else: - q = (self._find.findvalue == v) & (self._find.key == 'list') + q = (self._list.id == l.id) + self._joins.append(self._list.items) + if exclude: + q = ~q return q elif key_type == 'date': def parse_date(d): @@ -228,10 +223,13 @@ class Parser(object): #join query with operator qs = self._model.query #only include items that have hard metadata + self._joins = [] conditions = self.parse_conditions(data.get('query', {}).get('conditions', []), data.get('query', {}).get('operator', '&')) for c in conditions: - qs = qs.join(self._find).filter(c) + if self._joins: + qs = qs.join(self._joins.pop(0)) + qs = qs.filter(c) qs = qs.group_by(self._model.id) return qs diff --git a/oml/user/models.py b/oml/user/models.py index 56420da..ac7e4f4 100644 --- a/oml/user/models.py +++ b/oml/user/models.py @@ -108,8 +108,6 @@ class User(db.Model): i.users.remove(self) if not i.users: i.delete() - else: - i.update_lists() Changelog.query.filter_by(user_id=self.id).delete() self.save() if was_peering: @@ -219,12 +217,6 @@ class List(db.Model): i.update() db.session.add(self) db.session.commit() - for item_id in items: - i = Item.get(item_id) - if i: - i.update_lists() - db.session.add(i) - db.session.commit() if self.user_id == settings.USER_ID: Changelog.record(self.user, 'addlistitems', self.name, items) @@ -237,12 +229,6 @@ class List(db.Model): i.update() db.session.add(self) db.session.commit() - for item_id in items: - i = Item.get(item_id) - i.update_lists() - db.session.add(i) - db.session.commit() - db.session.commit() if self.user_id == settings.USER_ID: Changelog.record(self.user, 'removelistitems', self.name, items)