meta
This commit is contained in:
parent
edd42dfd76
commit
d385853186
48 changed files with 1344 additions and 488 deletions
|
|
@ -25,44 +25,21 @@ def find(request):
|
|||
data = json.loads(request.form['data']) if 'data' in request.form else {}
|
||||
q = query.parse(data)
|
||||
if 'group' in q:
|
||||
response['items'] = []
|
||||
'''
|
||||
items = 'items'
|
||||
item_qs = q['qs']
|
||||
order_by = query.order_by_group(q)
|
||||
qs = models.Facet.objects.filter(key=q['group']).filter(item__id__in=item_qs)
|
||||
qs = qs.values('value').annotate(items=Count('id')).order_by(*order_by)
|
||||
|
||||
if 'positions' in q:
|
||||
response['positions'] = {}
|
||||
ids = [j['value'] for j in qs]
|
||||
response['positions'] = utils.get_positions(ids, q['positions'])
|
||||
elif 'range' in data:
|
||||
qs = qs[q['range'][0]:q['range'][1]]
|
||||
response['items'] = [{'name': i['value'], 'items': i[items]} for i in qs]
|
||||
else:
|
||||
response['items'] = qs.count()
|
||||
'''
|
||||
_g = {}
|
||||
key = utils.get_by_id(settings.config['itemKeys'], q['group'])
|
||||
for item in q['qs']:
|
||||
i = item.json()
|
||||
if q['group'] in i:
|
||||
values = i[q['group']]
|
||||
if isinstance(values, basestring):
|
||||
values = [values]
|
||||
for value in values:
|
||||
if key.get('filterMap') and value:
|
||||
value = re.compile(key.get('filterMap')).findall(value)
|
||||
if value:
|
||||
value = value[0]
|
||||
else:
|
||||
continue
|
||||
if value not in _g:
|
||||
_g[value] = 0
|
||||
_g[value] += 1
|
||||
g = [{'name': k, 'items': _g[k]} for k in _g]
|
||||
if 'sort' in data: # parse adds default sort to q!
|
||||
names = {}
|
||||
groups = {}
|
||||
items = [i.id for i in q['qs']]
|
||||
qs = models.Find.query.filter_by(key=q['group'])
|
||||
if items:
|
||||
qs = qs.filter(models.Find.item_id.in_(items))
|
||||
for f in qs.values('value', 'findvalue'):
|
||||
value = f[0]
|
||||
findvalue = f[1]
|
||||
if findvalue not in groups:
|
||||
groups[findvalue] = 0
|
||||
groups[findvalue] += 1
|
||||
names[findvalue] = value
|
||||
g = [{'name': names[k], 'items': groups[k]} for k in groups]
|
||||
if 'sort' in q:
|
||||
g.sort(key=lambda k: k[q['sort'][0]['key']])
|
||||
if q['sort'][0]['operator'] == '-':
|
||||
g.reverse()
|
||||
|
|
@ -90,9 +67,11 @@ def find(request):
|
|||
j = i.json()
|
||||
response['items'].append({k:j[k] for k in j if not data['keys'] or k in data['keys']})
|
||||
else:
|
||||
items = [i.json() for i in q['qs']]
|
||||
response['items'] = len(items)
|
||||
response['size'] = sum([i.get('size',0) for i in items])
|
||||
response['items'] = q['qs'].count()
|
||||
#from sqlalchemy.sql import func
|
||||
#models.db.session.query(func.sum(models.Item.sort_size).label("size"))
|
||||
#response['size'] = x.scalar()
|
||||
response['size'] = sum([i.sort_size or 0 for i in q['qs']])
|
||||
return response
|
||||
actions.register(find)
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from user.models import User
|
|||
from person import get_sort_name
|
||||
|
||||
import media
|
||||
from meta import scraper
|
||||
import meta
|
||||
|
||||
import state
|
||||
import utils
|
||||
|
|
@ -201,6 +201,13 @@ class Item(db.Model):
|
|||
setattr(self, 'sort_%s' % key['id'], value)
|
||||
|
||||
def update_find(self):
|
||||
|
||||
def add(k, v):
|
||||
f = Find(item_id=self.id, key=k)
|
||||
f.findvalue = v.lower().strip()
|
||||
f.value = v
|
||||
db.session.add(f)
|
||||
|
||||
for key in config['itemKeys']:
|
||||
if key.get('find') or key.get('filter'):
|
||||
value = self.json().get(key['id'], None)
|
||||
|
|
@ -208,16 +215,11 @@ class Item(db.Model):
|
|||
value = re.compile(key.get('filterMap')).findall(value)
|
||||
if value: value = value[0]
|
||||
if value:
|
||||
if isinstance(value, list):
|
||||
Find.query.filter_by(item_id=self.id, key=key['id']).delete()
|
||||
for v in value:
|
||||
f = Find(item_id=self.id, key=key['id'])
|
||||
f.value = v.lower()
|
||||
db.session.add(f)
|
||||
else:
|
||||
f = Find.get_or_create(self.id, key['id'])
|
||||
f.value = value.lower()
|
||||
db.session.add(f)
|
||||
Find.query.filter_by(item_id=self.id, key=key['id']).delete()
|
||||
if not isinstance(value, list):
|
||||
value = [value]
|
||||
for v in value:
|
||||
add(key['id'], v)
|
||||
else:
|
||||
f = Find.get(self.id, key['id'])
|
||||
if f:
|
||||
|
|
@ -313,16 +315,9 @@ class Item(db.Model):
|
|||
def scrape(self):
|
||||
mainid = self.meta.get('mainid')
|
||||
print 'scrape', mainid, self.meta.get(mainid)
|
||||
if mainid == 'olid':
|
||||
scraper.update_ol(self)
|
||||
scraper.add_lookupbyisbn(self)
|
||||
elif mainid in ('isbn10', 'isbn13'):
|
||||
scraper.add_lookupbyisbn(self)
|
||||
elif mainid == 'lccn':
|
||||
import meta.lccn
|
||||
info = meta.lccn.info(self.meta[mainid])
|
||||
for key in info:
|
||||
self.meta[key] = info[key]
|
||||
if mainid:
|
||||
m = meta.lookup(mainid, self.meta[mainid])
|
||||
self.meta.update(m)
|
||||
else:
|
||||
print 'FIX UPDATE', mainid
|
||||
self.update()
|
||||
|
|
@ -380,7 +375,7 @@ for key in config['itemKeys']:
|
|||
|
||||
Item.id_keys = ['isbn10', 'isbn13', 'lccn', 'olid', 'oclc']
|
||||
Item.item_keys = config['itemKeys']
|
||||
Item.filter_keys = []
|
||||
Item.filter_keys = [k['id'] for k in config['itemKeys'] if k.get('filter')]
|
||||
|
||||
class Find(db.Model):
|
||||
id = db.Column(db.Integer(), primary_key=True)
|
||||
|
|
@ -388,9 +383,10 @@ class Find(db.Model):
|
|||
item = db.relationship('Item', backref=db.backref('find', lazy='dynamic'))
|
||||
key = db.Column(db.String(200), index=True)
|
||||
value = db.Column(db.Text())
|
||||
findvalue = db.Column(db.Text())
|
||||
|
||||
def __repr__(self):
|
||||
return (u'%s=%s' % (self.key, self.value)).encode('utf-8')
|
||||
return (u'%s=%s' % (self.key, self.findvalue)).encode('utf-8')
|
||||
|
||||
@classmethod
|
||||
def get(cls, item, key):
|
||||
|
|
|
|||
|
|
@ -11,7 +11,8 @@ from sqlalchemy.sql.expression import nullslast
|
|||
def parse(data):
|
||||
query = {}
|
||||
query['range'] = [0, 100]
|
||||
query['sort'] = [{'key':'title', 'operator':'+'}]
|
||||
if not 'group' in data:
|
||||
query['sort'] = [{'key':'title', 'operator':'+'}]
|
||||
for key in ('keys', 'group', 'list', 'range', 'sort', 'query'):
|
||||
if key in data:
|
||||
query[key] = data[key]
|
||||
|
|
@ -25,9 +26,7 @@ def parse(data):
|
|||
query['qs'] = models.Item.query.join(
|
||||
models.Find, models.Find.item_id==models.Item.id).filter(
|
||||
models.Find.value.contains(value))
|
||||
if 'group' in query:
|
||||
query['qs'] = order_by_group(query['qs'], query['sort'])
|
||||
else:
|
||||
if not 'group' in query:
|
||||
query['qs'] = order(query['qs'], query['sort'])
|
||||
return query
|
||||
|
||||
|
|
@ -61,23 +60,3 @@ def order(qs, sort, prefix='sort_'):
|
|||
order_by = _order_by
|
||||
qs = qs.order_by(*order_by)
|
||||
return qs
|
||||
|
||||
def order_by_group(qs, sort):
|
||||
return qs
|
||||
if 'sort' in query:
|
||||
if len(query['sort']) == 1 and query['sort'][0]['key'] == 'items':
|
||||
order_by = query['sort'][0]['operator'] == '-' and '-items' or 'items'
|
||||
if query['group'] == "year":
|
||||
secondary = query['sort'][0]['operator'] == '-' and '-sortvalue' or 'sortvalue'
|
||||
order_by = (order_by, secondary)
|
||||
elif query['group'] != "keyword":
|
||||
order_by = (order_by, 'sortvalue')
|
||||
else:
|
||||
order_by = (order_by, 'value')
|
||||
else:
|
||||
order_by = query['sort'][0]['operator'] == '-' and '-sortvalue' or 'sortvalue'
|
||||
order_by = (order_by, 'items')
|
||||
else:
|
||||
order_by = ('-sortvalue', 'items')
|
||||
return order_by
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue