From f81fa3a7d643c402dbafc0f4c22b2884f5422241 Mon Sep 17 00:00:00 2001 From: j Date: Thu, 31 May 2018 21:21:34 +0200 Subject: [PATCH 1/5] use ox.api.signin --- ontology/update_keywords.py | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/ontology/update_keywords.py b/ontology/update_keywords.py index f0a5b07..595f8d7 100755 --- a/ontology/update_keywords.py +++ b/ontology/update_keywords.py @@ -5,26 +5,10 @@ import json import sys import ox -import ox.web.auth +import ox.api site = 'pandora.cinemusespace.com' -api = ox.API('https://%s/api/' % site) -update = False -try: - credentials = ox.web.auth.get(site) -except: - credentials = {} - print('Please provide your username and password for %s:' % site) - credentials['username'] = input('Username: ') - credentials['password'] = getpass.getpass('Password: ') - update = True -r = api.signin(**credentials) -if 'errors' in r.get('data', {}): - for kv in r['data']['errors'].items(): - print('%s: %s' % kv) - sys.exit(1) -if update: - ox.web.auth.update(site, credentials) +api = ox.api.signin('https://%s/api/' % site) keywords = set() for annotation in api.findAnnotations({ From de5368e47a2b1477e9835ffd580f558f0c2eac84 Mon Sep 17 00:00:00 2001 From: j Date: Thu, 31 May 2018 21:32:56 +0200 Subject: [PATCH 2/5] get keyword size --- ontology/update.py | 26 +++++++++++++------------- ontology/update_keywords.py | 14 +++++++------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/ontology/update.py b/ontology/update.py index 47125c6..5b8aa64 100755 --- a/ontology/update.py +++ b/ontology/update.py @@ -3,10 +3,15 @@ import json import os from collections import defaultdict +base = os.path.abspath(os.path.dirname(__file__)) + +keywords = json.load(open(os.path.join(base, 'keywords.json'))) +ontology = json.load(open(os.path.join(base, 'ontology.json'))) + def find_path(parent, root=None, path=None): if root is None: root = ontology - if path == None: + if path is None: path = [] for key in root: if key == parent: @@ -16,32 +21,27 @@ def find_path(parent, root=None, path=None): if r: return r -def get_node(name, children): +def get_node(name, children, parent=None): node = { "size": len(children) + 100, "name": name, - "children": [get_node(child, children[child]) for child in children] + "children": [get_node(child, children[child], name) for child in children] } if not node['children']: del node['children'] + key = '%s: %s' % (parent, name) + if key in keywords: + node['size'] = keywords[key] return node if __name__ == '__main__': - base = os.path.abspath(os.path.dirname(__file__)) os.chdir(base) - keywords = json.load(open('keywords.json')) - ontology = json.load(open('ontology.json')) - tree = defaultdict(dict) for keyword in keywords: - if ': ' not in keyword: - parent = 'other' - child = keyword - else: - parent, child = keyword.split(': ') + parent, child = keyword.split(': ') path = find_path(parent) if path: p = tree @@ -61,7 +61,7 @@ if __name__ == '__main__': } for name in tree: children = tree[name] - child = get_node(name, tree[name]) + child = get_node(name, tree[name], name) sized_ontology['children'].append(child) with open('../static/ontology/sized_ontology.json', 'w') as fd: diff --git a/ontology/update_keywords.py b/ontology/update_keywords.py index 595f8d7..0a16852 100755 --- a/ontology/update_keywords.py +++ b/ontology/update_keywords.py @@ -1,8 +1,6 @@ #!/usr/bin/python3 - -import getpass +import collections import json -import sys import ox import ox.api @@ -10,7 +8,7 @@ import ox.api site = 'pandora.cinemusespace.com' api = ox.api.signin('https://%s/api/' % site) -keywords = set() +keywords = collections.Counter() for annotation in api.findAnnotations({ 'query': { 'conditions': [{ @@ -23,8 +21,10 @@ for annotation in api.findAnnotations({ 'keys': ['id', 'in', 'out', 'value', 'user', 'created'], 'range': [0, 500000] })['data']['items']: - keywords.add(annotation['value']) - + keyword = annotation['value'] + if ': ' not in keyword: + keyword = 'other: ' + keyword + keywords[keyword] += 1 with open('keywords.json', 'w') as fd: - json.dump(list(sorted(keywords)), fd, indent=4) + json.dump(keywords, fd, indent=4, ensure_ascii=False, sort_keys=True) From e3b4cda2a6d2f1db2b8ebcb566ac81962ba92f28 Mon Sep 17 00:00:00 2001 From: j Date: Thu, 31 May 2018 21:37:16 +0200 Subject: [PATCH 3/5] include missing --- ontology/update.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ontology/update.py b/ontology/update.py index 5b8aa64..b0acdab 100755 --- a/ontology/update.py +++ b/ontology/update.py @@ -51,6 +51,9 @@ if __name__ == '__main__': p = p[part] p[child] = {} else: + if parent not in tree['missing']: + tree['missing'][parent] = {} + tree['missing'][parent][child] = {} print('missing root - %s: %s' % (parent, child)) #print(json.dumps(tree, indent=4, sort_keys=True)) From 6150d3323d45fde01e5938f5856d08bf6205000d Mon Sep 17 00:00:00 2001 From: j Date: Mon, 6 Aug 2018 19:19:20 +0100 Subject: [PATCH 4/5] other --- ontology/update.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ontology/update.py b/ontology/update.py index b0acdab..de51c31 100755 --- a/ontology/update.py +++ b/ontology/update.py @@ -41,7 +41,11 @@ if __name__ == '__main__': tree = defaultdict(dict) for keyword in keywords: - parent, child = keyword.split(': ') + if ': ' not in keyword: + parent = 'other' + child = keyword + else: + parent, child = keyword.split(': ', 1) path = find_path(parent) if path: p = tree From eb9edf2da785e31e0e5ffe64fff927e24b1c3cec Mon Sep 17 00:00:00 2001 From: j Date: Mon, 6 Aug 2018 19:19:57 +0100 Subject: [PATCH 5/5] write to public path --- ontology/ontology.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ontology/ontology.py b/ontology/ontology.py index 1a3945a..8be2173 100644 --- a/ontology/ontology.py +++ b/ontology/ontology.py @@ -94,4 +94,4 @@ for string in sorted(strings): ) html.append(': '.join(parts_)) template = open('ontology_template.html').read() -open('ontology.html', 'w').write(template.replace('{TREE}', '
\n '.join(html))) +open('../static/ontology/ontology.html', 'w').write(template.replace('{TREE}', '
\n '.join(html)))