local app
This commit is contained in:
parent
d255ed6a03
commit
a775ed3055
7 changed files with 314 additions and 3 deletions
0
__init__.py
Normal file
0
__init__.py
Normal file
123
add_website_as_pdf.py
Normal file
123
add_website_as_pdf.py
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
#!/usr/bin/python3
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from urllib.parse import urlparse
|
||||
from datetime import datetime
|
||||
|
||||
import ox
|
||||
import ox.api
|
||||
import ox.cache
|
||||
|
||||
CHUNK_SIZE = 1024*1024*5
|
||||
|
||||
api = ox.api.signin('https://amp.0x2620.org/api/')
|
||||
|
||||
def url2pdf(url, pdf):
|
||||
cmd = ['chromium-browser', '--headless', '--disable-gpu', '--print-to-pdf=' + pdf, url]
|
||||
cmd += ['--timeout=10000']
|
||||
return subprocess.call(cmd) == 0
|
||||
|
||||
def upload_chunks(api, url, filename, data=None):
|
||||
form = ox.MultiPartForm()
|
||||
if data:
|
||||
for key in data:
|
||||
form.add_field(key, data[key])
|
||||
data = api._json_request(url, form)
|
||||
|
||||
def full_url(path):
|
||||
if path.startswith('/'):
|
||||
u = urlparse(url)
|
||||
path = '%s://%s%s' % (u.scheme, u.netloc, path)
|
||||
return path
|
||||
|
||||
if 'uploadUrl' in data:
|
||||
uploadUrl = full_url(data['uploadUrl'])
|
||||
f = open(filename, 'rb')
|
||||
fsize = os.stat(filename).st_size
|
||||
done = 0
|
||||
start = time.mktime(time.localtime())
|
||||
if 'offset' in data and data['offset'] < fsize:
|
||||
done = data['offset']
|
||||
f.seek(done)
|
||||
resume_offset = done
|
||||
else:
|
||||
resume_offset = 0
|
||||
chunk = f.read(CHUNK_SIZE)
|
||||
fname = os.path.basename(filename)
|
||||
if not isinstance(fname, bytes):
|
||||
fname = fname.encode('utf-8')
|
||||
while chunk:
|
||||
sys.stdout.flush()
|
||||
form = ox.MultiPartForm()
|
||||
form.add_file('chunk', fname, chunk)
|
||||
if len(chunk) < CHUNK_SIZE or f.tell() == fsize:
|
||||
form.add_field('done', '1')
|
||||
form.add_field('offset', str(done))
|
||||
try:
|
||||
data = api._json_request(uploadUrl, form)
|
||||
except KeyboardInterrupt:
|
||||
print("\ninterrupted by user.")
|
||||
sys.exit(1)
|
||||
except:
|
||||
print("uploading chunk failed, will try again in 5 seconds\r", end='')
|
||||
sys.stdout.flush()
|
||||
data = {'result': -1}
|
||||
time.sleep(5)
|
||||
if data and 'status' in data:
|
||||
if data['status']['code'] == 403:
|
||||
print("login required")
|
||||
return False
|
||||
if data['status']['code'] != 200:
|
||||
print("request returned error, will try again in 5 seconds")
|
||||
if DEBUG:
|
||||
print(data)
|
||||
time.sleep(5)
|
||||
if data and data.get('result') == 1:
|
||||
done += len(chunk)
|
||||
if data.get('offset') not in (None, done):
|
||||
print('server offset out of sync, continue from', data['offset'])
|
||||
done = data['offset']
|
||||
f.seek(done)
|
||||
chunk = f.read(CHUNK_SIZE)
|
||||
if data and 'result' in data and data.get('result') == 1:
|
||||
return data.get('id', True)
|
||||
else:
|
||||
return False
|
||||
return False
|
||||
|
||||
def upload_document(pdf):
|
||||
document_id = upload_chunks(api, url, pdf)
|
||||
|
||||
|
||||
def import_url(url):
|
||||
meta = {
|
||||
'description': 'PDF snapshot of %s from %s' % (url, datetime.now().strftime('%Y-%m-%d %H:%M:%S')),
|
||||
'title': 'Untitled'
|
||||
}
|
||||
data = ox.cache.read_url(url, unicode=True)
|
||||
title = re.compile('<title>(.*?)</title>').findall(data)
|
||||
if title:
|
||||
meta['title'] = title[0]
|
||||
author= re.compile('<meta name="author" content="(.*?)"').findall(data)
|
||||
if author:
|
||||
meta['author'] = author
|
||||
fd, pdf = tempfile.mkstemp('.pdf')
|
||||
if url2pdf(url, pdf):
|
||||
url = api.url + 'upload/document/'
|
||||
did = upload_chunks(api, url, pdf, {
|
||||
'filename': meta['title'] + '.pdf'
|
||||
})
|
||||
meta['id'] = did
|
||||
r = api.editDocument(meta)
|
||||
print(r['data']['id'])
|
||||
os.unlink(pdf)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
url = sys.argv[1]
|
||||
import_url(url)
|
||||
28
install.py
28
install.py
|
|
@ -52,6 +52,28 @@ for root, folders, files in os.walk(join(base, 'scripts')):
|
|||
os.unlink(target)
|
||||
os.symlink(rel_src, target)
|
||||
|
||||
# todo
|
||||
# custom python module etc
|
||||
# local_settings.py?
|
||||
if os.path.exists('__init__.py'):
|
||||
# make module available to pandora
|
||||
target = os.path.join('/srv/pandora/pandora/', name)
|
||||
rel_src = os.path.relpath(base, dirname(target))
|
||||
if os.path.exists(target):
|
||||
os.unlink(target)
|
||||
os.symlink(rel_src, target)
|
||||
|
||||
# include module in local settings
|
||||
local_settings_py = '/srv/pandora/pandora/local_settings.py'
|
||||
with open(local_settings_py) as fd:
|
||||
local_settings_changed = False
|
||||
local_settings = fd.read()
|
||||
if 'LOCAL_APPS' not in local_settings:
|
||||
local_settings += '\nLOCAL_APPS = ["%s"]\n' % name
|
||||
local_settings_changed = True
|
||||
else:
|
||||
apps = re.compile('(LOCAL_APPS.*?)\]', re.DOTALL).findall(local_settings)[0]
|
||||
if name not in apps:
|
||||
new_apps = apps.strip() + ',\n"%s"\n' % name
|
||||
local_settings = local_settings.replace(apps, new_apps)
|
||||
local_settings_changed = True
|
||||
if local_settings_changed:
|
||||
with open(local_settings_py, 'w') as fd:
|
||||
fd.write(local_settings)
|
||||
|
|
|
|||
86
static/js/importDocumentsDialog.amp.js
Normal file
86
static/js/importDocumentsDialog.amp.js
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
'use strict';
|
||||
|
||||
pandora.ui.importDocumentsDialog = function() {
|
||||
|
||||
var dialogHeight = 100,
|
||||
dialogWidth = 512 + 16,
|
||||
formWidth = getFormWidth(),
|
||||
|
||||
$button,
|
||||
$content = Ox.Element(),
|
||||
value,
|
||||
$input = [
|
||||
Ox.TextArea({
|
||||
labelWidth: 96,
|
||||
width: 512
|
||||
}).css({
|
||||
margin: '8px'
|
||||
}).bindEvent({
|
||||
change: function(data) {
|
||||
$button.options({disabled: !data.value});
|
||||
value = data.value
|
||||
}
|
||||
}).appendTo($content),
|
||||
],
|
||||
|
||||
that = Ox.Dialog({
|
||||
buttons: [
|
||||
Ox.Button({
|
||||
id: 'cancel',
|
||||
title: Ox._('Cancel')
|
||||
})
|
||||
.bindEvent({
|
||||
click: function() {
|
||||
that.close();
|
||||
}
|
||||
}),
|
||||
$button = Ox.Button({
|
||||
disabled: true,
|
||||
id: 'import',
|
||||
title: Ox._('Import URLs')
|
||||
})
|
||||
.bindEvent({
|
||||
click: importDocuments
|
||||
})
|
||||
],
|
||||
closeButton: true,
|
||||
content: $content,
|
||||
height: dialogHeight,
|
||||
removeOnClose: true,
|
||||
title: Ox._('Import Documents'),
|
||||
width: dialogWidth
|
||||
})
|
||||
.bindEvent({
|
||||
resize: setSize
|
||||
});
|
||||
|
||||
function getFormWidth() {
|
||||
return dialogWidth - 32 - Ox.UI.SCROLLBAR_SIZE;
|
||||
}
|
||||
|
||||
function setSize(data) {
|
||||
dialogHeight = data.height;
|
||||
dialogWidth = data.width;
|
||||
formWidth = getFormWidth();
|
||||
$input.forEach(function($element) {
|
||||
$element.options({width: formWidth});
|
||||
});
|
||||
}
|
||||
|
||||
function importDocuments() {
|
||||
that.options({content: Ox.LoadingScreen().start()});
|
||||
var urls = value.trim().split('\n');
|
||||
pandora.api.importDocuments({
|
||||
urls: urls
|
||||
}, function(result) {
|
||||
if (result.data.taskId) {
|
||||
pandora.wait(result.data.taskId, function(result) {
|
||||
that.close();
|
||||
}
|
||||
} else {
|
||||
that.options({content: 'Failed'});
|
||||
}
|
||||
})
|
||||
}
|
||||
return that;
|
||||
};
|
||||
53
static/js/localInit.amp.js
Normal file
53
static/js/localInit.amp.js
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
pandora.localInit = function() {
|
||||
var plugins = [];
|
||||
|
||||
plugins.push(ExtrasMenu());
|
||||
|
||||
plugins.length && load();
|
||||
|
||||
function load() {
|
||||
patchReload();
|
||||
plugins.forEach(function(plugin) { plugin.load() });
|
||||
}
|
||||
|
||||
function patchReload() {
|
||||
var reload = pandora.$ui.appPanel.reload;
|
||||
pandora.$ui.appPanel.reload = function() {
|
||||
reload();
|
||||
load();
|
||||
}
|
||||
}
|
||||
|
||||
function ExtrasMenu() {
|
||||
var that = {};
|
||||
|
||||
var css = {
|
||||
//margin: '2px',
|
||||
},
|
||||
$item = Ox.MenuButton({
|
||||
items: [
|
||||
].concat(pandora.user.level == 'admin' ? [
|
||||
] : [], [
|
||||
{id: 'import_documents', title: 'Import Documents...'},
|
||||
]),
|
||||
style: 'rounded',
|
||||
title: 'set',
|
||||
tooltip: Ox._('Extras'),
|
||||
type: 'image'
|
||||
}).css(css).bindEvent({
|
||||
click: function(data) {
|
||||
if (data.id == 'import_documents') {
|
||||
pandora.ui.importDocumentsDialog().open()
|
||||
}
|
||||
},
|
||||
}),
|
||||
plugins = [];
|
||||
|
||||
that.load = function() {
|
||||
pandora.$ui.mainMenu.find('.OxExtras').prepend($item);
|
||||
pandora.$ui.extraItem = $item;
|
||||
};
|
||||
return that;
|
||||
}
|
||||
|
||||
}
|
||||
12
tasks.py
Normal file
12
tasks.py
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
import os
|
||||
import subprocess
|
||||
|
||||
from celery.task import task
|
||||
|
||||
base = os.path.dirname(__file__)
|
||||
|
||||
@task(queue="encoding")
|
||||
def import_documents(urls):
|
||||
for url in urls:
|
||||
if url.startswith('http'):
|
||||
subprocess.call(['/srv/pandora/bin/python', os.path.join(base, 'add_website_as_pdf.py'), url])
|
||||
15
views.py
Normal file
15
views.py
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import division, print_function, absolute_import
|
||||
|
||||
from oxdjango.decorators import login_required_json
|
||||
from oxdjango.shortcuts import render_to_json_response
|
||||
from oxdjango.api import actions
|
||||
|
||||
from .tasks import import_documents
|
||||
|
||||
@login_required_json
|
||||
def importDocuments(request, data):
|
||||
t = tasks.import_documents.delay(urls=data['urls'])
|
||||
response['data']['taskId'] = t.task_id
|
||||
return render_to_json_response(response)
|
||||
actions.register(importDocuments)
|
||||
Loading…
Reference in a new issue