From a775ed3055a787d532f506672c8ded3113f27d6b Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 5 Nov 2018 20:45:42 +0100
Subject: [PATCH] local app

---
 __init__.py                            |   0
 add_website_as_pdf.py                  | 123 +++++++++++++++++++++++++
 install.py                             |  28 +++++-
 static/js/importDocumentsDialog.amp.js |  86 +++++++++++++++++
 static/js/localInit.amp.js             |  53 +++++++++++
 tasks.py                               |  12 +++
 views.py                               |  15 +++
 7 files changed, 314 insertions(+), 3 deletions(-)
 create mode 100644 __init__.py
 create mode 100644 add_website_as_pdf.py
 create mode 100644 static/js/importDocumentsDialog.amp.js
 create mode 100644 static/js/localInit.amp.js
 create mode 100644 tasks.py
 create mode 100644 views.py

diff --git a/__init__.py b/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/add_website_as_pdf.py b/add_website_as_pdf.py
new file mode 100644
index 0000000..3d82f62
--- /dev/null
+++ b/add_website_as_pdf.py
@@ -0,0 +1,123 @@
+#!/usr/bin/python3
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import time
+from urllib.parse import urlparse
+from datetime import datetime
+
+import ox
+import ox.api
+import ox.cache
+
+CHUNK_SIZE = 1024*1024*5
+
+api = ox.api.signin('https://amp.0x2620.org/api/')
+
+def url2pdf(url, pdf):
+    cmd = ['chromium-browser', '--headless', '--disable-gpu', '--print-to-pdf=' + pdf, url]
+    cmd += ['--timeout=10000']
+    return subprocess.call(cmd) == 0
+
+def upload_chunks(api, url, filename, data=None):
+    form = ox.MultiPartForm()
+    if data:
+        for key in data:
+            form.add_field(key, data[key])
+    data = api._json_request(url, form)
+
+    def full_url(path):
+        if path.startswith('/'):
+            u = urlparse(url)
+            path = '%s://%s%s' % (u.scheme, u.netloc, path)
+        return path
+
+    if 'uploadUrl' in data:
+        uploadUrl = full_url(data['uploadUrl'])
+        f = open(filename, 'rb')
+        fsize = os.stat(filename).st_size
+        done = 0
+        start = time.mktime(time.localtime())
+        if 'offset' in data and data['offset'] < fsize:
+            done = data['offset']
+            f.seek(done)
+            resume_offset = done
+        else:
+            resume_offset = 0
+        chunk = f.read(CHUNK_SIZE)
+        fname = os.path.basename(filename)
+        if not isinstance(fname, bytes):
+            fname = fname.encode('utf-8')
+        while chunk:
+            sys.stdout.flush()
+            form = ox.MultiPartForm()
+            form.add_file('chunk', fname, chunk)
+            if len(chunk) < CHUNK_SIZE or f.tell() == fsize:
+                form.add_field('done', '1')
+            form.add_field('offset', str(done))
+            try:
+                data = api._json_request(uploadUrl, form)
+            except KeyboardInterrupt:
+                print("\ninterrupted by user.")
+                sys.exit(1)
+            except:
+                print("uploading chunk failed, will try again in 5 seconds\r", end='')
+                sys.stdout.flush()
+                data = {'result': -1}
+                time.sleep(5)
+            if data and 'status' in data:
+                if data['status']['code'] == 403:
+                    print("login required")
+                    return False
+                if data['status']['code'] != 200:
+                    print("request returned error, will try again in 5 seconds")
+                    if DEBUG:
+                        print(data)
+                    time.sleep(5)
+            if data and data.get('result') == 1:
+                done += len(chunk)
+                if data.get('offset') not in (None, done):
+                    print('server offset out of sync, continue from', data['offset'])
+                    done = data['offset']
+                    f.seek(done)
+                chunk = f.read(CHUNK_SIZE)
+        if data and 'result' in data and data.get('result') == 1:
+            return data.get('id', True)
+        else:
+            return False
+    return False
+
+def upload_document(pdf):
+    document_id = upload_chunks(api, url, pdf)
+
+
+def import_url(url):
+    meta = {
+        'description': 'PDF snapshot of %s from %s' % (url, datetime.now().strftime('%Y-%m-%d %H:%M:%S')),
+        'title': 'Untitled'
+    }
+    data = ox.cache.read_url(url, unicode=True)
+    title = re.compile('<title>(.*?)</title>').findall(data)
+    if title:
+        meta['title'] = title[0]
+    author= re.compile('<meta name="author" content="(.*?)"').findall(data)
+    if author:
+        meta['author'] = author
+    fd, pdf = tempfile.mkstemp('.pdf')
+    if url2pdf(url, pdf):
+        url = api.url + 'upload/document/'
+        did = upload_chunks(api, url, pdf, {
+            'filename': meta['title'] + '.pdf'
+        })
+        meta['id'] = did
+        r = api.editDocument(meta)
+        print(r['data']['id'])
+        os.unlink(pdf)
+
+
+if __name__ == '__main__':
+    import sys
+    url = sys.argv[1]
+    import_url(url)
diff --git a/install.py b/install.py
index 01921a0..26e3dc6 100755
--- a/install.py
+++ b/install.py
@@ -52,6 +52,28 @@ for root, folders, files in os.walk(join(base, 'scripts')):
             os.unlink(target)
         os.symlink(rel_src, target)
 
-# todo
-# custom python module etc
-# local_settings.py?
+if os.path.exists('__init__.py'):
+    # make module available to pandora
+    target = os.path.join('/srv/pandora/pandora/', name)
+    rel_src = os.path.relpath(base, dirname(target))
+    if os.path.exists(target):
+        os.unlink(target)
+    os.symlink(rel_src, target)
+
+    # include module in local settings
+    local_settings_py = '/srv/pandora/pandora/local_settings.py'
+    with open(local_settings_py) as fd:
+        local_settings_changed = False
+        local_settings = fd.read()
+        if 'LOCAL_APPS' not in local_settings:
+            local_settings += '\nLOCAL_APPS = ["%s"]\n' % name
+            local_settings_changed = True
+        else:
+            apps = re.compile('(LOCAL_APPS.*?)\]', re.DOTALL).findall(local_settings)[0]
+            if name not in apps:
+                new_apps = apps.strip() + ',\n"%s"\n' % name
+                local_settings = local_settings.replace(apps, new_apps)
+                local_settings_changed = True
+    if local_settings_changed:
+        with open(local_settings_py, 'w') as fd:
+            fd.write(local_settings)
diff --git a/static/js/importDocumentsDialog.amp.js b/static/js/importDocumentsDialog.amp.js
new file mode 100644
index 0000000..fccd08c
--- /dev/null
+++ b/static/js/importDocumentsDialog.amp.js
@@ -0,0 +1,86 @@
+'use strict';
+
+pandora.ui.importDocumentsDialog = function() {
+
+    var dialogHeight = 100,
+        dialogWidth = 512 + 16,
+        formWidth = getFormWidth(),
+
+        $button,
+        $content = Ox.Element(),
+        value,
+        $input = [
+            Ox.TextArea({
+                labelWidth: 96,
+                width: 512
+            }).css({
+                margin: '8px'
+            }).bindEvent({
+                change: function(data) {
+                    $button.options({disabled: !data.value});
+                    value = data.value
+                }
+            }).appendTo($content),
+        ],
+
+        that = Ox.Dialog({
+                buttons: [
+                    Ox.Button({
+                            id: 'cancel',
+                            title: Ox._('Cancel')
+                        })
+                        .bindEvent({
+                            click: function() {
+                                that.close();
+                            }
+                        }),
+                    $button = Ox.Button({
+                            disabled: true,
+                            id: 'import',
+                            title: Ox._('Import URLs')
+                        })
+                        .bindEvent({
+                            click: importDocuments
+                        })
+                ],
+                closeButton: true,
+                content: $content,
+                height: dialogHeight,
+                removeOnClose: true,
+                title: Ox._('Import Documents'),
+                width: dialogWidth
+            })
+            .bindEvent({
+                resize: setSize
+            });
+
+    function getFormWidth() {
+        return dialogWidth - 32 - Ox.UI.SCROLLBAR_SIZE;
+    }
+
+    function setSize(data) {
+        dialogHeight = data.height;
+        dialogWidth = data.width;
+        formWidth = getFormWidth();
+        $input.forEach(function($element) {
+            $element.options({width: formWidth});
+        });
+    }
+
+    function importDocuments() {
+        that.options({content: Ox.LoadingScreen().start()});
+        var urls = value.trim().split('\n');
+        pandora.api.importDocuments({
+            urls: urls
+        }, function(result) {
+            if (result.data.taskId) {
+                pandora.wait(result.data.taskId, function(result) {
+                    that.close();
+                }
+            } else {
+                that.options({content: 'Failed'});
+            }
+        })
+    }
+    return that;
+};
diff --git a/static/js/localInit.amp.js b/static/js/localInit.amp.js
new file mode 100644
index 0000000..f75dfee
--- /dev/null
+++ b/static/js/localInit.amp.js
@@ -0,0 +1,53 @@
+pandora.localInit = function() {
+    var plugins = [];
+
+    plugins.push(ExtrasMenu());
+
+    plugins.length && load();
+
+    function load() {
+        patchReload();
+        plugins.forEach(function(plugin) { plugin.load() });
+    }
+
+    function patchReload() {
+        var reload = pandora.$ui.appPanel.reload;
+        pandora.$ui.appPanel.reload = function() {
+            reload();
+            load();
+        }
+    }
+
+    function ExtrasMenu() {
+        var that = {};
+
+        var css = {
+                //margin: '2px',
+            },
+            $item = Ox.MenuButton({
+                items: [
+                ].concat(pandora.user.level == 'admin' ? [
+                ] : [], [
+                    {id: 'import_documents', title: 'Import Documents...'},
+                ]),
+                style: 'rounded',
+                title: 'set',
+                tooltip: Ox._('Extras'),
+                type: 'image'
+            }).css(css).bindEvent({
+                click: function(data) {
+                    if (data.id == 'import_documents') {
+                        pandora.ui.importDocumentsDialog().open()
+                    }
+                },
+            }),
+            plugins = [];
+
+        that.load = function() {
+            pandora.$ui.mainMenu.find('.OxExtras').prepend($item);
+            pandora.$ui.extraItem = $item;
+        };
+        return that;
+    }
+
+}
diff --git a/tasks.py b/tasks.py
new file mode 100644
index 0000000..1dd8a6f
--- /dev/null
+++ b/tasks.py
@@ -0,0 +1,12 @@
+import os
+import subprocess
+
+from celery.task import task
+
+base = os.path.dirname(__file__)
+
+@task(queue="encoding")
+def import_documents(urls):
+    for url in urls:
+        if url.startswith('http'):
+            subprocess.call(['/srv/pandora/bin/python', os.path.join(base, 'add_website_as_pdf.py'), url])
diff --git a/views.py b/views.py
new file mode 100644
index 0000000..68d0ef7
--- /dev/null
+++ b/views.py
@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+from __future__ import division, print_function, absolute_import
+
+from oxdjango.decorators import login_required_json
+from oxdjango.shortcuts import render_to_json_response
+from oxdjango.api import actions
+
+from .tasks import import_documents
+
+@login_required_json
+def importDocuments(request, data):
+    t = tasks.import_documents.delay(urls=data['urls'])
+    response['data']['taskId'] = t.task_id
+    return render_to_json_response(response)
+actions.register(importDocuments)