From 8e4ba42fc1b6c015531d30e204d51f652a73fd8e Mon Sep 17 00:00:00 2001 From: j Date: Thu, 2 Nov 2017 21:08:17 +0000 Subject: [PATCH] use python3-venv --- README.md | 3 +- ctl | 2 +- pandora/item/utils.py | 1 + pandora/manage.py | 56 ++++++++++++++------- pandora/settings.py | 3 ++ scripts/item_icon.pandora.py | 14 ++---- scripts/list_icon.pandora.py | 14 ++---- scripts/poster.0xdb.py | 12 +---- scripts/poster.indiancinema.py | 12 +---- scripts/poster.padma.py | 13 ++--- scripts/poster.pandora.py | 12 +---- update.py | 92 ++++++++++++++++++++++------------ vm/pandora_install.sh | 3 +- 13 files changed, 119 insertions(+), 118 deletions(-) diff --git a/README.md b/README.md index e885c375..c9d7e351 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,7 @@ 2) install all required packages apt-get install git \ - python3-setuptools python3-pip python3-virtualenv \ - virtualenv ipython3 \ + python3-setuptools python3-pip python3-venv ipython3 \ python3-dev python3-pil python3-numpy python3-psycopg2 \ python3-pyinotify python3-simplejson \ python3-geoip python3-html5lib python3-lxml \ diff --git a/ctl b/ctl index 71bd31a9..d3643fab 100755 --- a/ctl +++ b/ctl @@ -9,7 +9,7 @@ fi if [ "$action" = "init" ]; then cd "`dirname "$0"`" BASE=`pwd` - virtualenv --system-site-packages -p /usr/bin/python3 . + python3 -m venv --system-site-packages . if [ ! -d static/oxjs ]; then git clone --depth 1 https://git.0x2620.org/oxjs.git static/oxjs fi diff --git a/pandora/item/utils.py b/pandora/item/utils.py index 8c48c8e6..c6b6cfa4 100644 --- a/pandora/item/utils.py +++ b/pandora/item/utils.py @@ -9,6 +9,7 @@ import ox from ox import sort_string from six import PY2 + def safe_filename(filename): filename = filename.replace(': ', '_ ') filename = filename.replace('/', '_') diff --git a/pandora/manage.py b/pandora/manage.py index 50800657..d33da6f9 100755 --- a/pandora/manage.py +++ b/pandora/manage.py @@ -3,6 +3,28 @@ import os import signal import sys + +def activate_venv(base): + if os.path.exists(base): + old_os_path = os.environ.get('PATH', '') + bin_path = os.path.join(base, 'bin') + if bin_path not in old_os_path: + os.environ['PATH'] = os.path.join(base, 'bin') + os.pathsep + old_os_path + site_packages = os.path.join(base, 'lib', 'python%s' % sys.version[:3], 'site-packages') + prev_sys_path = list(sys.path) + import site + site.addsitedir(site_packages) + sys.real_prefix = sys.prefix + sys.prefix = base + # Move the added items to the front of the path: + new_sys_path = [] + for item in list(sys.path): + if item not in prev_sys_path: + new_sys_path.append(item) + sys.path.remove(item) + sys.path[:0] = new_sys_path + + class DelayedSignalHandler(object): def __init__(self, managed_signals): self.managed_signals = managed_signals @@ -27,26 +49,22 @@ class DelayedSignalHandler(object): ''' -root_dir = os.path.normpath(os.path.abspath(os.path.dirname(__file__))) -os.chdir(root_dir) - -# use python3 from virtualenv -python3 = os.path.normpath(os.path.join(root_dir, '..', 'bin', 'python3')) -if os.path.exists(python3) and sys.version_info[0] == 2: - import subprocess - cmd = [python3] + sys.argv - with DelayedSignalHandler((signal.SIGINT, signal.SIGTERM, signal.SIGHUP)): - exit_value = subprocess.call(cmd) - sys.exit(exit_value) - -# using virtualenv's activate_this.py to reorder sys.path -activate_this = os.path.normpath(os.path.join(root_dir, '..', 'bin', 'activate_this.py')) -with open(activate_this) as f: - code = compile(f.read(), activate_this, 'exec') - exec(code, dict(__file__=activate_this)) - - if __name__ == "__main__": + root_dir = os.path.normpath(os.path.abspath(os.path.dirname(__file__))) + + # use python3 from venv + venv_dir = os.path.normpath(os.path.join(root_dir, '..')) + python3 = os.path.join(venv_dir, 'bin', 'python3') + if os.path.exists(python3) and sys.version_info[0] == 2: + import subprocess + cmd = [python3] + sys.argv + with DelayedSignalHandler((signal.SIGINT, signal.SIGTERM, signal.SIGHUP)): + exit_value = subprocess.call(cmd) + sys.exit(exit_value) + + os.chdir(root_dir) + activate_venv(venv_dir) + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") from django.core.management import execute_from_command_line import settings diff --git a/pandora/settings.py b/pandora/settings.py index 0e68b631..d854004d 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -11,6 +11,9 @@ import djcelery djcelery.setup_loader() BASE_DIR = PROJECT_ROOT = normpath(dirname(__file__)) +BIN_DIR = normpath(join(PROJECT_ROOT, '..', 'bin')) +if BIN_DIR not in os.environ['PATH']: + os.environ['PATH'] = BIN_DIR + os.pathsep + os.environ['PATH'] DEBUG = False JSON_DEBUG = False diff --git a/scripts/item_icon.pandora.py b/scripts/item_icon.pandora.py index 4681c555..0cace0fb 100755 --- a/scripts/item_icon.pandora.py +++ b/scripts/item_icon.pandora.py @@ -1,25 +1,17 @@ -#!/usr/bin/python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 from __future__ import division import os -root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - -# using virtualenv's activate_this.py to reorder sys.path -activate_this = os.path.join(root_dir, 'bin', 'activate_this.py') -if os.path.exists(activate_this): - with open(activate_this) as f: - code = compile(f.read(), activate_this, 'exec') - exec(code, dict(__file__=activate_this)) - from PIL import Image from optparse import OptionParser import sys - +root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) static_root = os.path.join(os.path.dirname(__file__), 'data') + def render_icon(frame, timeline, icon): icon_width = 1024 icon_height = 1024 diff --git a/scripts/list_icon.pandora.py b/scripts/list_icon.pandora.py index f3928095..0a65e07c 100755 --- a/scripts/list_icon.pandora.py +++ b/scripts/list_icon.pandora.py @@ -1,26 +1,20 @@ -#!/usr/bin/python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 from __future__ import division import os -root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - -# using virtualenv's activate_this.py to reorder sys.path -activate_this = os.path.join(root_dir, 'bin', 'activate_this.py') -if os.path.exists(activate_this): - with open(activate_this) as f: - code = compile(f.read(), activate_this, 'exec') - exec(code, dict(__file__=activate_this)) - from PIL import Image from PIL import ImageDraw from optparse import OptionParser from ox.image import drawText, wrapText import sys + +root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) static_root = os.path.join(os.path.dirname(__file__), 'data') + def render_list_icon(frames, icon): icon_width = 256 icon_height = 256 diff --git a/scripts/poster.0xdb.py b/scripts/poster.0xdb.py index dd925f16..f040c6ed 100755 --- a/scripts/poster.0xdb.py +++ b/scripts/poster.0xdb.py @@ -1,18 +1,9 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 from __future__ import division import os -root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - -# using virtualenv's activate_this.py to reorder sys.path -activate_this = os.path.join(root_dir, 'bin', 'activate_this.py') -if os.path.exists(activate_this): - with open(activate_this) as f: - code = compile(f.read(), activate_this, 'exec') - exec(code, dict(__file__=activate_this)) - from PIL import Image from PIL import ImageDraw import json @@ -22,6 +13,7 @@ from ox.image import drawText, getRGB, getTextSize, wrapText import subprocess import sys +root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) static_root = os.path.join(os.path.dirname(__file__), 'data') def get_frame(id, height, position): diff --git a/scripts/poster.indiancinema.py b/scripts/poster.indiancinema.py index 6947f9cc..40d61b38 100755 --- a/scripts/poster.indiancinema.py +++ b/scripts/poster.indiancinema.py @@ -1,18 +1,9 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 from __future__ import division import os -root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - -# using virtualenv's activate_this.py to reorder sys.path -activate_this = os.path.join(root_dir, 'bin', 'activate_this.py') -if os.path.exists(activate_this): - with open(activate_this) as f: - code = compile(f.read(), activate_this, 'exec') - exec(code, dict(__file__=activate_this)) - from PIL import Image from PIL import ImageDraw import json @@ -21,6 +12,7 @@ import ox from ox.image import getRGB, drawText, wrapText import sys +root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) static_root = os.path.join(os.path.dirname(__file__), 'data') def render_poster(data, poster): diff --git a/scripts/poster.padma.py b/scripts/poster.padma.py index 0434b5c1..a9372462 100755 --- a/scripts/poster.padma.py +++ b/scripts/poster.padma.py @@ -1,18 +1,9 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 from __future__ import division import os -root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - -# using virtualenv's activate_this.py to reorder sys.path -activate_this = os.path.join(root_dir, 'bin', 'activate_this.py') -if os.path.exists(activate_this): - with open(activate_this) as f: - code = compile(f.read(), activate_this, 'exec') - exec(code, dict(__file__=activate_this)) - from PIL import Image from PIL import ImageDraw import json @@ -21,6 +12,8 @@ import ox from ox.image import drawText, wrapText import sys + +root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) static_root = os.path.join(os.path.dirname(__file__), 'data') def render_poster(data, poster): diff --git a/scripts/poster.pandora.py b/scripts/poster.pandora.py index ccd43767..1524bc72 100755 --- a/scripts/poster.pandora.py +++ b/scripts/poster.pandora.py @@ -1,18 +1,9 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 from __future__ import division import os -root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - -# using virtualenv's activate_this.py to reorder sys.path -activate_this = os.path.join(root_dir, 'bin', 'activate_this.py') -if os.path.exists(activate_this): - with open(activate_this) as f: - code = compile(f.read(), activate_this, 'exec') - exec(code, dict(__file__=activate_this)) - from PIL import Image from PIL import ImageDraw import json @@ -21,6 +12,7 @@ import ox from ox.image import drawText, wrapText import sys +root_dir = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) static_root = os.path.join(os.path.dirname(__file__), 'data') def render_poster(data, poster): diff --git a/update.py b/update.py index 0ddd4239..9f961819 100755 --- a/update.py +++ b/update.py @@ -1,39 +1,74 @@ -#!/usr/bin/python +#!/usr/bin/env python from __future__ import print_function +import json import os - -base = os.path.normpath(os.path.abspath(os.path.dirname(__file__))) -os.chdir(base) - -# using virtualenv's activate_this.py to reorder sys.path -activate_this = os.path.join(base, 'bin', 'activate_this.py') -with open(activate_this) as f: - code = compile(f.read(), activate_this, 'exec') - exec(code, dict(__file__=activate_this)) - -import sys import shutil import subprocess +import sys try: from urllib.request import urlopen except: from urllib2 import urlopen -import json from os.path import join, exists + +repos = { + "pandora": { + "url": "https://git.0x2620.org/pandora.git", + "path": ".", + }, + "oxjs": { + "url": "https://git.0x2620.org/oxjs.git", + "path": "./static/oxjs", + }, + "oxtimelines": { + "url": "https://git.0x2620.org/oxtimelines.git", + "path": "./src/oxtimelines", + }, + "python-ox": { + "url": "https://git.0x2620.org/python-ox.git", + "path": "./src/python-ox", + } +} + + +def activate_venv(base): + if os.path.exists(base): + old_os_path = os.environ.get('PATH', '') + bin_path = os.path.join(base, 'bin') + if bin_path not in old_os_path: + os.environ['PATH'] = os.path.join(base, 'bin') + os.pathsep + old_os_path + site_packages = os.path.join(base, 'lib', 'python%s' % sys.version[:3], 'site-packages') + prev_sys_path = list(sys.path) + import site + site.addsitedir(site_packages) + sys.real_prefix = sys.prefix + sys.prefix = base + # Move the added items to the front of the path: + new_sys_path = [] + for item in list(sys.path): + if item not in prev_sys_path: + new_sys_path.append(item) + sys.path.remove(item) + sys.path[:0] = new_sys_path + + def run(*cmd): p = subprocess.Popen(cmd) p.wait() return p.returncode + def get(*cmd): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, error = p.communicate() return stdout.decode() + def get_json(url): return json.loads(urlopen(url).read().decode()) + def get_release(): if os.path.exists('.release'): url = open('.release').read().strip() @@ -45,28 +80,11 @@ def get_release(): print("Failed to load %s check your internet connection." % url) sys.exit(1) -repos = { - "pandora": { - "url": "https://git.0x2620.org/pandora.git", - "path": ".", - }, - "oxjs": { - "url": "https://git.0x2620.org/oxjs.git", - "path": "./static/oxjs", - }, - "oxtimelines": { - "url": "https://git.0x2620.org/oxtimelines.git", - "path": "./src/oxtimelines", - }, - "python-ox": { - "url": "https://git.0x2620.org/python-ox.git", - "path": "./src/python-ox", - } -} def reload_notice(base): print('\nPlease restart pan.do/ra to finish the update:\n\tsudo %s/ctl reload\n' % base) + def check_services(base): services = "pandora pandora-tasks pandora-encoding pandora-cron pandora-websocketd".split() for service in services: @@ -83,24 +101,32 @@ def check_services(base): print('\tsudo service %s start' % service) print('') + def update_service(service): print('Please install new init script for "%s" service:' % service) - if os.path.exists('/etc/init/%s.conf'%service): + if os.path.exists('/etc/init/%s.conf' % service): print('\tsudo cp %s/etc/init/%s.conf /etc/init/' % (base, service)) if os.path.exists('/bin/systemctl'): print('\tsudo cp %s/etc/systemd/system/%s.service /etc/systemd/system/' % (base, service)) print('\tsudo systemctl daemon-reload') print('\tsudo service %s restart' % service) + def run_git(path, *args): cmd = ['git'] + list(args) env = {'GIT_DIR': '%s/.git' % path} return subprocess.check_output(cmd, env=env).decode().strip() + def get_version(path): return run_git(path, 'rev-list', 'HEAD', '--count') + if __name__ == "__main__": + base = os.path.normpath(os.path.abspath(os.path.dirname(__file__))) + os.chdir(base) + activate_venv(base) + if len(sys.argv) == 2 and sys.argv[1] in ('database', 'db'): os.chdir(join(base, 'pandora')) print('\nRunning "./manage.py migrate"\n') @@ -144,7 +170,7 @@ if __name__ == "__main__": import pandora.settings with open('pandora/local_settings.py', 'r') as f: local_settings = f.read() - if not 'BROKER_URL' in local_settings: + if 'BROKER_URL' not in local_settings: broker_url = 'amqp://%s:%s@%s:%s/%s' % ( getattr(pandora.settings, 'BROKER_USER', 'pandora'), getattr(pandora.settings, 'BROKER_PASSWORD', 'box'), diff --git a/vm/pandora_install.sh b/vm/pandora_install.sh index 8a9dbb92..4e3977b8 100755 --- a/vm/pandora_install.sh +++ b/vm/pandora_install.sh @@ -69,8 +69,7 @@ apt-get install -y \ git \ python3-setuptools \ python3-pip \ - virtualenv \ - python3-virtualenv \ + python3-venv \ python3-dev \ python3-pil \ python3-numpy \