pandora/update.py

378 lines
16 KiB
Python
Raw Normal View History

2019-08-19 09:27:11 +00:00
#!/usr/bin/env python3
2017-11-02 21:08:17 +00:00
import json
import os
2015-11-14 14:22:40 +00:00
import shutil
import subprocess
2017-11-02 21:08:17 +00:00
import sys
2017-02-16 17:31:46 +00:00
try:
from urllib.request import urlopen
except:
from urllib2 import urlopen
from os.path import join, exists
2017-11-02 21:08:17 +00:00
repos = {
"pandora": {
"url": "https://git.0x2620.org/pandora.git",
"path": ".",
},
"oxjs": {
"url": "https://git.0x2620.org/oxjs.git",
"path": "./static/oxjs",
},
"oxtimelines": {
"url": "https://git.0x2620.org/oxtimelines.git",
"path": "./src/oxtimelines",
},
"python-ox": {
"url": "https://git.0x2620.org/python-ox.git",
"path": "./src/python-ox",
}
}
def activate_venv(base):
if os.path.exists(base):
old_os_path = os.environ.get('PATH', '')
bin_path = os.path.join(base, 'bin')
if bin_path not in old_os_path:
os.environ['PATH'] = os.path.join(base, 'bin') + os.pathsep + old_os_path
site_packages = os.path.join(base, 'lib', 'python%s' % sys.version[:3], 'site-packages')
prev_sys_path = list(sys.path)
import site
site.addsitedir(site_packages)
sys.real_prefix = sys.prefix
sys.prefix = base
# Move the added items to the front of the path:
new_sys_path = []
for item in list(sys.path):
if item not in prev_sys_path:
new_sys_path.append(item)
sys.path.remove(item)
sys.path[:0] = new_sys_path
def run(*cmd):
p = subprocess.Popen(cmd)
p.wait()
return p.returncode
def get(*cmd):
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, error = p.communicate()
2017-02-16 14:22:21 +00:00
return stdout.decode()
2017-11-02 21:08:17 +00:00
def get_json(url):
2017-02-16 17:31:46 +00:00
return json.loads(urlopen(url).read().decode())
2017-11-02 21:08:17 +00:00
def get_release():
2015-11-14 14:22:40 +00:00
if os.path.exists('.release'):
url = open('.release').read().strip()
else:
url = 'https://pan.do/json/release-stable.json'
try:
2015-11-14 14:22:40 +00:00
return get_json(url)
except:
2016-02-18 10:49:26 +00:00
print("Failed to load %s check your internet connection." % url)
sys.exit(1)
def reload_notice(base):
print('\nPlease restart pan.do/ra to finish the update:\n\tsudo pandoractl reload\n')
2017-11-02 21:08:17 +00:00
2015-04-30 13:01:27 +00:00
def check_services(base):
services = "pandora pandora-tasks pandora-encoding pandora-cron pandora-websocketd".split()
for service in services:
2015-04-30 13:07:24 +00:00
cmd = ['service', service, 'status']
2015-11-16 13:21:43 +00:00
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p.wait()
if p.returncode != 0:
2016-02-18 10:49:26 +00:00
print('Please install init script for "%s" service:' % service)
2016-03-08 07:48:35 +00:00
if os.path.exists('/etc/init') and not os.path.exists('/bin/systemctl'):
2016-02-18 10:49:26 +00:00
print('\tsudo cp %s/etc/init/%s.conf /etc/init/' % (base, service))
if os.path.exists('/bin/systemctl'):
print('\tsudo cp %s/etc/systemd/system/%s.service /etc/systemd/system/' % (base, service))
print('\tsudo systemctl daemon-reload')
2016-02-18 10:49:26 +00:00
print('\tsudo service %s start' % service)
print('')
2015-04-30 13:01:27 +00:00
2017-11-02 21:08:17 +00:00
def update_service(service):
print('Please install new init script for "%s" service:' % service)
2017-11-02 21:08:17 +00:00
if os.path.exists('/etc/init/%s.conf' % service):
print('\tsudo cp %s/etc/init/%s.conf /etc/init/' % (base, service))
if os.path.exists('/bin/systemctl'):
print('\tsudo cp %s/etc/systemd/system/%s.service /etc/systemd/system/' % (base, service))
print('\tsudo systemctl daemon-reload')
print('\tsudo service %s restart' % service)
2017-11-02 21:08:17 +00:00
2015-11-14 14:22:40 +00:00
def run_git(path, *args):
cmd = ['git'] + list(args)
env = {'GIT_DIR': '%s/.git' % path}
2017-02-16 14:22:21 +00:00
return subprocess.check_output(cmd, env=env).decode().strip()
2015-11-14 14:22:40 +00:00
2017-11-02 21:08:17 +00:00
2020-05-29 22:05:50 +00:00
def run_sql(sql):
cmd = [join(base, 'pandora/manage.py'), 'dbshell']
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
stdout, error = p.communicate(input=sql.encode())
return stdout.decode()
2015-11-14 14:22:40 +00:00
def get_version(path):
return run_git(path, 'rev-list', 'HEAD', '--count')
def get_branch(path=None):
if not path:
path = '.'
return get('cat', os.path.join(path, '.git/HEAD')).strip().split('/')[-1]
2017-11-02 21:08:17 +00:00
if __name__ == "__main__":
if os.stat(__file__).st_uid != os.getuid() or os.getuid() == 0:
print('you must run update.py as the pandora user')
sys.exit(1)
2017-11-02 21:08:17 +00:00
base = os.path.normpath(os.path.abspath(os.path.dirname(__file__)))
os.chdir(base)
activate_venv(base)
2013-04-30 17:17:48 +00:00
if len(sys.argv) == 2 and sys.argv[1] in ('database', 'db'):
os.chdir(join(base, 'pandora'))
2016-02-19 18:20:18 +00:00
print('\nRunning "./manage.py migrate"\n')
2019-06-24 11:42:22 +00:00
r = get('./manage.py', 'migrate', '--noinput')
r = r.replace("Your models have changes that are not yet reflected in a migration, and so won't be applied.", '')
r = r.replace("Run 'manage.py makemigrations' to make new migrations, and then re-run 'manage.py migrate' to apply them.", '')
print(r)
2016-02-19 18:20:18 +00:00
run('./manage.py', 'sqlfindindex')
run('./manage.py', 'sync_itemsort')
run('./manage.py', 'sync_documentsort')
2016-02-19 18:20:18 +00:00
reload_notice(base)
2013-05-29 12:41:13 +00:00
elif len(sys.argv) == 2 and sys.argv[1] == 'static':
os.chdir(join(base, 'pandora'))
run('./manage.py', 'update_static')
2013-05-30 11:17:48 +00:00
elif len(sys.argv) == 4 and sys.argv[1] == 'postupdate':
os.chdir(base)
2015-11-14 15:16:58 +00:00
old = sys.argv[2].strip()
new = sys.argv[3].strip()
2015-11-14 14:22:40 +00:00
if old.isdigit():
old = int(old)
if new.isdigit():
new = int(new)
2016-02-18 10:49:26 +00:00
print('Post Update from %s to %s' % (old, new))
2013-05-30 11:17:48 +00:00
if old < 3111:
2013-05-30 11:18:48 +00:00
run('bzr', 'resolved', 'pandora/monkey_patch', 'pandora/monkey_patch/migrations')
2013-05-30 11:17:48 +00:00
if os.path.exists('pandora/monkey_patch'):
run('rm', '-r', 'pandora/monkey_patch')
if old < 3448:
if os.path.exists('static/pandora'):
run('bzr', 'resolved', 'static/pandora')
2013-09-19 12:01:40 +00:00
if old < 3651:
if os.path.exists('src/django/.git'):
os.chdir(os.path.join(base, 'src/django'))
run('git', 'checkout', 'stable/1.4.x')
run('git', 'pull')
os.chdir(base)
if old < 3666:
run('./bin/pip', 'install', '-r', 'requirements.txt')
if old < 3770:
run('./bin/pip', 'install', '-r', 'requirements.txt')
if old < 4379:
run('./bin/pip', 'install', '-r', 'requirements.txt')
2014-11-26 20:03:27 +00:00
if old < 4549:
import pandora.settings
with open('pandora/local_settings.py', 'r') as f:
local_settings = f.read()
2017-11-02 21:08:17 +00:00
if 'BROKER_URL' not in local_settings:
2014-11-26 20:03:27 +00:00
broker_url = 'amqp://%s:%s@%s:%s/%s' % (
getattr(pandora.settings, 'BROKER_USER', 'pandora'),
getattr(pandora.settings, 'BROKER_PASSWORD', 'box'),
getattr(pandora.settings, 'BROKER_HOST', '127.0.0.1'),
getattr(pandora.settings, 'BROKER_PORT', 5672),
getattr(pandora.settings, 'BROKER_VHOST', '/pandora'),
)
local_settings = [
l for l in local_settings.split('\n') if not l.startswith('BROKER_')
] + [
'BROKER_URL = "%s"' % broker_url, ''
]
with open('pandora/local_settings.py', 'w') as f:
f.write('\n'.join(local_settings))
if old < 4947:
run('./bin/pip', 'install', 'tornado==4.1')
2015-04-30 13:01:27 +00:00
check_services(base)
2015-11-14 14:22:40 +00:00
if old < 5074:
for component in ('oxtimelines', 'python-ox'):
if not os.path.exists('./src/%s/.git' % component):
run('./bin/pip', 'install', '-e',
'git+https://git.0x2620.org/%s.git#egg=%s' % (component, component),
'--exists-action', 'w')
if not os.path.exists('./static/oxjs/.git'):
if os.path.exists('static/oxjs'):
shutil.move('static/oxjs', 'static/oxjs_bzr')
2016-02-12 08:32:24 +00:00
run('git', 'clone', '--depth', '1', 'https://git.0x2620.org/oxjs.git', 'static/oxjs')
2015-11-14 14:22:40 +00:00
run('./pandora/manage.py', 'update_static')
if os.path.exists('static/oxjs_bzr'):
shutil.rmtree('static/oxjs_bzr')
if os.path.exists('REPOSITORY_MOVED_TO_GIT'):
os.unlink('REPOSITORY_MOVED_TO_GIT')
if os.path.exists('.bzr'):
shutil.rmtree('.bzr')
run('git', 'checkout', 'update.py')
if old <= 5389:
2016-02-20 12:36:05 +00:00
for path in (
'lib/python2.7/site-packages/Django.egg-link',
2016-03-03 08:19:22 +00:00
'lib/python2.7/site-packages/django-extensions.egg-link',
'lib/python2.7/site-packages/django-devserver.egg-link'
2016-02-20 12:36:05 +00:00
):
if os.path.exists(path):
os.unlink(path)
2016-03-03 08:19:22 +00:00
for path in (
'src/django',
'src/django-devserver',
'src/django-extensions',
):
if os.path.exists(path):
shutil.rmtree(path)
if os.path.exists('contrib'):
shutil.rmtree('contrib')
run('./bin/pip', 'install', '-r', 'requirements.txt')
run('./pandora/manage.py', 'migrate', '--fake-initial', '--noinput')
update_service('pandora')
2016-03-04 07:20:44 +00:00
if old <= 5432:
import pandora.settings
run('./bin/pip', 'install', '-r', 'requirements.txt')
path = os.path.join(pandora.settings.GEOIP_PATH, 'GeoLite2-City.mmdb')
if not os.path.exists(path):
run('./pandora/manage.py', 'update_geoip')
if old <= 5443:
gunicorn_config = 'pandora/gunicorn_config.py'
if not os.path.exists(gunicorn_config):
shutil.copy('%s.in' % gunicorn_config, gunicorn_config)
if os.path.exists('/etc/init/pandora.conf'):
with open('/etc/init/pandora.conf') as fd:
data = fd.read()
if '0.0.0.0:2620' in data:
run('sed', '-i', 's/127.0.0.1:2620/0.0.0.0:2620/g', gunicorn_config)
if old > 5389:
update_service('pandora')
2016-03-17 09:47:08 +00:00
if old <= 5452:
run('./bin/pip', 'install', '-r', 'requirements.txt')
update_service('pandora-encoding')
update_service('pandora-tasks')
2018-07-09 13:29:06 +00:00
if old < 5975:
2018-06-19 18:48:18 +00:00
run('./bin/pip', 'install', '-r', 'requirements.txt')
2018-11-15 10:30:17 +00:00
if old <= 6064:
run('./bin/pip', 'install', '-r', 'requirements.txt')
run('./pandora/manage.py', 'createcachetable')
2019-04-30 13:04:39 +00:00
if old <= 6108:
run('./bin/pip', 'install', '-r', 'requirements.txt')
if old <= 6160:
run('./bin/pip', 'install', '-r', 'requirements.txt')
with open('pandora/local_settings.py', 'r') as f:
local_settings = f.read()
if 'BROKER_URL' in local_settings and 'CELERY_BROKER_URL' not in local_settings:
local_settings = [
'CELERY_' + l if l.startswith('BROKER_URL') else l
for l in local_settings.split('\n')
]
with open('pandora/local_settings.py', 'w') as f:
f.write('\n'.join(local_settings))
if old <= 6313:
2020-05-29 23:50:32 +00:00
if sys.version_info[0:2] < (3, 6):
print('Python 3.6 or late is required now, upgrade your system and run:')
print('')
print('./update.py postupdate %s %s' % (6313, new))
print('')
sys.exit(1)
run('./bin/pip', 'uninstall', 'django-celery', '-y')
run('./bin/pip', 'install', '-r', 'requirements.txt')
2020-05-29 22:05:50 +00:00
if old <= 6315:
for sql in [
"INSERT INTO django_migrations (app, name, applied) VALUES ('system', '0001_initial', CURRENT_TIMESTAMP)",
"UPDATE django_content_type SET app_label = 'system' WHERE app_label = 'auth' and model = 'user'",
]:
run_sql(sql)
run(join(base, 'pandora/manage.py'), 'migrate', 'system')
2020-05-30 12:00:08 +00:00
run(join(base, 'pandora/manage.py'), 'update_geoip')
2021-05-15 12:22:24 +00:00
if old <= 6383:
run('./bin/pip', 'install', '-r', 'requirements.txt')
2022-03-18 16:53:19 +00:00
if old < 6442:
run('./bin/pip', 'install', 'yt-dlp>=2022.3.8.2')
else:
if len(sys.argv) == 1:
branch = get_branch()
development = branch == 'master'
elif len(sys.argv) == 3 and sys.argv[1] == 'switch':
branch = sys.argv[2]
2020-05-30 10:37:08 +00:00
development = branch == 'master'
2012-11-14 17:52:26 +00:00
else:
branch = 'master'
development = True
os.chdir(base)
current = ''
new = ''
2013-07-03 21:14:11 +00:00
for repo in sorted(repos, key=lambda r: repos[r]['path']):
path = os.path.join(base, repos[repo]['path'])
if exists(path):
os.chdir(path)
current_branch = get_branch(path)
2020-05-30 12:35:09 +00:00
revno = get_version(path)
2013-05-30 11:17:48 +00:00
if repo == 'pandora':
pandora_old_revno = revno
2020-05-30 12:35:09 +00:00
current += revno
2020-05-30 12:00:08 +00:00
if current_branch != branch:
run('git', 'remote', 'set-branches', 'origin', '*')
run('git', 'fetch', 'origin')
run('git', 'checkout', branch)
url = repos[repo]['url']
print('Checking', repo)
run('git', 'pull')
2015-11-14 14:22:40 +00:00
revno = get_version(path)
new += revno
2013-05-30 11:17:48 +00:00
if repo == 'pandora':
pandora_new_revno = revno
else:
os.chdir(os.path.dirname(path))
run('git', 'clone', '--depth', '1', repos[repo]['url'])
setup = os.path.join(base, repos[repo]['path'], 'setup.py')
if repo in ('python-ox', 'oxtimelines') and os.path.exists(setup):
os.chdir(os.path.dirname(setup))
run(os.path.join(base, 'bin', 'python'), 'setup.py', 'develop')
new += '+'
os.chdir(join(base, 'pandora'))
2013-10-22 20:03:42 +00:00
if pandora_old_revno != pandora_new_revno:
os.chdir(base)
run('./update.py', 'postupdate', pandora_old_revno, pandora_new_revno)
2013-10-28 11:11:58 +00:00
os.chdir(join(base, 'pandora'))
2016-02-20 12:26:14 +00:00
if current != new:
run('./manage.py', 'update_static')
2016-03-03 09:34:54 +00:00
run('./manage.py', 'compile_pyc', '-p', '.')
2016-02-20 12:26:14 +00:00
os.chdir(join(base, 'pandora'))
diff = get('./manage.py', 'sqldiff', '-a').strip()
2019-07-16 11:33:24 +00:00
for row in [
'-- Model missing for table: djcelery_periodictasks\n',
'-- Model missing for table: celery_taskmeta\n',
'-- Model missing for table: celery_tasksetmeta\n',
'-- Model missing for table: djcelery_crontabschedule\n',
'-- Model missing for table: djcelery_periodictask\n',
'-- Model missing for table: djcelery_intervalschedule\n',
'-- Model missing for table: djcelery_workerstate\n',
'-- Model missing for table: djcelery_taskstate\n',
2019-07-22 09:11:34 +00:00
'-- Model missing for table: south_migrationhistory\n',
2019-07-16 11:33:24 +00:00
'-- Model missing for table: cache\n',
]:
if row in diff:
diff = diff.replace(row, '')
2018-11-16 15:57:05 +00:00
if diff not in [
'-- No differences',
2019-07-16 11:33:24 +00:00
'BEGIN;\nCOMMIT;'
2018-11-16 15:57:05 +00:00
]:
2016-02-18 10:49:26 +00:00
print('Database has changed, please make a backup and run %s db' % sys.argv[0])
elif branch != 'master':
print('pan.do/ra is at the latest release,\nyou can run "%s switch master" to switch to the development version' % sys.argv[0])
2013-10-22 20:03:42 +00:00
elif current != new:
reload_notice(base)