From 893f2e54235aa067f60625f525fda9017c0fe57c Mon Sep 17 00:00:00 2001 From: j Date: Sat, 30 Jul 2016 02:49:31 +0200 Subject: [PATCH] add importMediaDialog --- pandora/archive/external.py | 122 +++++++++++++++++++++++++++ pandora/archive/tasks.py | 5 ++ pandora/archive/views.py | 46 +++++++++++ static/js/importMediaDialog.js | 146 +++++++++++++++++++++++++++++++++ vm/pandora_install.sh | 1 + 5 files changed, 320 insertions(+) create mode 100644 pandora/archive/external.py create mode 100644 static/js/importMediaDialog.js diff --git a/pandora/archive/external.py b/pandora/archive/external.py new file mode 100644 index 00000000..5dac990e --- /dev/null +++ b/pandora/archive/external.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from __future__ import division, print_function + +import json +import subprocess +import shutil +import tempfile +import os + +import ox +from django.conf import settings + +from item.models import Item +from item.tasks import load_subtitles + +import models + +info_keys = [ + 'title', + 'description', + 'duration', + 'width', + 'height', + 'webpage_url', + 'thumbnail', + 'ext', + 'uploader', + 'subtitles', + 'tags' +] + +info_key_map = { + 'webpage_url': 'url', + 'ext': 'extension', +} + +def get_info(url): + cmd = ['youtube-dl', '-j', '--all-subs', url] + p = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, close_fds=True) + stdout, stderr = p.communicate() + stdout = stdout.decode().strip() + info = [] + if stdout: + for line in stdout.split('\n'): + i = json.loads(line) + if not i.get('is_live'): + info.append({ + info_key_map.get(k, k): i[k] + for k in info_keys + if k in i and i[k] + }) + return info + +def add_subtitles(item, media, tmp): + for language in media.get('subtitles', {}): + for subtitle in media['subtitles'][language]: + if subtitle['ext'] in ('vtt', 'srt'): + data = ox.cache.read_url(subtitle['url']) + srt = os.path.join(tmp, 'media.' + subtitle['ext']) + with open(srt, 'wb') as fd: + fd.write(data) + oshash = ox.oshash(srt) + sub, created = models.File.objects.get_or_create(oshash=oshash) + if created: + sub.item = item + sub.data.name = sub.get_path('data.' + subtitle['ext']) + ox.makedirs(os.path.dirname(sub.data.path)) + shutil.move(srt, sub.data.path) + sub.path = '.'.join([media['title'], language, subtitle['ext']]) + sub.info = ox.avinfo(sub.data.path) + if 'path' in sub.info: + del sub.info['path'] + sub.info['extension'] = subtitle['ext'] + sub.info['language'] = language + sub.parse_info() + sub.selected = True + sub.save() + +def download(item_id, url): + item = Item.objects.get(public_id=item_id) + info = get_info(url) + if len(info) != 1: + return '%s contains %d videos' % (url, len(info)) + media = info[0] + cdir = os.path.abspath(os.curdir) + tmp = tempfile.mkdtemp().decode('utf-8') + os.chdir(tmp) + cmd = ['youtube-dl', '-q', media['url']] + p = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, close_fds=True) + stdout, stderr = p.communicate() + fname = list(os.listdir(tmp)) + if fname: + fname = os.path.join(tmp, fname[0]) + oshash = ox.oshash(fname) + f, created = models.File.objects.get_or_create(oshash=oshash) + if created: + f.data.name = f.get_path('data.' + fname.split('.')[-1]) + ox.makedirs(os.path.dirname(f.data.path)) + shutil.move(fname, f.data.path) + f.item = item + f.info = ox.avinfo(f.data.path) + f.info['extension'] = media['extension'] + f.path = '%(title)s.%(extension)s' % media + f.parse_info() + f.selected = True + f.save() + f.item.save() + f.extract_stream() + status = True + else: + status = 'file exists' + add_subtitles(f.item, media, tmp) + else: + status = 'download failed' + os.chdir(cdir) + shutil.rmtree(tmp) + return status diff --git a/pandora/archive/tasks.py b/pandora/archive/tasks.py index 41172cf7..8c61726f 100644 --- a/pandora/archive/tasks.py +++ b/pandora/archive/tasks.py @@ -10,6 +10,7 @@ from django.db.models import Q from item.models import Item import models import extract +import external _INSTANCE_KEYS = ('mtime', 'path') @@ -169,3 +170,7 @@ def update_stream(id): for c in s.file.item.clips.all(): c.update_calculated_values() c.save() + +@task(queue="encoding") +def download_media(item_id, url): + return external.download(item_id, url) diff --git a/pandora/archive/views.py b/pandora/archive/views.py index bde0ce88..905e8dda 100644 --- a/pandora/archive/views.py +++ b/pandora/archive/views.py @@ -24,6 +24,7 @@ from changelog.models import add_changelog from . import models from . import queue from . import tasks +from . import external from .chunk import process_chunk @@ -718,3 +719,48 @@ def getEncodingStatus(request, data): response['data']['status'] = queue.status() return render_to_json_response(response) actions.register(getEncodingStatus, cache=False) + +@login_required_json +def getMediaUrlInfo(request, data): + ''' + Get info (title, duration,...) about given media url, + if url is a playlist, result has info about each item. + + takes { + url: string // url + } + returns { + items: [{title, url,...}] // info for each url found + + } + ''' + if not request.user.profile.capability('canAddItems'): + response = json_response(status=403, text='permission denied') + else: + response = json_response() + response['data']['items'] = external.get_info(data['url']) + return render_to_json_response(response) +actions.register(getMediaUrlInfo, cache=False) + +@login_required_json +def addMediaUrl(request, data): + ''' + Import video from url and add to item + + takes { + url: string, // url + item: string // item + } + returns { + taskId: string, // taskId + } + ''' + if not request.user.profile.capability('canAddItems'): + response = json_response(status=403, text='permission denied') + else: + response = json_response() + t = tasks.download_media.delay(data['item'], data['url']) + response['data']['taskId'] = t.task_id + add_changelog(request, data, data['item']) + return render_to_json_response(response) +actions.register(addMediaUrl, cache=False) diff --git a/static/js/importMediaDialog.js b/static/js/importMediaDialog.js new file mode 100644 index 00000000..2f0d11e1 --- /dev/null +++ b/static/js/importMediaDialog.js @@ -0,0 +1,146 @@ +'use strict'; + +pandora.ui.importMediaDialog = function(options) { + var + $content = Ox.Element().css({margin: '16px'}), + $url = Ox.Input({ + label: Ox._('Url'), + labelWidth: 32, + width: 384 + }) + .css({ + marginTop: '16px' + }) + .bindEvent({ + change: function(data) { + $info.empty(); + that.disableButton('import'); + if (data.value) { + $info.append(loadingIcon()); + getInfo(data.value, function(items) { + $info.empty(); + if (items.length) { + // FIXME: support playlists / multiple items + var info = items[0]; + that.enableButton('import'); + $info.append($('').css({ + 'max-width': '128px', + margin: '4px', + float: 'left' + }).attr('src', info.thumbnail)); + $info.append($('
').css({ + 'font-weight': 'bold' + }).html(info.title)); + if (info.duration) { + $info.append($('
').html(Ox.formatDuration(info.duration))); + } + $info.append($('
').html(info.description)); + } else { + $info.empty(); + that.disableButton('import'); + } + }); + } + } + }) + .appendTo($content), + $info = Ox.Element() + .css({margin: '8px'}) + .html('Enter a url from another site (like YouTube) to create a new item.') + .appendTo($content), + that = Ox.Dialog({ + buttons: [ + Ox.Button({ + id: 'cancel', + title: Ox._('Cancel') + }) + .bindEvent({ + click: function() { + that.close(); + } + }), + Ox.Button({ + disabled: true, + id: 'import', + title: Ox._('Import') + }).bindEvent({ + click: importMedia + }) + ], + closeButton: true, + content: $content, + fixedSize: true, + height: 176, + keys: { + escape: 'cancel' + }, + removeOnClose: true, + title: Ox._('Import Media'), + width: 416 + }); + window.$info = $info; + function getInfo(url, callback) { + pandora.api.getMediaUrlInfo({url: url}, function(result) { + callback(result.data.items); + }); + } + + function addMedia(url, callback) { + pandora.api.getMediaUrlInfo({url: url}, function(result) { + result.data.items.forEach(function(info) { + pandora.api.add({title: info.title}, function(result) { + var edit = { + director: info.uploader ? [info.uploader] : [], + id: result.data.id, + notes: info.url, + summary: info.description, + topic: info.tags + }; + pandora.api.edit(edit, function(result) { + pandora.api.addMediaUrl({ + url: info.url, + item: edit.id + }, function(result) { + if (result.data.taskId) { + pandora.wait(result.data.taskId, function(result) { + Ox.print('status?', result); + callback(edit.id); + }); + } else { + callback(edit.id); + } + }); + }); + }); + }); + }); + }; + + function importMedia() { + var url = $url.value(); + $info.empty(); + $info.append(loadingIcon()); + that.disableButton('import'); + that.disableButton('cancel'); + addMedia(url, function(item) { + if (item) { + that.close(); + Ox.Request.clearCache(); + pandora.URL.push('/'+item+'/media'); + } else { + $info.empty().html('Import failed'); + that.enableButton('cancel'); + } + }); + } + + function loadingIcon() { + return Ox.LoadingIcon().css({ + margin: 'auto', + marginTop: '64px', + width: '100%' + }).start(); + } + + return that; +}; diff --git a/vm/pandora_install.sh b/vm/pandora_install.sh index 3d36a996..6949e248 100755 --- a/vm/pandora_install.sh +++ b/vm/pandora_install.sh @@ -69,6 +69,7 @@ apt-get install -y \ gpac \ imagemagick \ poppler-utils \ + youtube-dl \ ipython \ postfix \ postgresql \