diff --git a/pandora/archive/external.py b/pandora/archive/external.py new file mode 100644 index 00000000..5dac990e --- /dev/null +++ b/pandora/archive/external.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 +from __future__ import division, print_function + +import json +import subprocess +import shutil +import tempfile +import os + +import ox +from django.conf import settings + +from item.models import Item +from item.tasks import load_subtitles + +import models + +info_keys = [ + 'title', + 'description', + 'duration', + 'width', + 'height', + 'webpage_url', + 'thumbnail', + 'ext', + 'uploader', + 'subtitles', + 'tags' +] + +info_key_map = { + 'webpage_url': 'url', + 'ext': 'extension', +} + +def get_info(url): + cmd = ['youtube-dl', '-j', '--all-subs', url] + p = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, close_fds=True) + stdout, stderr = p.communicate() + stdout = stdout.decode().strip() + info = [] + if stdout: + for line in stdout.split('\n'): + i = json.loads(line) + if not i.get('is_live'): + info.append({ + info_key_map.get(k, k): i[k] + for k in info_keys + if k in i and i[k] + }) + return info + +def add_subtitles(item, media, tmp): + for language in media.get('subtitles', {}): + for subtitle in media['subtitles'][language]: + if subtitle['ext'] in ('vtt', 'srt'): + data = ox.cache.read_url(subtitle['url']) + srt = os.path.join(tmp, 'media.' + subtitle['ext']) + with open(srt, 'wb') as fd: + fd.write(data) + oshash = ox.oshash(srt) + sub, created = models.File.objects.get_or_create(oshash=oshash) + if created: + sub.item = item + sub.data.name = sub.get_path('data.' + subtitle['ext']) + ox.makedirs(os.path.dirname(sub.data.path)) + shutil.move(srt, sub.data.path) + sub.path = '.'.join([media['title'], language, subtitle['ext']]) + sub.info = ox.avinfo(sub.data.path) + if 'path' in sub.info: + del sub.info['path'] + sub.info['extension'] = subtitle['ext'] + sub.info['language'] = language + sub.parse_info() + sub.selected = True + sub.save() + +def download(item_id, url): + item = Item.objects.get(public_id=item_id) + info = get_info(url) + if len(info) != 1: + return '%s contains %d videos' % (url, len(info)) + media = info[0] + cdir = os.path.abspath(os.curdir) + tmp = tempfile.mkdtemp().decode('utf-8') + os.chdir(tmp) + cmd = ['youtube-dl', '-q', media['url']] + p = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, close_fds=True) + stdout, stderr = p.communicate() + fname = list(os.listdir(tmp)) + if fname: + fname = os.path.join(tmp, fname[0]) + oshash = ox.oshash(fname) + f, created = models.File.objects.get_or_create(oshash=oshash) + if created: + f.data.name = f.get_path('data.' + fname.split('.')[-1]) + ox.makedirs(os.path.dirname(f.data.path)) + shutil.move(fname, f.data.path) + f.item = item + f.info = ox.avinfo(f.data.path) + f.info['extension'] = media['extension'] + f.path = '%(title)s.%(extension)s' % media + f.parse_info() + f.selected = True + f.save() + f.item.save() + f.extract_stream() + status = True + else: + status = 'file exists' + add_subtitles(f.item, media, tmp) + else: + status = 'download failed' + os.chdir(cdir) + shutil.rmtree(tmp) + return status diff --git a/pandora/archive/tasks.py b/pandora/archive/tasks.py index 41172cf7..8c61726f 100644 --- a/pandora/archive/tasks.py +++ b/pandora/archive/tasks.py @@ -10,6 +10,7 @@ from django.db.models import Q from item.models import Item import models import extract +import external _INSTANCE_KEYS = ('mtime', 'path') @@ -169,3 +170,7 @@ def update_stream(id): for c in s.file.item.clips.all(): c.update_calculated_values() c.save() + +@task(queue="encoding") +def download_media(item_id, url): + return external.download(item_id, url) diff --git a/pandora/archive/views.py b/pandora/archive/views.py index bde0ce88..905e8dda 100644 --- a/pandora/archive/views.py +++ b/pandora/archive/views.py @@ -24,6 +24,7 @@ from changelog.models import add_changelog from . import models from . import queue from . import tasks +from . import external from .chunk import process_chunk @@ -718,3 +719,48 @@ def getEncodingStatus(request, data): response['data']['status'] = queue.status() return render_to_json_response(response) actions.register(getEncodingStatus, cache=False) + +@login_required_json +def getMediaUrlInfo(request, data): + ''' + Get info (title, duration,...) about given media url, + if url is a playlist, result has info about each item. + + takes { + url: string // url + } + returns { + items: [{title, url,...}] // info for each url found + + } + ''' + if not request.user.profile.capability('canAddItems'): + response = json_response(status=403, text='permission denied') + else: + response = json_response() + response['data']['items'] = external.get_info(data['url']) + return render_to_json_response(response) +actions.register(getMediaUrlInfo, cache=False) + +@login_required_json +def addMediaUrl(request, data): + ''' + Import video from url and add to item + + takes { + url: string, // url + item: string // item + } + returns { + taskId: string, // taskId + } + ''' + if not request.user.profile.capability('canAddItems'): + response = json_response(status=403, text='permission denied') + else: + response = json_response() + t = tasks.download_media.delay(data['item'], data['url']) + response['data']['taskId'] = t.task_id + add_changelog(request, data, data['item']) + return render_to_json_response(response) +actions.register(addMediaUrl, cache=False) diff --git a/static/js/importMediaDialog.js b/static/js/importMediaDialog.js new file mode 100644 index 00000000..2f0d11e1 --- /dev/null +++ b/static/js/importMediaDialog.js @@ -0,0 +1,146 @@ +'use strict'; + +pandora.ui.importMediaDialog = function(options) { + var + $content = Ox.Element().css({margin: '16px'}), + $url = Ox.Input({ + label: Ox._('Url'), + labelWidth: 32, + width: 384 + }) + .css({ + marginTop: '16px' + }) + .bindEvent({ + change: function(data) { + $info.empty(); + that.disableButton('import'); + if (data.value) { + $info.append(loadingIcon()); + getInfo(data.value, function(items) { + $info.empty(); + if (items.length) { + // FIXME: support playlists / multiple items + var info = items[0]; + that.enableButton('import'); + $info.append($('').css({ + 'max-width': '128px', + margin: '4px', + float: 'left' + }).attr('src', info.thumbnail)); + $info.append($('