From b59a9c3122702dba481392a7a7e721610324c44c Mon Sep 17 00:00:00 2001 From: j Date: Tue, 9 Aug 2016 15:58:48 +0200 Subject: [PATCH 1/2] set canImportItems in config.*.jsonc --- pandora/config.0xdb.jsonc | 13 +++++++++++++ pandora/config.indiancinema.jsonc | 14 ++++++++++++++ pandora/config.padma.jsonc | 9 +++++++++ 3 files changed, 36 insertions(+) diff --git a/pandora/config.0xdb.jsonc b/pandora/config.0xdb.jsonc index 621e0047..da5140f2 100644 --- a/pandora/config.0xdb.jsonc +++ b/pandora/config.0xdb.jsonc @@ -51,6 +51,7 @@ "canEditUsers": {"admin": true}, "canExportAnnotations": {"friend": true, "staff": true, "admin": true}, "canImportAnnotations": {"staff": true, "admin": true}, + "canImportItems": {}, "canManageDocuments": {"staff": true, "admin": true}, "canManageEntities": {"staff": true, "admin": true}, "canManagePlacesAndEvents": {"staff": true, "admin": true}, @@ -132,6 +133,18 @@ {"id": "embeds", "title": "Embeds"} ], /* + "importMetadata" defines what fields (as defined in "itemKeys") will get + populated with imported metadata. There is metadata for "title", + "description", "uploader", "date", "tags", "id", "url", and the value must + be a format string (like "{title} ({id})"). The type of the itemKeys must + be "string", ["string"], "text", "date" or "year". The last two only work + with "{date}". + */ + "importMetadata": { + "summary": "{description}", + "title": "{title}" + }, + /* "itemKeys" defines the metadata associated with each item. Required keys are "*", "id" and "title". Annotation layers can be referenced too, which makes them available in the find element. Adding a key with "annotations" diff --git a/pandora/config.indiancinema.jsonc b/pandora/config.indiancinema.jsonc index 3ca4de2e..0fc624a9 100644 --- a/pandora/config.indiancinema.jsonc +++ b/pandora/config.indiancinema.jsonc @@ -52,6 +52,8 @@ "canEditUsers": {"staff": true, "admin": true}, "canExportAnnotations": {"member": true, "researcher": true, "staff": true, "admin": true}, "canImportAnnotations": {"researcher": true, "staff": true, "admin": true}, + // import needs to handle itemRequiresVideo=false first + "canImportItems": {}, "canManageDocuments": {"member": true, "researcher": true, "staff": true, "admin": true}, "canManageEntities": {"member": true, "researcher": true, "staff": true, "admin": true}, "canManagePlacesAndEvents": {"member": true, "researcher": true, "staff": true, "admin": true}, @@ -137,6 +139,18 @@ {"id": "embeds", "title": "Embeds"} ], /* + "importMetadata" defines what fields (as defined in "itemKeys") will get + populated with imported metadata. There is metadata for "title", + "description", "uploader", "date", "tags", "id", "url", and the value must + be a format string (like "{title} ({id})"). The type of the itemKeys must + be "string", ["string"], "text", "date" or "year". The last two only work + with "{date}". + */ + "importMetadata": { + "summary": "{description}", + "title": "{title}" + }, + /* "itemKeys" defines the metadata associated with each item. Required keys are "*", "id" and "title". Annotation layers can be referenced too, which makes them available in the find element. Adding a key with "annotations" diff --git a/pandora/config.padma.jsonc b/pandora/config.padma.jsonc index 0b55196f..7825e33c 100644 --- a/pandora/config.padma.jsonc +++ b/pandora/config.padma.jsonc @@ -51,6 +51,7 @@ "canEditUsers": {"admin": true}, "canExportAnnotations": {"member": true, "staff": true, "admin": true}, "canImportAnnotations": {"member": true, "staff": true, "admin": true}, + "canImportItems": {"member": true, "staff": true, "admin": true}, "canManageDocuments": {"member": true, "staff": true, "admin": true}, "canManageEntities": {"member": true, "staff": true, "admin": true}, "canManagePlacesAndEvents": {"member": true, "staff": true, "admin": true}, @@ -135,6 +136,14 @@ {"id": "texts", "title": "Texts"}, {"id": "embeds", "title": "Embeds"} ], + /* + "importMetadata" defines what fields (as defined in "itemKeys") will get + populated with imported metadata. There is metadata for "title", + "description", "uploader", "date", "tags", "id", "url", and the value must + be a format string (like "{title} ({id})"). The type of the itemKeys must + be "string", ["string"], "text", "date" or "year". The last two only work + with "{date}". + */ "importMetadata": { "date": "{date}", "topic": "{tags}", From 8c928860e186fb4b7b3c0ac0af7a931166ef7058 Mon Sep 17 00:00:00 2001 From: j Date: Wed, 10 Aug 2016 10:49:29 +0200 Subject: [PATCH 2/2] import url celanup - only import first url - pass empty string instead of undefined - page can contain multiple videos, import as parts --- pandora/archive/external.py | 54 ++++++++++-------- static/js/importMediaDialog.js | 100 ++++++++++++++++----------------- 2 files changed, 80 insertions(+), 74 deletions(-) diff --git a/pandora/archive/external.py b/pandora/archive/external.py index b9bce934..3683b2ed 100644 --- a/pandora/archive/external.py +++ b/pandora/archive/external.py @@ -89,8 +89,8 @@ def add_subtitles(item, media, tmp): def download(item_id, url): item = Item.objects.get(public_id=item_id) info = get_info(url) - if len(info) != 1: - return '%s contains %d videos' % (url, len(info)) + if not len(info): + return '%s contains no videos' % url media = info[0] cdir = os.path.abspath(os.curdir) tmp = tempfile.mkdtemp().decode('utf-8') @@ -100,28 +100,34 @@ def download(item_id, url): stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) stdout, stderr = p.communicate() - fname = list(os.listdir(tmp)) - if fname: - fname = os.path.join(tmp, fname[0]) - oshash = ox.oshash(fname) - f, created = models.File.objects.get_or_create(oshash=oshash) - if created: - f.data.name = f.get_path('data.' + fname.split('.')[-1]) - ox.makedirs(os.path.dirname(f.data.path)) - shutil.move(fname, f.data.path) - f.item = item - f.info = ox.avinfo(f.data.path) - f.info['extension'] = media['extension'] - f.path = '%(title)s.%(extension)s' % media - f.parse_info() - f.selected = True - f.save() - f.item.save() - f.extract_stream() - status = True - else: - status = 'file exists' - add_subtitles(f.item, media, tmp) + parts = list(os.listdir(tmp)) + if parts: + part = 1 + for name in parts: + name = os.path.join(tmp, name) + oshash = ox.oshash(name) + f, created = models.File.objects.get_or_create(oshash=oshash) + if created: + f.data.name = f.get_path('data.' + name.split('.')[-1]) + ox.makedirs(os.path.dirname(f.data.path)) + shutil.move(name, f.data.path) + f.item = item + f.info = ox.avinfo(f.data.path) + f.info['extension'] = media['extension'] + f.path = '%(title)s.%(extension)s' % media + f.parse_info() + f.selected = True + if len(parts) > 1: + f.part = part + part += 1 + f.save() + f.item.save() + f.extract_stream() + status = True + else: + status = 'file exists' + if len(parts) == 1: + add_subtitles(f.item, media, tmp) else: status = 'download failed' os.chdir(cdir) diff --git a/static/js/importMediaDialog.js b/static/js/importMediaDialog.js index 4ea21e3a..458bcbe7 100644 --- a/static/js/importMediaDialog.js +++ b/static/js/importMediaDialog.js @@ -84,57 +84,57 @@ pandora.ui.importMediaDialog = function(options) { function addMedia(url, callback) { pandora.api.getMediaUrlInfo({url: url}, function(result) { - result.data.items.forEach(function(info) { - var infoKeys = [ - 'date', 'description', 'id', 'tags', - 'title', 'uploader', 'url' - ]; - var values = Ox.map(pandora.site.importMetadata, function(value, key) { - var isArray = Ox.isArray( - Ox.getObjectById(pandora.site.itemKeys, key).type - ); - if (isArray && value == '{tags}') { - value = info.tags; - } else { - infoKeys.forEach(function(infoKey) { - var infoValue = info[infoKey]; - if (key == 'year' && infoKey == 'date') { - infoValue = infoValue.substr(0, 4); - } - if (infoKey == 'tags') { - infoValue = infoValue.join(', '); - } - value = value.replace( - new RegExp('\{' + infoKey + '\}', 'g'), infoValue - ); - }); - // For example: director -> uploader - if (isArray) { - value = [value]; + // FIXME: support playlists / multiple items + var info = result.data.items[0]; + var infoKeys = [ + 'date', 'description', 'id', 'tags', + 'title', 'uploader', 'url' + ]; + var values = Ox.map(pandora.site.importMetadata, function(value, key) { + var isArray = Ox.isArray( + Ox.getObjectById(pandora.site.itemKeys, key).type + ); + if (isArray && value == '{tags}') { + value = info.tags; + } else { + infoKeys.forEach(function(infoKey) { + var infoValue = info[infoKey] || ''; + if (key == 'year' && infoKey == 'date') { + infoValue = infoValue.substr(0, 4); } + if (infoKey == 'tags') { + infoValue = infoValue.join(', '); + } + value = value.replace( + new RegExp('\{' + infoKey + '\}', 'g'), infoValue + ); + }); + // For example: director -> uploader + if (isArray) { + value = [value]; } - return value; - }); - pandora.api.add({title: values.title || info.title}, function(result) { - var edit = Ox.extend( - Ox.filter(values, function(value, key) { - return key != 'title'; - }), - {'id': result.data.id} - ); - pandora.api.edit(edit, function(result) { - pandora.api.addMediaUrl({ - url: info.url, - item: edit.id - }, function(result) { - if (result.data.taskId) { - pandora.wait(result.data.taskId, function(result) { - callback(edit.id); - }); - } else { + } + return value; + }); + pandora.api.add({title: values.title || info.title}, function(result) { + var edit = Ox.extend( + Ox.filter(values, function(value, key) { + return key != 'title'; + }), + {'id': result.data.id} + ); + pandora.api.edit(edit, function(result) { + pandora.api.addMediaUrl({ + url: info.url, + item: edit.id + }, function(result) { + if (result.data.taskId) { + pandora.wait(result.data.taskId, function(result) { callback(edit.id); - } - }); + }); + } else { + callback(edit.id); + } }); }); }); @@ -183,7 +183,7 @@ pandora.ui.importMediaDialog = function(options) { if (items.length) { // FIXME: support playlists / multiple items var info = items[0]; - $info.append($('').css({ + info.thumbnail && $info.append($('').css({ position: 'absolute', width: '248px' }).attr('src', info.thumbnail)); @@ -196,7 +196,7 @@ pandora.ui.importMediaDialog = function(options) { width: '248px' }).html( '' + info.title - + '

' + info.description + + '

' + (info.description || '') )); that.enableButton('import'); } else {