0x2620/pandora_render
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

render form site

Browse source
This commit is contained in:
j 2018-02-09 12:20:28 +01:00
parent 8325174995
commit 748e6b3d6d
2 changed files with 240 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

195
edit.py
View file

@ -1,20 +1,50 @@
#!/usr/bin/python3 #!/usr/bin/python3
import os from argparse import ArgumentParser
import getpass
import json import json
import math import math
import os
import ox import ox
import ox.web.auth import ox.web.auth
import sys
import urllib.parse
import urllib.request
base_url = None
stream_resolution = 480
prefix = '/mnt'
render = './cache'
pandora_client_config = {}
use_local = False
class API(ox.API):
def save_url(self, url, filename, overwrite=False):
if not os.path.exists(filename) or overwrite:
dirname = os.path.dirname(filename)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
chunk_size = 16 * 1024
request = urllib.request.Request(url)
remote = self._opener.open(request)
with open(filename, 'wb') as f:
for chunk in iter(lambda: remote.read(chunk_size), b''):
f.write(chunk)
base_url = 'https://0xdb.org'
credentials = ox.web.auth.get('0xdb.org')
prefix = '/Cinema'
if os.path.exists('files.json'): if os.path.exists('files.json'):
files = json.load(open('files.json')) files = json.load(open('files.json'))
else: else:
files = {} files = {}
def get_info(api, oshash): def get_volume_prefix(name):
path = pandora_client_config.get('volumes', {}).get(name)
if path:
return path
else:
return os.path.join(prefix, name)
def get_info(api, oshash, item, part):
if oshash not in files: if oshash not in files:
r = api.findMedia({ r = api.findMedia({
'query': { 'query': {
@ -23,12 +53,15 @@ def get_info(api, oshash):
'keys': ['id', 'instances', 'resolution'] 'keys': ['id', 'instances', 'resolution']
})['data'] })['data']
if not r['items'][0]['instances']: if not r['items'][0]['instances']:
print(r) print(r, item, part)
raise Exception('item without instance') if use_local:
raise Exception('item without instance')
files[oshash] = { files[oshash] = {
'path': os.path.join(prefix, r['items'][0]['instances'][0]['path']),
'resolution': r['items'][0]['resolution'] 'resolution': r['items'][0]['resolution']
} }
if r['items'][0]['instances']:
volume_prefix = get_volume_prefix(r['items'][0]['instances'][0]['volume'])
files[oshash]['path'] = os.path.join(volume_prefix, r['items'][0]['instances'][0]['path'])
return files[oshash] return files[oshash]
def normalize(name): def normalize(name):
@ -45,7 +78,7 @@ def sort_clips(edit, sort):
s = reversed(s) s = reversed(s)
elif sort in [ elif sort in [
'id', 'index', 'in', 'out', 'duration', 'id', 'index', 'in', 'out', 'duration',
'title', 'director', 'year', 'videoRatio' 'title', 'director', 'year', 'videoRatio',
]: ]:
s = sorted(clips, key=lambda c: (str(c.get(sort, last)), c['title'], c['in'], c['out'])) s = sorted(clips, key=lambda c: (str(c.get(sort, last)), c['title'], c['in'], c['out']))
if reverse: if reverse:
@ -55,31 +88,91 @@ def sort_clips(edit, sort):
'edit': edit['id'], 'edit': edit['id'],
'sort': [{'key': sort, 'operator': '-' if reverse else '+'}] 'sort': [{'key': sort, 'operator': '-' if reverse else '+'}]
})['data']['clips'] })['data']['clips']
print(set(c['id'] for c in clips) - set(ids)) #print(set(c['id'] for c in clips) - set(ids))
print(set(ids) - set(c['id'] for c in clips)) #print(set(ids) - set(c['id'] for c in clips))
s = sorted(clips, key=lambda c: ids.index(c['id']) if c['id'] in ids else -1) s = sorted(clips, key=lambda c: ids.index(c['id']) if c['id'] in ids else -1)
return s return s
def cache_clips(api, videos):
for clip in videos:
out = '%s/%s.mp4' % (render, clip['oshash'])
if 'path' in clip:
clip['src'] = clip['path']
clip['path'] = out
if not os.path.exists(out):
print(clip['url'], out)
api.save_url(clip['url'], out)
if __name__ == '__main__': if __name__ == '__main__':
import sys usage = "usage: %(prog)s [options] edit-url"
edit_id = sys.argv[1] parser = ArgumentParser(usage=usage)
if len(sys.argv) > 2: parser.add_argument('-p', '--prefix', dest='prefix', type=str,
sort_by = sys.argv[2] help="prefix to use instead of pandora_client config", default='')
else: parser.add_argument('-s', '--source', dest='source', type=str,
sort_by = 'year' help="source, local or site", default='site')
parser.add_argument('-r', '--resolution', dest='stream_resolution', type=int,
help="resolution of streams to download i.e. 480, 240, 96 default 480", default=480)
parser.add_argument('-c', '--config', dest='config',
help='config.json containing config',
default='~/.ox/client.json')
parser.add_argument('url', metavar='url', type=str,
help='edit url')
opts = parser.parse_args()
edit_url = opts.url
use_local = opts.source == 'local'
prefix = opts.prefix
parts = edit_url.split('/')
site = parts[2]
base_url = '/'.join(parts[:3])
edit_id = urllib.parse.unquote(parts[4]).replace('_', ' ').replace('\t', '_')
sort_by = parts[6] if len(parts) >= 7 else 'year'
stream_resolution = opts.stream_resolution
api = ox.API(base_url + '/api/') credentials = None
api.signin(**credentials) config = os.path.expanduser(opts.config)
edit = api.getEdit(id=edit_id)['data'] if config and os.path.exists(config):
with open(config) as fd:
data = json.load(fd)
if data['url'].startswith(base_url):
pandora_client_config = data
credentials = {
'username': data['username'],
'password': data['password']
}
if not credentials:
try:
credentials = ox.web.auth.get(site)
except:
credentials = {}
credentials['username'] = input('Username: ')
credentials['password'] = getpass.getpass('Password: ')
ox.web.auth.update(site, credentials)
api = API(base_url + '/api/')
r = api.signin(**credentials)
if 'errors' in r.get('data', {}):
for kv in r['data']['errors'].items():
print('%s: %s' % kv)
sys.exit(1)
print('Edit:', edit_id, 'Sort:', sort_by)
r = api.getEdit(id=edit_id)
if 'data' not in r:
print(r)
sys.exit(1)
if r.get('status', {}).get('code') == 404:
print(r.get('status', {}).get('text'))
sys.exit(1)
edit = r['data']
videos = [] videos = []
subtitles = [] subtitles = []
position = 0 position = 0
for clip in sort_clips(edit, sort_by): for clip in sort_clips(edit, sort_by):
clip_subtitles = [] clip_subtitles = []
for sub in clip['layers']['subtitles']: for sub in clip['layers'].get('subtitles', []):
subtitles.append({ subtitles.append({
'in': position, 'in': position,
'out': position + (sub['out'] - sub['in']), 'out': position + (sub['out'] - sub['in']),
@ -99,42 +192,58 @@ if __name__ == '__main__':
stream_out = math.ceil(stream_out / (1/25)) * 1/25 stream_out = math.ceil(stream_out / (1/25)) * 1/25
part_pos += duration part_pos += duration
info = get_info(api, clip['streams'][i]) info = get_info(api, clip['streams'][i], clip['item'], i+1)
videos.append({ if stream_out > stream_in:
'oshash': clip['streams'][i], videos.append({
'path': os.path.join(prefix, info['path']), 'oshash': clip['streams'][i],
'resolution': info['resolution'], 'url': '%s/%s/%sp%s.mp4' % (base_url, clip['item'], stream_resolution, i + 1),
'in': stream_in, 'item': clip['item'],
'out': stream_out, 'annotation': clip.get('annotation'),
'subtitles': '\n'.join(clip_subtitles) 'resolution': info['resolution'],
}) 'in': stream_in,
'out': stream_out,
})
if 'path' in info:
videos[-1]['path'] = os.path.join(prefix, info['path'])
if clip_subtitles:
videos[-1]['subtitles'] = '\n'.join(clip_subtitles)
elif clip['out'] > part_pos: elif clip['out'] > part_pos:
stream_in = part_pos stream_in = 0
stream_out = min(clip['out'] - part_pos, duration) stream_out = min(clip['out'] - part_pos, duration)
stream_in = math.ceil(stream_in / (1/25)) * 1/25 stream_in = math.ceil(stream_in / (1/25)) * 1/25
stream_out = math.ceil(stream_out / (1/25)) * 1/25 stream_out = math.ceil(stream_out / (1/25)) * 1/25
part_pos += duration part_pos += duration
info = get_info(api, clip['streams'][i]) info = get_info(api, clip['streams'][i], clip['item'], i+1)
videos.append({ if stream_out > stream_in:
'oshash': clip['streams'][i], videos.append({
'path': os.path.join(prefix, info['path']), 'oshash': clip['streams'][i],
'resolution': info['resolution'], 'url': '%s/%s/%sp%s.mp4' % (base_url, clip['item'], stream_resolution, i),
'in': stream_in, 'resolution': info['resolution'],
'out': stream_out, 'item': clip['item'],
'subtitles': '\n'.join(clip_subtitles) 'annotation': clip.get('annotation'),
}) 'in': stream_in,
'out': stream_out,
})
if 'path' in info:
videos[-1]['path'] = os.path.join(prefix, info['path'])
if clip_subtitles:
videos[-1]['subtitles'] = '\n'.join(clip_subtitles)
position += clip['duration'] position += clip['duration']
position = math.ceil(position / (1/25)) * 1/25 position = math.ceil(position / (1/25)) * 1/25
if not use_local:
cache_clips(api, videos)
name = normalize(edit_id) name = normalize(edit_id)
if sort_by != 'year': if sort_by != 'year':
name += '_' + sort_by name += '_' + sort_by
with open('%s.srt' % name, 'wb') as fd: if subtitles:
fd.write(ox.srt.encode(subtitles)) with open('%s.srt' % name, 'wb') as fd:
fd.write(ox.srt.encode(subtitles))
with open('%s.json' % name, 'w') as fd: with open('%s.json' % name, 'w') as fd:
json.dump(videos, fd, indent=4, ensure_ascii=False) json.dump(videos, fd, indent=4, ensure_ascii=False)
with open('files.json', 'w') as fd: with open('files.json', 'w') as fd:
json.dump(files, fd, indent=4, ensure_ascii=False) json.dump(files, fd, indent=4, ensure_ascii=False)
print('created: %s' % '%s.json' % name)

130
ffmpeg.py
View file

@ -1,4 +1,5 @@
#!/usr/bin/python3 #!/usr/bin/python3
from argparse import ArgumentParser
import math import math
import json import json
import os import os
@ -11,13 +12,32 @@ def run(cmd):
subprocess.call(cmd) subprocess.call(cmd)
edit_json = sys.argv[1] usage = "usage: %(prog)s [options] edit.json"
parser = ArgumentParser(usage=usage)
parser.add_argument('-r', '--resolution', dest='height', type=int,
help="output height, default 480", default=480)
parser.add_argument('-a', '--aspect', dest='aspect', type=str,
help="aspect ratio, default 16/9 (float or /)", default='16/9')
parser.add_argument('-c', '--config', dest='config',
help='config.json containing config',
default='~/.ox/client.json')
parser.add_argument('path', metavar='path', type=str,
help='edit.json path')
opts = parser.parse_args()
edit_json = opts.path
edit = json.load(open(edit_json)) edit = json.load(open(edit_json))
render = './cache' render = './cache'
output = os.path.splitext(edit_json)[0] + '.mp4' output = os.path.splitext(edit_json)[0] + '.mp4'
height = 480 height = opts.height
aspect = 16/9 if '/' in opts.aspect:
aspect = [int(p) for p in opts.aspect.split('/')]
aspect = aspect[0] / aspect[1]
else:
aspect = float(opts.aspect)
width = int(height * aspect) width = int(height * aspect)
width -= width % 2 width -= width % 2
@ -31,18 +51,25 @@ position = 0
for clip in edit: for clip in edit:
out = render + '/%s_%0.3f-%0.3f.ts' % (clip['oshash'], clip['in'], clip['out']) out = render + '/%s_%0.3f-%0.3f.ts' % (clip['oshash'], clip['in'], clip['out'])
edit_duration += (clip['out']-clip['in']) edit_duration += (clip['out']-clip['in'])
files.append(out)
if os.path.exists(out): if os.path.exists(out):
duration = ox.avinfo(out)['duration'] try:
if clip['subtitles']: duration = ox.avinfo(out)['duration']
subtitles.append({ except KeyError:
'in': position, os.unlink(out)
'out': position+duration, duration = None
'value': clip['subtitles'] if duration:
}) files.append(out)
position += duration src_duration = clip['out']-clip['in']
print(out, duration, (clip['out'] - clip['in'])) if abs(src_duration-duration) > 1:
continue print(clip.get('annotation', clip['item']), src_duration, duration)
if clip.get('subtitles'):
subtitles.append({
'in': position,
'out': position+duration,
'value': clip['subtitles']
})
position += duration
continue
clip_aspect = clip['resolution'][0] / clip['resolution'][1] clip_aspect = clip['resolution'][0] / clip['resolution'][1]
if clip_aspect < aspect: if clip_aspect < aspect:
x = width x = width
@ -65,7 +92,6 @@ for clip in edit:
'-c:v', 'libx264', '-c:v', 'libx264',
'-b:v', '2M', '-b:v', '2M',
'-preset:v', 'medium', '-profile:v', 'high', '-level:v', '4.0', '-preset:v', 'medium', '-profile:v', 'high', '-level:v', '4.0',
'-map', '0:0,0:0', '-map', '0:1,0:1',
'-r', '25', '-r', '25',
'-c:a', 'aac', '-c:a', 'aac',
'-ar', '48000', '-ar', '48000',
@ -73,39 +99,59 @@ for clip in edit:
'-b:a', '192k', '-b:a', '192k',
] ]
clip_duration = math.ceil((clip['out'] - clip['in']) / (1/25)) * 1/25 clip_duration = math.ceil((clip['out'] - clip['in']) / (1/25)) * 1/25
if not clip_duration:
cmd = [ print('skip empty clip', clip)
'ffmpeg', else:
'-nostats', '-loglevel', 'error', files.append(out)
'-ss', str(clip['in']), src_info = ox.avinfo(clip['path'])
'-i', clip['path'] if not src_info['audio']:
] + options + [ audio = ['-f', 'lavfi', '-i', 'anullsrc=channel_layout=stereo:sample_rate=48000']
'-t', str(clip_duration), audio_map = []
out else:
] audio = []
run(cmd) audio_map = ['-map', '0:0,0:0', '-map', '0:1,0:1']
duration = ox.avinfo(out)['duration'] cmd = [
if clip['subtitles']: 'ffmpeg',
subtitles.append({ '-nostats', '-loglevel', 'error',
'in': position, '-ss', str(clip['in']),
'out': position+duration, ] + audio + [
'value': clip['subtitles'] '-i', clip['path']
}) ] + audio_map + options + [
position += duration '-t', str(clip_duration),
print(out, duration, (clip['out'] - clip['in'])) out
]
run(cmd)
try:
duration = ox.avinfo(out)['duration']
except:
print('invalid file:', out)
print('try again?:')
print(' '.join(cmd).replace('-nostats -loglevel error', ''))
sys.exit(1)
print(clip, duration, clip['out']-clip['in'])
if clip.get('subtitles'):
subtitles.append({
'in': position,
'out': position+duration,
'value': clip['subtitles']
})
position += duration
print(out, duration, (clip['out'] - clip['in']))
txt = output + '.txt' txt = output + '.txt'
with open(txt, 'w') as fd: with open(txt, 'w') as fd:
fd.write('file ' + '\nfile '.join(files)) fd.write('file ' + '\nfile '.join(files))
cmd = ['ffmpeg', '-y', '-f', 'concat', '-safe', '0', '-i', txt, '-c', 'copy', output] cmd = ['ffmpeg',
'-nostats', '-loglevel', 'error',
'-y', '-f', 'concat', '-safe', '0', '-i', txt, '-c', 'copy', output]
run(cmd) run(cmd)
os.unlink(txt) os.unlink(txt)
srt = output.replace('.mp4', '.srt') if subtitles:
srt = output.replace('.mp4', '.srt')
with open(srt, 'wb') as fd: with open(srt, 'wb') as fd:
fd.write(ox.srt.encode(subtitles)) fd.write(ox.srt.encode(subtitles))
duration = ox.avinfo(output)['duration'] duration = ox.avinfo(output)['duration']
print('file is %d, edit should be %d' % (duration, edit_duration)) print('file duration is %d, edit should be %d' % (duration, edit_duration))
print('created:', output)