render form site

This commit is contained in:
j 2018-02-09 12:20:28 +01:00
parent 8325174995
commit 748e6b3d6d
2 changed files with 240 additions and 85 deletions

195
edit.py
View file

@ -1,20 +1,50 @@
#!/usr/bin/python3
import os
from argparse import ArgumentParser
import getpass
import json
import math
import os
import ox
import ox.web.auth
import sys
import urllib.parse
import urllib.request
base_url = None
stream_resolution = 480
prefix = '/mnt'
render = './cache'
pandora_client_config = {}
use_local = False
class API(ox.API):
def save_url(self, url, filename, overwrite=False):
if not os.path.exists(filename) or overwrite:
dirname = os.path.dirname(filename)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
chunk_size = 16 * 1024
request = urllib.request.Request(url)
remote = self._opener.open(request)
with open(filename, 'wb') as f:
for chunk in iter(lambda: remote.read(chunk_size), b''):
f.write(chunk)
base_url = 'https://0xdb.org'
credentials = ox.web.auth.get('0xdb.org')
prefix = '/Cinema'
if os.path.exists('files.json'):
files = json.load(open('files.json'))
else:
files = {}
def get_info(api, oshash):
def get_volume_prefix(name):
path = pandora_client_config.get('volumes', {}).get(name)
if path:
return path
else:
return os.path.join(prefix, name)
def get_info(api, oshash, item, part):
if oshash not in files:
r = api.findMedia({
'query': {
@ -23,12 +53,15 @@ def get_info(api, oshash):
'keys': ['id', 'instances', 'resolution']
})['data']
if not r['items'][0]['instances']:
print(r)
raise Exception('item without instance')
print(r, item, part)
if use_local:
raise Exception('item without instance')
files[oshash] = {
'path': os.path.join(prefix, r['items'][0]['instances'][0]['path']),
'resolution': r['items'][0]['resolution']
}
if r['items'][0]['instances']:
volume_prefix = get_volume_prefix(r['items'][0]['instances'][0]['volume'])
files[oshash]['path'] = os.path.join(volume_prefix, r['items'][0]['instances'][0]['path'])
return files[oshash]
def normalize(name):
@ -45,7 +78,7 @@ def sort_clips(edit, sort):
s = reversed(s)
elif sort in [
'id', 'index', 'in', 'out', 'duration',
'title', 'director', 'year', 'videoRatio'
'title', 'director', 'year', 'videoRatio',
]:
s = sorted(clips, key=lambda c: (str(c.get(sort, last)), c['title'], c['in'], c['out']))
if reverse:
@ -55,31 +88,91 @@ def sort_clips(edit, sort):
'edit': edit['id'],
'sort': [{'key': sort, 'operator': '-' if reverse else '+'}]
})['data']['clips']
print(set(c['id'] for c in clips) - set(ids))
print(set(ids) - set(c['id'] for c in clips))
#print(set(c['id'] for c in clips) - set(ids))
#print(set(ids) - set(c['id'] for c in clips))
s = sorted(clips, key=lambda c: ids.index(c['id']) if c['id'] in ids else -1)
return s
def cache_clips(api, videos):
for clip in videos:
out = '%s/%s.mp4' % (render, clip['oshash'])
if 'path' in clip:
clip['src'] = clip['path']
clip['path'] = out
if not os.path.exists(out):
print(clip['url'], out)
api.save_url(clip['url'], out)
if __name__ == '__main__':
import sys
edit_id = sys.argv[1]
if len(sys.argv) > 2:
sort_by = sys.argv[2]
else:
sort_by = 'year'
usage = "usage: %(prog)s [options] edit-url"
parser = ArgumentParser(usage=usage)
parser.add_argument('-p', '--prefix', dest='prefix', type=str,
help="prefix to use instead of pandora_client config", default='')
parser.add_argument('-s', '--source', dest='source', type=str,
help="source, local or site", default='site')
parser.add_argument('-r', '--resolution', dest='stream_resolution', type=int,
help="resolution of streams to download i.e. 480, 240, 96 default 480", default=480)
parser.add_argument('-c', '--config', dest='config',
help='config.json containing config',
default='~/.ox/client.json')
parser.add_argument('url', metavar='url', type=str,
help='edit url')
opts = parser.parse_args()
edit_url = opts.url
use_local = opts.source == 'local'
prefix = opts.prefix
parts = edit_url.split('/')
site = parts[2]
base_url = '/'.join(parts[:3])
edit_id = urllib.parse.unquote(parts[4]).replace('_', ' ').replace('\t', '_')
sort_by = parts[6] if len(parts) >= 7 else 'year'
stream_resolution = opts.stream_resolution
api = ox.API(base_url + '/api/')
api.signin(**credentials)
edit = api.getEdit(id=edit_id)['data']
credentials = None
config = os.path.expanduser(opts.config)
if config and os.path.exists(config):
with open(config) as fd:
data = json.load(fd)
if data['url'].startswith(base_url):
pandora_client_config = data
credentials = {
'username': data['username'],
'password': data['password']
}
if not credentials:
try:
credentials = ox.web.auth.get(site)
except:
credentials = {}
credentials['username'] = input('Username: ')
credentials['password'] = getpass.getpass('Password: ')
ox.web.auth.update(site, credentials)
api = API(base_url + '/api/')
r = api.signin(**credentials)
if 'errors' in r.get('data', {}):
for kv in r['data']['errors'].items():
print('%s: %s' % kv)
sys.exit(1)
print('Edit:', edit_id, 'Sort:', sort_by)
r = api.getEdit(id=edit_id)
if 'data' not in r:
print(r)
sys.exit(1)
if r.get('status', {}).get('code') == 404:
print(r.get('status', {}).get('text'))
sys.exit(1)
edit = r['data']
videos = []
subtitles = []
position = 0
for clip in sort_clips(edit, sort_by):
clip_subtitles = []
for sub in clip['layers']['subtitles']:
for sub in clip['layers'].get('subtitles', []):
subtitles.append({
'in': position,
'out': position + (sub['out'] - sub['in']),
@ -99,42 +192,58 @@ if __name__ == '__main__':
stream_out = math.ceil(stream_out / (1/25)) * 1/25
part_pos += duration
info = get_info(api, clip['streams'][i])
videos.append({
'oshash': clip['streams'][i],
'path': os.path.join(prefix, info['path']),
'resolution': info['resolution'],
'in': stream_in,
'out': stream_out,
'subtitles': '\n'.join(clip_subtitles)
})
info = get_info(api, clip['streams'][i], clip['item'], i+1)
if stream_out > stream_in:
videos.append({
'oshash': clip['streams'][i],
'url': '%s/%s/%sp%s.mp4' % (base_url, clip['item'], stream_resolution, i + 1),
'item': clip['item'],
'annotation': clip.get('annotation'),
'resolution': info['resolution'],
'in': stream_in,
'out': stream_out,
})
if 'path' in info:
videos[-1]['path'] = os.path.join(prefix, info['path'])
if clip_subtitles:
videos[-1]['subtitles'] = '\n'.join(clip_subtitles)
elif clip['out'] > part_pos:
stream_in = part_pos
stream_in = 0
stream_out = min(clip['out'] - part_pos, duration)
stream_in = math.ceil(stream_in / (1/25)) * 1/25
stream_out = math.ceil(stream_out / (1/25)) * 1/25
part_pos += duration
info = get_info(api, clip['streams'][i])
videos.append({
'oshash': clip['streams'][i],
'path': os.path.join(prefix, info['path']),
'resolution': info['resolution'],
'in': stream_in,
'out': stream_out,
'subtitles': '\n'.join(clip_subtitles)
})
info = get_info(api, clip['streams'][i], clip['item'], i+1)
if stream_out > stream_in:
videos.append({
'oshash': clip['streams'][i],
'url': '%s/%s/%sp%s.mp4' % (base_url, clip['item'], stream_resolution, i),
'resolution': info['resolution'],
'item': clip['item'],
'annotation': clip.get('annotation'),
'in': stream_in,
'out': stream_out,
})
if 'path' in info:
videos[-1]['path'] = os.path.join(prefix, info['path'])
if clip_subtitles:
videos[-1]['subtitles'] = '\n'.join(clip_subtitles)
position += clip['duration']
position = math.ceil(position / (1/25)) * 1/25
if not use_local:
cache_clips(api, videos)
name = normalize(edit_id)
if sort_by != 'year':
name += '_' + sort_by
with open('%s.srt' % name, 'wb') as fd:
fd.write(ox.srt.encode(subtitles))
if subtitles:
with open('%s.srt' % name, 'wb') as fd:
fd.write(ox.srt.encode(subtitles))
with open('%s.json' % name, 'w') as fd:
json.dump(videos, fd, indent=4, ensure_ascii=False)
with open('files.json', 'w') as fd:
json.dump(files, fd, indent=4, ensure_ascii=False)
print('created: %s' % '%s.json' % name)

130
ffmpeg.py
View file

@ -1,4 +1,5 @@
#!/usr/bin/python3
from argparse import ArgumentParser
import math
import json
import os
@ -11,13 +12,32 @@ def run(cmd):
subprocess.call(cmd)
edit_json = sys.argv[1]
usage = "usage: %(prog)s [options] edit.json"
parser = ArgumentParser(usage=usage)
parser.add_argument('-r', '--resolution', dest='height', type=int,
help="output height, default 480", default=480)
parser.add_argument('-a', '--aspect', dest='aspect', type=str,
help="aspect ratio, default 16/9 (float or /)", default='16/9')
parser.add_argument('-c', '--config', dest='config',
help='config.json containing config',
default='~/.ox/client.json')
parser.add_argument('path', metavar='path', type=str,
help='edit.json path')
opts = parser.parse_args()
edit_json = opts.path
edit = json.load(open(edit_json))
render = './cache'
output = os.path.splitext(edit_json)[0] + '.mp4'
height = 480
aspect = 16/9
height = opts.height
if '/' in opts.aspect:
aspect = [int(p) for p in opts.aspect.split('/')]
aspect = aspect[0] / aspect[1]
else:
aspect = float(opts.aspect)
width = int(height * aspect)
width -= width % 2
@ -31,18 +51,25 @@ position = 0
for clip in edit:
out = render + '/%s_%0.3f-%0.3f.ts' % (clip['oshash'], clip['in'], clip['out'])
edit_duration += (clip['out']-clip['in'])
files.append(out)
if os.path.exists(out):
duration = ox.avinfo(out)['duration']
if clip['subtitles']:
subtitles.append({
'in': position,
'out': position+duration,
'value': clip['subtitles']
})
position += duration
print(out, duration, (clip['out'] - clip['in']))
continue
try:
duration = ox.avinfo(out)['duration']
except KeyError:
os.unlink(out)
duration = None
if duration:
files.append(out)
src_duration = clip['out']-clip['in']
if abs(src_duration-duration) > 1:
print(clip.get('annotation', clip['item']), src_duration, duration)
if clip.get('subtitles'):
subtitles.append({
'in': position,
'out': position+duration,
'value': clip['subtitles']
})
position += duration
continue
clip_aspect = clip['resolution'][0] / clip['resolution'][1]
if clip_aspect < aspect:
x = width
@ -65,7 +92,6 @@ for clip in edit:
'-c:v', 'libx264',
'-b:v', '2M',
'-preset:v', 'medium', '-profile:v', 'high', '-level:v', '4.0',
'-map', '0:0,0:0', '-map', '0:1,0:1',
'-r', '25',
'-c:a', 'aac',
'-ar', '48000',
@ -73,39 +99,59 @@ for clip in edit:
'-b:a', '192k',
]
clip_duration = math.ceil((clip['out'] - clip['in']) / (1/25)) * 1/25
cmd = [
'ffmpeg',
'-nostats', '-loglevel', 'error',
'-ss', str(clip['in']),
'-i', clip['path']
] + options + [
'-t', str(clip_duration),
out
]
run(cmd)
duration = ox.avinfo(out)['duration']
if clip['subtitles']:
subtitles.append({
'in': position,
'out': position+duration,
'value': clip['subtitles']
})
position += duration
print(out, duration, (clip['out'] - clip['in']))
if not clip_duration:
print('skip empty clip', clip)
else:
files.append(out)
src_info = ox.avinfo(clip['path'])
if not src_info['audio']:
audio = ['-f', 'lavfi', '-i', 'anullsrc=channel_layout=stereo:sample_rate=48000']
audio_map = []
else:
audio = []
audio_map = ['-map', '0:0,0:0', '-map', '0:1,0:1']
cmd = [
'ffmpeg',
'-nostats', '-loglevel', 'error',
'-ss', str(clip['in']),
] + audio + [
'-i', clip['path']
] + audio_map + options + [
'-t', str(clip_duration),
out
]
run(cmd)
try:
duration = ox.avinfo(out)['duration']
except:
print('invalid file:', out)
print('try again?:')
print(' '.join(cmd).replace('-nostats -loglevel error', ''))
sys.exit(1)
print(clip, duration, clip['out']-clip['in'])
if clip.get('subtitles'):
subtitles.append({
'in': position,
'out': position+duration,
'value': clip['subtitles']
})
position += duration
print(out, duration, (clip['out'] - clip['in']))
txt = output + '.txt'
with open(txt, 'w') as fd:
fd.write('file ' + '\nfile '.join(files))
cmd = ['ffmpeg', '-y', '-f', 'concat', '-safe', '0', '-i', txt, '-c', 'copy', output]
cmd = ['ffmpeg',
'-nostats', '-loglevel', 'error',
'-y', '-f', 'concat', '-safe', '0', '-i', txt, '-c', 'copy', output]
run(cmd)
os.unlink(txt)
srt = output.replace('.mp4', '.srt')
with open(srt, 'wb') as fd:
fd.write(ox.srt.encode(subtitles))
if subtitles:
srt = output.replace('.mp4', '.srt')
with open(srt, 'wb') as fd:
fd.write(ox.srt.encode(subtitles))
duration = ox.avinfo(output)['duration']
print('file is %d, edit should be %d' % (duration, edit_duration))
print('file duration is %d, edit should be %d' % (duration, edit_duration))
print('created:', output)