272 lines
9.4 KiB
Python
Executable File
272 lines
9.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
from argparse import ArgumentParser
|
|
import getpass
|
|
import json
|
|
import math
|
|
import os
|
|
import sys
|
|
import urllib.parse
|
|
import urllib.request
|
|
from glob import glob
|
|
|
|
import ox
|
|
import ox.web.auth
|
|
|
|
|
|
base_url = None
|
|
prefix = '/mnt'
|
|
render = './cache'
|
|
pandora_client_config = {}
|
|
use_local = False
|
|
|
|
|
|
|
|
if os.path.exists('files.json'):
|
|
files = json.load(open('files.json'))
|
|
else:
|
|
files = {}
|
|
|
|
def get_volume_prefix(name):
|
|
path = pandora_client_config.get('volumes', {}).get(name)
|
|
if path:
|
|
return path
|
|
else:
|
|
return os.path.join(prefix, name)
|
|
|
|
def get_info(api, oshash, item, part):
|
|
if oshash not in files:
|
|
r = api.findMedia({
|
|
'query': {
|
|
'conditions': [{'key': 'oshash', 'value': oshash}]
|
|
},
|
|
'keys': ['id', 'instances', 'resolution']
|
|
})['data']
|
|
if not r['items'][0]['instances']:
|
|
if use_local:
|
|
print(r, item, part)
|
|
raise Exception('item without instance')
|
|
files[oshash] = {
|
|
'resolution': r['items'][0]['resolution']
|
|
}
|
|
if r['items'][0]['instances']:
|
|
volume_prefix = get_volume_prefix(r['items'][0]['instances'][0]['volume'])
|
|
files[oshash]['path'] = os.path.join(volume_prefix, r['items'][0]['instances'][0]['path'])
|
|
return files[oshash]
|
|
|
|
def normalize(name):
|
|
return name.replace(':', '_').replace('/', '_')
|
|
|
|
def sort_clips(edit, sort):
|
|
clips = edit['clips']
|
|
idx = 0
|
|
for clip in clips:
|
|
clip['index'] = idx
|
|
idx += 1
|
|
reverse = False
|
|
if sort.startswith('+'):
|
|
reverse = False
|
|
sort = sort[1:]
|
|
elif sort.startswith('-'):
|
|
reverse = True
|
|
sort = sort[1:]
|
|
last = '' if reverse else 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
|
|
if sort == 'manual':
|
|
s = clips
|
|
if reverse:
|
|
s = reversed(s)
|
|
elif sort in [
|
|
'id', 'index', 'in', 'out', 'duration',
|
|
'title', 'director', 'year', 'videoRatio',
|
|
]:
|
|
s = sorted(clips, key=lambda c: (ox.sort_string(str(c.get(sort, last))), c['title'], c['in'], c['out']))
|
|
if reverse:
|
|
s = reversed(s)
|
|
else:
|
|
ids = api.sortClips({
|
|
'edit': edit['id'],
|
|
'sort': [{'key': sort, 'operator': '-' if reverse else '+'}]
|
|
})['data']['clips']
|
|
#print(set(c['id'] for c in clips) - set(ids))
|
|
#print(set(ids) - set(c['id'] for c in clips))
|
|
|
|
s = sorted(clips, key=lambda c: ids.index(c['id']) if c['id'] in ids else -1)
|
|
return s
|
|
|
|
def get_pandora_media_path(oshash):
|
|
h = oshash
|
|
path = '/srv/pandora/data/media/' + '/'.join([h[:2], h[2:4], h[4:6], h[6:]])
|
|
path = glob('%s/data.*' % path)
|
|
if path:
|
|
path = path[0]
|
|
else:
|
|
path = ox.sorted_strings(glob('%s/*.mp4' % path))[-1]
|
|
return path
|
|
|
|
def cache_clips(api, videos, use_source=False, use_pandora=False):
|
|
for clip in videos:
|
|
out = '%s/%s.mp4' % (render, clip['oshash'])
|
|
if 'path' in clip:
|
|
clip['src'] = clip['path']
|
|
clip['path'] = out
|
|
if not os.path.exists(out):
|
|
if use_pandora:
|
|
os.symlink(get_pandora_media_path(clip['oshash']), out)
|
|
else:
|
|
url = clip['url']
|
|
if use_source:
|
|
name = url.split('/')[-1].split('.')[0]
|
|
resolution, part = name.split('p')
|
|
if part and part.isdigit():
|
|
part = int(part)
|
|
else:
|
|
part = 1
|
|
url = '/'.join(url.split('/')[:-1] + ['download', 'source', str(part)])
|
|
print(url, out)
|
|
api.save_url(url, out)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
usage = "usage: %(prog)s [options] edit-url"
|
|
parser = ArgumentParser(usage=usage)
|
|
parser.add_argument('-p', '--prefix', dest='prefix', type=str,
|
|
help="prefix to use instead of pandora_client config", default='')
|
|
parser.add_argument('-s', '--source', dest='source', type=str,
|
|
help="source, local or site", default='site')
|
|
parser.add_argument('-r', '--resolution', dest='stream_resolution', type=int,
|
|
help="resolution of streams to download i.e. 480, 240, 96 default 480", default=480)
|
|
parser.add_argument('-c', '--config', dest='config',
|
|
help='config.json containing config',
|
|
default='~/.ox/client.json')
|
|
parser.add_argument('-o', '--output', dest='output',
|
|
help='json output',
|
|
default='')
|
|
parser.add_argument('url', metavar='url', type=str,
|
|
help='edit url')
|
|
opts = parser.parse_args()
|
|
edit_url = opts.url
|
|
use_local = opts.source == 'local'
|
|
use_source = opts.source == 'source'
|
|
use_pandora = opts.source == 'pandora'
|
|
prefix = opts.prefix
|
|
parts = edit_url.split('/')
|
|
site = parts[2]
|
|
base_url = '/'.join(parts[:3])
|
|
edit_id = urllib.parse.unquote(parts[4]).replace('_', ' ').replace('\t', '_')
|
|
sort_by = parts[6] if len(parts) >= 7 else 'year'
|
|
stream_resolution = opts.stream_resolution
|
|
|
|
credentials = None
|
|
config = os.path.expanduser(opts.config)
|
|
if config and os.path.exists(config):
|
|
with open(config) as fd:
|
|
data = json.load(fd)
|
|
if data['url'].startswith(base_url):
|
|
pandora_client_config = data
|
|
credentials = {
|
|
'username': data['username'],
|
|
'password': data['password']
|
|
}
|
|
|
|
update = False
|
|
if not credentials:
|
|
try:
|
|
credentials = ox.web.auth.get(site)
|
|
except:
|
|
credentials = {}
|
|
print('Please provide your username and password for %s:' % site)
|
|
credentials['username'] = input('Username: ')
|
|
credentials['password'] = getpass.getpass('Password: ')
|
|
update = True
|
|
|
|
api = ox.API(base_url + '/api/')
|
|
r = api.signin(**credentials)
|
|
if 'errors' in r.get('data', {}):
|
|
for kv in r['data']['errors'].items():
|
|
print('%s: %s' % kv)
|
|
sys.exit(1)
|
|
if update:
|
|
ox.web.auth.update(site, credentials)
|
|
print('Edit:', edit_id)
|
|
print('Sort:', sort_by)
|
|
r = api.getEdit(id=edit_id)
|
|
if 'data' not in r:
|
|
print(r)
|
|
sys.exit(1)
|
|
if r.get('status', {}).get('code') == 404:
|
|
print(r.get('status', {}).get('text'))
|
|
sys.exit(1)
|
|
edit = r['data']
|
|
videos = []
|
|
subtitles = []
|
|
position = 0
|
|
for clip in sort_clips(edit, sort_by):
|
|
clip_out = position + clip['duration']
|
|
clip_subtitles = []
|
|
for sub in clip['layers'].get('subtitles', []):
|
|
value = sub['value'].replace('<br/>', '\n').replace('<br>', '\n').replace('\n\n', '\n')
|
|
subtitles.append({
|
|
'in': max(position, position + sub['in'] - clip['in']),
|
|
'out': min(position + sub['out'] - clip['in'], clip_out),
|
|
'value': value,
|
|
})
|
|
clip_subtitles.append({
|
|
'in': max(0, sub['in'] - clip['in']),
|
|
'out': min(sub['out'] - clip['in'], clip['out'] - clip['in']),
|
|
'value': value,
|
|
})
|
|
|
|
part_pos = 0
|
|
for i, duration in enumerate(clip['durations']):
|
|
stream_out = stream_in = None
|
|
if part_pos + duration < clip['in']:
|
|
part_pos += duration
|
|
elif part_pos <= clip['in']:
|
|
stream_in = clip['in'] - part_pos
|
|
stream_out = min(clip['out'] - part_pos, duration)
|
|
elif clip['out'] > part_pos:
|
|
stream_in = 0
|
|
stream_out = min(clip['out'] - part_pos, duration)
|
|
|
|
if stream_in is not None and stream_out is not None:
|
|
stream_in = math.ceil(stream_in / (1/25)) * 1/25
|
|
stream_out = math.ceil(stream_out / (1/25)) * 1/25
|
|
info = get_info(api, clip['streams'][i], clip['item'], i+1)
|
|
clip_duration = stream_out - stream_in
|
|
if clip_duration > 0:
|
|
videos.append({
|
|
'oshash': clip['streams'][i],
|
|
'url': '%s/%s/%sp%s.mp4' % (base_url, clip['item'], stream_resolution, i + 1),
|
|
'item': clip['item'],
|
|
'annotation': clip.get('annotation'),
|
|
'resolution': info['resolution'],
|
|
'in': stream_in,
|
|
'out': stream_out,
|
|
'volume': clip.get('volume', 1),
|
|
'subtitles': clip_subtitles,
|
|
})
|
|
if 'path' in info:
|
|
videos[-1]['path'] = os.path.join(prefix, info['path'])
|
|
part_pos += duration
|
|
|
|
position += clip['duration']
|
|
position = math.ceil(position / (1/25)) * 1/25
|
|
|
|
if not use_local:
|
|
cache_clips(api, videos, use_source, use_pandora)
|
|
|
|
if opts.output:
|
|
name = opts.output.replace('.json', '')
|
|
else:
|
|
name = normalize(edit_id)
|
|
if sort_by != 'year':
|
|
name += '_' + sort_by
|
|
if subtitles:
|
|
with open('%s.srt' % name, 'wb') as fd:
|
|
fd.write(ox.srt.encode(subtitles))
|
|
with open('%s.json' % name, 'w') as fd:
|
|
json.dump(videos, fd, indent=4, ensure_ascii=False)
|
|
|
|
with open('files.json', 'w') as fd:
|
|
json.dump(files, fd, indent=4, ensure_ascii=False)
|
|
print('created: %s' % '%s.json' % name)
|