pandora_render/edit.py

#!/usr/bin/env python3
from argparse import ArgumentParser
import getpass
import json
import math
import os
import sys
import urllib.parse
import urllib.request
from glob import glob

import ox
import ox.web.auth


base_url = None
prefix = '/mnt'
render = './cache'
pandora_client_config = {}
use_local = False


class API(ox.API):

    def save_url(self, url, filename, overwrite=False):
        if not os.path.exists(filename) or overwrite:
            dirname = os.path.dirname(filename)
            if dirname and not os.path.exists(dirname):
                os.makedirs(dirname)
            chunk_size = 16 * 1024
            request = urllib.request.Request(url)
            remote = self._opener.open(request)
            with open(filename, 'wb') as f:
                for chunk in iter(lambda: remote.read(chunk_size), b''):
                    f.write(chunk)


if os.path.exists('files.json'):
    files = json.load(open('files.json'))
else:
    files = {}

def get_volume_prefix(name):
    path = pandora_client_config.get('volumes', {}).get(name)
    if path:
        return path
    else:
        return os.path.join(prefix, name)

def get_info(api, oshash, item, part):
    if oshash not in files:
        r = api.findMedia({
            'query': {
                'conditions': [{'key': 'oshash', 'value': oshash}]
            },
            'keys': ['id', 'instances', 'resolution']
        })['data']
        if not r['items'][0]['instances']:
            if use_local:
                print(r, item, part)
                raise Exception('item without instance')
        files[oshash] = {
            'resolution': r['items'][0]['resolution']
        }
        if r['items'][0]['instances']:
            volume_prefix = get_volume_prefix(r['items'][0]['instances'][0]['volume'])
            files[oshash]['path'] = os.path.join(volume_prefix, r['items'][0]['instances'][0]['path'])
    return files[oshash]

def normalize(name):
    return name.replace(':', '_').replace('/', '_')

def sort_clips(edit, sort):
    clips = edit['clips']
    idx = 0
    for clip in clips:
        clip['index'] = idx
        idx += 1
    reverse = False
    if sort.startswith('+'):
        reverse = False
        sort = sort[1:]
    elif sort.startswith('-'):
        reverse = True
        sort = sort[1:]
    last = '' if reverse else 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
    if sort == 'manual':
        s = clips
        if reverse:
            s = reversed(s)
    elif sort in [
        'id', 'index', 'in', 'out', 'duration',
        'title', 'director', 'year', 'videoRatio',
    ]:
        s = sorted(clips, key=lambda c: (ox.sort_string(str(c.get(sort, last))), c['title'], c['in'], c['out']))
        if reverse:
            s = reversed(s)
    else:
        ids = api.sortClips({
            'edit': edit['id'],
            'sort': [{'key': sort, 'operator': '-' if reverse else '+'}]
        })['data']['clips']
        #print(set(c['id'] for c in clips) - set(ids))
        #print(set(ids) - set(c['id'] for c in clips))

        s = sorted(clips, key=lambda c: ids.index(c['id']) if c['id'] in ids else -1)
    return s

def get_pandora_media_path(oshash):
    h = oshash
    path = '/srv/pandora/data/media/' + '/'.join([h[:2], h[2:4], h[4:6], h[6:]])
    path = glob('%s/data.*' % path)
    if path:
        path = path[0]
    else:
        path = ox.sorted_strings(glob('%s/*.mp4' % path))[-1]
    return path

def cache_clips(api, videos, use_source=False, use_pandora=False):
    for clip in videos:
        out = '%s/%s.mp4' % (render, clip['oshash'])
        if 'path' in clip:
            clip['src'] = clip['path']
        clip['path'] = out
        if not os.path.exists(out):
            if use_pandora:
                os.symlink(get_pandora_media_path(clip['oshash']), out)
            else:
                url = clip['url']
                if use_source:
                    name = url.split('/')[-1].split('.')[0]
                    resolution, part = name.split('p')
                    if part and part.isdigit():
                        part = int(part)
                    else:
                        part = 1
                    url = '/'.join(url.split('/')[:-1] + ['download', 'source', str(part)])
                print(url, out)
                api.save_url(url, out)


if __name__ == '__main__':
    usage = "usage: %(prog)s [options] edit-url"
    parser = ArgumentParser(usage=usage)
    parser.add_argument('-p', '--prefix', dest='prefix', type=str,
                        help="prefix to use instead of pandora_client config", default='')
    parser.add_argument('-s', '--source', dest='source', type=str,
                        help="source, local or site", default='site')
    parser.add_argument('-r', '--resolution', dest='stream_resolution', type=int,
                        help="resolution of streams to download i.e. 480, 240, 96 default 480", default=480)
    parser.add_argument('-c', '--config', dest='config',
                        help='config.json containing config',
                        default='~/.ox/client.json')
    parser.add_argument('-o', '--output', dest='output',
                        help='json output',
                        default='')
    parser.add_argument('url', metavar='url', type=str,
                        help='edit url')
    opts = parser.parse_args()
    edit_url = opts.url
    use_local = opts.source == 'local'
    use_source = opts.source == 'source'
    use_pandora = opts.source == 'pandora'
    prefix = opts.prefix
    parts = edit_url.split('/')
    site = parts[2]
    base_url = '/'.join(parts[:3])
    edit_id = urllib.parse.unquote(parts[4]).replace('_', ' ').replace('\t', '_')
    sort_by = parts[6] if len(parts) >= 7 else 'year'
    stream_resolution = opts.stream_resolution

    credentials = None
    config = os.path.expanduser(opts.config)
    if config and os.path.exists(config):
        with open(config) as fd:
            data = json.load(fd)
            if data['url'].startswith(base_url):
                pandora_client_config = data
                credentials = {
                    'username': data['username'],
                    'password': data['password']
                }

    update = False
    if not credentials:
        try:
            credentials = ox.web.auth.get(site)
        except:
            credentials = {}
            print('Please provide your username and password for %s:' % site)
            credentials['username'] = input('Username: ')
            credentials['password'] = getpass.getpass('Password: ')
            update = True

    api = API(base_url + '/api/')
    r = api.signin(**credentials)
    if 'errors' in r.get('data', {}):
        for kv in r['data']['errors'].items():
            print('%s: %s' % kv)
        sys.exit(1)
    if update:
        ox.web.auth.update(site, credentials)
    print('Edit:', edit_id)
    print('Sort:', sort_by)
    r = api.getEdit(id=edit_id)
    if 'data' not in r:
        print(r)
        sys.exit(1)
    if r.get('status', {}).get('code') == 404:
        print(r.get('status', {}).get('text'))
        sys.exit(1)
    edit = r['data']
    videos = []
    subtitles = []
    position = 0
    for clip in sort_clips(edit, sort_by):
        clip_out = position + clip['duration']
        clip_subtitles = []
        for sub in clip['layers'].get('subtitles', []):
            value = sub['value'].replace('<br/>', '\n').replace('<br>', '\n').replace('\n\n', '\n')
            subtitles.append({
                'in': max(position, position + sub['in'] - clip['in']),
                'out': min(position + sub['out'] - clip['in'], clip_out),
                'value': value,
            })
            clip_subtitles.append({
                'in': max(0, sub['in'] - clip['in']),
                'out': min(sub['out'] - clip['in'], clip['out'] - clip['in']),
                'value': value,
            })

        part_pos = 0
        for i, duration in enumerate(clip['durations']):
            stream_out = stream_in = None
            if part_pos + duration < clip['in']:
                part_pos += duration
            elif part_pos <= clip['in']:
                stream_in = clip['in'] - part_pos
                stream_out = min(clip['out'] - part_pos, duration)
            elif clip['out'] > part_pos:
                stream_in = 0
                stream_out = min(clip['out'] - part_pos, duration)

            if stream_in is not None and stream_out is not None:
                stream_in = math.ceil(stream_in / (1/25)) * 1/25
                stream_out = math.ceil(stream_out / (1/25)) * 1/25
                info = get_info(api, clip['streams'][i], clip['item'], i+1)
                clip_duration = stream_out - stream_in
                if clip_duration > 0:
                    videos.append({
                        'oshash': clip['streams'][i],
                        'url': '%s/%s/%sp%s.mp4' % (base_url, clip['item'], stream_resolution, i + 1),
                        'item': clip['item'],
                        'item-in': clip['in'],
                        'item-out': clip['out'],
                        'annotation': clip.get('annotation'),
                        'resolution': info['resolution'],
                        'in': stream_in,
                        'out': stream_out,
                        'volume': clip.get('volume', 1),
                        'subtitles': clip_subtitles
                    })
                    if 'path' in info:
                        videos[-1]['path'] = os.path.join(prefix, info['path'])
                part_pos += duration

        position += clip['duration']
        position = math.ceil(position / (1/25)) * 1/25

    if not use_local:
        cache_clips(api, videos, use_source, use_pandora)

    if opts.output:
        name = opts.output.replace('.json', '')
    else:
        name = normalize(edit_id)
        if sort_by != 'year':
            name += '_' + sort_by
    if subtitles:
        with open('%s.srt' % name, 'wb') as fd:
            fd.write(ox.srt.encode(subtitles))
    with open('%s.json' % name, 'w') as fd:
        json.dump(videos, fd, indent=4, ensure_ascii=False)

    with open('files.json', 'w') as fd:
        json.dump(files, fd, indent=4, ensure_ascii=False)
    print('created: %s' % '%s.json' % name)