1219 lines
44 KiB
Python
Executable File
1219 lines
44 KiB
Python
Executable File
#!/usr/bin/python
|
|
# -*- coding: utf-8 -*-
|
|
# vi:si:et:sw=4:sts=4:ts=4
|
|
# GPL 2012-2016
|
|
|
|
import getpass
|
|
from glob import glob
|
|
import importlib.util
|
|
import json
|
|
import math
|
|
import os
|
|
import socket
|
|
import sqlite3
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import pkg_resources
|
|
|
|
from urllib.parse import urlparse
|
|
|
|
import ox
|
|
|
|
from . import extract
|
|
from . import utils
|
|
|
|
|
|
DEBUG = False
|
|
|
|
__version__ = pkg_resources.require("pandora_client")[0].version
|
|
|
|
socket.setdefaulttimeout(300)
|
|
CHUNK_SIZE = 1024*1024*5
|
|
default_media_cache = os.environ.get('oxMEDIA', os.path.join(utils.basedir(), 'media'))
|
|
|
|
DOCUMENT_FORMATS = ('jpg', 'pdf', 'png')
|
|
sync_extensions = ()
|
|
|
|
def get_frames(filename, prefix, info, force=False):
|
|
oshash = info['oshash']
|
|
cache = os.path.join(prefix, os.path.join(*utils.hash_prefix(oshash)))
|
|
frames = []
|
|
for pos in utils.video_frame_positions(info['duration']):
|
|
frame_name = '%s.png' % pos
|
|
frame_f = os.path.join(cache, frame_name)
|
|
if force or not os.path.exists(frame_f):
|
|
print(frame_f)
|
|
extract.frame(filename, frame_f, pos)
|
|
frames.append(frame_f)
|
|
return frames
|
|
|
|
def encode(filename, prefix, profile, info=None, extract_frames=True):
|
|
if not info:
|
|
info = utils.avinfo(filename)
|
|
if 'oshash' not in info:
|
|
return None
|
|
oshash = info['oshash']
|
|
cache = os.path.join(prefix, os.path.join(*utils.hash_prefix(oshash)))
|
|
if info.get('video') and extract_frames:
|
|
frames = get_frames(filename, prefix, info)
|
|
else:
|
|
frames = []
|
|
if info.get('video') or info.get('audio'):
|
|
media_f = os.path.join(cache, profile)
|
|
if not os.path.exists(media_f) or os.stat(media_f).st_size == 0:
|
|
extract.video(filename, media_f, profile, info)
|
|
else:
|
|
print(info)
|
|
print(filename)
|
|
return None
|
|
return {
|
|
'info': info,
|
|
'oshash': oshash,
|
|
'frames': frames,
|
|
'media': media_f
|
|
}
|
|
|
|
def encode_cmd(filename, prefix, profile, info):
|
|
if not info:
|
|
info = utils.avinfo(filename)
|
|
if 'oshash' not in info:
|
|
return None
|
|
oshash = info['oshash']
|
|
cache = os.path.join(prefix, os.path.join(*utils.hash_prefix(oshash)))
|
|
media_f = os.path.join(cache, profile)
|
|
return extract.video_cmd(filename, media_f, profile, info)
|
|
|
|
def parse_path(client, path, prefix=None):
|
|
'''
|
|
args:
|
|
path - path without volume prefix
|
|
client - Client instance
|
|
prefix - volume prefix
|
|
return:
|
|
return None if file will not be used, dict with parsed item information otherwise
|
|
'''
|
|
if isinstance(path, bytes):
|
|
path = path.decode('utf-8')
|
|
path = path.replace(os.sep, '/')
|
|
parts = path.split('/')
|
|
if len(parts) >= client.folderdepth and parts[client.folderdepth-1] == 'Documents':
|
|
info = ox.movie.parse_path(u'/'.join(
|
|
parts[:client.folderdepth-1] + [parts[-1]]
|
|
))
|
|
else:
|
|
if len(parts) != client.folderdepth:
|
|
return None
|
|
info = ox.movie.parse_path(path)
|
|
if client.folderdepth == 3:
|
|
info['director'] = []
|
|
info['directorSort'] = []
|
|
return info
|
|
|
|
def example_path(client):
|
|
return '\t' + (client.folderdepth == 4 and 'L/Last, First/Title (Year)/Title.avi' or 'T/Title/Title.dv')
|
|
|
|
def ignore_file(client, path):
|
|
filename = os.path.basename(path)
|
|
if filename.startswith('._') \
|
|
or filename in ('.DS_Store', 'Thumbs.db') \
|
|
or filename.endswith('~') \
|
|
or 'Extras' + os.sep in path \
|
|
or 'Versions' + os.sep in path \
|
|
or not os.path.exists(path) \
|
|
or os.stat(path).st_size == 0:
|
|
return True
|
|
return False
|
|
|
|
def is_oshash(oshash):
|
|
try:
|
|
int(oshash, 16)
|
|
except:
|
|
return False
|
|
return len(oshash) == 16
|
|
|
|
def hide_cursor():
|
|
sys.stdout.write("\033[?25l")
|
|
sys.stdout.flush()
|
|
|
|
def show_cursor():
|
|
sys.stdout.write("\033[?25h")
|
|
sys.stdout.flush()
|
|
|
|
class Client(object):
|
|
_configfile = None
|
|
|
|
def __init__(self, config, offline=False):
|
|
if isinstance(config, str):
|
|
self._configfile = os.path.expanduser(config)
|
|
with open(config) as f:
|
|
try:
|
|
self._config = json.load(f)
|
|
except ValueError:
|
|
print("Failed to parse config at", config)
|
|
sys.exit(1)
|
|
base = self._config.get('plugin.d', os.path.join(utils.basedir(), 'client.d'))
|
|
self.load_plugins(base)
|
|
else:
|
|
self._config = config
|
|
if not self._config['url'].endswith('/'):
|
|
self._config['url'] = self._config['url'] + '/'
|
|
self.resolutions = list(reversed(sorted(self._config.get('resolutions', [480]))))
|
|
self.format = self._config.get('format', 'webm')
|
|
self.importFrames = False
|
|
|
|
if not offline:
|
|
self.online()
|
|
|
|
conn, c = self._conn()
|
|
|
|
c.execute(u'CREATE TABLE IF NOT EXISTS setting (key varchar(1024) unique, value text)')
|
|
|
|
if int(self.get('version', 0)) < 1:
|
|
self.set('version', 1)
|
|
db = [
|
|
u'''CREATE TABLE IF NOT EXISTS file (
|
|
path varchar(1024) unique,
|
|
oshash varchar(16),
|
|
atime FLOAT,
|
|
ctime FLOAT,
|
|
mtime FLOAT,
|
|
size INT,
|
|
info TEXT,
|
|
created INT,
|
|
modified INT,
|
|
deleted INT)''',
|
|
u'CREATE INDEX IF NOT EXISTS path_idx ON file (path)',
|
|
u'CREATE INDEX IF NOT EXISTS oshash_idx ON file (oshash)',
|
|
]
|
|
for i in db:
|
|
c.execute(i)
|
|
conn.commit()
|
|
if int(self.get('version', 0)) < 2:
|
|
self.set('version', 2)
|
|
db = [
|
|
u'''CREATE TABLE IF NOT EXISTS encode (
|
|
oshash varchar(16),
|
|
site varchar(255))''',
|
|
u'CREATE INDEX IF NOT EXISTS upload_site_idx ON encode (site)',
|
|
]
|
|
for i in db:
|
|
c.execute(i)
|
|
conn.commit()
|
|
if int(self.get('version', 0)) < 3:
|
|
self.set('version', 3)
|
|
db = [
|
|
u'ALTER TABLE file add sha1 varchar(42)'
|
|
]
|
|
for i in db:
|
|
c.execute(i)
|
|
conn.commit()
|
|
if int(self.get('version', 0)) < 4:
|
|
self.set('version', 4)
|
|
db = [
|
|
u'ALTER TABLE encode add status varchar(255)',
|
|
u'CREATE INDEX IF NOT EXISTS encode_status_idx ON encode (status)',
|
|
u'ALTER TABLE encode ADD modified INT DEFAULT 0',
|
|
]
|
|
for i in db:
|
|
c.execute(i)
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
def load_plugins(self, base=os.path.join(utils.basedir(), 'client.d')):
|
|
global parse_path, example_path, ignore_file, sync_extensions, encode
|
|
base = os.path.expanduser(base)
|
|
if not os.path.exists(base):
|
|
return
|
|
for name in sorted(os.listdir(base)):
|
|
if not name.endswith('.py'):
|
|
continue
|
|
path = os.path.join(base, name)
|
|
|
|
module_name = os.path.basename(path).split('.')[0]
|
|
spec = importlib.util.spec_from_file_location(module_name, path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
|
|
if hasattr(module, 'parse_path'):
|
|
parse_path = module.parse_path
|
|
if hasattr(module, 'example_path'):
|
|
example_path = module.example_path
|
|
if hasattr(module, 'ignore_file'):
|
|
ignore_file = module.ignore_file
|
|
if hasattr(module, 'sync_extensions'):
|
|
sync_extensions = module.sync_extensions
|
|
if hasattr(module, 'encode'):
|
|
encode = module.encode
|
|
|
|
def _conn(self):
|
|
db_conn = self._config['cache']
|
|
if isinstance(db_conn, bytes):
|
|
db_conn = db_conn.decode('utf-8')
|
|
db_conn = os.path.expanduser(db_conn)
|
|
if not os.path.exists(os.path.dirname(db_conn)):
|
|
os.makedirs(os.path.dirname(db_conn))
|
|
conn = sqlite3.connect(db_conn, timeout=10)
|
|
return conn, conn.cursor()
|
|
|
|
def media_cache(self):
|
|
path = self._config.get('media-cache', default_media_cache)
|
|
if path is None:
|
|
path = '/tmp/pandora_client_media_cache'
|
|
return os.path.expanduser(path)
|
|
|
|
def get(self, key, default=None):
|
|
conn, c = self._conn()
|
|
c.execute(u'SELECT value FROM setting WHERE key = ?', (key, ))
|
|
value = default
|
|
for row in c:
|
|
value = row[0]
|
|
break
|
|
conn.close()
|
|
return value
|
|
|
|
def set(self, key, value):
|
|
conn, c = self._conn()
|
|
c.execute(u'INSERT OR REPLACE INTO setting values (?, ?)', (key, str(value)))
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
def info(self, oshash):
|
|
conn, c = self._conn()
|
|
c.execute(u'SELECT info FROM file WHERE oshash = ?', (oshash, ))
|
|
info = None
|
|
for row in c:
|
|
info = json.loads(row[0])
|
|
break
|
|
conn.close()
|
|
return info
|
|
|
|
def get_info(self, oshash, prefix=None):
|
|
if prefix:
|
|
prefixes = [prefix]
|
|
else:
|
|
prefixes = self.active_volumes().values()
|
|
prefixes = [p.decode('utf-8') if isinstance(prefix, bytes) else p for p in prefixes]
|
|
_info = self.info(oshash)
|
|
for path in self.path(oshash):
|
|
for prefix in prefixes:
|
|
if path.startswith(prefix) and os.path.exists(path):
|
|
path = path[len(prefix):]
|
|
i = parse_path(self, path, prefix)
|
|
if i:
|
|
_info.update(i)
|
|
return _info
|
|
else:
|
|
print('failed to parse', path)
|
|
return
|
|
|
|
def get_info_for_ids(self, ids, prefix=None):
|
|
info = {}
|
|
for oshash in ids:
|
|
i = self.get_info(oshash, prefix)
|
|
if i:
|
|
info[oshash] = i
|
|
return info
|
|
|
|
def path(self, oshash):
|
|
conn, c = self._conn()
|
|
c.execute(u'SELECT path FROM file WHERE oshash = ?', (oshash, ))
|
|
paths = set()
|
|
for row in c:
|
|
path = row[0]
|
|
paths.add(path)
|
|
conn.close()
|
|
return list(paths)
|
|
|
|
def online(self):
|
|
self.api = API(self._config['url'], media_cache=self.media_cache())
|
|
self.api.DEBUG = DEBUG
|
|
if self.signin():
|
|
self.resolutions = list(reversed(sorted(self.api.site['video']['resolutions'])))
|
|
self.format = self.api.site['video']['formats'][0]
|
|
self.importFrames = self.api.site['media'].get('importFrames')
|
|
self.folderdepth = self._config.get('folderdepth', self.api.site['site'].get('folderdepth', 3))
|
|
|
|
def signin(self):
|
|
if 'username' in self._config:
|
|
r = self.api.signin(username=self._config['username'], password=self._config['password'])
|
|
if r['status']['code'] == 200 and 'errors' not in r['data']:
|
|
self.user = r['data']['user']
|
|
else:
|
|
self.user = False
|
|
if DEBUG:
|
|
print(r)
|
|
print('\nlogin failed! check config\n\n')
|
|
sys.exit(1)
|
|
r = self.api.init()
|
|
if r['status']['code'] == 200:
|
|
self.api.site = r['data']['site']
|
|
else:
|
|
print("\n init failed.", r['status'])
|
|
sys.exit(1)
|
|
return True
|
|
|
|
def set_encodes(self, site, files):
|
|
conn, c = self._conn()
|
|
c.execute(u'DELETE FROM encode WHERE site = ?', (site, ))
|
|
conn.commit()
|
|
conn.close()
|
|
self.add_encodes(site, files)
|
|
|
|
def get_encodes(self, site, status=''):
|
|
conn, c = self._conn()
|
|
sql = u'SELECT oshash FROM encode WHERE site = ? AND status = ?'
|
|
args = [site, status]
|
|
c.execute(sql, tuple(args))
|
|
encodes = [row[0] for row in c]
|
|
conn.close()
|
|
return encodes
|
|
|
|
def add_encodes(self, site, files):
|
|
conn, c = self._conn()
|
|
for oshash in files:
|
|
c.execute(u'INSERT INTO encode VALUES (?, ?, ?, 0)', (oshash, site, ''))
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
def update_encodes(self, add=False):
|
|
# send empty list to get updated list of requested info/files/data
|
|
site = self._config['url']
|
|
post = {'info': {}}
|
|
r = self.api.update(post)
|
|
files = r['data']['data']
|
|
if add:
|
|
conn, c = self._conn()
|
|
sql = u'SELECT oshash FROM encode WHERE site = ?'
|
|
c.execute(sql, (site, ))
|
|
known = [row[0] for row in c]
|
|
conn.close()
|
|
files = list(set(files) - set(known))
|
|
if files:
|
|
self.add_encodes(site, files)
|
|
else:
|
|
self.set_encodes(site, files)
|
|
|
|
def scan_file(self, path, rescan=False):
|
|
conn, c = self._conn()
|
|
|
|
update = True
|
|
modified = time.mktime(time.localtime())
|
|
created = modified
|
|
|
|
if isinstance(path, bytes):
|
|
path = path.decode('utf-8')
|
|
|
|
sql = u'SELECT atime, ctime, mtime, size, created, info FROM file WHERE deleted < 0 AND path=?'
|
|
c.execute(sql, [path])
|
|
stat = os.stat(path)
|
|
for row in c:
|
|
if stat.st_atime == row[0] and stat.st_ctime == row[1] and stat.st_mtime == row[2] and stat.st_size == row[3]:
|
|
created = row[4]
|
|
info = json.loads(row[5])
|
|
update = False
|
|
break
|
|
if update or rescan:
|
|
info = utils.avinfo(path, cached=not rescan)
|
|
if info['size'] > 0:
|
|
oshash = info['oshash']
|
|
sha1 = None
|
|
deleted = -1
|
|
t = (path, oshash, stat.st_atime, stat.st_ctime, stat.st_mtime,
|
|
stat.st_size, json.dumps(info), created, modified, deleted, sha1)
|
|
c.execute(u'INSERT OR REPLACE INTO file values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
|
|
t)
|
|
conn.commit()
|
|
conn.close()
|
|
return 'error' not in info
|
|
|
|
def get_resolution(self, info):
|
|
height = info['video'][0]['height'] if info.get('video') else None
|
|
for resolution in sorted(self.resolutions):
|
|
if height and height <= resolution:
|
|
return resolution
|
|
return resolution
|
|
|
|
def profile(self, info):
|
|
resolution = self.get_resolution(info)
|
|
profile = '%sp.%s' % (resolution, self.format)
|
|
return profile
|
|
|
|
def cmd(self, args):
|
|
filename = args[0]
|
|
if len(filename) == 16:
|
|
path = self.path(filename)
|
|
else:
|
|
path = [filename]
|
|
for p in path:
|
|
if os.path.exists(p):
|
|
info = utils.avinfo(p)
|
|
profile = self.profile(info)
|
|
cmds = encode_cmd(p, self.media_cache(), profile, info)
|
|
output = []
|
|
for cmd in cmds:
|
|
cmd = [' ' in c and u'"%s"' % c or c for c in cmd]
|
|
output.append(u' '.join(cmd))
|
|
print(' && '.join(output))
|
|
|
|
def save_config(self):
|
|
if not self._configfile:
|
|
raise Exception('Can not save temporary config')
|
|
with open(self._configfile, 'w') as f:
|
|
json.dump(self._config, f, indent=2)
|
|
|
|
def config(self, args):
|
|
print("Current Config:\n User: %s\n URL: %s\n" % (self._config['username'], self._config['url']))
|
|
print("Leave empty to keep current value\n")
|
|
username = input('Username: ')
|
|
if username:
|
|
self._config['username'] = username
|
|
password = getpass.getpass('Password: ')
|
|
if password:
|
|
self._config['password'] = password
|
|
url = input('Pan.do/ra URL (i.e. https://pandora.local/api/): ')
|
|
if url:
|
|
self._config['url'] = url
|
|
self.save_config()
|
|
print("\nconfiguration updated.")
|
|
|
|
def add_volume(self, args):
|
|
usage = "Usage: %s add_volume name path" % sys.argv[0]
|
|
if len(args) != 2:
|
|
print(usage)
|
|
sys.exit(1)
|
|
name = args[0]
|
|
path = os.path.abspath(args[1])
|
|
if not path.endswith(os.sep):
|
|
path += os.sep
|
|
if os.path.isdir(path):
|
|
if name in self._config['volumes']:
|
|
print("updated %s to %s" % (name, path))
|
|
else:
|
|
print("added %s %s" % (name, path))
|
|
self._config['volumes'][name] = path
|
|
self.save_config()
|
|
else:
|
|
print("'%s' does not exist" % path)
|
|
print(usage)
|
|
sys.exit(1)
|
|
|
|
def active_volumes(self):
|
|
volumes = {}
|
|
for name in sorted(self._config['volumes']):
|
|
path = self._config['volumes'][name]
|
|
path = os.path.normpath(path)
|
|
if not path.endswith(os.sep):
|
|
path += os.sep
|
|
if isinstance(path, bytes):
|
|
path = path.decode('utf-8')
|
|
if os.path.exists(path):
|
|
volumes[name] = path
|
|
return volumes
|
|
|
|
def scan(self, args):
|
|
rescan = 'rescan' in args
|
|
print("checking for new files ...")
|
|
volumes = self.active_volumes()
|
|
for name in sorted(volumes):
|
|
path = volumes[name]
|
|
conn, c = self._conn()
|
|
c.execute(u'SELECT path FROM file WHERE path LIKE ? AND deleted < 0', [u"%s%%" % path])
|
|
known_files = [r[0] for r in c.fetchall()]
|
|
conn.close()
|
|
|
|
files = []
|
|
unknown = []
|
|
ignored = []
|
|
unsupported = []
|
|
for dirpath, dirnames, filenames in os.walk(path, followlinks=True):
|
|
if filenames:
|
|
for filename in sorted(filenames):
|
|
file_path = os.path.join(dirpath, filename)
|
|
if not ignore_file(self, file_path):
|
|
files.append(file_path)
|
|
else:
|
|
ignored.append(file_path)
|
|
for f in files:
|
|
if not parse_path(self, f[len(path):], path):
|
|
unknown.append(f)
|
|
|
|
files = sorted(set(files) - set(unknown))
|
|
|
|
for f in files:
|
|
if os.path.splitext(f)[-1] not in sync_extensions and not self.scan_file(f, rescan):
|
|
unsupported.append(f)
|
|
|
|
if unknown:
|
|
example = example_path(self)
|
|
print('Files need to be in a folder structure like this:\n%s\n' % example)
|
|
print('The following files do not fit into the folder structure and will not be synced:')
|
|
print('\t' + '\n\t'.join([f[len(path):] for f in sorted(unknown)]))
|
|
print('')
|
|
|
|
if unsupported:
|
|
files = sorted(set(files) - set(unsupported))
|
|
print('The following files are in an unsupported format and will not be synced:')
|
|
print('\t' + '\n\t'.join([f[len(path):] for f in sorted(unsupported)]))
|
|
print('')
|
|
'''
|
|
'''
|
|
|
|
deleted_files = list(filter(lambda f: f not in files, known_files))
|
|
new_files = list(filter(lambda f: f not in known_files, files))
|
|
conn, c = self._conn()
|
|
if deleted_files:
|
|
deleted = time.mktime(time.localtime())
|
|
for f in deleted_files:
|
|
c.execute(u'UPDATE file SET deleted=? WHERE path=?', (deleted, f))
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
'''
|
|
print("scanned volume %s: %s files, %s new, %s deleted, %s ignored, %s unsupported" % (
|
|
name, len(files), len(new_files), len(deleted_files), len(ignored), len(unsupported)))
|
|
'''
|
|
print("scanned volume %s: %s files, %s new, %s deleted, %s ignored" % (
|
|
name, len(files), len(new_files), len(deleted_files), len(ignored)))
|
|
|
|
def extract(self, args):
|
|
if args:
|
|
if args[0] == 'offline':
|
|
files = self.get_encodes(self._config['url'])
|
|
elif args[0] == 'all':
|
|
files = []
|
|
for name in self._config['volumes']:
|
|
path = self._config['volumes'][name]
|
|
path = os.path.normpath(path)
|
|
if not path.endswith(os.sep):
|
|
path += os.sep
|
|
if os.path.exists(path):
|
|
files += self.files(path)['info']
|
|
else:
|
|
files = [f if len(f) == 16 else ox.oshash(f) for f in args]
|
|
else:
|
|
if not self.user:
|
|
print("you need to login or run pandora_client extract offline")
|
|
return
|
|
self.update_encodes()
|
|
files = self.get_encodes(self._config['url'])
|
|
|
|
for oshash in files:
|
|
info = self.info(oshash)
|
|
if 'error' not in info:
|
|
for path in self.path(oshash):
|
|
if os.path.exists(path):
|
|
profile = self.profile(info)
|
|
i = encode(path, self.media_cache(), profile, info,
|
|
self.importFrames)
|
|
break
|
|
|
|
def sync(self, args):
|
|
if not self.user:
|
|
print("you need to login")
|
|
return
|
|
conn, c = self._conn()
|
|
|
|
volumes = self.active_volumes()
|
|
if not volumes:
|
|
print("no active volumes found")
|
|
return
|
|
|
|
for name in sorted(volumes):
|
|
prefix = volumes[name]
|
|
files = self.files(prefix)
|
|
post = {}
|
|
post['files'] = files['files']
|
|
post['volume'] = name
|
|
print('sending list of files in %s (%s total)' % (name, len(post['files'])))
|
|
r = self.api.later('update', post)
|
|
# send empty list to get updated list of requested info/files/data
|
|
post = {'info': {}}
|
|
r = self.api.update(post)
|
|
|
|
if r['data']['info']:
|
|
r = self.update_info(r['data']['info'], prefix)
|
|
|
|
if 'data' not in r:
|
|
print(r)
|
|
return
|
|
|
|
if r['data']['data']:
|
|
files = []
|
|
for f in r['data']['data']:
|
|
for path in self.path(f):
|
|
if os.path.exists(path):
|
|
files.append(path)
|
|
break
|
|
if files:
|
|
print('\ncould encode and upload %s videos:\n' % len(files))
|
|
print('\n'.join(files))
|
|
if r['data']['file']:
|
|
files = []
|
|
for f in r['data']['file']:
|
|
for path in self.path(f):
|
|
if os.path.exists(path):
|
|
files.append(path)
|
|
break
|
|
if files:
|
|
print('\ncould upload %s subtitles:\n' % len(files))
|
|
print('\n'.join(files))
|
|
|
|
def upload(self, args):
|
|
if not self.user:
|
|
print("you need to login")
|
|
return
|
|
|
|
documents = []
|
|
if args:
|
|
data = []
|
|
for arg in args:
|
|
if os.path.exists(arg):
|
|
oshash = ox.oshash(arg)
|
|
self.scan_file(arg)
|
|
r = self.api.findMedia({'query': {
|
|
'conditions': [{'key': 'oshash', 'value': oshash}]
|
|
}})['data']['items']
|
|
if r == 0:
|
|
info = self.info(oshash)
|
|
filename = os.path.basename(arg)
|
|
r = self.api.addMedia({
|
|
'id': oshash,
|
|
'info': info,
|
|
'filename': filename
|
|
})
|
|
data.append(oshash)
|
|
elif not is_oshash(arg):
|
|
print('file not found "%s"' % arg)
|
|
sys.exit(1)
|
|
else:
|
|
data.append(arg)
|
|
files = []
|
|
info = []
|
|
else:
|
|
if not self.active_volumes():
|
|
print("no volumes found, mount volumes and run again")
|
|
return
|
|
# send empty list to get updated list of requested info/files/data
|
|
post = {'info': {}}
|
|
r = self.api.update(post)
|
|
data = r['data']['data']
|
|
files = r['data']['file']
|
|
info = r['data']['info']
|
|
documents = self._get_documents()
|
|
|
|
if info:
|
|
r = self.update_info(info)
|
|
data = r['data']['data']
|
|
files = r['data']['file']
|
|
|
|
if files:
|
|
print('uploading %s files' % len(files))
|
|
for oshash in files:
|
|
for path in self.path(oshash):
|
|
if os.path.exists(path):
|
|
self.api.uploadData(path, oshash)
|
|
break
|
|
if documents:
|
|
_documents = []
|
|
for oshash, item in documents:
|
|
for path in self.path(oshash):
|
|
if os.path.exists(path):
|
|
_documents.append([path, item])
|
|
break
|
|
print('uploading %s documents' % len(_documents))
|
|
for path, item in _documents:
|
|
self._add_document(path, item)
|
|
|
|
if data:
|
|
print('encoding and uploading %s videos' % len(data))
|
|
for oshash in data:
|
|
data = {}
|
|
info = self.info(oshash)
|
|
if info and 'error' not in info:
|
|
for path in self.path(oshash):
|
|
if os.path.exists(path):
|
|
if not self.api.uploadVideo(path, data,
|
|
self.profile(info), info):
|
|
print('video upload failed, giving up, please try again')
|
|
return
|
|
if 'rightsLevel' in self._config:
|
|
r = self.api.find({'query': {
|
|
'conditions': [
|
|
{'key': 'oshash', 'value': oshash, 'operator': '=='}
|
|
],
|
|
'keys': ['id'],
|
|
'range': [0, 1]
|
|
}})
|
|
if r['data']['items']:
|
|
item = r['data']['items'][0]['id']
|
|
r = self.api.edit({
|
|
'item': item,
|
|
'rightsLevel': self._config['rightsLevel']
|
|
})
|
|
break
|
|
|
|
def update_info(self, info, prefix=None):
|
|
if info:
|
|
print('sending info for %d files' % len(info))
|
|
post = {'info': {}, 'upload': True}
|
|
post['info'] = self.get_info_for_ids(info, prefix)
|
|
r = self.api.later('update', post)
|
|
# send empty list to get updated list of requested info/files/data
|
|
post = {'info': {}}
|
|
r = self.api.update(post)
|
|
return r
|
|
|
|
def upload_frames(self, args):
|
|
if not self.user:
|
|
print("you need to login")
|
|
return
|
|
for oshash in args:
|
|
info = self.info(oshash)
|
|
if info and 'error' not in info:
|
|
for path in self.path(oshash):
|
|
if os.path.exists(path):
|
|
frames = get_frames(path, self.api.media_cache, info, True)
|
|
i = {
|
|
'info': info,
|
|
'oshash': oshash,
|
|
'frames': frames,
|
|
}
|
|
r = self.api.uploadFrames(i, {})
|
|
if r.get('status', {}).get('code') != 200:
|
|
print(r)
|
|
|
|
def _get_documents(self):
|
|
files = []
|
|
for volume in self.active_volumes():
|
|
query = {
|
|
'conditions': [
|
|
{'key': 'list', 'value': volume, 'operator': '=='},
|
|
{
|
|
'conditions': [
|
|
{'key': 'filename', 'operator': '', 'value': value}
|
|
for value in DOCUMENT_FORMATS
|
|
],
|
|
'operator': '|'
|
|
}
|
|
],
|
|
'operator': '&'
|
|
}
|
|
n = self.api.findMedia({'query': query})['data']['items']
|
|
if n:
|
|
o = 0
|
|
chunk = 5000
|
|
while o < n:
|
|
files += [f for f in self.api.findMedia({
|
|
'query': query,
|
|
'keys': ['item', 'id', 'extension'],
|
|
'range': [o, o+chunk]
|
|
})['data']['items'] if f['extension'] in DOCUMENT_FORMATS]
|
|
o += chunk
|
|
missing = list(set((f['id'], f['item']) for f in files))
|
|
available = set()
|
|
total = len(missing)
|
|
ids = [m[0] for m in missing]
|
|
o = 0
|
|
chunk = 1000
|
|
while o < len(ids):
|
|
for d in self.api.findDocuments({
|
|
'query': {
|
|
'conditions': [
|
|
{'key': 'oshash', 'operator': '==', 'value': id}
|
|
for id in ids[o:o+chunk]
|
|
],
|
|
'operator': '|'
|
|
},
|
|
'keys': ['oshash'],
|
|
'range': [0, chunk]
|
|
})['data']['items']:
|
|
available.add(d['oshash'])
|
|
o += chunk
|
|
missing = [m for m in missing if m[0] not in available]
|
|
return missing
|
|
|
|
def find_document(self, oshash):
|
|
r = self.api.findDocuments({
|
|
'keys': ['id'],
|
|
'query': {
|
|
'conditions': [
|
|
{'key': 'oshash', 'value': oshash, 'operator': '=='}
|
|
],
|
|
'operator': '&'
|
|
}
|
|
})
|
|
if r['data']['items']:
|
|
return r['data']['items'][0]['id']
|
|
return None
|
|
|
|
def _add_document(self, f, item=None):
|
|
if f.split('.')[-1].lower() not in DOCUMENT_FORMATS:
|
|
print('skip, not a document', f)
|
|
return False
|
|
oshash = ox.oshash(f)
|
|
did = self.find_document(oshash)
|
|
if not did:
|
|
url = '%supload/document/' % self._config['url']
|
|
did = self.api.upload_chunks(url, f, {
|
|
'filename': os.path.basename(f)
|
|
})
|
|
if did and item:
|
|
r = self.api.addDocument({
|
|
'id': did,
|
|
'item': item
|
|
})
|
|
return did
|
|
|
|
def upload_document(self, args):
|
|
if not self.user:
|
|
print("you need to login")
|
|
return
|
|
for f in args:
|
|
r = self._add_document(f)
|
|
if not r:
|
|
print('unsupported format', f)
|
|
continue
|
|
|
|
def files(self, prefix):
|
|
if not prefix.endswith('/'):
|
|
prefix += '/'
|
|
conn, c = self._conn()
|
|
files = {}
|
|
files['info'] = {}
|
|
files['files'] = []
|
|
sql = u'SELECT path, oshash, info, atime, ctime, mtime FROM file WHERE deleted < 0 AND path LIKE ? ORDER BY path'
|
|
t = [u"%s%%" % prefix]
|
|
c.execute(sql, t)
|
|
for row in c:
|
|
path = row[0]
|
|
oshash = row[1]
|
|
info = json.loads(row[2])
|
|
if 'error' not in info:
|
|
for key in ('atime', 'ctime', 'mtime', 'path'):
|
|
if key in info:
|
|
del info[key]
|
|
files['info'][oshash] = info
|
|
files['files'].append({
|
|
'oshash': oshash,
|
|
'path': path[len(prefix):],
|
|
'atime': row[3],
|
|
'ctime': row[4],
|
|
'mtime': row[5],
|
|
})
|
|
conn.close()
|
|
return files
|
|
|
|
def clean(self, args):
|
|
if os.path.exists(self.api.media_cache):
|
|
if args and args[0] == 'all':
|
|
print("remove all cached videos in", self.api.media_cache)
|
|
|
|
# if os.path.exists(self.api.media_cache):
|
|
# shutil.rmtree(self.api.media_cache)
|
|
else:
|
|
nothing = False
|
|
for root, folders, files in os.walk(self.api.media_cache):
|
|
for f in sorted(files):
|
|
f = os.path.join(root, f)
|
|
if f.endswith('.webm'):
|
|
oshash = os.path.dirname(f)[len(self.api.media_cache):].replace('/', '')
|
|
remove = True
|
|
for path in self.path(oshash):
|
|
if os.path.exists(path):
|
|
remove = False
|
|
break
|
|
if remove:
|
|
nothing = False
|
|
os.unlink(f)
|
|
if nothing and folders:
|
|
print("No unused files found in cache, run \"%s clean all\" to remove the entire cache" % sys.argv[0])
|
|
else:
|
|
utils.cleanup_tree(self.api.media_cache)
|
|
|
|
def import_srt(self, args):
|
|
'''
|
|
import srt as annotations, usage:
|
|
pandora_client import_srt ITEMID layername /path/to/transcript.srt
|
|
i.e.
|
|
pandora_client ipmort_srt ABC transcripts /path/to/transcript.srt
|
|
'''
|
|
if not args:
|
|
print('Usage: pandora_client import_srt ABC transcripts /path/to/transcript.srt')
|
|
sys.exit(1)
|
|
item = args[0]
|
|
layer = args[1]
|
|
filename = args[2]
|
|
layers = [l['id'] for l in self.api.site['layers']]
|
|
if layer not in layers:
|
|
print("invalid layer name, choices are: ", ', '.join(layers))
|
|
sys.exit(1)
|
|
if filename.endswith('.vtt'):
|
|
load = ox.vtt.load
|
|
else:
|
|
load = ox.srt.load
|
|
annotations = [{
|
|
'in': s['in'],
|
|
'out': s['out'],
|
|
'value': s['value'].replace('\n', '<br>\n') if layer == 'subtitles' else s['value'],
|
|
} for s in load(filename)]
|
|
r = self.api.addAnnotations({
|
|
'item': item,
|
|
'layer': layer,
|
|
'annotations': annotations
|
|
})
|
|
if r['status']['code'] == 400:
|
|
print('failed')
|
|
sys.exit(1)
|
|
if r['status']['code'] == 403:
|
|
print('permission deinied')
|
|
sys.exit(1)
|
|
elif r['status']['code'] == 404:
|
|
print('item not found')
|
|
sys.exit(1)
|
|
|
|
def server(self, args):
|
|
from . import server
|
|
server.run(self, args)
|
|
|
|
def client(self, args):
|
|
threads = [t.split('=')[-1] for t in args if t.startswith('c=')]
|
|
if threads:
|
|
threads = int(threads[0])
|
|
else:
|
|
threads = 1
|
|
args = [a for a in args if not a.startswith('c=')]
|
|
urls = [u for u in args if u.startswith('http:')]
|
|
name = [u for u in args if u not in urls]
|
|
if not name:
|
|
name = '%s-%s' % (socket.gethostname(), int(time.time()))
|
|
else:
|
|
name = name[0]
|
|
if not urls:
|
|
from . import localnode
|
|
nodes = localnode.LocalNodes()
|
|
time.sleep(1)
|
|
found = len(nodes)
|
|
if not found:
|
|
print('usage: %s client <server_url>\n\ti.e. %s client http://192.168.1.1:8789' % (sys.argv[0], sys.argv[0]))
|
|
sys.exit(1)
|
|
elif found > 1:
|
|
print('found multiple servers, please select one, your options are:')
|
|
for id, url in nodes.items():
|
|
print('\t%s client %s' % (sys.argv[0], url))
|
|
sys.exit(1)
|
|
|
|
else:
|
|
for id, url in nodes.items():
|
|
break
|
|
print('connecting to %s (%s)' % (id, url))
|
|
else:
|
|
url = urls[0]
|
|
from . import client
|
|
c = client.DistributedClient(url, name, threads)
|
|
c.run()
|
|
|
|
class API(ox.API):
|
|
__name__ = 'pandora_client'
|
|
__version__ = __version__
|
|
|
|
def __init__(self, url, cj=None, media_cache=None):
|
|
super(API, self).__init__(url, cj)
|
|
|
|
self.media_cache = media_cache
|
|
if not self.media_cache:
|
|
self.media_cache = os.path.expanduser(default_media_cache)
|
|
netloc = urlparse(self.url).netloc
|
|
tmp = tempfile.gettempdir()
|
|
self._resume_file = os.path.join(tmp, 'pandora_client.%s.%s.json' % (os.environ.get('USER'), netloc))
|
|
|
|
if hasattr(self, 'taskStatus') and not hasattr(self, 'getTaskStatus'):
|
|
self.getTaskStatus = self.taskStatus
|
|
|
|
def later(self, action, data, interval=5):
|
|
t = r = getattr(self, action)(data)
|
|
if r['status']['code'] == 200:
|
|
# wait for async task to finish
|
|
if 'taskId' in r['data']:
|
|
t = self.getTaskStatus(task_id=r['data']['taskId'])
|
|
print('waiting for server ...')
|
|
while t['data'].get('status') == 'PENDING':
|
|
time.sleep(interval)
|
|
t = self.getTaskStatus(task_id=r['data']['taskId'])
|
|
return t
|
|
|
|
def uploadFrames(self, i, data):
|
|
# upload frames
|
|
if self.site['media'].get('importFrames') and i['frames']:
|
|
form = ox.MultiPartForm()
|
|
form.add_field('action', 'upload')
|
|
form.add_field('id', i['oshash'])
|
|
for key in data:
|
|
form.add_field(key, data[key])
|
|
for frame in i['frames']:
|
|
fname = os.path.basename(frame)
|
|
if os.path.exists(frame):
|
|
form.add_file('frame', fname, open(frame, 'rb'))
|
|
r = self._json_request(self.url, form)
|
|
return r
|
|
|
|
def uploadVideo(self, filename, data, profile, info=None):
|
|
i = encode(filename, self.media_cache, profile, info,
|
|
self.site['media'].get('importFrames'))
|
|
if not i:
|
|
print("failed")
|
|
return
|
|
|
|
# upload frames
|
|
r = self.uploadFrames(i, data)
|
|
|
|
# upload media
|
|
if os.path.exists(i['media']):
|
|
size = ox.format_bytes(os.path.getsize(i['media']))
|
|
name = os.path.basename(filename)
|
|
print(u"uploading %s of %s (%s)" % (profile, name, size))
|
|
url = self.url + 'upload/?profile=%s&id=%s' % (profile, i['oshash'])
|
|
if not self.upload_chunks(url, i['media'], data):
|
|
if DEBUG:
|
|
print("failed")
|
|
return False
|
|
else:
|
|
print("Failed")
|
|
return False
|
|
return True
|
|
|
|
def uploadData(self, filename, oshash):
|
|
if DEBUG:
|
|
print('upload', filename)
|
|
form = ox.MultiPartForm()
|
|
form.add_field('action', 'upload')
|
|
form.add_field('id', str(oshash))
|
|
fname = os.path.basename(filename)
|
|
if not isinstance(fname, bytes):
|
|
fname = fname.encode('utf-8')
|
|
form.add_file('file', fname, open(filename, 'rb'))
|
|
r = self._json_request(self.url, form)
|
|
return r
|
|
|
|
def upload_chunks(self, url, filename, data=None):
|
|
form = ox.MultiPartForm()
|
|
resume = None
|
|
if self._resume_file and os.path.exists(self._resume_file):
|
|
with open(self._resume_file) as f:
|
|
try:
|
|
resume = json.load(f)
|
|
except:
|
|
resume = {}
|
|
if resume.get('chunkUploadUrl') != url:
|
|
resume = None
|
|
if resume:
|
|
data = resume
|
|
else:
|
|
for key in data:
|
|
form.add_field(key, data[key])
|
|
data = self._json_request(url, form)
|
|
|
|
print(filename)
|
|
hide_cursor()
|
|
|
|
def full_url(path):
|
|
if path.startswith('/'):
|
|
u = urlparse(url)
|
|
path = '%s://%s%s' % (u.scheme, u.netloc, path)
|
|
return path
|
|
|
|
result_url = full_url(data.get('url'))
|
|
if 'uploadUrl' in data:
|
|
uploadUrl = full_url(data['uploadUrl'])
|
|
f = open(filename, 'rb')
|
|
fsize = os.stat(filename).st_size
|
|
done = 0
|
|
start = time.mktime(time.localtime())
|
|
if 'offset' in data and data['offset'] < fsize:
|
|
done = data['offset']
|
|
f.seek(done)
|
|
resume_offset = done
|
|
else:
|
|
resume_offset = 0
|
|
chunk = f.read(CHUNK_SIZE)
|
|
fname = os.path.basename(filename)
|
|
if not isinstance(fname, bytes):
|
|
fname = fname.encode('utf-8')
|
|
while chunk:
|
|
elapsed = time.mktime(time.localtime()) - start
|
|
remaining = ""
|
|
if done:
|
|
r = math.ceil((elapsed / (done/(fsize-resume_offset)) - elapsed)/60) * 60 * 1000
|
|
r = ox.format_duration(r, milliseconds=False, verbosity=2)
|
|
if r:
|
|
remaining = ", %s remaining" % r
|
|
msg = '%0.2f%% %s of %s done%s' % (
|
|
100 * done/fsize, ox.format_bytes(done), ox.format_bytes(fsize), remaining)
|
|
print(''.join([msg, ' ' * (80-len(msg)), '\r']), end='')
|
|
sys.stdout.flush()
|
|
form = ox.MultiPartForm()
|
|
form.add_file('chunk', fname, chunk)
|
|
if len(chunk) < CHUNK_SIZE or f.tell() == fsize:
|
|
form.add_field('done', '1')
|
|
form.add_field('offset', str(done))
|
|
try:
|
|
data = self._json_request(uploadUrl, form)
|
|
except KeyboardInterrupt:
|
|
print("\ninterrupted by user.")
|
|
sys.exit(1)
|
|
except:
|
|
print("uploading chunk failed, will try again in 5 seconds\r", end='')
|
|
sys.stdout.flush()
|
|
if DEBUG:
|
|
print('\n', uploadUrl)
|
|
import traceback
|
|
traceback.print_exc()
|
|
data = {'result': -1}
|
|
time.sleep(5)
|
|
if data and 'status' in data:
|
|
if data['status']['code'] == 403:
|
|
print("login required")
|
|
return False
|
|
if data['status']['code'] != 200:
|
|
print("request returned error, will try again in 5 seconds")
|
|
if DEBUG:
|
|
print(data)
|
|
time.sleep(5)
|
|
if data and data.get('result') == 1:
|
|
done += len(chunk)
|
|
if data.get('offset') not in (None, done):
|
|
print('server offset out of sync, continue from', data['offset'])
|
|
done = data['offset']
|
|
f.seek(done)
|
|
if self._resume_file:
|
|
with open(self._resume_file, 'w') as r:
|
|
json.dump({
|
|
'uploadUrl': uploadUrl,
|
|
'chunkUploadUrl': url,
|
|
'url': result_url,
|
|
'offset': done
|
|
}, r, indent=2)
|
|
chunk = f.read(CHUNK_SIZE)
|
|
if self._resume_file and os.path.exists(self._resume_file):
|
|
os.unlink(self._resume_file)
|
|
resume = None
|
|
if result_url:
|
|
print(result_url + (' ' * (80-len(result_url))))
|
|
else:
|
|
print(' ' * 80)
|
|
print('')
|
|
show_cursor()
|
|
if data and 'result' in data and data.get('result') == 1:
|
|
return data.get('id', True)
|
|
else:
|
|
return False
|
|
else:
|
|
if DEBUG:
|
|
if 'status' in data and data['status']['code'] == 401:
|
|
print("login required")
|
|
else:
|
|
print("failed to upload file to", url)
|
|
print(data)
|
|
return False
|
|
|