0x2620/pandora_client
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

flake8, print_function

Browse source
This commit is contained in:
j 2016-06-15 16:12:37 +02:00
parent bdca487883
commit adf409cc4c
6 changed files with 171 additions and 169 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
bin/pandora_client
View file

@ -6,6 +6,7 @@
import os import os
import sys import sys
from optparse import OptionParser from optparse import OptionParser
import json import json
root = os.path.join(os.path.abspath(os.path.dirname(__file__)), '..') root = os.path.join(os.path.abspath(os.path.dirname(__file__)), '..')

240
pandora_client/__init__.py
View file

@ -1,8 +1,8 @@
#!/usr/bin/python #!/usr/bin/python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4 # vi:si:et:sw=4:sts=4:ts=4
# GPL 2012 # GPL 2012-2016
from __future__ import division, with_statement from __future__ import division, with_statement, print_function
import getpass import getpass
from glob import glob from glob import glob
@ -42,7 +42,7 @@ def get_frames(filename, prefix, info, force=False):
frame_name = '%s.png' % pos frame_name = '%s.png' % pos
frame_f = os.path.join(cache, frame_name) frame_f = os.path.join(cache, frame_name)
if force or not os.path.exists(frame_f): if force or not os.path.exists(frame_f):
print frame_f print(frame_f)
extract.frame(filename, frame_f, pos) extract.frame(filename, frame_f, pos)
frames.append(frame_f) frames.append(frame_f)
return frames return frames
@ -50,7 +50,7 @@ def get_frames(filename, prefix, info, force=False):
def encode(filename, prefix, profile, info=None, extract_frames=True): def encode(filename, prefix, profile, info=None, extract_frames=True):
if not info: if not info:
info = utils.avinfo(filename) info = utils.avinfo(filename)
if not 'oshash' in info: if 'oshash' not in info:
return None return None
oshash = info['oshash'] oshash = info['oshash']
cache = os.path.join(prefix, os.path.join(*utils.hash_prefix(oshash))) cache = os.path.join(prefix, os.path.join(*utils.hash_prefix(oshash)))
@ -60,12 +60,11 @@ def encode(filename, prefix, profile, info=None, extract_frames=True):
frames = [] frames = []
if info.get('video') or info.get('audio'): if info.get('video') or info.get('audio'):
media_f = os.path.join(cache, profile) media_f = os.path.join(cache, profile)
if not os.path.exists(media_f) \ if not os.path.exists(media_f) or os.stat(media_f).st_size == 0:
or os.stat(media_f).st_size == 0:
extract.video(filename, media_f, profile, info) extract.video(filename, media_f, profile, info)
else: else:
print info print(info)
print filename print(filename)
return None return None
return { return {
'info': info, 'info': info,
@ -77,7 +76,7 @@ def encode(filename, prefix, profile, info=None, extract_frames=True):
def encode_cmd(filename, prefix, profile, info): def encode_cmd(filename, prefix, profile, info):
if not info: if not info:
info = utils.avinfo(filename) info = utils.avinfo(filename)
if not 'oshash' in info: if 'oshash' not in info:
return None return None
oshash = info['oshash'] oshash = info['oshash']
cache = os.path.join(prefix, os.path.join(*utils.hash_prefix(oshash))) cache = os.path.join(prefix, os.path.join(*utils.hash_prefix(oshash)))
@ -115,12 +114,12 @@ def example_path(client):
def ignore_file(client, path): def ignore_file(client, path):
filename = os.path.basename(path) filename = os.path.basename(path)
if filename.startswith('._') \ if filename.startswith('._') \
or filename in ('.DS_Store', 'Thumbs.db') \ or filename in ('.DS_Store', 'Thumbs.db') \
or filename.endswith('~') \ or filename.endswith('~') \
or 'Extras' + os.sep in path \ or 'Extras' + os.sep in path \
or 'Versions' + os.sep in path \ or 'Versions' + os.sep in path \
or not os.path.exists(path) \ or not os.path.exists(path) \
or os.stat(path).st_size == 0: or os.stat(path).st_size == 0:
return True return True
return False return False
@ -149,7 +148,7 @@ class Client(object):
try: try:
self._config = json.load(f) self._config = json.load(f)
except ValueError: except ValueError:
print "Failed to parse config at", config print("Failed to parse config at", config)
sys.exit(1) sys.exit(1)
base = self._config.get('plugin.d', os.path.join(utils.basedir(), 'client.d')) base = self._config.get('plugin.d', os.path.join(utils.basedir(), 'client.d'))
self.load_plugins(base) self.load_plugins(base)
@ -283,7 +282,7 @@ class Client(object):
_info.update(i) _info.update(i)
return _info return _info
else: else:
print 'failed to parse', path print('failed to parse', path)
return return
def get_info_for_ids(self, ids, prefix=None): def get_info_for_ids(self, ids, prefix=None):
@ -315,25 +314,25 @@ class Client(object):
def signin(self): def signin(self):
if 'username' in self._config: if 'username' in self._config:
r = self.api.signin(username=self._config['username'], password=self._config['password']) r = self.api.signin(username=self._config['username'], password=self._config['password'])
if r['status']['code'] == 200 and not 'errors' in r['data']: if r['status']['code'] == 200 and 'errors' not in r['data']:
self.user = r['data']['user'] self.user = r['data']['user']
else: else:
self.user = False self.user = False
if DEBUG: if DEBUG:
print r print(r)
print '\nlogin failed! check config\n\n' print('\nlogin failed! check config\n\n')
sys.exit(1) sys.exit(1)
r = self.api.init() r = self.api.init()
if r['status']['code'] == 200: if r['status']['code'] == 200:
self.api.site = r['data']['site'] self.api.site = r['data']['site']
else: else:
print "\n init failed.", r['status'] print("\n init failed.", r['status'])
sys.exit(1) sys.exit(1)
return True return True
def set_encodes(self, site, files): def set_encodes(self, site, files):
conn, c = self._conn() conn, c = self._conn()
c.execute(u'DELETE FROM encode WHERE site = ?' , (site, )) c.execute(u'DELETE FROM encode WHERE site = ?', (site, ))
conn.commit() conn.commit()
self.add_encodes(site, files) self.add_encodes(site, files)
@ -351,7 +350,7 @@ class Client(object):
conn.commit() conn.commit()
def update_encodes(self, add=False): def update_encodes(self, add=False):
#send empty list to get updated list of requested info/files/data # send empty list to get updated list of requested info/files/data
site = self._config['url'] site = self._config['url']
post = {'info': {}} post = {'info': {}}
r = self.api.update(post) r = self.api.update(post)
@ -422,7 +421,7 @@ class Client(object):
profile = self.profile(info) profile = self.profile(info)
cmd = encode_cmd(p, self.media_cache(), profile, info) cmd = encode_cmd(p, self.media_cache(), profile, info)
cmd = [' ' in c and u'"%s"' % c or c for c in cmd] cmd = [' ' in c and u'"%s"' % c or c for c in cmd]
print (u' '.join(cmd)).encode('utf-8') print(u' '.join(cmd)).encode('utf-8')
def save_config(self): def save_config(self):
if not self._configfile: if not self._configfile:
@ -431,8 +430,8 @@ class Client(object):
json.dump(self._config, f, indent=2) json.dump(self._config, f, indent=2)
def config(self, args): def config(self, args):
print "Current Config:\n User: %s\n URL: %s\n" %(self._config['username'], self._config['url']) print("Current Config:\n User: %s\n URL: %s\n" % (self._config['username'], self._config['url']))
print "Leave empty to keep current value\n" print("Leave empty to keep current value\n")
username = raw_input('Username: ') username = raw_input('Username: ')
if username: if username:
self._config['username'] = username self._config['username'] = username
@ -443,12 +442,12 @@ class Client(object):
if url: if url:
self._config['url'] = url self._config['url'] = url
self.save_config() self.save_config()
print "\nconfiguration updated." print("\nconfiguration updated.")
self.install_programs() self.install_programs()
def install_programs(self, args=[]): def install_programs(self, args=[]):
update = 'update' in args update = 'update' in args
#install required programs # install required programs
if sys.platform == 'darwin': if sys.platform == 'darwin':
osext = 'macosx' osext = 'macosx'
elif sys.platform.startswith('win'): elif sys.platform.startswith('win'):
@ -462,14 +461,14 @@ class Client(object):
if sys.platform.startswith('win'): if sys.platform.startswith('win'):
p += '.exe' p += '.exe'
if not os.path.exists(path) or update: if not os.path.exists(path) or update:
print "installing %s in %s" % (p, bindir) print("installing %s in %s" % (p, bindir))
ox.net.save_url('http://firefogg.org/bin/%s.%s' % (p, osext), path, True) ox.net.save_url('http://firefogg.org/bin/%s.%s' % (p, osext), path, True)
os.chmod(path, 0755) os.chmod(path, 0o755)
def add_volume(self, args): def add_volume(self, args):
usage = "Usage: %s add_volume name path" % sys.argv[0] usage = "Usage: %s add_volume name path" % sys.argv[0]
if len(args) != 2: if len(args) != 2:
print usage print(usage)
sys.exit(1) sys.exit(1)
name = args[0] name = args[0]
path = os.path.abspath(args[1]) path = os.path.abspath(args[1])
@ -477,14 +476,14 @@ class Client(object):
path += os.sep path += os.sep
if os.path.isdir(path): if os.path.isdir(path):
if name in self._config['volumes']: if name in self._config['volumes']:
print "updated %s to %s" % (name, path) print("updated %s to %s" % (name, path))
else: else:
print "added %s %s" % (name, path) print("added %s %s" % (name, path))
self._config['volumes'][name] = path self._config['volumes'][name] = path
self.save_config() self.save_config()
else: else:
print "'%s' does not exist" % path print("'%s' does not exist" % path)
print usage print(usage)
sys.exit(1) sys.exit(1)
def active_volumes(self): def active_volumes(self):
@ -502,7 +501,7 @@ class Client(object):
def scan(self, args): def scan(self, args):
rescan = 'rescan' in args rescan = 'rescan' in args
print "checking for new files ..." print("checking for new files ...")
volumes = self.active_volumes() volumes = self.active_volumes()
for name in sorted(volumes): for name in sorted(volumes):
path = volumes[name] path = volumes[name]
@ -526,7 +525,7 @@ class Client(object):
if not parse_path(self, f[len(path):]): if not parse_path(self, f[len(path):]):
unknown.append(f) unknown.append(f)
files = list(set(files) - set(unknown)) files = sorted(set(files) - set(unknown))
for f in files: for f in files:
if not self.scan_file(f, rescan): if not self.scan_file(f, rescan):
@ -534,18 +533,16 @@ class Client(object):
if unknown: if unknown:
example = example_path(self) example = example_path(self)
print 'Files need to be in a folder structure like this:\n%s\n' % example print('Files need to be in a folder structure like this:\n%s\n' % example)
print 'The following files do not fit into the folder structure and will not be synced:' print('The following files do not fit into the folder structure and will not be synced:')
print '\t', print('\t' + '\n\t'.join([f[len(path):].encode('utf-8') for f in unknown]))
print '\n\t'.join([f[len(path):].encode('utf-8') for f in unknown]) print('')
print ''
if unsupported: if unsupported:
files = list(set(files) - set(unsupported)) files = sorted(set(files) - set(unsupported))
print 'The following files are in an unsupported format and will not be synced:' print('The following files are in an unsupported format and will not be synced:')
print '\t', print('\t' + '\n\t'.join([f[len(path):].encode('utf-8') for f in unsupported]))
print '\n\t'.join([f[len(path):].encode('utf-8') for f in unsupported]) print('')
print ''
''' '''
''' '''
@ -559,12 +556,11 @@ class Client(object):
conn.commit() conn.commit()
''' '''
print "scanned volume %s: %s files, %s new, %s deleted, %s ignored, %s unsupported" % ( print("scanned volume %s: %s files, %s new, %s deleted, %s ignored, %s unsupported" % (
name, len(files), len(new_files), len(deleted_files), len(ignored), len(unsupported)) name, len(files), len(new_files), len(deleted_files), len(ignored), len(unsupported)))
''' '''
print "scanned volume %s: %s files, %s new, %s deleted, %s ignored" % ( print("scanned volume %s: %s files, %s new, %s deleted, %s ignored" % (
name, len(files), len(new_files), len(deleted_files), len(ignored)) name, len(files), len(new_files), len(deleted_files), len(ignored)))
def extract(self, args): def extract(self, args):
conn, c = self._conn() conn, c = self._conn()
@ -584,30 +580,30 @@ class Client(object):
files = [f if len(f) == 16 else ox.oshash(f) for f in args] files = [f if len(f) == 16 else ox.oshash(f) for f in args]
else: else:
if not self.user: if not self.user:
print "you need to login or run pandora_client extract offline" print("you need to login or run pandora_client extract offline")
return return
self.update_encodes() self.update_encodes()
files = self.get_encodes(self._config['url']) files = self.get_encodes(self._config['url'])
for oshash in files: for oshash in files:
info = self.info(oshash) info = self.info(oshash)
if not 'error' in info: if 'error' not in info:
for path in self.path(oshash): for path in self.path(oshash):
if os.path.exists(path): if os.path.exists(path):
profile = self.profile(info) profile = self.profile(info)
i = encode(path, self.media_cache(), profile, info, i = encode(path, self.media_cache(), profile, info,
self.importFrames) self.importFrames)
break break
def sync(self, args): def sync(self, args):
if not self.user: if not self.user:
print "you need to login" print("you need to login")
return return
conn, c = self._conn() conn, c = self._conn()
volumes = self.active_volumes() volumes = self.active_volumes()
if not volumes: if not volumes:
print "no active volumes found" print("no active volumes found")
return return
for name in sorted(volumes): for name in sorted(volumes):
@ -616,17 +612,17 @@ class Client(object):
post = {} post = {}
post['files'] = files['files'] post['files'] = files['files']
post['volume'] = name post['volume'] = name
print 'sending list of files in %s (%s total)' % (name, len(post['files'])) print('sending list of files in %s (%s total)' % (name, len(post['files'])))
r = self.api.async('update', post) r = self.api.async('update', post)
#send empty list to get updated list of requested info/files/data # send empty list to get updated list of requested info/files/data
post = {'info': {}} post = {'info': {}}
r = self.api.update(post) r = self.api.update(post)
if r['data']['info']: if r['data']['info']:
r = self.update_info(r['data']['info'], prefix) r = self.update_info(r['data']['info'], prefix)
if not 'data' in r: if 'data' not in r:
print r print(r)
return return
if r['data']['data']: if r['data']['data']:
@ -637,8 +633,8 @@ class Client(object):
files.append(path) files.append(path)
break break
if files: if files:
print '\ncould encode and upload %s videos:\n' % len(files) print('\ncould encode and upload %s videos:\n' % len(files))
print '\n'.join(files) print('\n'.join(files))
if r['data']['file']: if r['data']['file']:
files = [] files = []
for f in r['data']['file']: for f in r['data']['file']:
@ -647,12 +643,12 @@ class Client(object):
files.append(path) files.append(path)
break break
if files: if files:
print '\ncould upload %s subtitles:\n' % len(files) print('\ncould upload %s subtitles:\n' % len(files))
print '\n'.join(files) print('\n'.join(files))
def upload(self, args): def upload(self, args):
if not self.user: if not self.user:
print "you need to login" print("you need to login")
return return
conn, c = self._conn() conn, c = self._conn()
documents = [] documents = []
@ -675,14 +671,14 @@ class Client(object):
}) })
data.append(oshash) data.append(oshash)
elif not is_oshash(arg): elif not is_oshash(arg):
print 'file not found "%s"' % arg print('file not found "%s"' % arg)
sys.exit(1) sys.exit(1)
else: else:
data.append(arg) data.append(arg)
files = [] files = []
info = [] info = []
else: else:
#send empty list to get updated list of requested info/files/data # send empty list to get updated list of requested info/files/data
post = {'info': {}} post = {'info': {}}
r = self.api.update(post) r = self.api.update(post)
data = r['data']['data'] data = r['data']['data']
@ -696,30 +692,30 @@ class Client(object):
files = r['data']['file'] files = r['data']['file']
if files: if files:
print 'uploading %s files' % len(files) print('uploading %s files' % len(files))
for oshash in files: for oshash in files:
for path in self.path(oshash): for path in self.path(oshash):
if os.path.exists(path): if os.path.exists(path):
self.api.uploadData(path, oshash) self.api.uploadData(path, oshash)
break break
if documents: if documents:
print 'uploading %s documents' % len(documents) print('uploading %s documents' % len(documents))
for oshash, item in documents: for oshash, item in documents:
for path in self.path(oshash): for path in self.path(oshash):
if os.path.exists(path): if os.path.exists(path):
self._add_document(path, item) self._add_document(path, item)
if data: if data:
print 'encoding and uploading %s videos' % len(data) print('encoding and uploading %s videos' % len(data))
for oshash in data: for oshash in data:
data = {} data = {}
info = self.info(oshash) info = self.info(oshash)
if info and not 'error' in info: if info and 'error' not in info:
for path in self.path(oshash): for path in self.path(oshash):
if os.path.exists(path): if os.path.exists(path):
if not self.api.uploadVideo(path, if not self.api.uploadVideo(path, data,
data, self.profile(info), info): self.profile(info), info):
print 'video upload failed, giving up, please try again' print('video upload failed, giving up, please try again')
return return
if 'rightsLevel' in self._config: if 'rightsLevel' in self._config:
r = self.api.find({'query': { r = self.api.find({'query': {
@ -739,23 +735,23 @@ class Client(object):
def update_info(self, info, prefix=None): def update_info(self, info, prefix=None):
if info: if info:
print 'sending info for %d files' % len(info) print('sending info for %d files' % len(info))
post = {'info': {}, 'upload': True} post = {'info': {}, 'upload': True}
post['info'] = self.get_info_for_ids(info, prefix) post['info'] = self.get_info_for_ids(info, prefix)
r = self.api.async('update', post) r = self.api.async('update', post)
#send empty list to get updated list of requested info/files/data # send empty list to get updated list of requested info/files/data
post = {'info': {}} post = {'info': {}}
r = self.api.update(post) r = self.api.update(post)
return r return r
def upload_frames(self, args): def upload_frames(self, args):
if not self.user: if not self.user:
print "you need to login" print("you need to login")
return return
conn, c = self._conn() conn, c = self._conn()
for oshash in args: for oshash in args:
info = self.info(oshash) info = self.info(oshash)
if info and not 'error' in info: if info and 'error' not in info:
for path in self.path(oshash): for path in self.path(oshash):
if os.path.exists(path): if os.path.exists(path):
frames = get_frames(path, self.api.media_cache, info, True) frames = get_frames(path, self.api.media_cache, info, True)
@ -766,7 +762,7 @@ class Client(object):
} }
r = self.api.uploadFrames(i, {}) r = self.api.uploadFrames(i, {})
if r.get('status', {}).get('code') != 200: if r.get('status', {}).get('code') != 200:
print r print(r)
def _get_documents(self): def _get_documents(self):
files = self.api.findMedia({ files = self.api.findMedia({
@ -793,9 +789,8 @@ class Client(object):
})['data']['items'] })['data']['items']
available = set(f['oshash'] available = set(f['oshash']
for f in d if f['extension'] in DOCUMENT_FORMATS) for f in d if f['extension'] in DOCUMENT_FORMATS)
missing = [(f['id'], f['item']) for f in files \ missing = [(f['id'], f['item']) for f in files
if f['id'] not in available \ if f['id'] not in available and f['extension'] in DOCUMENT_FORMATS]
and f['extension'] in DOCUMENT_FORMATS]
return missing return missing
def _add_document(self, f, item=None): def _add_document(self, f, item=None):
@ -823,13 +818,13 @@ class Client(object):
def upload_document(self, args): def upload_document(self, args):
if not self.user: if not self.user:
print "you need to login" print("you need to login")
return return
conn, c = self._conn() conn, c = self._conn()
for f in args: for f in args:
r = self._add_document(f) r = self._add_document(f)
if not r: if not r:
print 'unsupported format', f print('unsupported format', f)
continue continue
def files(self, prefix): def files(self, prefix):
@ -840,13 +835,13 @@ class Client(object):
files['info'] = {} files['info'] = {}
files['files'] = [] files['files'] = []
sql = u'SELECT path, oshash, info, atime, ctime, mtime FROM file WHERE deleted < 0 AND path LIKE ? ORDER BY path' sql = u'SELECT path, oshash, info, atime, ctime, mtime FROM file WHERE deleted < 0 AND path LIKE ? ORDER BY path'
t = [u"%s%%"%prefix] t = [u"%s%%" % prefix]
c.execute(sql, t) c.execute(sql, t)
for row in c: for row in c:
path = row[0] path = row[0]
oshash = row[1] oshash = row[1]
info = json.loads(row[2]) info = json.loads(row[2])
if not 'error' in info: if 'error' not in info:
for key in ('atime', 'ctime', 'mtime', 'path'): for key in ('atime', 'ctime', 'mtime', 'path'):
if key in info: if key in info:
del info[key] del info[key]
@ -863,14 +858,14 @@ class Client(object):
def clean(self, args): def clean(self, args):
if os.path.exists(self.api.media_cache): if os.path.exists(self.api.media_cache):
if args and args[0] == 'all': if args and args[0] == 'all':
print "remove all cached videos in", self.api.media_cache print("remove all cached videos in", self.api.media_cache)
#if os.path.exists(self.api.media_cache): # if os.path.exists(self.api.media_cache):
# shutil.rmtree(self.api.media_cache) # shutil.rmtree(self.api.media_cache)
else: else:
nothing = False nothing = False
for root, folders, files in os.walk(self.api.media_cache): for root, folders, files in os.walk(self.api.media_cache):
for f in files: for f in sorted(files):
f = os.path.join(root, f) f = os.path.join(root, f)
if f.endswith('.webm'): if f.endswith('.webm'):
oshash = os.path.dirname(f)[len(self.api.media_cache):].replace('/', '') oshash = os.path.dirname(f)[len(self.api.media_cache):].replace('/', '')
@ -883,7 +878,7 @@ class Client(object):
nothing = False nothing = False
os.unlink(f) os.unlink(f)
if nothing and folders: if nothing and folders:
print "No unused files found in cache, run \"%s clean all\" to remove the entire cache" % sys.argv[0] print("No unused files found in cache, run \"%s clean all\" to remove the entire cache" % sys.argv[0])
else: else:
utils.cleanup_tree(self.api.media_cache) utils.cleanup_tree(self.api.media_cache)
@ -895,14 +890,14 @@ class Client(object):
pandora_client ABC transcripts /path/to/transcript.srt pandora_client ABC transcripts /path/to/transcript.srt
''' '''
if not args: if not args:
print 'Usage: pandora_client ABC transcripts /path/to/transcript.srt' print('Usage: pandora_client ABC transcripts /path/to/transcript.srt')
sys.exit(1) sys.exit(1)
item = args[0] item = args[0]
layer = args[1] layer = args[1]
filename = args[2] filename = args[2]
layers = [l['id'] for l in self.api.site['layers']] layers = [l['id'] for l in self.api.site['layers']]
if layer not in layers: if layer not in layers:
print "invalid layer name, choices are: ", ', '.join(layers) print("invalid layer name, choices are: ", ', '.join(layers))
sys.exit(1) sys.exit(1)
annotations = [{ annotations = [{
'in': s['in'], 'in': s['in'],
@ -915,13 +910,13 @@ class Client(object):
'annotations': annotations 'annotations': annotations
}) })
if r['status']['code'] == 400: if r['status']['code'] == 400:
print 'failed' print('failed')
sys.exit(1) sys.exit(1)
if r['status']['code'] == 403: if r['status']['code'] == 403:
print 'permission deinied' print('permission deinied')
sys.exit(1) sys.exit(1)
elif r['status']['code'] == 404: elif r['status']['code'] == 404:
print 'item not found' print('item not found')
sys.exit(1) sys.exit(1)
def server(self, args): def server(self, args):
@ -930,7 +925,7 @@ class Client(object):
def client(self, args): def client(self, args):
if not args: if not args:
print 'usage: %s client <server_url>\n\ti.e. %s client http://192.168.1.1:8789' % (sys.argv[0], sys.argv[0]) print('usage: %s client <server_url>\n\ti.e. %s client http://192.168.1.1:8789' % (sys.argv[0], sys.argv[0]))
sys.exit(1) sys.exit(1)
import client import client
url = args[0] url = args[0]
@ -964,14 +959,14 @@ class API(ox.API):
# wait for async task to finish # wait for async task to finish
if 'taskId' in r['data']: if 'taskId' in r['data']:
t = self.getTaskStatus(task_id=r['data']['taskId']) t = self.getTaskStatus(task_id=r['data']['taskId'])
print 'waiting for server ...' print('waiting for server ...')
while t['data'].get('status') == 'PENDING': while t['data'].get('status') == 'PENDING':
time.sleep(interval) time.sleep(interval)
t = self.getTaskStatus(task_id=r['data']['taskId']) t = self.getTaskStatus(task_id=r['data']['taskId'])
return t return t
def uploadFrames(self, i, data): def uploadFrames(self, i, data):
#upload frames # upload frames
if self.site['media'].get('importFrames') and i['frames']: if self.site['media'].get('importFrames') and i['frames']:
form = ox.MultiPartForm() form = ox.MultiPartForm()
form.add_field('action', 'upload') form.add_field('action', 'upload')
@ -989,35 +984,36 @@ class API(ox.API):
i = encode(filename, self.media_cache, profile, info, i = encode(filename, self.media_cache, profile, info,
self.site['media'].get('importFrames')) self.site['media'].get('importFrames'))
if not i: if not i:
print "failed" print("failed")
return return
#upload frames # upload frames
r = self.uploadFrames(i, data) r = self.uploadFrames(i, data)
#upload media # upload media
if os.path.exists(i['media']): if os.path.exists(i['media']):
size = ox.format_bytes(os.path.getsize(i['media'])) size = ox.format_bytes(os.path.getsize(i['media']))
name = os.path.basename(filename) name = os.path.basename(filename)
print (u"uploading %s of %s (%s)" % (profile, name, size)).encode('utf-8') print(u"uploading %s of %s (%s)" % (profile, name, size)).encode('utf-8')
url = self.url + 'upload/?profile=%s&id=%s' % (profile, i['oshash']) url = self.url + 'upload/?profile=%s&id=%s' % (profile, i['oshash'])
if not self.upload_chunks(url, i['media'], data): if not self.upload_chunks(url, i['media'], data):
if DEBUG: if DEBUG:
print "failed" print("failed")
return False return False
else: else:
print "Failed" print("Failed")
return False return False
return True return True
def uploadData(self, filename, oshash): def uploadData(self, filename, oshash):
if DEBUG: if DEBUG:
print 'upload', filename print('upload', filename)
form = ox.MultiPartForm() form = ox.MultiPartForm()
form.add_field('action', 'upload') form.add_field('action', 'upload')
form.add_field('id', str(oshash)) form.add_field('id', str(oshash))
fname = os.path.basename(filename) fname = os.path.basename(filename)
if isinstance(fname, unicode): fname = fname.encode('utf-8') if isinstance(fname, unicode):
fname = fname.encode('utf-8')
form.add_file('file', fname, open(filename, 'rb')) form.add_file('file', fname, open(filename, 'rb'))
r = self._json_request(self.url, form) r = self._json_request(self.url, form)
return r return r
@ -1040,7 +1036,7 @@ class API(ox.API):
form.add_field(key, data[key]) form.add_field(key, data[key])
data = self._json_request(url, form) data = self._json_request(url, form)
print filename print(filename)
hide_cursor() hide_cursor()
result_url = data.get('url') result_url = data.get('url')
if 'uploadUrl' in data: if 'uploadUrl' in data:
@ -1072,7 +1068,7 @@ class API(ox.API):
remaining = ", %s remaining" % r remaining = ", %s remaining" % r
msg = '%0.2f%% %s of %s done%s' % ( msg = '%0.2f%% %s of %s done%s' % (
100 * done/fsize, ox.format_bytes(done), ox.format_bytes(fsize), remaining) 100 * done/fsize, ox.format_bytes(done), ox.format_bytes(fsize), remaining)
print ''.join([msg, ' ' * (80-len(msg)), '\r']), print(''.join([msg, ' ' * (80-len(msg)), '\r']), end='')
sys.stdout.flush() sys.stdout.flush()
form = ox.MultiPartForm() form = ox.MultiPartForm()
form.add_file('chunk', fname, chunk) form.add_file('chunk', fname, chunk)
@ -1082,30 +1078,30 @@ class API(ox.API):
try: try:
data = self._json_request(uploadUrl, form) data = self._json_request(uploadUrl, form)
except KeyboardInterrupt: except KeyboardInterrupt:
print "\ninterrupted by user." print("\ninterrupted by user.")
sys.exit(1) sys.exit(1)
except: except:
print "uploading chunk failed, will try again in 5 seconds\r", print("uploading chunk failed, will try again in 5 seconds\r", end='')
sys.stdout.flush() sys.stdout.flush()
if DEBUG: if DEBUG:
print '\n', uploadUrl print('\n', uploadUrl)
import traceback import traceback
traceback.print_exc() traceback.print_exc()
data = {'result': -1} data = {'result': -1}
time.sleep(5) time.sleep(5)
if data and 'status' in data: if data and 'status' in data:
if data['status']['code'] == 403: if data['status']['code'] == 403:
print "login required" print("login required")
return False return False
if data['status']['code'] != 200: if data['status']['code'] != 200:
print "request returned error, will try again in 5 seconds" print("request returned error, will try again in 5 seconds")
if DEBUG: if DEBUG:
print data print(data)
time.sleep(5) time.sleep(5)
if data and data.get('result') == 1: if data and data.get('result') == 1:
done += len(chunk) done += len(chunk)
if data.get('offset') not in (None, done): if data.get('offset') not in (None, done):
print 'server offset out of sync, continue from', data['offset'] print('server offset out of sync, continue from', data['offset'])
done = data['offset'] done = data['offset']
f.seek(done) f.seek(done)
with open(self._resume_file, 'w') as r: with open(self._resume_file, 'w') as r:
@ -1120,18 +1116,18 @@ class API(ox.API):
os.unlink(self._resume_file) os.unlink(self._resume_file)
resume = None resume = None
if result_url: if result_url:
print result_url + (' ' * (80-len(result_url))) print(result_url + (' ' * (80-len(result_url))))
else: else:
print ' ' * 80 print(' ' * 80)
print '' print('')
show_cursor() show_cursor()
return data and 'result' in data and data.get('result') == 1 return data and 'result' in data and data.get('result') == 1
else: else:
if DEBUG: if DEBUG:
if 'status' in data and data['status']['code'] == 401: if 'status' in data and data['status']['code'] == 401:
print "login required" print("login required")
else: else:
print "failed to upload file to", url print("failed to upload file to", url)
print data print(data)
return False return False

9
pandora_client/client.py
View file

@ -1,5 +1,6 @@
# encoding: utf-8 # encoding: utf-8
# vi:si:et:sw=4:sts=4:ts=4 # vi:si:et:sw=4:sts=4:ts=4
from __future__ import print_function
import os import os
import json import json
import subprocess import subprocess
@ -23,7 +24,7 @@ class DistributedClient:
url = '%s/ping/%s/%s' % (self.url, oshash, self.name) url = '%s/ping/%s/%s' % (self.url, oshash, self.name)
requests.get(url) requests.get(url)
except: except:
print 'cound not ping server' print('cound not ping server')
def status(self, oshash, status): def status(self, oshash, status):
url = '%s/status/%s' % (self.url, oshash) url = '%s/status/%s' % (self.url, oshash)
@ -46,10 +47,10 @@ class DistributedClient:
def encode(self, oshash, cmd, output): def encode(self, oshash, cmd, output):
cmd[0] = extract.command('ffmpeg') cmd[0] = extract.command('ffmpeg')
if 'webm' in cmd and not self.supported_formats['webm']: if 'webm' in cmd and not self.supported_formats['webm']:
print "ffmpeg is compiled without WebM support" print("ffmpeg is compiled without WebM support")
return return
elif cmd[-1].endswith('.mp4') and not self.supported_formats['webm']: elif cmd[-1].endswith('.mp4') and not self.supported_formats['webm']:
print "ffmpeg is compiled without H.264 support" print("ffmpeg is compiled without H.264 support")
return return
try: try:
p = subprocess.Popen(cmd) p = subprocess.Popen(cmd)
@ -85,7 +86,7 @@ class DistributedClient:
if not self.next(): if not self.next():
if new: if new:
new = False new = False
print "currently no more files to encode, ctrl-c to quit" print("currently no more files to encode, ctrl-c to quit")
try: try:
time.sleep(60) time.sleep(60)
except KeyboardInterrupt: except KeyboardInterrupt:

50
pandora_client/extract.py
View file

@ -2,7 +2,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4 # vi:si:et:sw=4:sts=4:ts=4
# GPL 2010 # GPL 2010
from __future__ import division, with_statement from __future__ import division, with_statement, print_function
import os import os
import subprocess import subprocess
@ -28,21 +28,21 @@ def frame(video, target, position):
os.makedirs(fdir) os.makedirs(fdir)
''' '''
#oxframe # oxframe
cmd = ['oxframe', '-i', video, '-p', str(position), '-o', target] cmd = ['oxframe', '-i', video, '-p', str(position), '-o', target]
print cmd print(cmd)
r = run_command(cmd) r = run_command(cmd)
return r == 0 return r == 0
''' '''
''' '''
#mplayer # mplayer
cwd = os.getcwd() cwd = os.getcwd()
target = os.path.abspath(target) target = os.path.abspath(target)
framedir = tempfile.mkdtemp() framedir = tempfile.mkdtemp()
os.chdir(framedir) os.chdir(framedir)
cmd = ['mplayer', '-noautosub', video, '-ss', str(position), '-frames', '2', '-vo', 'png:z=9', '-ao', 'null'] cmd = ['mplayer', '-noautosub', video, '-ss', str(position), '-frames', '2', '-vo', 'png:z=9', '-ao', 'null']
print cmd print(cmd)
r = run_command(cmd) r = run_command(cmd)
images = glob('%s%s*.png' % (framedir, os.sep)) images = glob('%s%s*.png' % (framedir, os.sep))
if images: if images:
@ -54,24 +54,24 @@ def frame(video, target, position):
shutil.rmtree(framedir) shutil.rmtree(framedir)
return r == 0 return r == 0
''' '''
#ffmpeg # ffmpeg
pre = position - 2 pre = position - 2
if pre < 0: if pre < 0:
pre = 0 pre = 0
else: else:
position = 2 position = 2
cmd = [command('ffmpeg'), '-y', '-ss', str(pre), '-i', video, '-ss', str(position), cmd = [command('ffmpeg'), '-y', '-ss', str(pre), '-i', video, '-ss', str(position),
'-vf', 'scale=iw*sar:ih', '-vf', 'scale=iw*sar:ih',
'-an', '-vframes', '1', target] '-an', '-vframes', '1', target]
r = run_command(cmd) r = run_command(cmd)
return r == 0 return r == 0
def supported_formats(): def supported_formats():
p = subprocess.Popen([command('ffmpeg'), '-codecs'], p = subprocess.Popen([command('ffmpeg'), '-codecs'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate() stdout, stderr = p.communicate()
return { return {
#'ogg': 'libtheora' in stdout and 'libvorbis' in stdout, # 'ogg': 'libtheora' in stdout and 'libvorbis' in stdout,
'webm': 'libvpx' in stdout and 'libvorbis' in stdout, 'webm': 'libvpx' in stdout and 'libvorbis' in stdout,
'mp4': 'libx264' in stdout and 'libvo_aacenc' in stdout, 'mp4': 'libx264' in stdout and 'libvo_aacenc' in stdout,
} }
@ -179,15 +179,15 @@ def video_cmd(video, target, profile, info):
fps = min(float(fps), 30) fps = min(float(fps), 30)
bitrate = height*width*fps*bpp/1000 bitrate = height*width*fps*bpp/1000
aspect = dar.ratio aspect = dar.ratio
#use 1:1 pixel aspect ratio if dar is close to that # use 1:1 pixel aspect ratio if dar is close to that
if abs(width/height - dar) < 0.02: if abs(width/height - dar) < 0.02:
aspect = '%s:%s' % (width, height) aspect = '%s:%s' % (width, height)
video_settings = [ video_settings = [
'-vb', '%dk'%bitrate, '-vb', '%dk' % bitrate,
'-aspect', aspect, '-aspect', aspect,
'-g', '%d' % int(fps*5), '-g', '%d' % int(fps*5),
'-vf', 'yadif,hqdn3d,scale=%s:%s'%(width, height), '-vf', 'yadif,hqdn3d,scale=%s:%s' % (width, height),
] + extra ] + extra
if format == 'webm': if format == 'webm':
video_settings += [ video_settings += [
@ -198,7 +198,7 @@ def video_cmd(video, target, profile, info):
'-auto-alt-ref', '1', '-auto-alt-ref', '1',
] ]
if format == 'mp4': if format == 'mp4':
#quicktime does not support bpyramid # quicktime does not support bpyramid
''' '''
video_settings += [ video_settings += [
'-vcodec', 'libx264', '-vcodec', 'libx264',
@ -238,7 +238,7 @@ def video_cmd(video, target, profile, info):
'-qmin', '10', '-qmax', '51', '-qmin', '10', '-qmax', '51',
'-qdiff', '4' '-qdiff', '4'
] ]
video_settings += ['-map', '0:%s,0:0'%info['video'][0]['id']] video_settings += ['-map', '0:%s,0:0' % info['video'][0]['id']]
else: else:
video_settings = ['-vn'] video_settings = ['-vn']
@ -284,7 +284,7 @@ def video_cmd(video, target, profile, info):
if format == 'webm': if format == 'webm':
cmd += ['-f', 'webm', target] cmd += ['-f', 'webm', target]
elif format == 'mp4': elif format == 'mp4':
#mp4 needs postprocessing(qt-faststart), write to temp file # mp4 needs postprocessing(qt-faststart), write to temp file
cmd += ["%s.mp4" % target] cmd += ["%s.mp4" % target]
else: else:
cmd += [target] cmd += [target]
@ -294,7 +294,7 @@ def video(video, target, profile, info):
enc_target = target if target.endswith('.mp4') else target + '.tmp.webm' enc_target = target if target.endswith('.mp4') else target + '.tmp.webm'
cmd = video_cmd(video, enc_target, profile, info) cmd = video_cmd(video, enc_target, profile, info)
profile, format = profile.split('.') profile, format = profile.split('.')
#r = run_command(cmd, -1) # r = run_command(cmd, -1)
p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
try: try:
p.wait() p.wait()
@ -302,24 +302,24 @@ def video(video, target, profile, info):
if format == 'mp4': if format == 'mp4':
cmd = [command('qt-faststart'), "%s.mp4" % target, target] cmd = [command('qt-faststart'), "%s.mp4" % target, target]
p = subprocess.Popen(cmd, stdin=subprocess.PIPE, p = subprocess.Popen(cmd, stdin=subprocess.PIPE,
stdout=subprocess.PIPE if sys.platform.startswith('win') else open('/dev/null', 'w'), stdout=subprocess.PIPE if sys.platform.startswith('win') else open('/dev/null', 'w'),
stderr=subprocess.STDOUT) stderr=subprocess.STDOUT)
p.communicate() p.communicate()
os.unlink("%s.mp4" % target) os.unlink("%s.mp4" % target)
elif r == 0 and target != enc_target: elif r == 0 and target != enc_target:
shutil.move(enc_target, target) shutil.move(enc_target, target)
print 'Input:\t', video print('Input:\t', video)
print 'Output:\t', target print('Output:\t', target)
except KeyboardInterrupt: except KeyboardInterrupt:
p.kill() p.kill()
r = 1 r = 1
if os.path.exists(enc_target): if os.path.exists(enc_target):
print "\n\ncleanup unfinished encoding:\nremoving", enc_target print("\n\ncleanup unfinished encoding:\nremoving", enc_target)
print "\n" print("\n")
os.unlink(enc_target) os.unlink(enc_target)
if os.path.exists(target): if os.path.exists(target):
print "\n\ncleanup unfinished encoding:\nremoving", target print("\n\ncleanup unfinished encoding:\nremoving", target)
print "\n" print("\n")
os.unlink(target) os.unlink(target)
if format == 'mp4' and os.path.exists("%s.mp4" % target): if format == 'mp4' and os.path.exists("%s.mp4" % target):
os.unlink("%s.mp4" % target) os.unlink("%s.mp4" % target)

21
pandora_client/server.py
View file

@ -1,5 +1,7 @@
# encoding: utf-8 # encoding: utf-8
# vi:si:et:sw=4:sts=4:ts=4 # vi:si:et:sw=4:sts=4:ts=4
from __future__ import print_function
import os import os
import json import json
import shutil import shutil
@ -25,11 +27,11 @@ class UploadThread(Thread):
def run(self): def run(self):
while True: while True:
oshash = self.server.upload.get() oshash = self.server.upload.get()
print oshash print(oshash)
try: try:
self.server.client.upload([oshash]) self.server.client.upload([oshash])
except: except:
print 'failed to upload', oshash print('failed to upload', oshash)
self.server.upload.task_done() self.server.upload.task_done()
class Server(Resource): class Server(Resource):
@ -51,7 +53,7 @@ class Server(Resource):
args = [site, status, active] args = [site, status, active]
c.execute(sql, tuple(args)) c.execute(sql, tuple(args))
files = [row[0] for row in c] files = [row[0] for row in c]
#reset inactive encodes # reset inactive encodes
sql = 'UPDATE encode SET status = ? WHERE site = ? AND status = ? AND modified < ?' sql = 'UPDATE encode SET status = ? WHERE site = ? AND status = ? AND modified < ?'
c.execute(sql, ('', site, 'active', active)) c.execute(sql, ('', site, 'active', active))
conn.commit() conn.commit()
@ -59,7 +61,7 @@ class Server(Resource):
def is_available(self, oshash): def is_available(self, oshash):
info = self.client.info(oshash) info = self.client.info(oshash)
if info and not 'error' in info: if info and 'error' not in info:
for path in self.client.path(oshash): for path in self.client.path(oshash):
if os.path.exists(path): if os.path.exists(path):
return True return True
@ -95,7 +97,7 @@ class Server(Resource):
return json.dumps(response, indent=2) return json.dumps(response, indent=2)
def getChild(self, name, request): def getChild(self, name, request):
#make source media available via oshash # make source media available via oshash
if request.path.startswith('/get/'): if request.path.startswith('/get/'):
oshash = request.path.split('/')[-1] oshash = request.path.split('/')[-1]
for path in self.client.path(oshash): for path in self.client.path(oshash):
@ -155,12 +157,12 @@ class Server(Resource):
response['cmd'][0] = 'ffmpeg' response['cmd'][0] = 'ffmpeg'
response['output'] = output response['output'] = output
self.update_status(oshash, 'active') self.update_status(oshash, 'active')
print oshash, f print(oshash, f)
return self.render_json(request, response) return self.render_json(request, response)
return self.render_json(request, response) return self.render_json(request, response)
elif request.path.startswith('/ping/'): elif request.path.startswith('/ping/'):
parts = request.path.split('/') parts = request.path.split('/')
#FIXME: store client id somewhere # FIXME: store client id somewhere
client = parts[-1] client = parts[-1]
oshash = parts[-2] oshash = parts[-2]
self.update_status(oshash, 'active') self.update_status(oshash, 'active')
@ -185,7 +187,8 @@ class Server(Resource):
self.client.update_encodes(True) self.client.update_encodes(True)
def run(client, args=None): def run(client, args=None):
if not args: args = [] if not args:
args = []
root = Server(client) root = Server(client)
site = Site(root) site = Site(root)
port = 8789 port = 8789
@ -197,6 +200,6 @@ def run(client, args=None):
else: else:
port = int(args[0]) port = int(args[0])
reactor.listenTCP(port, site, interface=interface) reactor.listenTCP(port, site, interface=interface)
print 'listening on http://%s:%s' % (interface, port) print('listening on http://%s:%s' % (interface, port))
client.update_encodes() client.update_encodes()
reactor.run() reactor.run()

17
pandora_client/utils.py
View file

@ -2,7 +2,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4 # vi:si:et:sw=4:sts=4:ts=4
# GPL 2010 # GPL 2010
from __future__ import division, with_statement from __future__ import division, with_statement, print_function
import fractions import fractions
import os import os
@ -17,10 +17,11 @@ class AspectRatio(fractions.Fraction):
def __new__(cls, numerator, denominator=None): def __new__(cls, numerator, denominator=None):
if not denominator: if not denominator:
ratio = map(int, numerator.split(':')) ratio = map(int, numerator.split(':'))
if len(ratio) == 1: ratio.append(1) if len(ratio) == 1:
ratio.append(1)
numerator = ratio[0] numerator = ratio[0]
denominator = ratio[1] denominator = ratio[1]
#if its close enough to the common aspect ratios rather use that # if its close enough to the common aspect ratios rather use that
if abs(numerator/denominator - 4/3) < 0.03: if abs(numerator/denominator - 4/3) < 0.03:
numerator = 4 numerator = 4
denominator = 3 denominator = 3
@ -37,7 +38,7 @@ def avinfo(filename, cached=True):
if os.path.getsize(filename): if os.path.getsize(filename):
info = ox.avinfo(filename, cached=cached) info = ox.avinfo(filename, cached=cached)
if 'video' in info and info['video'] and 'width' in info['video'][0]: if 'video' in info and info['video'] and 'width' in info['video'][0]:
if not 'display_aspect_ratio' in info['video'][0]: if 'display_aspect_ratio' not in info['video'][0]:
dar = AspectRatio(info['video'][0]['width'], info['video'][0]['height']) dar = AspectRatio(info['video'][0]['width'], info['video'][0]['height'])
info['video'][0]['display_aspect_ratio'] = dar.ratio info['video'][0]['display_aspect_ratio'] = dar.ratio
del info['path'] del info['path']
@ -52,21 +53,21 @@ def hash_prefix(h):
return [h[:2], h[2:4], h[4:6], h[6:]] return [h[:2], h[2:4], h[4:6], h[6:]]
def run_command(cmd, timeout=25): def run_command(cmd, timeout=25):
#print cmd # print(cmd)
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
while timeout > 0: while timeout > 0:
time.sleep(0.2) time.sleep(0.2)
timeout -= 0.2 timeout -= 0.2
if p.poll() != None: if p.poll() is not None:
return p.returncode return p.returncode
if p.poll() == None: if p.poll() is None:
os.kill(p.pid, 9) os.kill(p.pid, 9)
killedpid, stat = os.waitpid(p.pid, os.WNOHANG) killedpid, stat = os.waitpid(p.pid, os.WNOHANG)
return p.returncode return p.returncode
def video_frame_positions(duration): def video_frame_positions(duration):
pos = duration / 2 pos = duration / 2
#return [pos/4, pos/2, pos/2+pos/4, pos, pos+pos/2, pos+pos/2+pos/4] # return [pos/4, pos/2, pos/2+pos/4, pos, pos+pos/2, pos+pos/2+pos/4]
return map(int, [pos/2, pos, pos+pos/2]) return map(int, [pos/2, pos, pos+pos/2])
def cleanup_tree(root): def cleanup_tree(root):