0x2620/pandora_client
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

use unicode for all sql statements and arguments

Browse source
This commit is contained in:
j 2014-04-15 14:40:28 +02:00
parent 77096ddb17
commit 392504e239
2 changed files with 33 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
bin/pandora_client
View file

@ -66,5 +66,6 @@ for more information visit https://wiki.0x2620.org/wiki/pandora_client''' % ', '
}, f, indent=2) }, f, indent=2)
pandora_client.DEBUG = opts.debug pandora_client.DEBUG = opts.debug
c = pandora_client.Client(opts.config, offline) c = pandora_client.Client(opts.config, offline)
getattr(c, action)(args[1:]) args = [a.decode('utf-8') if isinstance(a, str) else a for a in args[1:]]
getattr(c, action)(args)

60
pandora_client/__init__.py
View file

@ -148,12 +148,12 @@ class Client(object):
conn, c = self._conn() conn, c = self._conn()
c.execute('''CREATE TABLE IF NOT EXISTS setting (key varchar(1024) unique, value text)''') c.execute(u'CREATE TABLE IF NOT EXISTS setting (key varchar(1024) unique, value text)')
if int(self.get('version', 0)) < 1: if int(self.get('version', 0)) < 1:
self.set('version', 1) self.set('version', 1)
db = [ db = [
'''CREATE TABLE IF NOT EXISTS file ( u'''CREATE TABLE IF NOT EXISTS file (
path varchar(1024) unique, path varchar(1024) unique,
oshash varchar(16), oshash varchar(16),
atime FLOAT, atime FLOAT,
@ -164,8 +164,8 @@ class Client(object):
created INT, created INT,
modified INT, modified INT,
deleted INT)''', deleted INT)''',
'''CREATE INDEX IF NOT EXISTS path_idx ON file (path)''', u'CREATE INDEX IF NOT EXISTS path_idx ON file (path)',
'''CREATE INDEX IF NOT EXISTS oshash_idx ON file (oshash)''', u'CREATE INDEX IF NOT EXISTS oshash_idx ON file (oshash)',
] ]
for i in db: for i in db:
c.execute(i) c.execute(i)
@ -173,10 +173,10 @@ class Client(object):
if int(self.get('version', 0)) < 2: if int(self.get('version', 0)) < 2:
self.set('version', 2) self.set('version', 2)
db = [ db = [
'''CREATE TABLE IF NOT EXISTS encode ( u'''CREATE TABLE IF NOT EXISTS encode (
oshash varchar(16), oshash varchar(16),
site varchar(255))''', site varchar(255))''',
'''CREATE INDEX IF NOT EXISTS upload_site_idx ON encode (site)''', u'CREATE INDEX IF NOT EXISTS upload_site_idx ON encode (site)',
] ]
for i in db: for i in db:
c.execute(i) c.execute(i)
@ -184,7 +184,7 @@ class Client(object):
if int(self.get('version', 0)) < 3: if int(self.get('version', 0)) < 3:
self.set('version', 3) self.set('version', 3)
db = [ db = [
'''ALTER TABLE file add sha1 varchar(42)''' u'ALTER TABLE file add sha1 varchar(42)'
] ]
for i in db: for i in db:
c.execute(i) c.execute(i)
@ -192,9 +192,9 @@ class Client(object):
if int(self.get('version', 0)) < 4: if int(self.get('version', 0)) < 4:
self.set('version', 4) self.set('version', 4)
db = [ db = [
'''ALTER TABLE encode add status varchar(255)''', u'ALTER TABLE encode add status varchar(255)',
'''CREATE INDEX IF NOT EXISTS encode_status_idx ON encode (status)''', u'CREATE INDEX IF NOT EXISTS encode_status_idx ON encode (status)',
'''ALTER TABLE encode ADD modified INT DEFAULT 0''', u'ALTER TABLE encode ADD modified INT DEFAULT 0',
] ]
for i in db: for i in db:
c.execute(i) c.execute(i)
@ -220,7 +220,6 @@ class Client(object):
if not os.path.exists(os.path.dirname(db_conn)): if not os.path.exists(os.path.dirname(db_conn)):
os.makedirs(os.path.dirname(db_conn)) os.makedirs(os.path.dirname(db_conn))
conn = sqlite3.connect(db_conn, timeout=10) conn = sqlite3.connect(db_conn, timeout=10)
conn.text_factory = str
return conn, conn.cursor() return conn, conn.cursor()
def media_cache(self): def media_cache(self):
@ -228,7 +227,7 @@ class Client(object):
def get(self, key, default=None): def get(self, key, default=None):
conn, c = self._conn() conn, c = self._conn()
c.execute('SELECT value FROM setting WHERE key = ?', (key, )) c.execute(u'SELECT value FROM setting WHERE key = ?', (key, ))
for row in c: for row in c:
return row[0] return row[0]
return default return default
@ -240,7 +239,7 @@ class Client(object):
def info(self, oshash): def info(self, oshash):
conn, c = self._conn() conn, c = self._conn()
c.execute('SELECT info FROM file WHERE oshash = ?', (oshash, )) c.execute(u'SELECT info FROM file WHERE oshash = ?', (oshash, ))
info = None info = None
for row in c: for row in c:
info = json.loads(row[0]) info = json.loads(row[0])
@ -252,6 +251,7 @@ class Client(object):
prefixes = [prefix] prefixes = [prefix]
else: else:
prefixes = self.active_volumes().values() prefixes = self.active_volumes().values()
prefixes = [p.decode('utf-8') if isinstance(prefix, str) else p for p in prefixes]
_info = self.info(oshash) _info = self.info(oshash)
for path in self.path(oshash): for path in self.path(oshash):
for prefix in prefixes: for prefix in prefixes:
@ -275,10 +275,11 @@ class Client(object):
def path(self, oshash): def path(self, oshash):
conn, c = self._conn() conn, c = self._conn()
c.execute('SELECT path FROM file WHERE oshash = ?', (oshash, )) c.execute(u'SELECT path FROM file WHERE oshash = ?', (oshash, ))
paths = [] paths = []
for row in c: for row in c:
paths.append(row[0]) path = row[0]
paths.append(path)
return paths return paths
def online(self): def online(self):
@ -311,13 +312,13 @@ class Client(object):
def set_encodes(self, site, files): def set_encodes(self, site, files):
conn, c = self._conn() conn, c = self._conn()
c.execute('DELETE FROM encode WHERE site = ?' , (site, )) c.execute(u'DELETE FROM encode WHERE site = ?' , (site, ))
conn.commit() conn.commit()
self.add_encodes(site, files) self.add_encodes(site, files)
def get_encodes(self, site, status=''): def get_encodes(self, site, status=''):
conn, c = self._conn() conn, c = self._conn()
sql = 'SELECT oshash FROM encode WHERE site = ? AND status = ?' sql = u'SELECT oshash FROM encode WHERE site = ? AND status = ?'
args = [site, status] args = [site, status]
c.execute(sql, tuple(args)) c.execute(sql, tuple(args))
return [row[0] for row in c] return [row[0] for row in c]
@ -336,7 +337,7 @@ class Client(object):
files = r['data']['data'] files = r['data']['data']
if add: if add:
conn, c = self._conn() conn, c = self._conn()
sql = 'SELECT oshash FROM encode WHERE site = ?' sql = u'SELECT oshash FROM encode WHERE site = ?'
c.execute(sql, (site, )) c.execute(sql, (site, ))
known = [row[0] for row in c] known = [row[0] for row in c]
files = list(set(files) - set(known)) files = list(set(files) - set(known))
@ -351,7 +352,10 @@ class Client(object):
modified = time.mktime(time.localtime()) modified = time.mktime(time.localtime())
created = modified created = modified
sql = 'SELECT atime, ctime, mtime, size, created, info FROM file WHERE deleted < 0 AND path=?' if isinstance(path, str):
path = path.decode('utf-8')
sql = u'SELECT atime, ctime, mtime, size, created, info FROM file WHERE deleted < 0 AND path=?'
c.execute(sql, [path]) c.execute(sql, [path])
stat = os.stat(path) stat = os.stat(path)
for row in c: for row in c:
@ -395,7 +399,7 @@ class Client(object):
if os.path.exists(p): if os.path.exists(p):
info = utils.avinfo(p) info = utils.avinfo(p)
profile = self.profile(info) profile = self.profile(info)
cmd = encode_cmd(p.decode('utf-8'), self.media_cache(), profile, info) cmd = encode_cmd(p, self.media_cache(), profile, info)
cmd = [' ' in c and u'"%s"' % c or c for c in cmd] cmd = [' ' in c and u'"%s"' % c or c for c in cmd]
print (u' '.join(cmd)).encode('utf-8') print (u' '.join(cmd)).encode('utf-8')
@ -469,6 +473,8 @@ class Client(object):
path = os.path.normpath(path) path = os.path.normpath(path)
if not path.endswith('/'): if not path.endswith('/'):
path += '/' path += '/'
if isinstance(path, str):
path = path.decode('utf-8')
if os.path.exists(path): if os.path.exists(path):
volumes[name] = path volumes[name] = path
return volumes return volumes
@ -480,7 +486,7 @@ class Client(object):
for name in sorted(volumes): for name in sorted(volumes):
path = volumes[name] path = volumes[name]
conn, c = self._conn() conn, c = self._conn()
c.execute('SELECT path FROM file WHERE path LIKE ? AND deleted < 0', ["%s%%"%path]) c.execute(u'SELECT path FROM file WHERE path LIKE ? AND deleted < 0', [u"%s%%" % path])
known_files = [r[0] for r in c.fetchall()] known_files = [r[0] for r in c.fetchall()]
files = [] files = []
@ -488,12 +494,8 @@ class Client(object):
ignored = [] ignored = []
unsupported = [] unsupported = []
for dirpath, dirnames, filenames in os.walk(path, followlinks=True): for dirpath, dirnames, filenames in os.walk(path, followlinks=True):
if isinstance(dirpath, str):
dirpath = dirpath.decode('utf-8')
if filenames: if filenames:
for filename in sorted(filenames): for filename in sorted(filenames):
if isinstance(filename, str):
filename = filename.decode('utf-8')
file_path = os.path.join(dirpath, filename) file_path = os.path.join(dirpath, filename)
if not ignore_file(self, file_path): if not ignore_file(self, file_path):
files.append(file_path) files.append(file_path)
@ -532,7 +534,7 @@ class Client(object):
if deleted_files: if deleted_files:
deleted = time.mktime(time.localtime()) deleted = time.mktime(time.localtime())
for f in deleted_files: for f in deleted_files:
c.execute('UPDATE file SET deleted=? WHERE path=?', (deleted, f)) c.execute(u'UPDATE file SET deleted=? WHERE path=?', (deleted, f))
conn.commit() conn.commit()
''' '''
@ -558,7 +560,7 @@ class Client(object):
if os.path.exists(path): if os.path.exists(path):
files += self.files(path)['info'] files += self.files(path)['info']
else: else:
files = [len(f) == 16 and f or ox.oshash(f) for f in args] files = [f if len(f) == 16 else ox.oshash(f) for f in args]
else: else:
if not self.user: if not self.user:
print "you need to login or run pandora_client extract offline" print "you need to login or run pandora_client extract offline"
@ -797,7 +799,7 @@ class Client(object):
files = {} files = {}
files['info'] = {} files['info'] = {}
files['files'] = [] files['files'] = []
sql = 'SELECT path, oshash, info, atime, ctime, mtime FROM file WHERE deleted < 0 AND path LIKE ? ORDER BY path' sql = u'SELECT path, oshash, info, atime, ctime, mtime FROM file WHERE deleted < 0 AND path LIKE ? ORDER BY path'
t = [u"%s%%"%prefix] t = [u"%s%%"%prefix]
c.execute(sql, t) c.execute(sql, t)
for row in c: for row in c:
@ -939,7 +941,7 @@ class API(ox.API):
#upload media #upload media
if os.path.exists(i['media']): if os.path.exists(i['media']):
size = ox.format_bytes(os.path.getsize(i['media'])) size = ox.format_bytes(os.path.getsize(i['media']))
name = os.path.basename(filename).decode('utf-8') name = os.path.basename(filename)
print (u"uploading %s of %s (%s)" % (profile, name, size)).encode('utf-8') print (u"uploading %s of %s (%s)" % (profile, name, size)).encode('utf-8')
url = self.url + 'upload/?profile=%s&id=%s' % (profile, i['oshash']) url = self.url + 'upload/?profile=%s&id=%s' % (profile, i['oshash'])
if not self.upload_chunks(url, i['media'], data): if not self.upload_chunks(url, i['media'], data):