more str/bytes

This commit is contained in:
j 2014-09-08 21:17:35 +02:00
parent 8c6164e0c4
commit 461fe3b9cf
7 changed files with 25 additions and 14 deletions

View file

@ -55,6 +55,7 @@ class Changelog(db.Model):
c.revision = cls.query.filter_by(user_id=user.id).count() c.revision = cls.query.filter_by(user_id=user.id).count()
c.data = json.dumps([action] + list(args)) c.data = json.dumps([action] + list(args))
_data = str(c.revision) + str(c.timestamp) + c.data _data = str(c.revision) + str(c.timestamp) + c.data
_data = _data.encode('utf-8')
c.sig = settings.sk.sign(_data, encoding='base64') c.sig = settings.sk.sign(_data, encoding='base64')
state.db.session.add(c) state.db.session.add(c)
state.db.session.commit() state.db.session.commit()
@ -79,6 +80,7 @@ class Changelog(db.Model):
next_revision = last.revision + 1 if last else 0 next_revision = last.revision + 1 if last else 0
if revision == next_revision: if revision == next_revision:
_data = str(revision) + str(timestamp) + data _data = str(revision) + str(timestamp) + data
_data = _data.encode('utf-8')
if rebuild: if rebuild:
sig = settings.sk.sign(_data, encoding='base64') sig = settings.sk.sign(_data, encoding='base64')
if valid(user.id, _data, sig): if valid(user.id, _data, sig):
@ -110,12 +112,14 @@ class Changelog(db.Model):
def verify(self): def verify(self):
_data = str(self.revision) + str(self.timestamp) + self.data _data = str(self.revision) + str(self.timestamp) + self.data
_data = _data.encode('utf-8')
return valid(self.user_id, _data, self.sig) return valid(self.user_id, _data, self.sig)
@classmethod @classmethod
def _rebuild(cls): def _rebuild(cls):
for c in cls.query.filter_by(user_id=settings.USER_ID): for c in cls.query.filter_by(user_id=settings.USER_ID):
_data = str(c.revision) + str(c.timestamp) + c.data _data = str(c.revision) + str(c.timestamp) + c.data
_data = _data.encode('utf-8')
c.sig = settings.sk.sign(_data, encoding='base64') c.sig = settings.sk.sign(_data, encoding='base64')
state.db.session.add(c) state.db.session.add(c)
state.db.session.commit() state.db.session.commit()

View file

@ -16,8 +16,8 @@ logger = logging.getLogger('oml.directory')
base = settings.server['directory_service'] base = settings.server['directory_service']
def get(vk): def get(vk):
id = vk.to_ascii(encoding='base64') id = vk.to_ascii(encoding='base64').decode()
url ='%s/%s' % (base, id) url = '%s/%s' % (base, id)
headers = { headers = {
'User-Agent': settings.USER_AGENT 'User-Agent': settings.USER_AGENT
} }
@ -36,7 +36,7 @@ def get(vk):
return data return data
def put(sk, data): def put(sk, data):
id = sk.get_verifying_key().to_ascii(encoding='base64') id = sk.get_verifying_key().to_ascii(encoding='base64').decode()
data = json.dumps(data).encode('utf-8') data = json.dumps(data).encode('utf-8')
sig = sk.sign(data, encoding='base64') sig = sk.sign(data, encoding='base64')
url ='%s/%s' % (base, id) url ='%s/%s' % (base, id)

View file

@ -3,7 +3,7 @@
from datetime import datetime from datetime import datetime
from io import StringIO from io import StringIO, BytesIO
import base64 import base64
import hashlib import hashlib
import json import json
@ -164,7 +164,7 @@ class Item(db.Model):
def add(k, v): def add(k, v):
f = Find(item_id=self.id, key=k) f = Find(item_id=self.id, key=k)
if isinstance(v, str): if isinstance(v, bytes):
v = v.decode('utf-8') v = v.decode('utf-8')
f.findvalue = unicodedata.normalize('NFKD', v).lower() f.findvalue = unicodedata.normalize('NFKD', v).lower()
f.value = v f.value = v
@ -290,7 +290,7 @@ class Item(db.Model):
def update_icons(self): def update_icons(self):
def get_ratio(data): def get_ratio(data):
img = Image.open(StringIO(data)) img = Image.open(BytesIO(data))
return img.size[0]/img.size[1] return img.size[0]/img.size[1]
key = 'cover:%s'%self.id key = 'cover:%s'%self.id
cover = None cover = None

View file

@ -57,7 +57,7 @@ class LocalNodesBase(Thread):
'port': server['node_port'], 'port': server['node_port'],
'cert': server['cert'] 'cert': server['cert']
}) })
sig = sk.sign(message.encode('utf-8'), encoding='base64') sig = sk.sign(message.encode('utf-8'), encoding='base64').decode('utf-8')
packet = json.dumps([sig, USER_ID, message]).encode('utf-8') packet = json.dumps([sig, USER_ID, message]).encode('utf-8')
else: else:
packet = None packet = None

View file

@ -85,7 +85,7 @@ def info(epub):
return data return data
def extract_text(path): def extract_text(path):
data = '' data = b''
z = zipfile.ZipFile(path) z = zipfile.ZipFile(path)
for f in z.filelist: for f in z.filelist:
if f.filename.endswith('html'): if f.filename.endswith('html'):

View file

@ -44,7 +44,7 @@ class Node(Thread):
def __init__(self, nodes, user): def __init__(self, nodes, user):
self._nodes = nodes self._nodes = nodes
self.user_id = user.id self.user_id = user.id
key = str(user.id) key = user.id.encode()
self.vk = ed25519.VerifyingKey(key, encoding=ENCODING) self.vk = ed25519.VerifyingKey(key, encoding=ENCODING)
logger.debug('new Node %s online=%s', self.user_id, self.online) logger.debug('new Node %s online=%s', self.user_id, self.online)
self._q = Queue() self._q = Queue()
@ -126,7 +126,7 @@ class Node(Thread):
self.online = False self.online = False
return None return None
content = json.dumps([action, args]).encode('utf-8') content = json.dumps([action, args]).encode('utf-8')
sig = settings.sk.sign(content, encoding=ENCODING) sig = settings.sk.sign(content, encoding=ENCODING).decode()
headers = { headers = {
'User-Agent': settings.USER_AGENT, 'User-Agent': settings.USER_AGENT,
'X-Node-Protocol': settings.NODE_PROTOCOL, 'X-Node-Protocol': settings.NODE_PROTOCOL,
@ -137,6 +137,7 @@ class Node(Thread):
'X-Ed25519-Signature': sig, 'X-Ed25519-Signature': sig,
} }
self._opener.addheaders = list(zip(list(headers.keys()), list(headers.values()))) self._opener.addheaders = list(zip(list(headers.keys()), list(headers.values())))
logger.debug('headers: %s', self._opener.addheaders)
try: try:
self._opener.timeout = self.TIMEOUT self._opener.timeout = self.TIMEOUT
r = self._opener.open(url, data=content) r = self._opener.open(url, data=content)
@ -161,6 +162,7 @@ class Node(Thread):
self.online = False self.online = False
return None return None
data = r.read() data = r.read()
logger.debug('response data: %s', data)
if r.headers.get('content-encoding', None) == 'gzip': if r.headers.get('content-encoding', None) == 'gzip':
data = gzip.GzipFile(fileobj=StringIO(data)).read() data = gzip.GzipFile(fileobj=StringIO(data)).read()
@ -178,9 +180,12 @@ class Node(Thread):
else: else:
logger.debug('invalid signature %s', data) logger.debug('invalid signature %s', data)
response = None response = None
logger.debug('response: %s', response)
return response return response
def _valid(self, data, sig): def _valid(self, data, sig):
if isinstance(data, str):
data = data.encode('utf-8')
try: try:
self.vk.verify(sig, data, encoding=ENCODING) self.vk.verify(sig, data, encoding=ENCODING)
#except ed25519.BadSignatureError: #except ed25519.BadSignatureError:
@ -307,9 +312,10 @@ class Node(Thread):
if r.headers.get('content-encoding', None) == 'gzip': if r.headers.get('content-encoding', None) == 'gzip':
content = gzip.GzipFile(fileobj=r).read() content = gzip.GzipFile(fileobj=r).read()
else: else:
content = '' content = b''
ct = datetime.utcnow() ct = datetime.utcnow()
for chunk in iter(lambda: r.read(16*1024), ''): '''
for chunk in iter(lambda: r.read(16*1024), b''):
content += chunk content += chunk
if (datetime.utcnow() - ct).total_seconds() > 1: if (datetime.utcnow() - ct).total_seconds() > 1:
ct = datetime.utcnow() ct = datetime.utcnow()
@ -319,9 +325,10 @@ class Node(Thread):
trigger_event('transfer', { trigger_event('transfer', {
'id': item.id, 'progress': t.progress 'id': item.id, 'progress': t.progress
}) })
''' '''
content = r.read() content = r.read()
''' logger.debug('download done %s', item.id)
t2 = datetime.utcnow() t2 = datetime.utcnow()
duration = (t2-t1).total_seconds() duration = (t2-t1).total_seconds()

View file

@ -62,7 +62,7 @@ else:
fd.write(sk.to_bytes()) fd.write(sk.to_bytes())
os.chmod(key_path, 0o400) os.chmod(key_path, 0o400)
USER_ID = vk.to_ascii(encoding='base64') USER_ID = vk.to_ascii(encoding='base64').decode()
OML_UPDATE_KEY='K55EZpPYbP3X+3mA66cztlw1sSaUMqGwfTDKQyP2qOU' OML_UPDATE_KEY='K55EZpPYbP3X+3mA66cztlw1sSaUMqGwfTDKQyP2qOU'
if 'modules' in release and 'openmedialibrary' in release['modules']: if 'modules' in release and 'openmedialibrary' in release['modules']: