From 7ee959cd118d52d3b1e9405ee00225d368f69d47 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sat, 23 Feb 2019 17:11:50 +0530
Subject: [PATCH] all changelogs should be utf-8

---
 oml/changelog.py |  2 +-
 oml/settings.py  |  2 +-
 oml/update.py    | 26 +++++++++++++++++++++++++-
 3 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/oml/changelog.py b/oml/changelog.py
index 05296e0..f641fd4 100644
--- a/oml/changelog.py
+++ b/oml/changelog.py
@@ -446,7 +446,7 @@ class Changelog(db.Model):
                 listitems = data[2]
                 #remove from additemlists
                 removed = []
-                if list_id in changes.get('addlistitems',{}):
+                if list_id in changes.get('addlistitems', {}):
                     removed = [
                         i for i in changes['addlistitems'][list_id] if i in listitems
                     ]
diff --git a/oml/settings.py b/oml/settings.py
index 86888ee..5c06f74 100644
--- a/oml/settings.py
+++ b/oml/settings.py
@@ -95,4 +95,4 @@ FULLTEXT_SUPPORT = fulltext.platform_supported()
 if not FULLTEXT_SUPPORT:
     config['itemKeys'] = [k for k in config['itemKeys'] if k['id'] != 'fulltext']
 
-DB_VERSION = 19
+DB_VERSION = 20
diff --git a/oml/update.py b/oml/update.py
index 861a819..5f7f7fe 100644
--- a/oml/update.py
+++ b/oml/update.py
@@ -381,6 +381,8 @@ class Update(Thread):
                 db_version = migrate_18()
             if db_version < 19:
                 db_version = migrate_19()
+            if db_version < 20:
+                db_version = migrate_20()
             settings.server['db_version'] = db_version
 
     def run(self):
@@ -710,5 +712,27 @@ def migrate_19():
             peer = utils.get_peer(u.id)
             if not peer.info.get('revision') and os.path.exists(peer._logpath) and os.path.getsize(peer._logpath):
                 logger.debug('try to apply pending logs for %s', u.id)
-                peer.apply_log()
+                try:
+                    peer.apply_log()
+                except:
+                    logger.error('failed to apply log for %s', u.id)
     return 19
+
+def migrate_20():
+    from glob import glob
+    changed = False
+    for log in glob(os.path.join(settings.data_path, 'peers', '*.log')):
+        with open(log, 'rb') as fd:
+            data = fd.read()
+        try:
+            data.decode('utf-8')
+        except UnicodeDecodeError:
+            data = data.decode('Windows-1252')
+            logger.error('convert %s to utf-8', log)
+            with open(log, 'wb') as fd:
+                fd.write(data.encode('utf-8'))
+            changed = True
+
+    if changed:
+        migrate_19()
+    return 20