all changelogs should be utf-8

oml/changelog.py

@@ -446,7 +446,7 @@ class Changelog(db.Model):
                 listitems = data[2]
                 #remove from additemlists
                 removed = []
-                if list_id in changes.get('addlistitems',{}):
+                if list_id in changes.get('addlistitems', {}):
                     removed = [
                         i for i in changes['addlistitems'][list_id] if i in listitems
                     ]

oml/settings.py

@@ -95,4 +95,4 @@ FULLTEXT_SUPPORT = fulltext.platform_supported()
 if not FULLTEXT_SUPPORT:
     config['itemKeys'] = [k for k in config['itemKeys'] if k['id'] != 'fulltext']
 
-DB_VERSION = 19
+DB_VERSION = 20

oml/update.py

@@ -381,6 +381,8 @@ class Update(Thread):
                 db_version = migrate_18()
             if db_version < 19:
                 db_version = migrate_19()
+            if db_version < 20:
+                db_version = migrate_20()
             settings.server['db_version'] = db_version
 
     def run(self):
@@ -710,5 +712,27 @@ def migrate_19():
             peer = utils.get_peer(u.id)
             if not peer.info.get('revision') and os.path.exists(peer._logpath) and os.path.getsize(peer._logpath):
                 logger.debug('try to apply pending logs for %s', u.id)
-                peer.apply_log()
+                try:
+                    peer.apply_log()
+                except:
+                    logger.error('failed to apply log for %s', u.id)
     return 19
+
+def migrate_20():
+    from glob import glob
+    changed = False
+    for log in glob(os.path.join(settings.data_path, 'peers', '*.log')):
+        with open(log, 'rb') as fd:
+            data = fd.read()
+        try:
+            data.decode('utf-8')
+        except UnicodeDecodeError:
+            data = data.decode('Windows-1252')
+            logger.error('convert %s to utf-8', log)
+            with open(log, 'wb') as fd:
+                fd.write(data.encode('utf-8'))
+            changed = True
+
+    if changed:
+        migrate_19()
+    return 20