Browse Source

all changelogs should be utf-8

j 1 month ago
parent
commit
7ee959cd11
3 changed files with 27 additions and 3 deletions
  1. 1
    1
      oml/changelog.py
  2. 1
    1
      oml/settings.py
  3. 25
    1
      oml/update.py

+ 1
- 1
oml/changelog.py View File

@@ -446,7 +446,7 @@ class Changelog(db.Model):
446 446
                 listitems = data[2]
447 447
                 #remove from additemlists
448 448
                 removed = []
449
-                if list_id in changes.get('addlistitems',{}):
449
+                if list_id in changes.get('addlistitems', {}):
450 450
                     removed = [
451 451
                         i for i in changes['addlistitems'][list_id] if i in listitems
452 452
                     ]

+ 1
- 1
oml/settings.py View File

@@ -95,4 +95,4 @@ FULLTEXT_SUPPORT = fulltext.platform_supported()
95 95
 if not FULLTEXT_SUPPORT:
96 96
     config['itemKeys'] = [k for k in config['itemKeys'] if k['id'] != 'fulltext']
97 97
 
98
-DB_VERSION = 19
98
+DB_VERSION = 20

+ 25
- 1
oml/update.py View File

@@ -381,6 +381,8 @@ class Update(Thread):
381 381
                 db_version = migrate_18()
382 382
             if db_version < 19:
383 383
                 db_version = migrate_19()
384
+            if db_version < 20:
385
+                db_version = migrate_20()
384 386
             settings.server['db_version'] = db_version
385 387
 
386 388
     def run(self):
@@ -710,5 +712,27 @@ def migrate_19():
710 712
             peer = utils.get_peer(u.id)
711 713
             if not peer.info.get('revision') and os.path.exists(peer._logpath) and os.path.getsize(peer._logpath):
712 714
                 logger.debug('try to apply pending logs for %s', u.id)
713
-                peer.apply_log()
715
+                try:
716
+                    peer.apply_log()
717
+                except:
718
+                    logger.error('failed to apply log for %s', u.id)
714 719
     return 19
720
+
721
+def migrate_20():
722
+    from glob import glob
723
+    changed = False
724
+    for log in glob(os.path.join(settings.data_path, 'peers', '*.log')):
725
+        with open(log, 'rb') as fd:
726
+            data = fd.read()
727
+        try:
728
+            data.decode('utf-8')
729
+        except UnicodeDecodeError:
730
+            data = data.decode('Windows-1252')
731
+            logger.error('convert %s to utf-8', log)
732
+            with open(log, 'wb') as fd:
733
+                fd.write(data.encode('utf-8'))
734
+            changed = True
735
+
736
+    if changed:
737
+        migrate_19()
738
+    return 20