migrate html tags (again), also update find values

oml/setup.py

@@ -231,6 +231,50 @@ def upgrade_db(old, new=None):
                                 update_item = True
                             if update_item:
                                 i.save()
+        if old <= '20160110-583-59a3709' and new > '20160110-583-59a3709':
+            with db.session() as session:
+                import ox
+                from item.models import Item, Find, Metadata
+                from meta.utils import decode_html_data
+                def cleanup_description(data):
+                    if 'description' in data:
+                        description = data['description']
+                        description = description.replace('<br>', '\n').replace('</p><p>', '\n\n')
+                        data['description'] = ox.strip_tags(description).strip()
+                for m in Metadata.query:
+                    data = decode_html_data(m.data.copy())
+                    cleanup_description(data)
+                    if data != m.data:
+                        #print(m, m.data, '\n',data, '\n\n')
+                        m.data = data
+                        m.save()
+                for i in Item.query:
+                    changed = False
+                    meta = decode_html_data(i.meta.copy())
+                    cleanup_description(meta)
+                    if meta != i.meta:
+                        #print(i, i.meta, '\n', meta, '\n\n')
+                        i.meta = meta
+                        changed = True
+                    info = decode_html_data(i.info.copy())
+                    cleanup_description(info)
+                    if info != i.info:
+                        #print(i, i.info, '\n', info, '\n\n')
+                        i.info = info
+                        changed = True
+                    if changed:
+                        i.update_sort()
+                        i.update_find()
+                        i.save()
+                for tag in ('&apos;', '&quot;', '&#'):
+                    items = set([f.item_id for f in Find.query.filter(Find.value.ilike('%'+tag+'%'))])
+                    if items:
+                        for i in Item.query.filter(Item.id.in_(items)):
+                            #print(tag, i)
+                            i.update_sort()
+                            i.update_find()
+                session.commit()
+
 
     if old <= '20140527-120-3cb9819':
         run_sql('CREATE INDEX ix_find_findvalue ON find (findvalue)')
@@ -325,30 +369,6 @@ def upgrade_db(old, new=None):
             sql = 'CREATE UNIQUE INDEX IF NOT EXISTS useritem_index on useritem(user_id,item_id)'
             session.execute(sql)
             session.commit()
-    if old <= '20160108-555-d4a961f' and new and new < "20160108-565-d4a961f":
-        with db.session() as session:
-            import item.models
-            from meta.utils import decode_html_data
-            for m in item.models.Metadata.query:
-                data = decode_html_data(m.data.copy())
-                if data != m.data:
-                    #print(m, m.data, data)
-                    m.data = data
-                    m.save()
-            for i in item.models.Item.query:
-                changed = False
-                meta = decode_html_data(i.meta.copy())
-                if meta != i.meta:
-                    #print(i, i.meta, meta)
-                    i.meta = meta
-                    changed = True
-                info = decode_html_data(i.info.copy())
-                if info != i.info:
-                    #print(i, i.info, info)
-                    i.info = info
-                    changed = True
-                if changed:
-                    i.save()
 
 def create_default_lists(user_id=None):
     with db.session():