use postgres json field

pandora/archive/managers.py

@@ -4,7 +4,6 @@ from django.db.models import Q, Manager
 
 from oxdjango.managers import get_operator
 from oxdjango.query import QuerySet
-from oxdjango.fields import DictField
 
 
 keymap = {
@@ -36,8 +35,7 @@ def parseCondition(condition, user):
     if isinstance(v, bool):
         key = k
     elif k == 'url':
-        key = 'info' + get_operator('=', 'istr')
+        key = 'info__url' + get_operator('=', 'istr')
-        v = DictField.dumps({'url': v})[1:-1]
     elif k == 'list':
         q = Q(id=0)
         l = v.split(":")

pandora/archive/migrations/0004_jsonfield.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2018-06-19 17:23
+from __future__ import unicode_literals
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('archive', '0003_auto_20161104_1726'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            'ALTER TABLE "archive_file" ALTER COLUMN "info" TYPE jsonb USING "info"::text::jsonb'
+        ),
+        migrations.RunSQL(
+            'ALTER TABLE "archive_stream" ALTER COLUMN "info" TYPE jsonb USING "info"::text::jsonb'
+        ),
+    ]

pandora/archive/models.py

@@ -13,6 +13,7 @@ from django.contrib.auth.models import User
 from django.db import models
 from django.db.models.signals import pre_delete
 from django.utils.encoding import python_2_unicode_compatible
+from oxdjango.fields import JSONField
 
 from oxdjango import fields
 import ox
@@ -69,7 +70,7 @@ class File(models.Model):
     size = models.BigIntegerField(default=0)
     duration = models.FloatField(null=True)
 
-    info = fields.DictField(default={})
+    info = JSONField(default=dict, editable=False)
 
     video_codec = models.CharField(max_length=255)
     pixel_format = models.CharField(max_length=255)
@@ -687,7 +688,7 @@ class Stream(models.Model):
     source = models.ForeignKey('Stream', related_name='derivatives', default=None, null=True)
     available = models.BooleanField(default=False)
     oshash = models.CharField(max_length=16, null=True, db_index=True)
-    info = fields.DictField(default={})
+    info = JSONField(default=dict, editable=False)
     duration = models.FloatField(default=0)
     aspect_ratio = models.FloatField(default=0)
 

pandora/changelog/migrations/0002_jsonfield.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2018-06-19 17:23
+from __future__ import unicode_literals
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('changelog', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            'ALTER TABLE "changelog_changelog" ALTER COLUMN "value" TYPE jsonb USING "value"::text::jsonb'
+        ),
+        migrations.RunSQL(
+            'ALTER TABLE "changelog_log" ALTER COLUMN "data" TYPE jsonb USING "data"::text::jsonb'
+        ),
+    ]

pandora/changelog/models.py

@@ -6,8 +6,8 @@ from datetime import datetime
 from django.contrib.auth.models import User
 from django.db import models
 from django.utils.encoding import python_2_unicode_compatible
+from oxdjango.fields import JSONField
 
-from oxdjango import fields
 import ox
 
 import websocket
@@ -21,7 +21,7 @@ FIXME: remove this table more migrate to new ChangeLog
 class Changelog(models.Model):
     created = models.DateTimeField(auto_now_add=True)
     type = models.CharField(max_length=255, db_index=True)
-    value = fields.DictField(default={})
+    value = JSONField(default=dict, editable=False)
 
     def __str__(self):
         return u'%s %s' % (self.type, self.created)
@@ -52,7 +52,7 @@ def add_changelog(request, data, id=None):
 class Log(models.Model):
 
     action = models.CharField(max_length=255, db_index=True)
-    data = fields.DictField(default={})
+    data = JSONField(default=dict, editable=False)
     created = models.DateTimeField(db_index=True)
     user = models.ForeignKey(User, null=True, related_name='changelog')
     changeid = models.TextField()

pandora/document/migrations/0011_jsonfield.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2018-06-19 17:23
+from __future__ import unicode_literals
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('document', '0010_auto_20170126_1528'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            'ALTER TABLE "document_document" ALTER COLUMN "data" TYPE jsonb USING "data"::text::jsonb'
+        ),
+    ]

pandora/document/models.py

@@ -14,12 +14,12 @@ from django.contrib.auth.models import User, Group
 from django.db.models.signals import pre_delete
 from django.conf import settings
 from django.utils.encoding import python_2_unicode_compatible
+from oxdjango.fields import JSONField
 
 from PIL import Image
 import ox
 
 
-from oxdjango import fields
 from oxdjango.sortmodel import get_sort_field
 from person.models import get_name_sort
 from item.models import Item
@@ -65,7 +65,7 @@ class Document(models.Model):
     linked_documents = models.ManyToManyField('Document', related_name='linking_documents')
 
     rightslevel = models.IntegerField(db_index=True, default=0)
-    data = fields.DictField(default={})
+    data = JSONField(default=dict, editable=False)
 
     def update_access(self, user):
         if not user.is_authenticated():

pandora/documentcollection/migrations/0003_jsonfield.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2018-06-19 17:23
+from __future__ import unicode_literals
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documentcollection', '0002_collection_groups'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            'ALTER TABLE "documentcollection_collection" ALTER COLUMN "query" TYPE jsonb USING "query"::text::jsonb'
+        ),
+        migrations.RunSQL(
+            'ALTER TABLE "documentcollection_collection" ALTER COLUMN "sort" TYPE jsonb USING "sort"::text::jsonb'
+        ),
+        migrations.RunSQL(
+            'ALTER TABLE "documentcollection_collection" ALTER COLUMN "poster_frames" TYPE jsonb USING "poster_frames"::text::jsonb'
+        ),
+    ]

pandora/documentcollection/models.py

@@ -11,11 +11,10 @@ from django.db.models import Max
 from django.contrib.auth.models import User, Group
 from django.conf import settings
 from django.utils.encoding import python_2_unicode_compatible
+from oxdjango.fields import JSONField
 
 import ox
 
-from oxdjango.fields import DictField, TupleField
-
 from archive import extract
 from user.utils import update_groups
 
@@ -47,16 +46,16 @@ class Collection(models.Model):
     name = models.CharField(max_length=255)
     status = models.CharField(max_length=20, default='private')
     _status = ['private', 'public', 'featured']
-    query = DictField(default={"static": True})
+    query = JSONField(default=lambda: {"static": True}, editable=False)
     type = models.CharField(max_length=255, default='static')
     description = models.TextField(default='')
 
     icon = models.ImageField(default=None, blank=True, upload_to=get_icon_path)
 
     view = models.TextField(default=get_collectionview)
-    sort = TupleField(default=get_collectionsort, editable=False)
+    sort = JSONField(default=get_collectionsort, editable=False)
 
-    poster_frames = TupleField(default=[], editable=False)
+    poster_frames = JSONField(default=[], editable=False)
 
     #is through table still required?
     documents = models.ManyToManyField('document.Document', related_name='collections',

pandora/edit/migrations/0005_jsonfield.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2018-06-19 17:23
+from __future__ import unicode_literals
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('edit', '0004_edit_groups'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            'ALTER TABLE "edit_edit" ALTER COLUMN "query" TYPE jsonb USING "query"::text::jsonb'
+        ),
+        migrations.RunSQL(
+            'ALTER TABLE "edit_edit" ALTER COLUMN "poster_frames" TYPE jsonb USING "poster_frames"::text::jsonb'
+        ),
+    ]

pandora/edit/models.py

@@ -10,12 +10,12 @@ import tempfile
 
 from six.moves.urllib.parse import quote
 import ox
-from oxdjango.fields import DictField, TupleField
 from django.conf import settings
 from django.db import models, transaction
 from django.db.models import Max
 from django.contrib.auth.models import User, Group
 from django.utils.encoding import python_2_unicode_compatible
+from oxdjango.fields import JSONField
 
 from annotation.models import Annotation
 from item.models import Item
@@ -49,12 +49,12 @@ class Edit(models.Model):
     description = models.TextField(default='')
     rightslevel = models.IntegerField(db_index=True, default=0)
 
-    query = DictField(default={"static": True})
+    query = JSONField(default=lambda: {"static": True}, editable=False)
     type = models.CharField(max_length=255, default='static')
 
     icon = models.ImageField(default=None, blank=True, null=True, upload_to=get_icon_path)
 
-    poster_frames = TupleField(default=[], editable=False)
+    poster_frames = JSONField(default=[], editable=False)
     subscribed_users = models.ManyToManyField(User, related_name='subscribed_edits')
 
     def __str__(self):

pandora/entity/migrations/0005_jsonfield.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2018-06-19 17:23
+from __future__ import unicode_literals
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('entity', '0004_related_name_Document_documentproperties'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            'ALTER TABLE "entity_entity" ALTER COLUMN "data" TYPE jsonb USING "data"::text::jsonb'
+        ),
+        migrations.RunSQL(
+            'ALTER TABLE "entity_documentproperties" ALTER COLUMN "data" TYPE jsonb USING "data"::text::jsonb'
+        ),
+    ]

pandora/entity/models.py

@@ -14,6 +14,7 @@ from django.contrib.auth.models import User
 from django.db.models.signals import pre_delete, post_init
 from django.conf import settings
 from django.utils.encoding import python_2_unicode_compatible
+from oxdjango.fields import JSONField
 
 import ox
 from oxdjango import fields
@@ -40,7 +41,7 @@ class Entity(models.Model):
     name = models.CharField(max_length=255)
     alternativeNames = fields.TupleField(default=())
 
-    data = fields.DictField(default={}, editable=False)
+    data = JSONField(default=dict, editable=False)
     matches = models.IntegerField(default=0)
 
     objects = managers.EntityManager()
@@ -271,7 +272,7 @@ class DocumentProperties(models.Model):
     document = models.ForeignKey(Document, related_name='documentproperties')
     entity = models.ForeignKey(Entity, related_name='documentproperties')
     index = models.IntegerField(default=0)
-    data = fields.DictField(default={})
+    data = JSONField(default=dict, editable=False)
 
     def __str__(self):
         return u"%r-%r" % (self.document, self.entity)

pandora/home/migrations/0002_jsonfield.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2018-06-19 17:23
+from __future__ import unicode_literals
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('home', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            'ALTER TABLE "home_item" ALTER COLUMN "data" TYPE jsonb USING "data"::text::jsonb'
+        ),
+    ]

pandora/home/models.py

@@ -8,10 +8,10 @@ from django.db import models
 from django.db.models import Max
 from django.db.models.signals import pre_delete
 from django.utils.encoding import python_2_unicode_compatible
+from oxdjango.fields import JSONField
 
 import ox
 
-from oxdjango import fields
 from itemlist.models import List
 from edit.models import Edit
 from documentcollection.models import Collection
@@ -24,7 +24,7 @@ class Item(models.Model):
 
     active = models.BooleanField(default=True)
     index = models.IntegerField(default=-1)
-    data = fields.DictField(default={}, editable=False)
+    data = JSONField(default=dict, editable=False)
 
     def editable(self, user):
         return user.is_authenticated() and user.profile.capability("canManageHome")

pandora/item/migrations/0003_jsonfield.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2018-06-19 17:23
+from __future__ import unicode_literals
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('item', '0002_auto_20160219_1734'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            'ALTER TABLE "item_item" ALTER COLUMN "data" TYPE jsonb USING "data"::text::jsonb'
+        ),
+        migrations.RunSQL(
+            'ALTER TABLE "item_item" ALTER COLUMN "external_data" TYPE jsonb USING "external_data"::text::jsonb'
+        ),
+        migrations.RunSQL(
+            'ALTER TABLE "item_item" ALTER COLUMN "json" TYPE jsonb USING "json"::text::jsonb'
+        ),
+        migrations.RunSQL(
+            'ALTER TABLE "item_item" ALTER COLUMN "stream_info" TYPE jsonb USING "stream_info"::text::jsonb'
+        ),
+    ]

pandora/item/models.py

@@ -23,7 +23,7 @@ from django.utils import datetime_safe
 from django.utils.encoding import python_2_unicode_compatible
 
 import ox
-from oxdjango import fields
+from oxdjango.fields import JSONField, to_json
 from oxdjango.sortmodel import get_sort_field
 import ox.web.imdb
 import ox.image
@@ -175,9 +175,9 @@ class Item(models.Model):
 
     public_id = models.CharField(max_length=128, unique=True, blank=True)
     oxdbId = models.CharField(max_length=42, unique=True, blank=True, null=True)
-    external_data = fields.DictField(default={}, editable=False)
+    external_data = JSONField(default=dict, editable=False)
-    data = fields.DictField(default={}, editable=False)
+    data = JSONField(default=dict, editable=False)
-    json = fields.DictField(default={}, editable=False)
+    json = JSONField(default=dict, editable=False)
     poster = models.ImageField(default=None, blank=True, upload_to=get_poster_path)
     poster_source = models.TextField(blank=True)
     poster_height = models.IntegerField(default=0)
@@ -187,7 +187,7 @@ class Item(models.Model):
     icon = models.ImageField(default=None, blank=True, upload_to=get_icon_path)
 
     torrent = models.FileField(default=None, blank=True, max_length=1000, upload_to=get_torrent_path)
-    stream_info = fields.DictField(default={}, editable=False)
+    stream_info = JSONField(default=dict, editable=False)
 
     # stream related fields
     stream_aspect = models.FloatField(default=4/3)
@@ -1476,7 +1476,7 @@ class Item(models.Model):
         data = utils.normalize_dict('NFC', data)
         ox.makedirs(os.path.join(settings.MEDIA_ROOT, self.path()))
         p = subprocess.Popen(cmd, stdin=subprocess.PIPE, close_fds=True)
-        p.communicate(json.dumps(data, default=fields.to_json).encode('utf-8'))
+        p.communicate(json.dumps(data, default=to_json).encode('utf-8'))
         self.clear_poster_cache(poster)
         return poster
 

pandora/itemlist/migrations/0003_jsonfield.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2018-06-19 17:23
+from __future__ import unicode_literals
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('itemlist', '0002_list_groups'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            'ALTER TABLE "itemlist_list" ALTER COLUMN "query" TYPE jsonb USING "query"::text::jsonb'
+        ),
+        migrations.RunSQL(
+            'ALTER TABLE "itemlist_list" ALTER COLUMN "sort" TYPE jsonb USING "sort"::text::jsonb'
+        ),
+        migrations.RunSQL(
+            'ALTER TABLE "itemlist_list" ALTER COLUMN "poster_frames" TYPE jsonb USING "poster_frames"::text::jsonb'
+        ),
+    ]

pandora/itemlist/models.py

@@ -11,9 +11,9 @@ from django.db.models import Max
 from django.contrib.auth.models import User, Group
 from django.conf import settings
 from django.utils.encoding import python_2_unicode_compatible
-import ox
+from oxdjango.fields import JSONField
 
-from oxdjango.fields import DictField, TupleField
+import ox
 
 from archive import extract
 from user.utils import update_groups
@@ -39,16 +39,16 @@ class List(models.Model):
     name = models.CharField(max_length=255)
     status = models.CharField(max_length=20, default='private')
     _status = ['private', 'public', 'featured']
-    query = DictField(default={"static": True})
+    query = JSONField(default=lambda: {"static": True}, editable=False)
     type = models.CharField(max_length=255, default='static')
     description = models.TextField(default='')
 
     icon = models.ImageField(default=None, blank=True, upload_to=get_icon_path)
 
     view = models.TextField(default=get_listview)
-    sort = TupleField(default=get_listsort, editable=False)
+    sort = JSONField(default=get_listsort, editable=False)
 
-    poster_frames = TupleField(default=[], editable=False)
+    poster_frames = JSONField(default=[], editable=False)
 
     #is through table still required?
     items = models.ManyToManyField('item.Item', related_name='lists',

pandora/oxdjango/fields.py

@@ -5,10 +5,19 @@ import copy
 
 from django.db import models
 from django.utils import datetime_safe
+import django.contrib.postgres.fields
+from django.core.serializers.json import DjangoJSONEncoder
+
 from six import string_types
 
 from ox.utils import json
 
+class JSONField(django.contrib.postgres.fields.JSONField):
+
+    def __init__(self, *args, **kwargs):
+        if 'encoder' not in kwargs:
+            kwargs['encoder'] = DjangoJSONEncoder
+        super().__init__(*args, **kwargs)
 
 def to_json(python_object):
     if isinstance(python_object, datetime.datetime):

pandora/user/migrations/0004_jsonfield.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2018-06-19 17:23
+from __future__ import unicode_literals
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('user', '0003_sessiondata_numberofcollections')
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            'ALTER TABLE "user_sessiondata" ALTER COLUMN "info" TYPE jsonb USING "info"::text::jsonb'
+        ),
+        migrations.RunSQL(
+            'ALTER TABLE "user_userprofile" ALTER COLUMN "ui" TYPE jsonb USING "ui"::text::jsonb'
+        ),
+        migrations.RunSQL(
+            'ALTER TABLE "user_userprofile" ALTER COLUMN "preferences" TYPE jsonb USING "preferences"::text::jsonb'
+        ),
+    ]

pandora/user/models.py

@@ -9,9 +9,9 @@ from django.db import models
 from django.db.models import Max
 from django.conf import settings
 from django.utils.encoding import python_2_unicode_compatible
+from oxdjango.fields import JSONField
 
 import ox
-from oxdjango.fields import DictField
 from ox.utils import json
 
 from itemlist.models import List, Position
@@ -38,7 +38,7 @@ class SessionData(models.Model):
     useragent = models.CharField(max_length=4096, null=True)
     windowsize = models.CharField(max_length=255, null=True)
     screensize = models.CharField(max_length=255, null=True)
-    info = DictField(default={})
+    info = JSONField(default=dict, editable=False)
 
     location = models.CharField(max_length=255, null=True)
     location_sort = models.CharField(max_length=255, null=True)
@@ -178,8 +178,8 @@ class UserProfile(models.Model):
     level = models.IntegerField(default=1)
     files_updated = models.DateTimeField(default=datetime.now)
     newsletter = models.BooleanField(default=True)
-    ui = DictField(default={})
+    ui = JSONField(default=dict, editable=False)
-    preferences = DictField(default={})
+    preferences = JSONField(default=dict, editable=False)
 
     notes = models.TextField(default='')
 

requirements.txt

@@ -1,12 +1,12 @@
-Django==1.9.4
+Django==1.11.13
 simplejson
 chardet
-celery==3.1.23
+celery==3.1.26.post2
-django-celery==3.1.17
+django-celery==3.2.2
-django-extensions==1.6.1
+django-extensions==2.0.7
-gunicorn==19.4.5
+gunicorn==19.8.1
 html5lib
-requests==2.9.1
+requests==2.19.1
-tornado==4.1
+tornado==5.0.2
-geoip2==2.2.0
+geoip2==2.9.0
 youtube-dl

update.py

@@ -247,6 +247,8 @@ if __name__ == "__main__":
             run('./bin/pip', 'install', '-r', 'requirements.txt')
             update_service('pandora-encoding')
             update_service('pandora-tasks')
+        if old < 5972:
+            run('./bin/pip', 'install', '-r', 'requirements.txt')
     else:
         if len(sys.argv) == 1:
             release = get_release()