diff --git a/pandora/archive/managers.py b/pandora/archive/managers.py index eba14343..a5d176a3 100644 --- a/pandora/archive/managers.py +++ b/pandora/archive/managers.py @@ -4,7 +4,6 @@ from django.db.models import Q, Manager from oxdjango.managers import get_operator from oxdjango.query import QuerySet -from oxdjango.fields import DictField keymap = { @@ -36,8 +35,7 @@ def parseCondition(condition, user): if isinstance(v, bool): key = k elif k == 'url': - key = 'info' + get_operator('=', 'istr') - v = DictField.dumps({'url': v})[1:-1] + key = 'info__url' + get_operator('=', 'istr') elif k == 'list': q = Q(id=0) l = v.split(":") diff --git a/pandora/archive/migrations/0004_jsonfield.py b/pandora/archive/migrations/0004_jsonfield.py new file mode 100644 index 00000000..5e1f433c --- /dev/null +++ b/pandora/archive/migrations/0004_jsonfield.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.4 on 2018-06-19 17:23 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('archive', '0003_auto_20161104_1726'), + ] + + operations = [ + migrations.RunSQL( + 'ALTER TABLE "archive_file" ALTER COLUMN "info" TYPE jsonb USING "info"::text::jsonb' + ), + migrations.RunSQL( + 'ALTER TABLE "archive_stream" ALTER COLUMN "info" TYPE jsonb USING "info"::text::jsonb' + ), + ] diff --git a/pandora/archive/models.py b/pandora/archive/models.py index a406d4a4..4c756508 100644 --- a/pandora/archive/models.py +++ b/pandora/archive/models.py @@ -13,6 +13,7 @@ from django.contrib.auth.models import User from django.db import models from django.db.models.signals import pre_delete from django.utils.encoding import python_2_unicode_compatible +from oxdjango.fields import JSONField from oxdjango import fields import ox @@ -69,7 +70,7 @@ class File(models.Model): size = models.BigIntegerField(default=0) duration = models.FloatField(null=True) - info = fields.DictField(default={}) + info = JSONField(default=dict, editable=False) video_codec = models.CharField(max_length=255) pixel_format = models.CharField(max_length=255) @@ -687,7 +688,7 @@ class Stream(models.Model): source = models.ForeignKey('Stream', related_name='derivatives', default=None, null=True) available = models.BooleanField(default=False) oshash = models.CharField(max_length=16, null=True, db_index=True) - info = fields.DictField(default={}) + info = JSONField(default=dict, editable=False) duration = models.FloatField(default=0) aspect_ratio = models.FloatField(default=0) diff --git a/pandora/changelog/migrations/0002_jsonfield.py b/pandora/changelog/migrations/0002_jsonfield.py new file mode 100644 index 00000000..f2bd935f --- /dev/null +++ b/pandora/changelog/migrations/0002_jsonfield.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.4 on 2018-06-19 17:23 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('changelog', '0001_initial'), + ] + + operations = [ + migrations.RunSQL( + 'ALTER TABLE "changelog_changelog" ALTER COLUMN "value" TYPE jsonb USING "value"::text::jsonb' + ), + migrations.RunSQL( + 'ALTER TABLE "changelog_log" ALTER COLUMN "data" TYPE jsonb USING "data"::text::jsonb' + ), + ] diff --git a/pandora/changelog/models.py b/pandora/changelog/models.py index 1df9d728..31b955a2 100644 --- a/pandora/changelog/models.py +++ b/pandora/changelog/models.py @@ -6,8 +6,8 @@ from datetime import datetime from django.contrib.auth.models import User from django.db import models from django.utils.encoding import python_2_unicode_compatible +from oxdjango.fields import JSONField -from oxdjango import fields import ox import websocket @@ -21,7 +21,7 @@ FIXME: remove this table more migrate to new ChangeLog class Changelog(models.Model): created = models.DateTimeField(auto_now_add=True) type = models.CharField(max_length=255, db_index=True) - value = fields.DictField(default={}) + value = JSONField(default=dict, editable=False) def __str__(self): return u'%s %s' % (self.type, self.created) @@ -52,7 +52,7 @@ def add_changelog(request, data, id=None): class Log(models.Model): action = models.CharField(max_length=255, db_index=True) - data = fields.DictField(default={}) + data = JSONField(default=dict, editable=False) created = models.DateTimeField(db_index=True) user = models.ForeignKey(User, null=True, related_name='changelog') changeid = models.TextField() diff --git a/pandora/document/migrations/0011_jsonfield.py b/pandora/document/migrations/0011_jsonfield.py new file mode 100644 index 00000000..166d3731 --- /dev/null +++ b/pandora/document/migrations/0011_jsonfield.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.4 on 2018-06-19 17:23 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('document', '0010_auto_20170126_1528'), + ] + + operations = [ + migrations.RunSQL( + 'ALTER TABLE "document_document" ALTER COLUMN "data" TYPE jsonb USING "data"::text::jsonb' + ), + ] diff --git a/pandora/document/models.py b/pandora/document/models.py index 8b8b8cff..1a94a9bb 100644 --- a/pandora/document/models.py +++ b/pandora/document/models.py @@ -14,12 +14,12 @@ from django.contrib.auth.models import User, Group from django.db.models.signals import pre_delete from django.conf import settings from django.utils.encoding import python_2_unicode_compatible +from oxdjango.fields import JSONField from PIL import Image import ox -from oxdjango import fields from oxdjango.sortmodel import get_sort_field from person.models import get_name_sort from item.models import Item @@ -65,7 +65,7 @@ class Document(models.Model): linked_documents = models.ManyToManyField('Document', related_name='linking_documents') rightslevel = models.IntegerField(db_index=True, default=0) - data = fields.DictField(default={}) + data = JSONField(default=dict, editable=False) def update_access(self, user): if not user.is_authenticated(): diff --git a/pandora/documentcollection/migrations/0003_jsonfield.py b/pandora/documentcollection/migrations/0003_jsonfield.py new file mode 100644 index 00000000..e00bf45c --- /dev/null +++ b/pandora/documentcollection/migrations/0003_jsonfield.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.4 on 2018-06-19 17:23 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('documentcollection', '0002_collection_groups'), + ] + + operations = [ + migrations.RunSQL( + 'ALTER TABLE "documentcollection_collection" ALTER COLUMN "query" TYPE jsonb USING "query"::text::jsonb' + ), + migrations.RunSQL( + 'ALTER TABLE "documentcollection_collection" ALTER COLUMN "sort" TYPE jsonb USING "sort"::text::jsonb' + ), + migrations.RunSQL( + 'ALTER TABLE "documentcollection_collection" ALTER COLUMN "poster_frames" TYPE jsonb USING "poster_frames"::text::jsonb' + ), + ] diff --git a/pandora/documentcollection/models.py b/pandora/documentcollection/models.py index 3026cb7f..7a41e5d3 100644 --- a/pandora/documentcollection/models.py +++ b/pandora/documentcollection/models.py @@ -11,11 +11,10 @@ from django.db.models import Max from django.contrib.auth.models import User, Group from django.conf import settings from django.utils.encoding import python_2_unicode_compatible +from oxdjango.fields import JSONField import ox -from oxdjango.fields import DictField, TupleField - from archive import extract from user.utils import update_groups @@ -47,16 +46,16 @@ class Collection(models.Model): name = models.CharField(max_length=255) status = models.CharField(max_length=20, default='private') _status = ['private', 'public', 'featured'] - query = DictField(default={"static": True}) + query = JSONField(default=lambda: {"static": True}, editable=False) type = models.CharField(max_length=255, default='static') description = models.TextField(default='') icon = models.ImageField(default=None, blank=True, upload_to=get_icon_path) view = models.TextField(default=get_collectionview) - sort = TupleField(default=get_collectionsort, editable=False) + sort = JSONField(default=get_collectionsort, editable=False) - poster_frames = TupleField(default=[], editable=False) + poster_frames = JSONField(default=[], editable=False) #is through table still required? documents = models.ManyToManyField('document.Document', related_name='collections', diff --git a/pandora/edit/migrations/0005_jsonfield.py b/pandora/edit/migrations/0005_jsonfield.py new file mode 100644 index 00000000..745e5e33 --- /dev/null +++ b/pandora/edit/migrations/0005_jsonfield.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.4 on 2018-06-19 17:23 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('edit', '0004_edit_groups'), + ] + + operations = [ + migrations.RunSQL( + 'ALTER TABLE "edit_edit" ALTER COLUMN "query" TYPE jsonb USING "query"::text::jsonb' + ), + migrations.RunSQL( + 'ALTER TABLE "edit_edit" ALTER COLUMN "poster_frames" TYPE jsonb USING "poster_frames"::text::jsonb' + ), + ] diff --git a/pandora/edit/models.py b/pandora/edit/models.py index 28ef5da5..278e8804 100644 --- a/pandora/edit/models.py +++ b/pandora/edit/models.py @@ -10,12 +10,12 @@ import tempfile from six.moves.urllib.parse import quote import ox -from oxdjango.fields import DictField, TupleField from django.conf import settings from django.db import models, transaction from django.db.models import Max from django.contrib.auth.models import User, Group from django.utils.encoding import python_2_unicode_compatible +from oxdjango.fields import JSONField from annotation.models import Annotation from item.models import Item @@ -49,12 +49,12 @@ class Edit(models.Model): description = models.TextField(default='') rightslevel = models.IntegerField(db_index=True, default=0) - query = DictField(default={"static": True}) + query = JSONField(default=lambda: {"static": True}, editable=False) type = models.CharField(max_length=255, default='static') icon = models.ImageField(default=None, blank=True, null=True, upload_to=get_icon_path) - poster_frames = TupleField(default=[], editable=False) + poster_frames = JSONField(default=[], editable=False) subscribed_users = models.ManyToManyField(User, related_name='subscribed_edits') def __str__(self): diff --git a/pandora/entity/migrations/0005_jsonfield.py b/pandora/entity/migrations/0005_jsonfield.py new file mode 100644 index 00000000..07097872 --- /dev/null +++ b/pandora/entity/migrations/0005_jsonfield.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.4 on 2018-06-19 17:23 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('entity', '0004_related_name_Document_documentproperties'), + ] + + operations = [ + migrations.RunSQL( + 'ALTER TABLE "entity_entity" ALTER COLUMN "data" TYPE jsonb USING "data"::text::jsonb' + ), + migrations.RunSQL( + 'ALTER TABLE "entity_documentproperties" ALTER COLUMN "data" TYPE jsonb USING "data"::text::jsonb' + ), + ] diff --git a/pandora/entity/models.py b/pandora/entity/models.py index ad2a5fff..b094e9bf 100644 --- a/pandora/entity/models.py +++ b/pandora/entity/models.py @@ -14,6 +14,7 @@ from django.contrib.auth.models import User from django.db.models.signals import pre_delete, post_init from django.conf import settings from django.utils.encoding import python_2_unicode_compatible +from oxdjango.fields import JSONField import ox from oxdjango import fields @@ -40,7 +41,7 @@ class Entity(models.Model): name = models.CharField(max_length=255) alternativeNames = fields.TupleField(default=()) - data = fields.DictField(default={}, editable=False) + data = JSONField(default=dict, editable=False) matches = models.IntegerField(default=0) objects = managers.EntityManager() @@ -271,7 +272,7 @@ class DocumentProperties(models.Model): document = models.ForeignKey(Document, related_name='documentproperties') entity = models.ForeignKey(Entity, related_name='documentproperties') index = models.IntegerField(default=0) - data = fields.DictField(default={}) + data = JSONField(default=dict, editable=False) def __str__(self): return u"%r-%r" % (self.document, self.entity) diff --git a/pandora/home/migrations/0002_jsonfield.py b/pandora/home/migrations/0002_jsonfield.py new file mode 100644 index 00000000..b716fe13 --- /dev/null +++ b/pandora/home/migrations/0002_jsonfield.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.4 on 2018-06-19 17:23 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('home', '0001_initial'), + ] + + operations = [ + migrations.RunSQL( + 'ALTER TABLE "home_item" ALTER COLUMN "data" TYPE jsonb USING "data"::text::jsonb' + ), + ] diff --git a/pandora/home/models.py b/pandora/home/models.py index 2969816c..775e3714 100644 --- a/pandora/home/models.py +++ b/pandora/home/models.py @@ -8,10 +8,10 @@ from django.db import models from django.db.models import Max from django.db.models.signals import pre_delete from django.utils.encoding import python_2_unicode_compatible +from oxdjango.fields import JSONField import ox -from oxdjango import fields from itemlist.models import List from edit.models import Edit from documentcollection.models import Collection @@ -24,7 +24,7 @@ class Item(models.Model): active = models.BooleanField(default=True) index = models.IntegerField(default=-1) - data = fields.DictField(default={}, editable=False) + data = JSONField(default=dict, editable=False) def editable(self, user): return user.is_authenticated() and user.profile.capability("canManageHome") diff --git a/pandora/item/migrations/0003_jsonfield.py b/pandora/item/migrations/0003_jsonfield.py new file mode 100644 index 00000000..c98fb6f8 --- /dev/null +++ b/pandora/item/migrations/0003_jsonfield.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.4 on 2018-06-19 17:23 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('item', '0002_auto_20160219_1734'), + ] + + operations = [ + migrations.RunSQL( + 'ALTER TABLE "item_item" ALTER COLUMN "data" TYPE jsonb USING "data"::text::jsonb' + ), + migrations.RunSQL( + 'ALTER TABLE "item_item" ALTER COLUMN "external_data" TYPE jsonb USING "external_data"::text::jsonb' + ), + migrations.RunSQL( + 'ALTER TABLE "item_item" ALTER COLUMN "json" TYPE jsonb USING "json"::text::jsonb' + ), + migrations.RunSQL( + 'ALTER TABLE "item_item" ALTER COLUMN "stream_info" TYPE jsonb USING "stream_info"::text::jsonb' + ), + ] diff --git a/pandora/item/models.py b/pandora/item/models.py index 0897f5bd..c03b163a 100644 --- a/pandora/item/models.py +++ b/pandora/item/models.py @@ -23,7 +23,7 @@ from django.utils import datetime_safe from django.utils.encoding import python_2_unicode_compatible import ox -from oxdjango import fields +from oxdjango.fields import JSONField, to_json from oxdjango.sortmodel import get_sort_field import ox.web.imdb import ox.image @@ -175,9 +175,9 @@ class Item(models.Model): public_id = models.CharField(max_length=128, unique=True, blank=True) oxdbId = models.CharField(max_length=42, unique=True, blank=True, null=True) - external_data = fields.DictField(default={}, editable=False) - data = fields.DictField(default={}, editable=False) - json = fields.DictField(default={}, editable=False) + external_data = JSONField(default=dict, editable=False) + data = JSONField(default=dict, editable=False) + json = JSONField(default=dict, editable=False) poster = models.ImageField(default=None, blank=True, upload_to=get_poster_path) poster_source = models.TextField(blank=True) poster_height = models.IntegerField(default=0) @@ -187,7 +187,7 @@ class Item(models.Model): icon = models.ImageField(default=None, blank=True, upload_to=get_icon_path) torrent = models.FileField(default=None, blank=True, max_length=1000, upload_to=get_torrent_path) - stream_info = fields.DictField(default={}, editable=False) + stream_info = JSONField(default=dict, editable=False) # stream related fields stream_aspect = models.FloatField(default=4/3) @@ -1476,7 +1476,7 @@ class Item(models.Model): data = utils.normalize_dict('NFC', data) ox.makedirs(os.path.join(settings.MEDIA_ROOT, self.path())) p = subprocess.Popen(cmd, stdin=subprocess.PIPE, close_fds=True) - p.communicate(json.dumps(data, default=fields.to_json).encode('utf-8')) + p.communicate(json.dumps(data, default=to_json).encode('utf-8')) self.clear_poster_cache(poster) return poster diff --git a/pandora/itemlist/migrations/0003_jsonfield.py b/pandora/itemlist/migrations/0003_jsonfield.py new file mode 100644 index 00000000..020c4ff8 --- /dev/null +++ b/pandora/itemlist/migrations/0003_jsonfield.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.4 on 2018-06-19 17:23 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('itemlist', '0002_list_groups'), + ] + + operations = [ + migrations.RunSQL( + 'ALTER TABLE "itemlist_list" ALTER COLUMN "query" TYPE jsonb USING "query"::text::jsonb' + ), + migrations.RunSQL( + 'ALTER TABLE "itemlist_list" ALTER COLUMN "sort" TYPE jsonb USING "sort"::text::jsonb' + ), + migrations.RunSQL( + 'ALTER TABLE "itemlist_list" ALTER COLUMN "poster_frames" TYPE jsonb USING "poster_frames"::text::jsonb' + ), + ] diff --git a/pandora/itemlist/models.py b/pandora/itemlist/models.py index 29de3632..452aac25 100644 --- a/pandora/itemlist/models.py +++ b/pandora/itemlist/models.py @@ -11,9 +11,9 @@ from django.db.models import Max from django.contrib.auth.models import User, Group from django.conf import settings from django.utils.encoding import python_2_unicode_compatible -import ox +from oxdjango.fields import JSONField -from oxdjango.fields import DictField, TupleField +import ox from archive import extract from user.utils import update_groups @@ -39,16 +39,16 @@ class List(models.Model): name = models.CharField(max_length=255) status = models.CharField(max_length=20, default='private') _status = ['private', 'public', 'featured'] - query = DictField(default={"static": True}) + query = JSONField(default=lambda: {"static": True}, editable=False) type = models.CharField(max_length=255, default='static') description = models.TextField(default='') icon = models.ImageField(default=None, blank=True, upload_to=get_icon_path) view = models.TextField(default=get_listview) - sort = TupleField(default=get_listsort, editable=False) + sort = JSONField(default=get_listsort, editable=False) - poster_frames = TupleField(default=[], editable=False) + poster_frames = JSONField(default=[], editable=False) #is through table still required? items = models.ManyToManyField('item.Item', related_name='lists', diff --git a/pandora/oxdjango/fields.py b/pandora/oxdjango/fields.py index fcb749bb..e4f4ae43 100644 --- a/pandora/oxdjango/fields.py +++ b/pandora/oxdjango/fields.py @@ -5,10 +5,19 @@ import copy from django.db import models from django.utils import datetime_safe +import django.contrib.postgres.fields +from django.core.serializers.json import DjangoJSONEncoder + from six import string_types from ox.utils import json +class JSONField(django.contrib.postgres.fields.JSONField): + + def __init__(self, *args, **kwargs): + if 'encoder' not in kwargs: + kwargs['encoder'] = DjangoJSONEncoder + super().__init__(*args, **kwargs) def to_json(python_object): if isinstance(python_object, datetime.datetime): diff --git a/pandora/user/migrations/0004_jsonfield.py b/pandora/user/migrations/0004_jsonfield.py new file mode 100644 index 00000000..90a5548a --- /dev/null +++ b/pandora/user/migrations/0004_jsonfield.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.4 on 2018-06-19 17:23 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('user', '0003_sessiondata_numberofcollections') + ] + + operations = [ + migrations.RunSQL( + 'ALTER TABLE "user_sessiondata" ALTER COLUMN "info" TYPE jsonb USING "info"::text::jsonb' + ), + migrations.RunSQL( + 'ALTER TABLE "user_userprofile" ALTER COLUMN "ui" TYPE jsonb USING "ui"::text::jsonb' + ), + migrations.RunSQL( + 'ALTER TABLE "user_userprofile" ALTER COLUMN "preferences" TYPE jsonb USING "preferences"::text::jsonb' + ), + ] diff --git a/pandora/user/models.py b/pandora/user/models.py index 6c7208eb..7994651c 100644 --- a/pandora/user/models.py +++ b/pandora/user/models.py @@ -9,9 +9,9 @@ from django.db import models from django.db.models import Max from django.conf import settings from django.utils.encoding import python_2_unicode_compatible +from oxdjango.fields import JSONField import ox -from oxdjango.fields import DictField from ox.utils import json from itemlist.models import List, Position @@ -38,7 +38,7 @@ class SessionData(models.Model): useragent = models.CharField(max_length=4096, null=True) windowsize = models.CharField(max_length=255, null=True) screensize = models.CharField(max_length=255, null=True) - info = DictField(default={}) + info = JSONField(default=dict, editable=False) location = models.CharField(max_length=255, null=True) location_sort = models.CharField(max_length=255, null=True) @@ -178,8 +178,8 @@ class UserProfile(models.Model): level = models.IntegerField(default=1) files_updated = models.DateTimeField(default=datetime.now) newsletter = models.BooleanField(default=True) - ui = DictField(default={}) - preferences = DictField(default={}) + ui = JSONField(default=dict, editable=False) + preferences = JSONField(default=dict, editable=False) notes = models.TextField(default='') diff --git a/requirements.txt b/requirements.txt index d26eb648..4370f871 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ -Django==1.9.4 +Django==1.11.13 simplejson chardet -celery==3.1.23 -django-celery==3.1.17 -django-extensions==1.6.1 -gunicorn==19.4.5 +celery==3.1.26.post2 +django-celery==3.2.2 +django-extensions==2.0.7 +gunicorn==19.8.1 html5lib -requests==2.9.1 -tornado==4.1 -geoip2==2.2.0 +requests==2.19.1 +tornado==5.0.2 +geoip2==2.9.0 youtube-dl diff --git a/update.py b/update.py index 9f961819..136f10e3 100755 --- a/update.py +++ b/update.py @@ -247,6 +247,8 @@ if __name__ == "__main__": run('./bin/pip', 'install', '-r', 'requirements.txt') update_service('pandora-encoding') update_service('pandora-tasks') + if old < 5972: + run('./bin/pip', 'install', '-r', 'requirements.txt') else: if len(sys.argv) == 1: release = get_release()