0x2620/pandora
2
1
Fork 1
Code Issues 1.3k Pull requests Releases Wiki Activity

use postgres json field

Browse source
This commit is contained in:
j 2018-06-19 20:48:18 +02:00
parent bbaaeb7263
commit 1bac062a50
24 changed files with 284 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
pandora/archive/managers.py
View file

@ -4,7 +4,6 @@ from django.db.models import Q, Manager
from oxdjango.managers import get_operator
from oxdjango.query import QuerySet
from oxdjango.fields import DictField
keymap = {
@ -36,8 +35,7 @@ def parseCondition(condition, user):
if isinstance(v, bool):
key = k
elif k == 'url':
key = 'info' + get_operator('=', 'istr')
v = DictField.dumps({'url': v})[1:-1]
key = 'info__url' + get_operator('=', 'istr')
elif k == 'list':
q = Q(id=0)
l = v.split(":")

22
pandora/archive/migrations/0004_jsonfield.py Normal file
View file

@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.9.4 on 2018-06-19 17:23
from __future__ import unicode_literals
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('archive', '0003_auto_20161104_1726'),
]
operations = [
migrations.RunSQL(
'ALTER TABLE "archive_file" ALTER COLUMN "info" TYPE jsonb USING "info"::text::jsonb'
),
migrations.RunSQL(
'ALTER TABLE "archive_stream" ALTER COLUMN "info" TYPE jsonb USING "info"::text::jsonb'
),
]

5
pandora/archive/models.py
View file

@ -13,6 +13,7 @@ from django.contrib.auth.models import User
from django.db import models
from django.db.models.signals import pre_delete
from django.utils.encoding import python_2_unicode_compatible
from oxdjango.fields import JSONField
from oxdjango import fields
import ox
@ -69,7 +70,7 @@ class File(models.Model):
size = models.BigIntegerField(default=0)
duration = models.FloatField(null=True)
info = fields.DictField(default={})
info = JSONField(default=dict, editable=False)
video_codec = models.CharField(max_length=255)
pixel_format = models.CharField(max_length=255)
@ -687,7 +688,7 @@ class Stream(models.Model):
source = models.ForeignKey('Stream', related_name='derivatives', default=None, null=True)
available = models.BooleanField(default=False)
oshash = models.CharField(max_length=16, null=True, db_index=True)
info = fields.DictField(default={})
info = JSONField(default=dict, editable=False)
duration = models.FloatField(default=0)
aspect_ratio = models.FloatField(default=0)

22
pandora/changelog/migrations/0002_jsonfield.py Normal file
View file

@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.9.4 on 2018-06-19 17:23
from __future__ import unicode_literals
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('changelog', '0001_initial'),
]
operations = [
migrations.RunSQL(
'ALTER TABLE "changelog_changelog" ALTER COLUMN "value" TYPE jsonb USING "value"::text::jsonb'
),
migrations.RunSQL(
'ALTER TABLE "changelog_log" ALTER COLUMN "data" TYPE jsonb USING "data"::text::jsonb'
),
]

6
pandora/changelog/models.py
View file

@ -6,8 +6,8 @@ from datetime import datetime
from django.contrib.auth.models import User
from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from oxdjango.fields import JSONField
from oxdjango import fields
import ox
import websocket
@ -21,7 +21,7 @@ FIXME: remove this table more migrate to new ChangeLog
class Changelog(models.Model):
created = models.DateTimeField(auto_now_add=True)
type = models.CharField(max_length=255, db_index=True)
value = fields.DictField(default={})
value = JSONField(default=dict, editable=False)
def __str__(self):
return u'%s %s' % (self.type, self.created)
@ -52,7 +52,7 @@ def add_changelog(request, data, id=None):
class Log(models.Model):
action = models.CharField(max_length=255, db_index=True)
data = fields.DictField(default={})
data = JSONField(default=dict, editable=False)
created = models.DateTimeField(db_index=True)
user = models.ForeignKey(User, null=True, related_name='changelog')
changeid = models.TextField()

19
pandora/document/migrations/0011_jsonfield.py Normal file
View file

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.9.4 on 2018-06-19 17:23
from __future__ import unicode_literals
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('document', '0010_auto_20170126_1528'),
]
operations = [
migrations.RunSQL(
'ALTER TABLE "document_document" ALTER COLUMN "data" TYPE jsonb USING "data"::text::jsonb'
),
]

4
pandora/document/models.py
View file

@ -14,12 +14,12 @@ from django.contrib.auth.models import User, Group
from django.db.models.signals import pre_delete
from django.conf import settings
from django.utils.encoding import python_2_unicode_compatible
from oxdjango.fields import JSONField
from PIL import Image
import ox
from oxdjango import fields
from oxdjango.sortmodel import get_sort_field
from person.models import get_name_sort
from item.models import Item
@ -65,7 +65,7 @@ class Document(models.Model):
linked_documents = models.ManyToManyField('Document', related_name='linking_documents')
rightslevel = models.IntegerField(db_index=True, default=0)
data = fields.DictField(default={})
data = JSONField(default=dict, editable=False)
def update_access(self, user):
if not user.is_authenticated():

25
pandora/documentcollection/migrations/0003_jsonfield.py Normal file
View file

@ -0,0 +1,25 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.9.4 on 2018-06-19 17:23
from __future__ import unicode_literals
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('documentcollection', '0002_collection_groups'),
]
operations = [
migrations.RunSQL(
'ALTER TABLE "documentcollection_collection" ALTER COLUMN "query" TYPE jsonb USING "query"::text::jsonb'
),
migrations.RunSQL(
'ALTER TABLE "documentcollection_collection" ALTER COLUMN "sort" TYPE jsonb USING "sort"::text::jsonb'
),
migrations.RunSQL(
'ALTER TABLE "documentcollection_collection" ALTER COLUMN "poster_frames" TYPE jsonb USING "poster_frames"::text::jsonb'
),
]

9
pandora/documentcollection/models.py
View file

@ -11,11 +11,10 @@ from django.db.models import Max
from django.contrib.auth.models import User, Group
from django.conf import settings
from django.utils.encoding import python_2_unicode_compatible
from oxdjango.fields import JSONField
import ox
from oxdjango.fields import DictField, TupleField
from archive import extract
from user.utils import update_groups
@ -47,16 +46,16 @@ class Collection(models.Model):
name = models.CharField(max_length=255)
status = models.CharField(max_length=20, default='private')
_status = ['private', 'public', 'featured']
query = DictField(default={"static": True})
query = JSONField(default=lambda: {"static": True}, editable=False)
type = models.CharField(max_length=255, default='static')
description = models.TextField(default='')
icon = models.ImageField(default=None, blank=True, upload_to=get_icon_path)
view = models.TextField(default=get_collectionview)
sort = TupleField(default=get_collectionsort, editable=False)
sort = JSONField(default=get_collectionsort, editable=False)
poster_frames = TupleField(default=[], editable=False)
poster_frames = JSONField(default=[], editable=False)
#is through table still required?
documents = models.ManyToManyField('document.Document', related_name='collections',

22
pandora/edit/migrations/0005_jsonfield.py Normal file
View file

@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.9.4 on 2018-06-19 17:23
from __future__ import unicode_literals
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('edit', '0004_edit_groups'),
]
operations = [
migrations.RunSQL(
'ALTER TABLE "edit_edit" ALTER COLUMN "query" TYPE jsonb USING "query"::text::jsonb'
),
migrations.RunSQL(
'ALTER TABLE "edit_edit" ALTER COLUMN "poster_frames" TYPE jsonb USING "poster_frames"::text::jsonb'
),
]

6
pandora/edit/models.py
View file

@ -10,12 +10,12 @@ import tempfile
from six.moves.urllib.parse import quote
import ox
from oxdjango.fields import DictField, TupleField
from django.conf import settings
from django.db import models, transaction
from django.db.models import Max
from django.contrib.auth.models import User, Group
from django.utils.encoding import python_2_unicode_compatible
from oxdjango.fields import JSONField
from annotation.models import Annotation
from item.models import Item
@ -49,12 +49,12 @@ class Edit(models.Model):
description = models.TextField(default='')
rightslevel = models.IntegerField(db_index=True, default=0)
query = DictField(default={"static": True})
query = JSONField(default=lambda: {"static": True}, editable=False)
type = models.CharField(max_length=255, default='static')
icon = models.ImageField(default=None, blank=True, null=True, upload_to=get_icon_path)
poster_frames = TupleField(default=[], editable=False)
poster_frames = JSONField(default=[], editable=False)
subscribed_users = models.ManyToManyField(User, related_name='subscribed_edits')
def __str__(self):

22
pandora/entity/migrations/0005_jsonfield.py Normal file
View file

@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.9.4 on 2018-06-19 17:23
from __future__ import unicode_literals
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('entity', '0004_related_name_Document_documentproperties'),
]
operations = [
migrations.RunSQL(
'ALTER TABLE "entity_entity" ALTER COLUMN "data" TYPE jsonb USING "data"::text::jsonb'
),
migrations.RunSQL(
'ALTER TABLE "entity_documentproperties" ALTER COLUMN "data" TYPE jsonb USING "data"::text::jsonb'
),
]

5
pandora/entity/models.py
View file

@ -14,6 +14,7 @@ from django.contrib.auth.models import User
from django.db.models.signals import pre_delete, post_init
from django.conf import settings
from django.utils.encoding import python_2_unicode_compatible
from oxdjango.fields import JSONField
import ox
from oxdjango import fields
@ -40,7 +41,7 @@ class Entity(models.Model):
name = models.CharField(max_length=255)
alternativeNames = fields.TupleField(default=())
data = fields.DictField(default={}, editable=False)
data = JSONField(default=dict, editable=False)
matches = models.IntegerField(default=0)
objects = managers.EntityManager()
@ -271,7 +272,7 @@ class DocumentProperties(models.Model):
document = models.ForeignKey(Document, related_name='documentproperties')
entity = models.ForeignKey(Entity, related_name='documentproperties')
index = models.IntegerField(default=0)
data = fields.DictField(default={})
data = JSONField(default=dict, editable=False)
def __str__(self):
return u"%r-%r" % (self.document, self.entity)

19
pandora/home/migrations/0002_jsonfield.py Normal file
View file

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.9.4 on 2018-06-19 17:23
from __future__ import unicode_literals
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('home', '0001_initial'),
]
operations = [
migrations.RunSQL(
'ALTER TABLE "home_item" ALTER COLUMN "data" TYPE jsonb USING "data"::text::jsonb'
),
]

4
pandora/home/models.py
View file

@ -8,10 +8,10 @@ from django.db import models
from django.db.models import Max
from django.db.models.signals import pre_delete
from django.utils.encoding import python_2_unicode_compatible
from oxdjango.fields import JSONField
import ox
from oxdjango import fields
from itemlist.models import List
from edit.models import Edit
from documentcollection.models import Collection
@ -24,7 +24,7 @@ class Item(models.Model):
active = models.BooleanField(default=True)
index = models.IntegerField(default=-1)
data = fields.DictField(default={}, editable=False)
data = JSONField(default=dict, editable=False)
def editable(self, user):
return user.is_authenticated() and user.profile.capability("canManageHome")

28
pandora/item/migrations/0003_jsonfield.py Normal file
View file

@ -0,0 +1,28 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.9.4 on 2018-06-19 17:23
from __future__ import unicode_literals
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('item', '0002_auto_20160219_1734'),
]
operations = [
migrations.RunSQL(
'ALTER TABLE "item_item" ALTER COLUMN "data" TYPE jsonb USING "data"::text::jsonb'
),
migrations.RunSQL(
'ALTER TABLE "item_item" ALTER COLUMN "external_data" TYPE jsonb USING "external_data"::text::jsonb'
),
migrations.RunSQL(
'ALTER TABLE "item_item" ALTER COLUMN "json" TYPE jsonb USING "json"::text::jsonb'
),
migrations.RunSQL(
'ALTER TABLE "item_item" ALTER COLUMN "stream_info" TYPE jsonb USING "stream_info"::text::jsonb'
),
]

12
pandora/item/models.py
View file

@ -23,7 +23,7 @@ from django.utils import datetime_safe
from django.utils.encoding import python_2_unicode_compatible
import ox
from oxdjango import fields
from oxdjango.fields import JSONField, to_json
from oxdjango.sortmodel import get_sort_field
import ox.web.imdb
import ox.image
@ -175,9 +175,9 @@ class Item(models.Model):
public_id = models.CharField(max_length=128, unique=True, blank=True)
oxdbId = models.CharField(max_length=42, unique=True, blank=True, null=True)
external_data = fields.DictField(default={}, editable=False)
data = fields.DictField(default={}, editable=False)
json = fields.DictField(default={}, editable=False)
external_data = JSONField(default=dict, editable=False)
data = JSONField(default=dict, editable=False)
json = JSONField(default=dict, editable=False)
poster = models.ImageField(default=None, blank=True, upload_to=get_poster_path)
poster_source = models.TextField(blank=True)
poster_height = models.IntegerField(default=0)
@ -187,7 +187,7 @@ class Item(models.Model):
icon = models.ImageField(default=None, blank=True, upload_to=get_icon_path)
torrent = models.FileField(default=None, blank=True, max_length=1000, upload_to=get_torrent_path)
stream_info = fields.DictField(default={}, editable=False)
stream_info = JSONField(default=dict, editable=False)
# stream related fields
stream_aspect = models.FloatField(default=4/3)
@ -1476,7 +1476,7 @@ class Item(models.Model):
data = utils.normalize_dict('NFC', data)
ox.makedirs(os.path.join(settings.MEDIA_ROOT, self.path()))
p = subprocess.Popen(cmd, stdin=subprocess.PIPE, close_fds=True)
p.communicate(json.dumps(data, default=fields.to_json).encode('utf-8'))
p.communicate(json.dumps(data, default=to_json).encode('utf-8'))
self.clear_poster_cache(poster)
return poster

25
pandora/itemlist/migrations/0003_jsonfield.py Normal file
View file

@ -0,0 +1,25 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.9.4 on 2018-06-19 17:23
from __future__ import unicode_literals
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('itemlist', '0002_list_groups'),
]
operations = [
migrations.RunSQL(
'ALTER TABLE "itemlist_list" ALTER COLUMN "query" TYPE jsonb USING "query"::text::jsonb'
),
migrations.RunSQL(
'ALTER TABLE "itemlist_list" ALTER COLUMN "sort" TYPE jsonb USING "sort"::text::jsonb'
),
migrations.RunSQL(
'ALTER TABLE "itemlist_list" ALTER COLUMN "poster_frames" TYPE jsonb USING "poster_frames"::text::jsonb'
),
]

10
pandora/itemlist/models.py
View file

@ -11,9 +11,9 @@ from django.db.models import Max
from django.contrib.auth.models import User, Group
from django.conf import settings
from django.utils.encoding import python_2_unicode_compatible
import ox
from oxdjango.fields import JSONField
from oxdjango.fields import DictField, TupleField
import ox
from archive import extract
from user.utils import update_groups
@ -39,16 +39,16 @@ class List(models.Model):
name = models.CharField(max_length=255)
status = models.CharField(max_length=20, default='private')
_status = ['private', 'public', 'featured']
query = DictField(default={"static": True})
query = JSONField(default=lambda: {"static": True}, editable=False)
type = models.CharField(max_length=255, default='static')
description = models.TextField(default='')
icon = models.ImageField(default=None, blank=True, upload_to=get_icon_path)
view = models.TextField(default=get_listview)
sort = TupleField(default=get_listsort, editable=False)
sort = JSONField(default=get_listsort, editable=False)
poster_frames = TupleField(default=[], editable=False)
poster_frames = JSONField(default=[], editable=False)
#is through table still required?
items = models.ManyToManyField('item.Item', related_name='lists',

9
pandora/oxdjango/fields.py
View file

@ -5,10 +5,19 @@ import copy
from django.db import models
from django.utils import datetime_safe
import django.contrib.postgres.fields
from django.core.serializers.json import DjangoJSONEncoder
from six import string_types
from ox.utils import json
class JSONField(django.contrib.postgres.fields.JSONField):
def __init__(self, *args, **kwargs):
if 'encoder' not in kwargs:
kwargs['encoder'] = DjangoJSONEncoder
super().__init__(*args, **kwargs)
def to_json(python_object):
if isinstance(python_object, datetime.datetime):

25
pandora/user/migrations/0004_jsonfield.py Normal file
View file

@ -0,0 +1,25 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.9.4 on 2018-06-19 17:23
from __future__ import unicode_literals
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('user', '0003_sessiondata_numberofcollections')
]
operations = [
migrations.RunSQL(
'ALTER TABLE "user_sessiondata" ALTER COLUMN "info" TYPE jsonb USING "info"::text::jsonb'
),
migrations.RunSQL(
'ALTER TABLE "user_userprofile" ALTER COLUMN "ui" TYPE jsonb USING "ui"::text::jsonb'
),
migrations.RunSQL(
'ALTER TABLE "user_userprofile" ALTER COLUMN "preferences" TYPE jsonb USING "preferences"::text::jsonb'
),
]

8
pandora/user/models.py
View file

@ -9,9 +9,9 @@ from django.db import models
from django.db.models import Max
from django.conf import settings
from django.utils.encoding import python_2_unicode_compatible
from oxdjango.fields import JSONField
import ox
from oxdjango.fields import DictField
from ox.utils import json
from itemlist.models import List, Position
@ -38,7 +38,7 @@ class SessionData(models.Model):
useragent = models.CharField(max_length=4096, null=True)
windowsize = models.CharField(max_length=255, null=True)
screensize = models.CharField(max_length=255, null=True)
info = DictField(default={})
info = JSONField(default=dict, editable=False)
location = models.CharField(max_length=255, null=True)
location_sort = models.CharField(max_length=255, null=True)
@ -178,8 +178,8 @@ class UserProfile(models.Model):
level = models.IntegerField(default=1)
files_updated = models.DateTimeField(default=datetime.now)
newsletter = models.BooleanField(default=True)
ui = DictField(default={})
preferences = DictField(default={})
ui = JSONField(default=dict, editable=False)
preferences = JSONField(default=dict, editable=False)
notes = models.TextField(default='')

16
requirements.txt
View file

@ -1,12 +1,12 @@
Django==1.9.4
Django==1.11.13
simplejson
chardet
celery==3.1.23
django-celery==3.1.17
django-extensions==1.6.1
gunicorn==19.4.5
celery==3.1.26.post2
django-celery==3.2.2
django-extensions==2.0.7
gunicorn==19.8.1
html5lib
requests==2.9.1
tornado==4.1
geoip2==2.2.0
requests==2.19.1
tornado==5.0.2
geoip2==2.9.0
youtube-dl

2
update.py
View file

@ -247,6 +247,8 @@ if __name__ == "__main__":
run('./bin/pip', 'install', '-r', 'requirements.txt')
update_service('pandora-encoding')
update_service('pandora-tasks')
if old < 5972:
run('./bin/pip', 'install', '-r', 'requirements.txt')
else:
if len(sys.argv) == 1:
release = get_release()