From 032b843bac2798ab837b27fae16828e4ba5d8e29 Mon Sep 17 00:00:00 2001 From: j Date: Tue, 28 Apr 2020 15:32:02 +0200 Subject: [PATCH 01/37] add note about BRANCH --- README.md | 1 + vm/LXC_README.md | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 902b3f46..71963b6a 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ cd /root curl -sL https://pan.do/ra-install > pandora_install.sh chmod +x pandora_install.sh +BRANCH=stable # change to 'master' to get current developement version ./pandora_install.sh 2>&1 | tee pandora_install.log ``` diff --git a/vm/LXC_README.md b/vm/LXC_README.md index 92cd6262..e8152239 100644 --- a/vm/LXC_README.md +++ b/vm/LXC_README.md @@ -4,7 +4,7 @@ # Installing pan.do/ra inside LXC -1) Install lxc on the host (Ubuntu 18.04 or later): +1) Install lxc on the host (Ubuntu 18.04): sudo apt-get install lxc @@ -41,5 +41,6 @@ cd /root curl -sL https://pan.do/ra-install > pandora_install.sh chmod +x pandora_install.sh + BRANCH=stable # or master ./pandora_install.sh 2>&1 | tee pandora_install.log From b18a6ab3cae6ca7cc3b839c131dbd00f1097b5e7 Mon Sep 17 00:00:00 2001 From: j Date: Tue, 12 May 2020 15:15:26 +0200 Subject: [PATCH 02/37] stream might have been removed --- pandora/archive/models.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pandora/archive/models.py b/pandora/archive/models.py index 75d25de7..e9bd62a8 100644 --- a/pandora/archive/models.py +++ b/pandora/archive/models.py @@ -814,8 +814,12 @@ class Stream(models.Model): # file could have been moved while encoding # get current version from db and update - self.refresh_from_db() - self.update_status(ok, error) + try: + self.refresh_from_db() + except archive.models.DoesNotExist: + pass + else: + self.update_status(ok, error) def get_index(self): index = 1 From 7ba58d0b63a3525c48e5c11f3994d15b5a8184fb Mon Sep 17 00:00:00 2001 From: j Date: Fri, 15 May 2020 18:48:58 +0200 Subject: [PATCH 03/37] add way to change version --- pandora/app/config.py | 12 ++++-------- pandora/settings.py | 2 ++ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pandora/app/config.py b/pandora/app/config.py index 36dfdfe3..7d158e4d 100644 --- a/pandora/app/config.py +++ b/pandora/app/config.py @@ -29,18 +29,14 @@ RUN_RELOADER = True NOTIFIER = None def get_version(): - info = join(dirname(dirname(dirname(__file__))), '.bzr', 'branch', 'last-revision') git_dir = join(dirname(dirname(dirname(__file__))), '.git') if exists(git_dir): env = {'GIT_DIR': git_dir} cmd = ['git', 'rev-list', 'HEAD', '--count'] - return subprocess.check_output(cmd, env=env).strip().decode('utf-8') - elif exists(info): - f = open(info) - rev = int(f.read().split()[0]) - f.close() - if rev: - return u'%s' % rev + version = subprocess.check_output(cmd, env=env).strip().decode('utf-8') + if settings.VERSION_EPOCH: + version = settings.VERSION_EPOCH + version + return version else: return u'unknown' diff --git a/pandora/settings.py b/pandora/settings.py index 83dfb602..fd1cf7a0 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -36,6 +36,8 @@ LANGUAGE_CODE = 'en-us' SITE_ID = 1 +VERSION_EPOCH = None + # If you set this to False, Django will make some optimizations so as not # to load the internationalization machinery. USE_I18N = True From 71a0a82aee70533204028a2c3ce450c26ec49fb2 Mon Sep 17 00:00:00 2001 From: j Date: Fri, 15 May 2020 19:25:07 +0200 Subject: [PATCH 04/37] not in collection --- pandora/document/managers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandora/document/managers.py b/pandora/document/managers.py index fa0cd64a..c3b19f08 100644 --- a/pandora/document/managers.py +++ b/pandora/document/managers.py @@ -132,6 +132,8 @@ def buildCondition(k, op, v, user, exclude=False, owner=None): q = Q(id__in=l.documents.all()) else: q = Q(id=0) + if exclude: + q = ~q return q elif key_config.get('fulltext'): qs = models.Document.find_fulltext_ids(v) From 545e1ba57d977c62b2433486b688e7ed0fa06b75 Mon Sep 17 00:00:00 2001 From: j Date: Fri, 15 May 2020 20:47:09 +0200 Subject: [PATCH 05/37] add files to current list if it is editable --- static/js/utils.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/static/js/utils.js b/static/js/utils.js index a08f49a0..d4d0f610 100644 --- a/static/js/utils.js +++ b/static/js/utils.js @@ -425,6 +425,13 @@ pandora.uploadDroppedFiles = function(files) { Ox.Request.clearCache('findDocuments'); if (pandora.user.ui.document || pandora.user.ui.section != 'documents') { pandora.UI.set({section: 'documents', document: ''}); + } else if (pandora.user.ui._collection && pandora.getListData().editable) { + pandora.api.addCollectionItems({ + items: files.ids, + collection: pandora.user.ui._collection + }, function(results) { + pandora.$ui.list && pandora.$ui.list.reloadList(); + }) } else { pandora.$ui.list && pandora.$ui.list.reloadList(); } From b9b109a2c88c2f22a3b13533680183a6787c6d39 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 16 May 2020 10:43:45 +0200 Subject: [PATCH 06/37] use autocompleteDocuments for collections --- static/js/filterForm.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/static/js/filterForm.js b/static/js/filterForm.js index 2240876b..53305293 100644 --- a/static/js/filterForm.js +++ b/static/js/filterForm.js @@ -73,7 +73,11 @@ pandora.ui.filterForm = function(options) { }); function autocompleteFunction(key) { return key.autocomplete ? function(value, callback) { - pandora.api.autocomplete({ + pandora.api[ + pandora.user.ui.sections == 'documents' + ? 'autocompleteDocuments' + : 'autocomplete' + ]({ key: key.id, query: { conditions: [], From d2eeadf86035f84d8ee6b0b21f59a42ccfa41171 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 16 May 2020 10:46:31 +0200 Subject: [PATCH 07/37] typo --- static/js/filterForm.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/js/filterForm.js b/static/js/filterForm.js index 53305293..4413869f 100644 --- a/static/js/filterForm.js +++ b/static/js/filterForm.js @@ -74,7 +74,7 @@ pandora.ui.filterForm = function(options) { function autocompleteFunction(key) { return key.autocomplete ? function(value, callback) { pandora.api[ - pandora.user.ui.sections == 'documents' + pandora.user.ui.section == 'documents' ? 'autocompleteDocuments' : 'autocomplete' ]({ From 30f025f13f3b0785e17b21c80f9e1f0071ffa823 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 16 May 2020 10:51:37 +0200 Subject: [PATCH 08/37] really use autocompleteDocuments for collections --- static/js/documentFilterForm.js | 2 +- static/js/filterForm.js | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/static/js/documentFilterForm.js b/static/js/documentFilterForm.js index a1fb25c9..a7353eb0 100644 --- a/static/js/documentFilterForm.js +++ b/static/js/documentFilterForm.js @@ -72,7 +72,7 @@ pandora.ui.documentFilterForm = function(options) { }); function autocompleteFunction(key) { return key.autocomplete ? function(value, callback) { - pandora.api.autocomplete({ + pandora.api.autocompleteDocuments({ key: key.id, query: { conditions: [], diff --git a/static/js/filterForm.js b/static/js/filterForm.js index 4413869f..2240876b 100644 --- a/static/js/filterForm.js +++ b/static/js/filterForm.js @@ -73,11 +73,7 @@ pandora.ui.filterForm = function(options) { }); function autocompleteFunction(key) { return key.autocomplete ? function(value, callback) { - pandora.api[ - pandora.user.ui.section == 'documents' - ? 'autocompleteDocuments' - : 'autocomplete' - ]({ + pandora.api.autocomplete({ key: key.id, query: { conditions: [], From 008ba18d5de4be8ddf3bf8cc78748c1f60bda6b7 Mon Sep 17 00:00:00 2001 From: j Date: Fri, 22 May 2020 21:56:56 +0200 Subject: [PATCH 09/37] decode sort title --- pandora/item/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandora/item/utils.py b/pandora/item/utils.py index 9d0beb66..be884bb9 100644 --- a/pandora/item/utils.py +++ b/pandora/item/utils.py @@ -58,6 +58,7 @@ def sort_title(title): title = title.replace(u'Æ', 'Ae') if isinstance(title, bytes): title = title.decode('utf-8') + title = ox.decode_html(title) title = sort_string(title) #title From ed977686d17450ab527aabfa6264a2681cbf3266 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 23 May 2020 16:10:33 +0200 Subject: [PATCH 10/37] group permissions for documents --- pandora/document/models.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pandora/document/models.py b/pandora/document/models.py index 4d85f785..f95f8d5b 100644 --- a/pandora/document/models.py +++ b/pandora/document/models.py @@ -27,6 +27,7 @@ from annotation.models import Annotation from archive.extract import resize_image from archive.chunk import save_chunk from user.models import Group +from user.utils import update_groups from . import managers from . import utils @@ -334,6 +335,7 @@ class Document(models.Model, FulltextMixin): if not user or user.is_anonymous(): return False if self.user == user or \ + self.groups.filter(id__in=user.groups.all()).count() > 0 or \ user.is_staff or \ user.profile.capability('canEditDocuments') is True or \ (item and item.editable(user)): @@ -347,6 +349,9 @@ class Document(models.Model, FulltextMixin): p.description = ox.sanitize_html(data['description']) p.save() else: + if 'groups' in data: + groups = data.pop('groups') + update_groups(self, groups) for key in data: k = list(filter(lambda i: i['id'] == key, settings.CONFIG['documentKeys'])) ktype = k and k[0].get('type') or '' @@ -436,6 +441,7 @@ class Document(models.Model, FulltextMixin): 'matches', 'size', 'user', + 'groups', 'referenced', ] if self.extension in ('html', 'txt'): @@ -455,6 +461,8 @@ class Document(models.Model, FulltextMixin): response[key] = self.editable(user) elif key == 'user': response[key] = self.user.username + elif key == 'groups': + response[key] = [g.name for g in self.groups.all()] elif key == 'accessed': response[key] = self.accessed.aggregate(Max('access'))['access__max'] elif key == 'timesaccessed': From 80390a1f9b1f63cb65c4b377e2e1c88a9544dcfa Mon Sep 17 00:00:00 2001 From: j Date: Sat, 23 May 2020 16:17:35 +0200 Subject: [PATCH 11/37] groups dialog for documents --- static/js/groupsDialog.js | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/static/js/groupsDialog.js b/static/js/groupsDialog.js index 6065a519..cf3973a9 100644 --- a/static/js/groupsDialog.js +++ b/static/js/groupsDialog.js @@ -23,6 +23,7 @@ pandora.ui.groupsDialog = function(options) { canManageGroups = pandora.hasCapability('canManageUsers'), isItem = options.type == 'item', + isDocument = options.type == 'document', $content, $label, @@ -40,7 +41,11 @@ pandora.ui.groupsDialog = function(options) { selectedGroups = options.groups; renderGroups(); } else { - pandora.api[isItem ? 'get' : 'getUser']({ + pandora.api[ + isItem ? 'get' + : isDocument ? 'getDocument' + : 'getUser' + ]({ id: options.id, keys: ['groups'] }, function(result) { @@ -70,8 +75,16 @@ pandora.ui.groupsDialog = function(options) { return group.name; }); // disableElements(); - Ox.Request.clearCache(isItem ? 'get' : 'getUser'); - pandora.api[isItem ? 'edit' : 'editUser']({ + Ox.Request.clearCache( + isItem ? 'get' + : isDocument ? 'getDocument' + : 'getUser' + ); + pandora.api[ + isItem ? 'edit' + : isDocument ? 'editDocument' + : 'editUser' + ]({ id: options.id, groups: selectedGroups }, function(result) { @@ -198,7 +211,9 @@ pandora.ui.groupsDialog = function(options) { $label = Ox.Label({ textAlign: 'center', title: Ox._( - isItem ? pandora.site.itemName.singular : 'User' + isItem ? pandora.site.itemName.singular + : isDocument ? 'Document' + : 'User' ) + ': ' + options.name, width: 552 }) From 844382b1e8f78fa255f1f80e833e812ed906d4aa Mon Sep 17 00:00:00 2001 From: j Date: Fri, 29 May 2020 00:20:18 +0200 Subject: [PATCH 12/37] drop python2 support, upgrade to django 3 --- pandora/annotation/admin.py | 1 - .../management/commands/import_srt.py | 1 - pandora/annotation/managers.py | 2 +- pandora/annotation/models.py | 13 ++- pandora/annotation/tasks.py | 1 - pandora/annotation/views.py | 1 - pandora/app/admin.py | 1 - pandora/app/config.py | 3 +- pandora/app/documentation.py | 1 - pandora/app/management/commands/init_db.py | 1 - .../management/commands/south_installed.py | 1 - .../commands/update_config_documentation.py | 1 - pandora/app/models.py | 4 - pandora/app/monkey_patch.py | 1 - pandora/app/tasks.py | 1 - pandora/app/views.py | 1 - pandora/archive/admin.py | 1 - pandora/archive/external.py | 1 - pandora/archive/extract.py | 1 - .../management/commands/import_streams.py | 1 - .../management/commands/migrate_timelines.py | 1 - .../management/commands/orphaned_media.py | 1 - pandora/archive/models.py | 39 ++++----- pandora/archive/queue.py | 1 - pandora/archive/tasks.py | 1 - pandora/archive/views.py | 1 - pandora/changelog/models.py | 10 +-- pandora/changelog/recover.py | 55 ++++++------ pandora/changelog/views.py | 1 - pandora/clip/managers.py | 2 +- pandora/clip/models.py | 13 ++- pandora/clip/views.py | 5 +- .../commands/rebuild_documentfind.py | 1 - .../management/commands/sync_documentsort.py | 1 - pandora/document/managers.py | 2 +- pandora/document/models.py | 50 +++++------ pandora/document/sync_sort.py | 1 - pandora/document/views.py | 1 - pandora/documentcollection/managers.py | 2 +- pandora/documentcollection/models.py | 27 +++--- pandora/documentcollection/views.py | 5 +- pandora/edit/managers.py | 2 +- pandora/edit/models.py | 33 +++---- pandora/edit/views.py | 6 +- pandora/entity/models.py | 42 ++++----- pandora/entity/views.py | 5 +- pandora/event/admin.py | 1 - pandora/event/models.py | 9 +- pandora/event/tasks.py | 1 - pandora/event/views.py | 5 +- pandora/home/models.py | 7 +- pandora/item/admin.py | 1 - pandora/item/management/commands/get_frame.py | 1 - .../management/commands/rebuild_filter.py | 1 - .../item/management/commands/rebuild_find.py | 1 - .../management/commands/rebuild_indexes.py | 1 - .../item/management/commands/rebuild_sort.py | 1 - .../management/commands/rebuild_timelines.py | 1 - .../item/management/commands/rebuildcache.py | 1 - .../item/management/commands/sqlfindindex.py | 9 +- .../item/management/commands/sync_itemsort.py | 1 - .../management/commands/update_external.py | 1 - .../management/commands/update_itemsfolder.py | 5 +- pandora/item/managers.py | 9 +- pandora/item/models.py | 60 ++++++------- pandora/item/site.py | 51 +++++++++++ pandora/item/tasks.py | 1 - pandora/item/timelines.py | 1 - pandora/item/urls.py | 47 ---------- pandora/item/utils.py | 4 +- pandora/item/views.py | 35 ++++---- pandora/itemlist/managers.py | 2 +- pandora/itemlist/models.py | 27 +++--- pandora/itemlist/views.py | 5 +- pandora/log/models.py | 7 +- pandora/log/tasks.py | 1 - pandora/log/utils.py | 3 +- pandora/log/views.py | 3 +- pandora/news/admin.py | 1 - pandora/news/models.py | 7 +- pandora/news/views.py | 1 - pandora/oxdjango/api/actions.py | 4 +- pandora/oxdjango/api/{urls.py => site.py} | 10 ++- pandora/oxdjango/api/views.py | 4 +- pandora/oxdjango/decorators.py | 4 +- pandora/oxdjango/fields.py | 2 +- pandora/oxdjango/middleware.py | 7 +- pandora/oxdjango/shortcuts.py | 3 +- pandora/person/models.py | 5 +- pandora/person/tasks.py | 1 - pandora/person/views.py | 1 - pandora/place/admin.py | 1 - pandora/place/models.py | 9 +- pandora/place/tasks.py | 1 - pandora/place/views.py | 7 +- pandora/sequence/managers.py | 1 - pandora/sequence/models.py | 7 +- pandora/sequence/tasks.py | 1 - pandora/sequence/views.py | 1 - pandora/settings.py | 2 +- pandora/taskqueue/models.py | 7 +- pandora/taskqueue/views.py | 1 - pandora/text/managers.py | 2 +- pandora/text/models.py | 18 ++-- pandora/text/views.py | 8 +- pandora/title/models.py | 5 +- pandora/title/views.py | 1 - pandora/translation/tasks.py | 1 - pandora/tv/models.py | 16 ++-- pandora/tv/tasks.py | 1 - pandora/tv/views.py | 1 - pandora/urlalias/models.py | 13 +-- pandora/urlalias/views.py | 1 - pandora/urls.py | 87 +++++++++---------- pandora/user/decorators.py | 2 +- pandora/user/middleware.py | 6 +- pandora/user/models.py | 26 +++--- pandora/user/tasks.py | 1 - pandora/user/views.py | 25 +++--- requirements.txt | 14 +-- scripts/poster.0xdb.py | 4 +- scripts/poster.indiancinema.py | 2 +- scripts/poster.pandora.py | 2 +- update.py | 3 + 124 files changed, 413 insertions(+), 563 deletions(-) create mode 100644 pandora/item/site.py delete mode 100644 pandora/item/urls.py rename pandora/oxdjango/api/{urls.py => site.py} (56%) diff --git a/pandora/annotation/admin.py b/pandora/annotation/admin.py index 663a1396..baf4f500 100644 --- a/pandora/annotation/admin.py +++ b/pandora/annotation/admin.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from django.contrib import admin diff --git a/pandora/annotation/management/commands/import_srt.py b/pandora/annotation/management/commands/import_srt.py index 7c86ee17..881aa186 100644 --- a/pandora/annotation/management/commands/import_srt.py +++ b/pandora/annotation/management/commands/import_srt.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function from django.conf import settings from django.contrib.auth import get_user_model diff --git a/pandora/annotation/managers.py b/pandora/annotation/managers.py index a0c32c04..3b6ee2fe 100644 --- a/pandora/annotation/managers.py +++ b/pandora/annotation/managers.py @@ -158,7 +158,7 @@ class AnnotationManager(Manager): #anonymous can only see public items public_layers = self.model.public_layers() - if user.is_anonymous(): + if user.is_anonymous: qs = qs.filter(layer__in=public_layers) #users can see public and own else: diff --git a/pandora/annotation/models.py b/pandora/annotation/models.py index 0326e37d..e1e9d553 100644 --- a/pandora/annotation/models.py +++ b/pandora/annotation/models.py @@ -1,10 +1,8 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import re import unicodedata -from django.utils.encoding import python_2_unicode_compatible from django.db import models, transaction from django.db.models import Q from django.contrib.auth import get_user_model @@ -83,16 +81,15 @@ def get_matches(obj, model, layer_type, qs=None): matches = [-1] return Annotation.objects.filter(id__in=matches) -@python_2_unicode_compatible class Annotation(models.Model): objects = managers.AnnotationManager() #FIXME: here having a item,start index would be good created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) - user = models.ForeignKey(User, related_name='annotations') - item = models.ForeignKey('item.Item', related_name='annotations') - clip = models.ForeignKey('clip.Clip', null=True, related_name='annotations') + user = models.ForeignKey(User, related_name='annotations', on_delete=models.CASCADE) + item = models.ForeignKey('item.Item', related_name='annotations', on_delete=models.CASCADE) + clip = models.ForeignKey('clip.Clip', null=True, related_name='annotations', on_delete=models.CASCADE) public_id = models.CharField(max_length=128, unique=True) #seconds @@ -107,7 +104,7 @@ class Annotation(models.Model): languages = models.CharField(max_length=255, null=True, blank=True) def editable(self, user): - if user.is_authenticated(): + if user.is_authenticated: if user.profile.capability('canEditAnnotations') or \ self.user == user or \ user.groups.filter(id__in=self.item.groups.all()).count() > 0: @@ -400,7 +397,7 @@ class Annotation(models.Model): return j def __str__(self): - return u"%s %s-%s" % (self.public_id, self.start, self.end) + return "%s %s-%s" % (self.public_id, self.start, self.end) def cleanup_related(sender, **kwargs): kwargs['instance'].cleanup_undefined_relations() diff --git a/pandora/annotation/tasks.py b/pandora/annotation/tasks.py index 88087b0f..8054c6e3 100644 --- a/pandora/annotation/tasks.py +++ b/pandora/annotation/tasks.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from django.conf import settings from django.contrib.auth import get_user_model diff --git a/pandora/annotation/views.py b/pandora/annotation/views.py index d090de56..8bfe7006 100644 --- a/pandora/annotation/views.py +++ b/pandora/annotation/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from django.conf import settings from django.db.models import Count, Sum, F, Value diff --git a/pandora/app/admin.py b/pandora/app/admin.py index 0394dbc9..e4a5bc96 100644 --- a/pandora/app/admin.py +++ b/pandora/app/admin.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from django.contrib import admin diff --git a/pandora/app/config.py b/pandora/app/config.py index 7d158e4d..93714552 100644 --- a/pandora/app/config.py +++ b/pandora/app/config.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import codecs import os @@ -38,7 +37,7 @@ def get_version(): version = settings.VERSION_EPOCH + version return version else: - return u'unknown' + return 'unknown' def load_config(init=False): with open(settings.SITE_CONFIG) as f: diff --git a/pandora/app/documentation.py b/pandora/app/documentation.py index 2cba4e64..89abf100 100644 --- a/pandora/app/documentation.py +++ b/pandora/app/documentation.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function import re import ox.jsonc diff --git a/pandora/app/management/commands/init_db.py b/pandora/app/management/commands/init_db.py index e4378cf3..92ac750f 100644 --- a/pandora/app/management/commands/init_db.py +++ b/pandora/app/management/commands/init_db.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function from django.core.management.base import BaseCommand from django.conf import settings diff --git a/pandora/app/management/commands/south_installed.py b/pandora/app/management/commands/south_installed.py index e753dbdf..d3957807 100644 --- a/pandora/app/management/commands/south_installed.py +++ b/pandora/app/management/commands/south_installed.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function from django.core.management.base import BaseCommand from django.db import connection diff --git a/pandora/app/management/commands/update_config_documentation.py b/pandora/app/management/commands/update_config_documentation.py index de7c7332..665f7528 100644 --- a/pandora/app/management/commands/update_config_documentation.py +++ b/pandora/app/management/commands/update_config_documentation.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function from django.core.management.base import BaseCommand from django.conf import settings from ... import documentation diff --git a/pandora/app/models.py b/pandora/app/models.py index c7232863..3798eea7 100644 --- a/pandora/app/models.py +++ b/pandora/app/models.py @@ -1,16 +1,13 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import json from django.db import models -from django.utils.encoding import python_2_unicode_compatible from . import monkey_patch from . import tasks -@python_2_unicode_compatible class Page(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) @@ -20,7 +17,6 @@ class Page(models.Model): def __str__(self): return self.name -@python_2_unicode_compatible class Settings(models.Model): created = models.DateTimeField(auto_now_add=True) diff --git a/pandora/app/monkey_patch.py b/pandora/app/monkey_patch.py index 8f48af00..b578a01e 100644 --- a/pandora/app/monkey_patch.py +++ b/pandora/app/monkey_patch.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from django.contrib.auth import get_user_model from django.contrib.auth.models import Group diff --git a/pandora/app/tasks.py b/pandora/app/tasks.py index c380f1e8..03b26778 100644 --- a/pandora/app/tasks.py +++ b/pandora/app/tasks.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import datetime diff --git a/pandora/app/views.py b/pandora/app/views.py index 8bd0fbfa..1ebc219c 100644 --- a/pandora/app/views.py +++ b/pandora/app/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import copy from datetime import datetime diff --git a/pandora/archive/admin.py b/pandora/archive/admin.py index bf10ba42..2261bdf5 100644 --- a/pandora/archive/admin.py +++ b/pandora/archive/admin.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from django.contrib import admin diff --git a/pandora/archive/external.py b/pandora/archive/external.py index 69179e4f..510d3f70 100644 --- a/pandora/archive/external.py +++ b/pandora/archive/external.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import json import subprocess diff --git a/pandora/archive/extract.py b/pandora/archive/extract.py index 90422d47..875759d4 100644 --- a/pandora/archive/extract.py +++ b/pandora/archive/extract.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import os from os.path import exists diff --git a/pandora/archive/management/commands/import_streams.py b/pandora/archive/management/commands/import_streams.py index 4d0d18f3..6b23af51 100644 --- a/pandora/archive/management/commands/import_streams.py +++ b/pandora/archive/management/commands/import_streams.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function from django.core.management.base import BaseCommand from django.conf import settings diff --git a/pandora/archive/management/commands/migrate_timelines.py b/pandora/archive/management/commands/migrate_timelines.py index 9d9e5e59..d4ce903d 100644 --- a/pandora/archive/management/commands/migrate_timelines.py +++ b/pandora/archive/management/commands/migrate_timelines.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function import os import re diff --git a/pandora/archive/management/commands/orphaned_media.py b/pandora/archive/management/commands/orphaned_media.py index a6e831ab..756602a0 100644 --- a/pandora/archive/management/commands/orphaned_media.py +++ b/pandora/archive/management/commands/orphaned_media.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function from django.core.management.base import BaseCommand from django.conf import settings diff --git a/pandora/archive/models.py b/pandora/archive/models.py index e9bd62a8..391393ca 100644 --- a/pandora/archive/models.py +++ b/pandora/archive/models.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import json import os.path @@ -12,7 +11,6 @@ from django.conf import settings from django.contrib.auth import get_user_model from django.db import models from django.db.models.signals import pre_delete -from django.utils.encoding import python_2_unicode_compatible from oxdjango.fields import JSONField from oxdjango import fields @@ -36,7 +34,6 @@ if not PY2: def data_path(f, x): return f.get_path('data.bin') -@python_2_unicode_compatible class File(models.Model): AV_INFO = ( 'duration', 'video', 'audio', 'oshash', 'size', @@ -55,7 +52,7 @@ class File(models.Model): modified = models.DateTimeField(auto_now=True) oshash = models.CharField(max_length=16, unique=True) - item = models.ForeignKey("item.Item", related_name='files', null=True) + item = models.ForeignKey("item.Item", related_name='files', null=True, on_delete=models.CASCADE) path = models.CharField(max_length=2048, default="") # canoncial path/file sort_path = models.CharField(max_length=2048, default="") # sort name @@ -483,7 +480,7 @@ class File(models.Model): if k not in keys: del data[k] can_see_media = False - if user and not user.is_anonymous(): + if user and not user.is_anonymous: can_see_media = user.profile.capability('canSeeMedia') or \ user.is_staff or \ self.item.user == user or \ @@ -598,15 +595,15 @@ class File(models.Model): status = {} if self.encoding: for s in self.streams.all(): - status[s.name()] = u'done' if s.available else u'encoding' + status[s.name()] = 'done' if s.available else 'encoding' config = settings.CONFIG['video'] max_resolution = self.streams.get(source=None).resolution for resolution in sorted(config['resolutions'], reverse=True): if resolution <= max_resolution: for f in config['formats']: - name = u'%sp.%s' % (resolution, f) + name = '%sp.%s' % (resolution, f) if name not in status: - status[name] = u'queued' + status[name] = 'queued' return status def delete_frames(self): @@ -627,7 +624,6 @@ def delete_file(sender, **kwargs): f.delete_files() pre_delete.connect(delete_file, sender=File) -@python_2_unicode_compatible class Volume(models.Model): class Meta: @@ -636,11 +632,11 @@ class Volume(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) - user = models.ForeignKey(User, related_name='volumes') + user = models.ForeignKey(User, related_name='volumes', on_delete=models.CASCADE) name = models.CharField(max_length=1024) def __str__(self): - return u"%s's %s" % (self.user, self.name) + return "%s's %s" % (self.user, self.name) def json(self): return { @@ -652,7 +648,6 @@ class Volume(models.Model): def inttime(): return int(time.time()) -@python_2_unicode_compatible class Instance(models.Model): class Meta: @@ -668,11 +663,11 @@ class Instance(models.Model): path = models.CharField(max_length=2048) ignore = models.BooleanField(default=False) - file = models.ForeignKey(File, related_name='instances') - volume = models.ForeignKey(Volume, related_name='files') + file = models.ForeignKey(File, related_name='instances', on_delete=models.CASCADE) + volume = models.ForeignKey(Volume, related_name='files', on_delete=models.CASCADE) def __str__(self): - return u"%s's %s <%s>" % (self.volume.user, self.path, self.file.oshash) + return "%s's %s <%s>" % (self.volume.user, self.path, self.file.oshash) @property def public_id(self): @@ -691,14 +686,13 @@ def frame_path(frame, name): name = "%s%s" % (frame.position, ext) return frame.file.get_path(name) -@python_2_unicode_compatible class Frame(models.Model): class Meta: unique_together = ("file", "position") created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) - file = models.ForeignKey(File, related_name="frames") + file = models.ForeignKey(File, related_name="frames", on_delete=models.CASCADE) position = models.FloatField() frame = models.ImageField(default=None, null=True, upload_to=frame_path) width = models.IntegerField(default=0) @@ -711,7 +705,7 @@ class Frame(models.Model): super(Frame, self).save(*args, **kwargs) def __str__(self): - return u'%s/%s' % (self.file, self.position) + return '%s/%s' % (self.file, self.position) def delete_frame(sender, **kwargs): f = kwargs['instance'] @@ -722,18 +716,17 @@ pre_delete.connect(delete_frame, sender=Frame) def stream_path(f, x): return f.path(x) -@python_2_unicode_compatible class Stream(models.Model): class Meta: unique_together = ("file", "resolution", "format") - file = models.ForeignKey(File, related_name='streams') + file = models.ForeignKey(File, related_name='streams', on_delete=models.CASCADE) resolution = models.IntegerField(default=96) format = models.CharField(max_length=255, default='webm') media = models.FileField(default=None, blank=True, upload_to=stream_path) - source = models.ForeignKey('Stream', related_name='derivatives', default=None, null=True) + source = models.ForeignKey('Stream', related_name='derivatives', default=None, null=True, on_delete=models.CASCADE) available = models.BooleanField(default=False) oshash = models.CharField(max_length=16, null=True, db_index=True) info = JSONField(default=dict, editable=False) @@ -753,10 +746,10 @@ class Stream(models.Model): return os.path.join(settings.MEDIA_ROOT, self.path(), 'timeline') def name(self): - return u"%sp.%s" % (self.resolution, self.format) + return "%sp.%s" % (self.resolution, self.format) def __str__(self): - return u"%s/%s" % (self.file, self.name()) + return "%s/%s" % (self.file, self.name()) def get(self, resolution, format): streams = [] diff --git a/pandora/archive/queue.py b/pandora/archive/queue.py index 6b917c20..8a8ff4ce 100644 --- a/pandora/archive/queue.py +++ b/pandora/archive/queue.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function from datetime import datetime from time import time diff --git a/pandora/archive/tasks.py b/pandora/archive/tasks.py index 92446d58..8013d1ff 100644 --- a/pandora/archive/tasks.py +++ b/pandora/archive/tasks.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from glob import glob diff --git a/pandora/archive/views.py b/pandora/archive/views.py index 0b8abddd..c7d80ff0 100644 --- a/pandora/archive/views.py +++ b/pandora/archive/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import os.path from datetime import datetime diff --git a/pandora/changelog/models.py b/pandora/changelog/models.py index bad1a858..67d1cb0c 100644 --- a/pandora/changelog/models.py +++ b/pandora/changelog/models.py @@ -1,11 +1,9 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from datetime import datetime from django.contrib.auth import get_user_model from django.db import models -from django.utils.encoding import python_2_unicode_compatible from oxdjango.fields import JSONField import ox @@ -19,14 +17,13 @@ User = get_user_model() ''' FIXME: remove this table more migrate to new ChangeLog ''' -@python_2_unicode_compatible class Changelog(models.Model): created = models.DateTimeField(auto_now_add=True) type = models.CharField(max_length=255, db_index=True) value = JSONField(default=dict, editable=False) def __str__(self): - return u'%s %s' % (self.type, self.created) + return '%s %s' % (self.type, self.created) def json(self): return self.value @@ -50,19 +47,18 @@ def add_changelog(request, data, id=None): 'user': c.user.username, }) -@python_2_unicode_compatible class Log(models.Model): action = models.CharField(max_length=255, db_index=True) data = JSONField(default=dict, editable=False) created = models.DateTimeField(db_index=True) - user = models.ForeignKey(User, null=True, related_name='changelog') + user = models.ForeignKey(User, null=True, related_name='changelog', on_delete=models.CASCADE) changeid = models.TextField() objects = managers.LogManager() def __str__(self): - return u'%s %s %s' % (self.created, self.action, self.changeid) + return '%s %s %s' % (self.created, self.action, self.changeid) def get_id(self): return ox.toAZ(self.id) diff --git a/pandora/changelog/recover.py b/pandora/changelog/recover.py index 425af0f9..53be924c 100644 --- a/pandora/changelog/recover.py +++ b/pandora/changelog/recover.py @@ -1,4 +1,3 @@ -from __future__ import print_function import models import item.models @@ -18,33 +17,33 @@ def recover_item(id): created = old.value['created'] i.user = user.models.User.objects.get(username=i.data['user']) for key in [ - u'rendered', - u'random', - u'cuts', - u'duration', - u'id', - u'size', - u'posterFrame', - u'parts', - u'cutsperminute', - u'hue', - u'numberofcuts', - u'durations', - u'volume', - u'user', - u'words', - u'videoRatio', - u'aspectratio', - u'bitrate', - u'pixels', - u'created', - u'numberoffiles', - u'modified', - u'timesaccessed', - u'accessed', - u'resolution', - u'wordsperminute', - u'posterRatio' + 'rendered', + 'random', + 'cuts', + 'duration', + 'id', + 'size', + 'posterFrame', + 'parts', + 'cutsperminute', + 'hue', + 'numberofcuts', + 'durations', + 'volume', + 'user', + 'words', + 'videoRatio', + 'aspectratio', + 'bitrate', + 'pixels', + 'created', + 'numberoffiles', + 'modified', + 'timesaccessed', + 'accessed', + 'resolution', + 'wordsperminute', + 'posterRatio' ]: if key in i.data: del i.data[key] diff --git a/pandora/changelog/views.py b/pandora/changelog/views.py index 677653b4..f32c3626 100644 --- a/pandora/changelog/views.py +++ b/pandora/changelog/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import ox diff --git a/pandora/clip/managers.py b/pandora/clip/managers.py index 729624d5..52f18e5d 100644 --- a/pandora/clip/managers.py +++ b/pandora/clip/managers.py @@ -215,7 +215,7 @@ class ClipManager(Manager): for l in list(filter(lambda k: k in layer_ids, data['keys'])): qs = qs.filter(**{l: True}) #anonymous can only see public clips - if not user or user.is_anonymous(): + if not user or user.is_anonymous: allowed_level = settings.CONFIG['capabilities']['canSeeItem']['guest'] qs = qs.filter(sort__rightslevel__lte=allowed_level) #users can see public clips, there own clips and clips of there groups diff --git a/pandora/clip/models.py b/pandora/clip/models.py index 37641c15..9736adb1 100644 --- a/pandora/clip/models.py +++ b/pandora/clip/models.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from django.db import models from django.conf import settings -from django.utils.encoding import python_2_unicode_compatible import ox @@ -15,7 +13,7 @@ from . import managers def get_layers(item, interval=None, user=None): from annotation.models import Annotation - if user and user.is_anonymous(): + if user and user.is_anonymous: user = None layers = {} @@ -45,7 +43,6 @@ def get_layers(item, interval=None, user=None): return layers -@python_2_unicode_compatible class MetaClip(object): def update_calculated_values(self): start = self.start @@ -184,7 +181,7 @@ class MetaClip(object): @property def public_id(self): - return u"%s/%0.03f-%0.03f" % (self.item.public_id, float(self.start), float(self.end)) + return "%s/%0.03f-%0.03f" % (self.item.public_id, float(self.start), float(self.end)) def __str__(self): return self.public_id @@ -200,8 +197,8 @@ attrs = { 'modified': models.DateTimeField(auto_now=True), 'aspect_ratio': models.FloatField(default=0), - 'item': models.ForeignKey('item.Item', related_name='clips'), - 'sort': models.ForeignKey('item.ItemSort', related_name='matching_clips'), + 'item': models.ForeignKey('item.Item', related_name='clips', on_delete=models.CASCADE), + 'sort': models.ForeignKey('item.ItemSort', related_name='matching_clips', on_delete=models.CASCADE), 'user': models.IntegerField(db_index=True, null=True), #seconds @@ -226,4 +223,4 @@ Clip = type('Clip', (MetaClip, models.Model), attrs) class ClipRandom(models.Model): id = models.BigIntegerField(primary_key=True) - clip = models.OneToOneField(Clip) + clip = models.OneToOneField(Clip, on_delete=models.CASCADE) diff --git a/pandora/clip/views.py b/pandora/clip/views.py index 7c2fcd23..416e4a10 100644 --- a/pandora/clip/views.py +++ b/pandora/clip/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from django.conf import settings import ox @@ -154,7 +153,7 @@ def findClips(request, data): add_annotations(layer, aqs) elif 'position' in query: qs = order_query(qs, query['sort']) - ids = [u'%s/%0.03f-%0.03f' % (c['item__public_id'], c['start'], c['end']) + ids = ['%s/%0.03f-%0.03f' % (c['item__public_id'], c['start'], c['end']) for c in qs.values('item__public_id', 'start', 'end')] data['conditions'] = data['conditions'] + { 'value': data['position'], @@ -167,7 +166,7 @@ def findClips(request, data): response['data']['position'] = utils.get_positions(ids, [qs[0].public_id])[0] elif 'positions' in data: qs = order_query(qs, query['sort']) - ids = [u'%s/%0.03f-%0.03f' % (c['item__public_id'], c['start'], c['end']) + ids = ['%s/%0.03f-%0.03f' % (c['item__public_id'], c['start'], c['end']) for c in qs.values('item__public_id', 'start', 'end')] response['data']['positions'] = utils.get_positions(ids, data['positions']) else: diff --git a/pandora/document/management/commands/rebuild_documentfind.py b/pandora/document/management/commands/rebuild_documentfind.py index d374ffaf..af56b748 100644 --- a/pandora/document/management/commands/rebuild_documentfind.py +++ b/pandora/document/management/commands/rebuild_documentfind.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function from django.core.management.base import BaseCommand from django.db import connection, transaction diff --git a/pandora/document/management/commands/sync_documentsort.py b/pandora/document/management/commands/sync_documentsort.py index 0bdb9909..fa6cb5d8 100644 --- a/pandora/document/management/commands/sync_documentsort.py +++ b/pandora/document/management/commands/sync_documentsort.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function from django.core.management.base import BaseCommand from django.db import connection, transaction diff --git a/pandora/document/managers.py b/pandora/document/managers.py index c3b19f08..fb862f79 100644 --- a/pandora/document/managers.py +++ b/pandora/document/managers.py @@ -283,7 +283,7 @@ class DocumentManager(Manager): qs = qs.distinct() #anonymous can only see public items - if not user or user.is_anonymous(): + if not user or user.is_anonymous: level = 'guest' allowed_level = settings.CONFIG['capabilities']['canSeeDocument'][level] qs = qs.filter(rightslevel__lte=allowed_level) diff --git a/pandora/document/models.py b/pandora/document/models.py index f95f8d5b..a5a4f6b4 100644 --- a/pandora/document/models.py +++ b/pandora/document/models.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import os import re @@ -13,7 +12,6 @@ from django.db.models import Q, Sum, Max from django.contrib.auth import get_user_model from django.db.models.signals import pre_delete from django.conf import settings -from django.utils.encoding import python_2_unicode_compatible from oxdjango.fields import JSONField from PIL import Image @@ -42,13 +40,12 @@ if not PY2: def get_path(f, x): return f.path(x) -@python_2_unicode_compatible class Document(models.Model, FulltextMixin): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) - user = models.ForeignKey(User, related_name='documents') + user = models.ForeignKey(User, related_name='documents', on_delete=models.CASCADE) groups = models.ManyToManyField(Group, blank=True, related_name='documents') extension = models.CharField(max_length=255) @@ -74,7 +71,7 @@ class Document(models.Model, FulltextMixin): data = JSONField(default=dict, editable=False) def update_access(self, user): - if not user.is_authenticated(): + if not user.is_authenticated: user = None access, created = Access.objects.get_or_create(document=self, user=user) if not created: @@ -161,7 +158,7 @@ class Document(models.Model, FulltextMixin): elif i not in ('*', 'dimensions') and i not in self.facet_keys: value = data.get(i) if isinstance(value, list): - value = u'\n'.join(value) + value = '\n'.join(value) save(i, value) base_keys = ('id', 'size', 'dimensions', 'extension', 'matches') @@ -192,11 +189,11 @@ class Document(models.Model, FulltextMixin): s.dimensions = ox.sort_string('%d' % prefix) + ox.sort_string('%d' % value) def sortNames(values): - sort_value = u'' + sort_value = '' if values: - sort_value = u'; '.join([get_name_sort(name) for name in values]) + sort_value = '; '.join([get_name_sort(name) for name in values]) if not sort_value: - sort_value = u'' + sort_value = '' return sort_value.lower() def set_value(s, name, value): @@ -229,7 +226,7 @@ class Document(models.Model, FulltextMixin): if isinstance(sort_type, list): sort_type = sort_type[0] if sort_type == 'title': - value = self.get_value(source, u'Untitled') + value = self.get_value(source, 'Untitled') value = utils.sort_title(value)[:955] set_value(s, name, value) elif sort_type == 'person': @@ -237,9 +234,9 @@ class Document(models.Model, FulltextMixin): value = utils.sort_string(value)[:955] set_value(s, name, value) elif sort_type == 'string': - value = self.get_value(source, u'') + value = self.get_value(source, '') if isinstance(value, list): - value = u','.join(value) + value = ','.join(value) if not isinstance(value, str): value = str(value) value = utils.sort_string(value)[:955] @@ -319,7 +316,7 @@ class Document(models.Model, FulltextMixin): return ox.toAZ(self.id) def access(self, user): - if user.is_anonymous(): + if user.is_anonymous: level = 'guest' else: level = user.profile.get_level() @@ -332,7 +329,7 @@ class Document(models.Model, FulltextMixin): return False def editable(self, user, item=None): - if not user or user.is_anonymous(): + if not user or user.is_anonymous: return False if self.user == user or \ self.groups.filter(id__in=user.groups.all()).count() > 0 or \ @@ -693,8 +690,8 @@ class ItemProperties(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) - item = models.ForeignKey(Item) - document = models.ForeignKey(Document, related_name='descriptions') + item = models.ForeignKey(Item, on_delete=models.CASCADE) + document = models.ForeignKey(Document, related_name='descriptions', on_delete=models.CASCADE) description = models.TextField(default="") index = models.IntegerField(default=0) @@ -709,14 +706,13 @@ class ItemProperties(models.Model): super(ItemProperties, self).save(*args, **kwargs) -@python_2_unicode_compatible class Access(models.Model): class Meta: unique_together = ("document", "user") access = models.DateTimeField(auto_now=True) - document = models.ForeignKey(Document, related_name='accessed') - user = models.ForeignKey(User, null=True, related_name='accessed_documents') + document = models.ForeignKey(Document, related_name='accessed', on_delete=models.CASCADE) + user = models.ForeignKey(User, null=True, related_name='accessed_documents', on_delete=models.CASCADE) accessed = models.IntegerField(default=0) def save(self, *args, **kwargs): @@ -729,10 +725,9 @@ class Access(models.Model): def __str__(self): if self.user: - return u"%s/%s/%s" % (self.user, self.document, self.access) - return u"%s/%s" % (self.item, self.access) + return "%s/%s/%s" % (self.user, self.document, self.access) + return "%s/%s" % (self.item, self.access) -@python_2_unicode_compatible class Facet(models.Model): ''' used for keys that can have multiple values like people, languages etc. @@ -743,13 +738,13 @@ class Facet(models.Model): class Meta: unique_together = ("document", "key", "value") - document = models.ForeignKey('Document', related_name='facets') + document = models.ForeignKey('Document', related_name='facets', on_delete=models.CASCADE) key = models.CharField(max_length=200, db_index=True) value = models.CharField(max_length=1000, db_index=True) sortvalue = models.CharField(max_length=1000, db_index=True) def __str__(self): - return u"%s=%s" % (self.key, self.value) + return "%s=%s" % (self.key, self.value) def save(self, *args, **kwargs): if not self.sortvalue: @@ -768,18 +763,17 @@ for key in settings.CONFIG['itemKeys']: if key.get('sortType') == 'person': Document.person_keys.append(key['id']) -@python_2_unicode_compatible class Find(models.Model): class Meta: unique_together = ('document', 'key') - document = models.ForeignKey('Document', related_name='find', db_index=True) + document = models.ForeignKey('Document', related_name='find', db_index=True, on_delete=models.CASCADE) key = models.CharField(max_length=200, db_index=True) value = models.TextField(blank=True, db_index=settings.DB_GIN_TRGM) def __str__(self): - return u'%s=%s' % (self.key, self.value) + return '%s=%s' % (self.key, self.value) ''' Sort @@ -787,7 +781,7 @@ table constructed based on info in settings.CONFIG['documentKeys'] ''' attrs = { '__module__': 'document.models', - 'document': models.OneToOneField('Document', related_name='sort', primary_key=True), + 'document': models.OneToOneField('Document', related_name='sort', primary_key=True, on_delete=models.CASCADE), 'created': models.DateTimeField(null=True, blank=True, db_index=True), } for key in list(filter(lambda k: k.get('sort', False) or k['type'] in ('integer', 'time', 'float', 'date', 'enum'), settings.CONFIG['documentKeys'])): diff --git a/pandora/document/sync_sort.py b/pandora/document/sync_sort.py index 4fe44881..4a0dff2d 100644 --- a/pandora/document/sync_sort.py +++ b/pandora/document/sync_sort.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function from django.core.management.base import BaseCommand from django.db import connection, transaction diff --git a/pandora/document/views.py b/pandora/document/views.py index 2cc080e6..0798913a 100644 --- a/pandora/document/views.py +++ b/pandora/document/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import os import re diff --git a/pandora/documentcollection/managers.py b/pandora/documentcollection/managers.py index 30ead9f0..45d5cbaa 100644 --- a/pandora/documentcollection/managers.py +++ b/pandora/documentcollection/managers.py @@ -124,7 +124,7 @@ class CollectionManager(Manager): if conditions: qs = qs.filter(conditions) - if user.is_anonymous(): + if user.is_anonymous: qs = qs.filter(Q(status='public') | Q(status='featured')) else: qs = qs.filter(Q(status='public') | Q(status='featured') | Q(user=user)) diff --git a/pandora/documentcollection/models.py b/pandora/documentcollection/models.py index e0295c47..8504ae47 100644 --- a/pandora/documentcollection/models.py +++ b/pandora/documentcollection/models.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import os import re @@ -10,7 +9,6 @@ from django.db import models from django.db.models import Max from django.contrib.auth import get_user_model from django.conf import settings -from django.utils.encoding import python_2_unicode_compatible from oxdjango.fields import JSONField import ox @@ -36,7 +34,6 @@ def get_collectionview(): def get_collectionsort(): return tuple(settings.CONFIG['user']['ui']['collectionSort']) -@python_2_unicode_compatible class Collection(models.Model): class Meta: @@ -44,7 +41,7 @@ class Collection(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) - user = models.ForeignKey(User, related_name='collections') + user = models.ForeignKey(User, related_name='collections', on_delete=models.CASCADE) groups = models.ManyToManyField(Group, blank=True, related_name='collections') name = models.CharField(max_length=255) status = models.CharField(max_length=20, default='private') @@ -58,7 +55,7 @@ class Collection(models.Model): view = models.TextField(default=get_collectionview) sort = JSONField(default=get_collectionsort, editable=False) - poster_frames = JSONField(default=[], editable=False) + poster_frames = JSONField(default=list, editable=False) #is through table still required? documents = models.ManyToManyField('document.Document', related_name='collections', @@ -117,13 +114,13 @@ class Collection(models.Model): return self.get_id() def get_id(self): - return u'%s:%s' % (self.user.username, self.name) + return '%s:%s' % (self.user.username, self.name) def accessible(self, user): return self.user == user or self.status in ('public', 'featured') def editable(self, user): - if not user or user.is_anonymous(): + if not user or user.is_anonymous: return False if self.user == user or \ self.groups.filter(id__in=user.groups.all()).count() > 0 or \ @@ -251,7 +248,7 @@ class Collection(models.Model): elif key == 'subscribers': response[key] = self.subscribed_users.all().count() elif key == 'subscribed': - if user and not user.is_anonymous(): + if user and not user.is_anonymous: response[key] = self.subscribed_users.filter(id=user.id).exists() else: response[key] = getattr(self, { @@ -318,28 +315,26 @@ class Collection(models.Model): path = source return path -@python_2_unicode_compatible class CollectionDocument(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) - collection = models.ForeignKey(Collection) + collection = models.ForeignKey(Collection, on_delete=models.CASCADE) index = models.IntegerField(default=0) - document = models.ForeignKey('document.Document') + document = models.ForeignKey('document.Document', on_delete=models.CASCADE) def __str__(self): - return u'%s in %s' % (self.document, self.collection) + return '%s in %s' % (self.document, self.collection) -@python_2_unicode_compatible class Position(models.Model): class Meta: unique_together = ("user", "collection", "section") - collection = models.ForeignKey(Collection, related_name='position') - user = models.ForeignKey(User, related_name='collection_positions') + collection = models.ForeignKey(Collection, related_name='position', on_delete=models.CASCADE) + user = models.ForeignKey(User, related_name='collection_positions', on_delete=models.CASCADE) section = models.CharField(max_length=255) position = models.IntegerField(default=0) def __str__(self): - return u'%s/%s/%s' % (self.section, self.position, self.collection) + return '%s/%s/%s' % (self.section, self.position, self.collection) diff --git a/pandora/documentcollection/views.py b/pandora/documentcollection/views.py index 170168de..66fa746d 100644 --- a/pandora/documentcollection/views.py +++ b/pandora/documentcollection/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import os import re @@ -75,7 +74,7 @@ def findCollections(request, data): query = parse_query(data, request.user) #order - is_section_request = query['sort'] == [{u'operator': u'+', u'key': u'position'}] + is_section_request = query['sort'] == [{'operator': '+', 'key': 'position'}] def is_featured_condition(x): return x['key'] == 'status' and \ @@ -89,7 +88,7 @@ def findCollections(request, data): if is_section_request: qs = query['qs'] - if not is_featured and not request.user.is_anonymous(): + if not is_featured and not request.user.is_anonymous: qs = qs.filter(position__in=models.Position.objects.filter(user=request.user)) qs = qs.order_by('position__position') else: diff --git a/pandora/edit/managers.py b/pandora/edit/managers.py index 1bfaea00..37909ad4 100644 --- a/pandora/edit/managers.py +++ b/pandora/edit/managers.py @@ -124,7 +124,7 @@ class EditManager(Manager): if conditions: qs = qs.filter(conditions) - if user.is_anonymous(): + if user.is_anonymous: qs = qs.filter(Q(status='public') | Q(status='featured')) else: qs = qs.filter(Q(status='public') | Q(status='featured') | Q(user=user)) diff --git a/pandora/edit/models.py b/pandora/edit/models.py index 4ce6ab29..9d22247d 100644 --- a/pandora/edit/models.py +++ b/pandora/edit/models.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import re import os @@ -15,7 +14,6 @@ from django.db import models, transaction from django.db.models import Max from django.contrib.auth import get_user_model -from django.utils.encoding import python_2_unicode_compatible from oxdjango.fields import JSONField from annotation.models import Annotation @@ -35,7 +33,6 @@ User = get_user_model() def get_path(f, x): return f.path(x) def get_icon_path(f, x): return get_path(f, 'icon.jpg') -@python_2_unicode_compatible class Edit(models.Model): class Meta: @@ -45,7 +42,7 @@ class Edit(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) - user = models.ForeignKey(User, related_name='edits') + user = models.ForeignKey(User, related_name='edits', on_delete=models.CASCADE) groups = models.ManyToManyField(Group, blank=True, related_name='edits') name = models.CharField(max_length=255) @@ -59,11 +56,11 @@ class Edit(models.Model): icon = models.ImageField(default=None, blank=True, null=True, upload_to=get_icon_path) - poster_frames = JSONField(default=[], editable=False) + poster_frames = JSONField(default=list, editable=False) subscribed_users = models.ManyToManyField(User, related_name='subscribed_edits') def __str__(self): - return u'%s (%s)' % (self.name, self.user) + return '%s (%s)' % (self.name, self.user) @classmethod def get(cls, id): @@ -73,7 +70,7 @@ class Edit(models.Model): return cls.objects.get(user__username=username, name=name) def get_id(self): - return u'%s:%s' % (self.user.username, self.name) + return '%s:%s' % (self.user.username, self.name) def get_absolute_url(self): return ('/edits/%s' % quote(self.get_id())).replace('%3A', ':') @@ -137,7 +134,7 @@ class Edit(models.Model): return self.user == user or self.status in ('public', 'featured') def editable(self, user): - if not user or user.is_anonymous(): + if not user or user.is_anonymous: return False if self.user == user or \ self.groups.filter(id__in=user.groups.all()).count() > 0 or \ @@ -403,7 +400,7 @@ class Edit(models.Model): elif key == 'subscribers': response[key] = self.subscribed_users.all().count() elif key == 'subscribed': - if user and not user.is_anonymous(): + if user and not user.is_anonymous: response[key] = self.subscribed_users.filter(id=user.id).exists() elif hasattr(self, _map.get(key, key)): response[key] = getattr(self, _map.get(key, key)) @@ -430,15 +427,14 @@ class Edit(models.Model): #p.wait() shutil.rmtree(tmp) -@python_2_unicode_compatible class Clip(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) - edit = models.ForeignKey(Edit, related_name='clips') + edit = models.ForeignKey(Edit, related_name='clips', on_delete=models.CASCADE) index = models.IntegerField(default=0) - item = models.ForeignKey(Item, null=True, default=None, related_name='editclip') - annotation = models.ForeignKey(Annotation, null=True, default=None, related_name='editclip') + item = models.ForeignKey(Item, null=True, default=None, related_name='editclip', on_delete=models.CASCADE) + annotation = models.ForeignKey(Annotation, null=True, default=None, related_name='editclip', on_delete=models.CASCADE) start = models.FloatField(default=0) end = models.FloatField(default=0) duration = models.FloatField(default=0) @@ -454,8 +450,8 @@ class Clip(models.Model): def __str__(self): if self.annotation: - return u'%s' % self.annotation.public_id - return u'%s/%0.3f-%0.3f' % (self.item.public_id, self.start, self.end) + return '%s' % self.annotation.public_id + return '%s/%0.3f-%0.3f' % (self.item.public_id, self.start, self.end) def get_id(self): return ox.toAZ(self.id) @@ -541,17 +537,16 @@ class Clip(models.Model): return clip.models.get_layers(item=item, interval=(start, end), user=user) -@python_2_unicode_compatible class Position(models.Model): class Meta: unique_together = ("user", "edit", "section") - edit = models.ForeignKey(Edit, related_name='position') - user = models.ForeignKey(User, related_name='edit_position') + edit = models.ForeignKey(Edit, related_name='position', on_delete=models.CASCADE) + user = models.ForeignKey(User, related_name='edit_position', on_delete=models.CASCADE) section = models.CharField(max_length=255) position = models.IntegerField(default=0) def __str__(self): - return u'%s/%s/%s' % (self.section, self.position, self.edit) + return '%s/%s/%s' % (self.section, self.position, self.edit) diff --git a/pandora/edit/views.py b/pandora/edit/views.py index 57abed30..09261310 100644 --- a/pandora/edit/views.py +++ b/pandora/edit/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import os import re @@ -404,7 +403,8 @@ def findEdits(request, data): query = parse_query(data, request.user) #order - is_section_request = query['sort'] == [{u'operator': u'+', u'key': u'position'}] + is_section_request = query['sort'] == [{'operator': '+', 'key': 'position'}] + def is_featured_condition(x): return x['key'] == 'status' and \ x['value'] == 'featured' and \ @@ -414,7 +414,7 @@ def findEdits(request, data): if is_section_request: qs = query['qs'] - if not is_featured and not request.user.is_anonymous(): + if not is_featured and not request.user.is_anonymous: qs = qs.filter(position__in=models.Position.objects.filter(user=request.user)) qs = qs.order_by('position__position') else: diff --git a/pandora/entity/models.py b/pandora/entity/models.py index bb58cc53..c39baccb 100644 --- a/pandora/entity/models.py +++ b/pandora/entity/models.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import os import re @@ -14,7 +13,6 @@ from django.contrib.auth import get_user_model from django.db.models.signals import pre_delete, post_init from django.conf import settings -from django.utils.encoding import python_2_unicode_compatible from oxdjango.fields import JSONField import ox @@ -28,7 +26,6 @@ from . import managers User = get_user_model() -@python_2_unicode_compatible class Entity(models.Model): class ValueError(ValueError): '''Raised if a field name or value is invalid (based on the "entities" @@ -38,7 +35,7 @@ class Entity(models.Model): class Meta: unique_together = ("type", "name") - user = models.ForeignKey(User, related_name='entities', null=True, default=None) + user = models.ForeignKey(User, related_name='entities', null=True, default=None, on_delete=models.CASCADE) created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) @@ -64,7 +61,7 @@ class Entity(models.Model): self.name = self.name.decode('utf-8') self.name_sort = get_name_sort(self.name)[:255].lower() else: - self.name_sort = ox.sort_string(self.name or u'')[:255].lower() or None + self.name_sort = ox.sort_string(self.name or '')[:255].lower() or None self.name_find = '||' + '||'.join((self.name,) + self.alternativeNames) + '||' self.name_find = self.name_find.lower() super(Entity, self).save(*args, **kwargs) @@ -88,11 +85,11 @@ class Entity(models.Model): @classmethod def get_by_name(cls, name, type): - return cls.objects.get(name_find__contains=u'|%s|' % name.lower(), type=type) + return cls.objects.get(name_find__contains='|%s|' % name.lower(), type=type) @classmethod def get_or_create(model, name): - qs = model.objects.filter(name_find__contains=u'|%s|' % name.lower()) + qs = model.objects.filter(name_find__contains='|%s|' % name.lower()) if qs.count() == 0: instance = model(name=name) instance.save() @@ -117,7 +114,7 @@ class Entity(models.Model): DocumentProperties.objects.filter(document=document, entity=self).delete() def editable(self, user, item=None): - if not user or user.is_anonymous(): + if not user or user.is_anonymous: return False if user.is_staff or \ user.profile.capability('canEditEntities') == True or \ @@ -140,7 +137,7 @@ class Entity(models.Model): data['name'] = "Unnamed" name = data['name'] n = 1 - while Entity.objects.filter(name_find__contains=u'|%s|' % name.lower()).exclude(id=self.id).count() > 0: + while Entity.objects.filter(name_find__contains='|%s|' % name.lower()).exclude(id=self.id).count() > 0: n += 1 name = data['name'] + ' [%d]' % n self.name = name @@ -155,7 +152,7 @@ class Entity(models.Model): name_ = name n = 1 while name in used_names or \ - Entity.objects.filter(name_find__contains=u'|%s|' % name.lower()).exclude(id=self.id).count() > 0: + Entity.objects.filter(name_find__contains='|%s|' % name.lower()).exclude(id=self.id).count() > 0: n += 1 name = name_ + ' [%d]' % n names.append(name) @@ -268,7 +265,7 @@ class Entity(models.Model): return response def annotation_value(self): - #return u'%s' % (self.get_id(), ox.escape_html(self.name)) + #return '%s' % (self.get_id(), ox.escape_html(self.name)) return ox.escape_html(self.name) def update_find(self): @@ -292,10 +289,10 @@ class Entity(models.Model): for key in entity['keys']: value = self.data.get(key['id']) if isinstance(value, list): - value = u'\n'.join(value) + value = '\n'.join(value) save(key['id'], value) ids.append(key['id']) - save('name', u'\n'.join([self.name] + list(self.alternativeNames))) + save('name', '\n'.join([self.name] + list(self.alternativeNames))) self.find.exclude(key__in=ids).delete() def update_matches(self): @@ -344,7 +341,6 @@ post_init.connect( ) -@python_2_unicode_compatible class DocumentProperties(models.Model): class Meta: @@ -353,42 +349,40 @@ class DocumentProperties(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) - document = models.ForeignKey(Document, related_name='documentproperties') - entity = models.ForeignKey(Entity, related_name='documentproperties') + document = models.ForeignKey(Document, related_name='documentproperties', on_delete=models.CASCADE) + entity = models.ForeignKey(Entity, related_name='documentproperties', on_delete=models.CASCADE) index = models.IntegerField(default=0) data = JSONField(default=dict, editable=False) def __str__(self): - return u"%r-%r" % (self.document, self.entity) + return "%r-%r" % (self.document, self.entity) def save(self, *args, **kwargs): super(DocumentProperties, self).save(*args, **kwargs) -@python_2_unicode_compatible class Find(models.Model): class Meta: unique_together = ("entity", "key") - entity = models.ForeignKey('Entity', related_name='find', db_index=True) + entity = models.ForeignKey('Entity', related_name='find', db_index=True, on_delete=models.CASCADE) key = models.CharField(max_length=200, db_index=True) value = models.TextField(blank=True, db_index=settings.DB_GIN_TRGM) def __str__(self): - return u"%s=%s" % (self.key, self.value) + return "%s=%s" % (self.key, self.value) -@python_2_unicode_compatible class Link(models.Model): '''Models entity fields of type "entity".''' class Meta: unique_together = ("source", "key", "target") - source = models.ForeignKey(Entity, related_name='links') + source = models.ForeignKey(Entity, related_name='links', on_delete=models.CASCADE) key = models.CharField(max_length=200) - target = models.ForeignKey(Entity, related_name='backlinks') + target = models.ForeignKey(Entity, related_name='backlinks', on_delete=models.CASCADE) def __str__(self): - return u"%s-[%s]->%s" % (self.source, self.key, self.target) + return "%s-[%s]->%s" % (self.source, self.key, self.target) diff --git a/pandora/entity/views.py b/pandora/entity/views.py index a42ec38d..483bf1a9 100644 --- a/pandora/entity/views.py +++ b/pandora/entity/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from six import string_types import ox @@ -60,7 +59,7 @@ def addEntity(request, data): for name in names: name = ox.decode_html(name) if models.Entity.objects.filter(type=data['type'], - name_find__icontains=u'|%s|'%name).count() != 0: + name_find__icontains='|%s|'%name).count() != 0: exists = True existing_names.append(name) if not exists: @@ -87,7 +86,7 @@ def addEntity(request, data): type = data['type'] name = 'Unnamed' num = 1 - while models.Entity.objects.filter(name_find__icontains=u'|%s|'%name).count() > 0: + while models.Entity.objects.filter(name_find__icontains='|%s|'%name).count() > 0: num += 1 name = 'Unnamed [%d]' % num entity = models.Entity(name=name, type=type) diff --git a/pandora/event/admin.py b/pandora/event/admin.py index 51597291..adb737ea 100644 --- a/pandora/event/admin.py +++ b/pandora/event/admin.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from django.contrib import admin diff --git a/pandora/event/models.py b/pandora/event/models.py index e66c86fe..f35c599a 100644 --- a/pandora/event/models.py +++ b/pandora/event/models.py @@ -1,10 +1,8 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from django.db import models, transaction from django.contrib.auth import get_user_model -from django.utils.encoding import python_2_unicode_compatible import ox from oxdjango import fields @@ -19,7 +17,6 @@ from . import managers User = get_user_model() -@python_2_unicode_compatible class Event(models.Model): ''' Events are events in time that can be once or recurring, @@ -33,7 +30,7 @@ class Event(models.Model): modified = models.DateTimeField(auto_now=True) defined = models.BooleanField(default=False) - user = models.ForeignKey(User, null=True, related_name='events') + user = models.ForeignKey(User, null=True, related_name='events', on_delete=models.CASCADE) name = models.CharField(null=True, max_length=255, unique=True) name_sort = models.CharField(null=True, max_length=255, db_index=True) @@ -66,7 +63,7 @@ class Event(models.Model): @classmethod def get_or_create(model, name): - qs = model.objects.filter(name_find__contains=u'|%s|' % name.lower()) + qs = model.objects.filter(name_find__contains='|%s|' % name.lower()) if qs.count() == 0: instance = model(name=name) instance.save() @@ -75,7 +72,7 @@ class Event(models.Model): return instance def editable(self, user): - if user and not user.is_anonymous() \ + if user and not user.is_anonymous \ and (not self.user or \ self.user == user or \ user.profile.capability('canEditEvents')): diff --git a/pandora/event/tasks.py b/pandora/event/tasks.py index 85f30dff..234dd5a7 100644 --- a/pandora/event/tasks.py +++ b/pandora/event/tasks.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from celery.task import task diff --git a/pandora/event/views.py b/pandora/event/views.py index 47744fa5..d4d63e6d 100644 --- a/pandora/event/views.py +++ b/pandora/event/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from django.db.models import Count from django.conf import settings @@ -37,7 +36,7 @@ def addEvent(request, data): for name in names: name = ox.decode_html(name) if models.Event.objects.filter(defined=True, - name_find__icontains=u'|%s|'%name).count() != 0: + name_find__icontains='|%s|'%name).count() != 0: exists = True existing_names.append(name) if not exists: @@ -93,7 +92,7 @@ def editEvent(request, data): names = [data.get('name', event.name)] + data.get('alternativeNames', []) for name in names: if models.Event.objects.filter(defined=True, - name_find__icontains=u'|%s|'%name).exclude(id=event.id).count() != 0: + name_find__icontains='|%s|'%name).exclude(id=event.id).count() != 0: conflict = True conflict_names.append(name) if not conflict: diff --git a/pandora/home/models.py b/pandora/home/models.py index 775e3714..49b08f73 100644 --- a/pandora/home/models.py +++ b/pandora/home/models.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from six import string_types from six.moves.urllib.parse import quote @@ -7,7 +6,6 @@ from six.moves.urllib.parse import quote from django.db import models from django.db.models import Max from django.db.models.signals import pre_delete -from django.utils.encoding import python_2_unicode_compatible from oxdjango.fields import JSONField import ox @@ -17,7 +15,6 @@ from edit.models import Edit from documentcollection.models import Collection -@python_2_unicode_compatible class Item(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) @@ -27,7 +24,7 @@ class Item(models.Model): data = JSONField(default=dict, editable=False) def editable(self, user): - return user.is_authenticated() and user.profile.capability("canManageHome") + return user.is_authenticated and user.profile.capability("canManageHome") def edit(self, data): changed = False @@ -153,7 +150,7 @@ class Item(models.Model): return j def __str__(self): - return u"%s" % (self.get_id()) + return "%s" % (self.get_id()) def delete_item(type, contentid): for home in Item.objects.all(): diff --git a/pandora/item/admin.py b/pandora/item/admin.py index 5cfebaf2..a94cac7f 100644 --- a/pandora/item/admin.py +++ b/pandora/item/admin.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from django.contrib import admin diff --git a/pandora/item/management/commands/get_frame.py b/pandora/item/management/commands/get_frame.py index 69e78e4e..1aaba624 100644 --- a/pandora/item/management/commands/get_frame.py +++ b/pandora/item/management/commands/get_frame.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function from django.core.management.base import BaseCommand from django.conf import settings diff --git a/pandora/item/management/commands/rebuild_filter.py b/pandora/item/management/commands/rebuild_filter.py index 2ff7e2df..eb52b13b 100644 --- a/pandora/item/management/commands/rebuild_filter.py +++ b/pandora/item/management/commands/rebuild_filter.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function from django.core.management.base import BaseCommand diff --git a/pandora/item/management/commands/rebuild_find.py b/pandora/item/management/commands/rebuild_find.py index e74e1afe..8418fc2f 100644 --- a/pandora/item/management/commands/rebuild_find.py +++ b/pandora/item/management/commands/rebuild_find.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function from django.core.management.base import BaseCommand from django.db import connection, transaction diff --git a/pandora/item/management/commands/rebuild_indexes.py b/pandora/item/management/commands/rebuild_indexes.py index be885e07..5ad61971 100644 --- a/pandora/item/management/commands/rebuild_indexes.py +++ b/pandora/item/management/commands/rebuild_indexes.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function from django.core.management.base import BaseCommand from django.db import connection, transaction diff --git a/pandora/item/management/commands/rebuild_sort.py b/pandora/item/management/commands/rebuild_sort.py index 71a69986..5e29ade8 100644 --- a/pandora/item/management/commands/rebuild_sort.py +++ b/pandora/item/management/commands/rebuild_sort.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function from django.core.management.base import BaseCommand diff --git a/pandora/item/management/commands/rebuild_timelines.py b/pandora/item/management/commands/rebuild_timelines.py index d7c03355..f45559c4 100644 --- a/pandora/item/management/commands/rebuild_timelines.py +++ b/pandora/item/management/commands/rebuild_timelines.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function import os from glob import glob diff --git a/pandora/item/management/commands/rebuildcache.py b/pandora/item/management/commands/rebuildcache.py index a15c67d5..50c3f20e 100644 --- a/pandora/item/management/commands/rebuildcache.py +++ b/pandora/item/management/commands/rebuildcache.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function import time diff --git a/pandora/item/management/commands/sqlfindindex.py b/pandora/item/management/commands/sqlfindindex.py index 3ecba6a8..42031e67 100644 --- a/pandora/item/management/commands/sqlfindindex.py +++ b/pandora/item/management/commands/sqlfindindex.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function from django.core.management.base import BaseCommand @@ -44,7 +43,13 @@ class Command(BaseCommand): (document.models.Find._meta.db_table, 'value'), # Document Find ): cursor = connection.cursor() - indexes = connection.introspection.get_indexes(cursor, table) + contraints = connection.introspection.get_constraints(cursor, table) + indexes = { + ','.join(c['columns']): {'primary_key': c['primary_key'], 'unique': c['unique']} + for k, c in contraints.items() + if c['index'] or c['primary_key'] or c['unique'] + } + #indexes = connection.introspection.get_indexes(cursor, table) drop = [] if column in indexes: for sql in ( diff --git a/pandora/item/management/commands/sync_itemsort.py b/pandora/item/management/commands/sync_itemsort.py index 8af5e14a..4640229d 100644 --- a/pandora/item/management/commands/sync_itemsort.py +++ b/pandora/item/management/commands/sync_itemsort.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function from django.core.management.base import BaseCommand from django.db import connection, transaction diff --git a/pandora/item/management/commands/update_external.py b/pandora/item/management/commands/update_external.py index 4dbd1cbd..49f3d842 100644 --- a/pandora/item/management/commands/update_external.py +++ b/pandora/item/management/commands/update_external.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function from django.core.management.base import BaseCommand diff --git a/pandora/item/management/commands/update_itemsfolder.py b/pandora/item/management/commands/update_itemsfolder.py index 38f0ea8d..95862428 100644 --- a/pandora/item/management/commands/update_itemsfolder.py +++ b/pandora/item/management/commands/update_itemsfolder.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function import os @@ -38,10 +37,10 @@ class Command(BaseCommand): versions = os.path.join(prefix, os.path.dirname(path), 'Versions') s = f.streams.filter(source=None)[0] basename = os.path.basename(path).rsplit('.', 1)[0] - target = os.path.join(versions, u'%s.%s' % (basename, s.format)) + target = os.path.join(versions, '%s.%s' % (basename, s.format)) link(s.media.path, target) else: s = f.streams.filter(source=None)[0] basename = path.rsplit('.', 1)[0] - target = os.path.join(prefix, u'%s.%s' % (basename, s.format)) + target = os.path.join(prefix, '%s.%s' % (basename, s.format)) link(s.media.path, target) diff --git a/pandora/item/managers.py b/pandora/item/managers.py index be409e5f..bcb1d8cc 100644 --- a/pandora/item/managers.py +++ b/pandora/item/managers.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from datetime import datetime import unicodedata @@ -81,7 +80,7 @@ def parseCondition(condition, user, owner=None): if (not exclude and op == '=' or op in ('$', '^')) and v == '': return Q() - elif k == 'filename' and (user.is_anonymous() or not user.profile.capability('canSeeMedia')): + elif k == 'filename' and (user.is_anonymous or not user.profile.capability('canSeeMedia')): return Q(id=0) elif k == 'oshash': return Q(files__oshash=v) @@ -100,7 +99,7 @@ def parseCondition(condition, user, owner=None): q = ~q return q elif k in ('canplayvideo', 'canplayclips'): - level = user.is_anonymous() and 'guest' or user.profile.get_level() + level = user.is_anonymous and 'guest' or user.profile.get_level() allowed_level = settings.CONFIG['capabilities'][{ 'canplayvideo': 'canPlayVideo', 'canplayclips': 'canPlayClips' @@ -249,7 +248,7 @@ class ItemManager(Manager): if l != "*": l = l.split(":") only_public = True - if not user.is_anonymous(): + if not user.is_anonymous: if len(l) == 1: l = [user.username] + l if user.username == l[0]: @@ -305,7 +304,7 @@ class ItemManager(Manager): qs = qs.distinct() #anonymous can only see public items - if not user or user.is_anonymous(): + if not user or user.is_anonymous: level = 'guest' allowed_level = settings.CONFIG['capabilities']['canSeeItem'][level] qs = qs.filter(level__lte=allowed_level) diff --git a/pandora/item/models.py b/pandora/item/models.py index b9dd78ee..0519584d 100644 --- a/pandora/item/models.py +++ b/pandora/item/models.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import json import os @@ -21,7 +20,6 @@ from django.core.files.temp import NamedTemporaryFile from django.db import models, transaction, connection from django.db.models import Q, Sum, Max from django.db.models.signals import pre_delete -from django.utils.encoding import python_2_unicode_compatible from django.utils import datetime_safe import ox @@ -164,12 +162,11 @@ def get_poster_path(f, x): def get_torrent_path(f, x): return get_path(f, 'torrent.torrent') -@python_2_unicode_compatible class Item(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) - user = models.ForeignKey(User, null=True, related_name='items') + user = models.ForeignKey(User, null=True, related_name='items', on_delete=models.CASCADE) groups = models.ManyToManyField(Group, blank=True, related_name='items') # while metadata is updated, files are set to rendered=False @@ -221,7 +218,7 @@ class Item(models.Model): return default def access(self, user): - if user.is_anonymous(): + if user.is_anonymous: level = 'guest' else: level = user.profile.get_level() @@ -236,7 +233,7 @@ class Item(models.Model): return False def editable(self, user): - if user.is_anonymous(): + if user.is_anonymous: return False if user.profile.capability('canEditMetadata') or \ user.is_staff or \ @@ -350,10 +347,10 @@ class Item(models.Model): def __str__(self): year = self.get('year') if year: - string = u'%s (%s)' % (ox.decode_html(self.get('title', 'Untitled')), self.get('year')) + string = '%s (%s)' % (ox.decode_html(self.get('title', 'Untitled')), self.get('year')) else: - string = self.get('title', u'Untitled') - return u'[%s] %s' % (self.public_id, string) + string = self.get('title', 'Untitled') + return '[%s] %s' % (self.public_id, string) def get_absolute_url(self): return '/%s' % self.public_id @@ -400,7 +397,7 @@ class Item(models.Model): title = self.get(key, 'Untitled') while q.count() != 0: n += 1 - self.data[key] = u'%s [%d]' % (title, n) + self.data[key] = '%s [%d]' % (title, n) oxdbId = self.oxdb_id() q = Item.objects.filter(oxdbId=oxdbId).exclude(id=self.id) self.oxdbId = oxdbId @@ -821,7 +818,7 @@ class Item(models.Model): for key in settings.CONFIG['itemKeys']: i = key['id'] if i == 'title': - save(i, u'\n'.join(get_titles())) + save(i, '\n'.join(get_titles())) elif i == 'rightslevel': save(i, self.level) elif i == 'filename': @@ -830,17 +827,17 @@ class Item(models.Model): qs = Annotation.objects.filter(item=self) qs = qs.filter(layer__in=Annotation.public_layers()).exclude(findvalue=None) qs = qs.order_by('start') - save(i, u'\n'.join([l.findvalue for l in qs])) + save(i, '\n'.join([l.findvalue for l in qs])) elif key['type'] == 'layer': qs = Annotation.objects.filter(item=self).exclude(findvalue=None) qs = qs.filter(layer=i) qs = qs.order_by('start') - save(i, u'\n'.join(list(filter(None, [l.findvalue for l in qs])))) + save(i, '\n'.join(list(filter(None, [l.findvalue for l in qs])))) layer_keys.append(i) elif i != '*' and i not in self.facet_keys: value = self.get(i) if isinstance(value, list): - value = u'\n'.join(value) + value = '\n'.join(value) save(i, value) for key in self.facet_keys: @@ -911,11 +908,11 @@ class Item(models.Model): s = ItemSort(item=self) def sortNames(values): - sort_value = u'' + sort_value = '' if values: - sort_value = u'; '.join([get_name_sort(name) for name in values]) + sort_value = '; '.join([get_name_sort(name) for name in values]) if not sort_value: - sort_value = u'' + sort_value = '' return sort_value.lower() def set_value(s, name, value): @@ -1019,7 +1016,7 @@ class Item(models.Model): sort_type = sort_type[0] if name not in self.base_keys: if sort_type == 'title': - value = get_title_sort(self.get(source, u'Untitled')) + value = get_title_sort(self.get(source, 'Untitled')) value = utils.sort_title(value)[:955] set_value(s, name, value) elif sort_type == 'person': @@ -1027,9 +1024,9 @@ class Item(models.Model): value = utils.sort_string(value)[:955] set_value(s, name, value) elif sort_type == 'string': - value = self.get(source, u'') + value = self.get(source, '') if isinstance(value, list): - value = u','.join(value) + value = ','.join(value) value = utils.sort_string(value)[:955] set_value(s, name, value) elif sort_type == 'words': @@ -1784,7 +1781,6 @@ for key in settings.CONFIG['itemKeys']: if key.get('sortType') == 'person': Item.person_keys.append(key['id']) -@python_2_unicode_compatible class ItemFind(models.Model): """ used to find items, @@ -1795,19 +1791,19 @@ class ItemFind(models.Model): class Meta: unique_together = ("item", "key") - item = models.ForeignKey('Item', related_name='find', db_index=True) + item = models.ForeignKey('Item', related_name='find', db_index=True, on_delete=models.CASCADE) key = models.CharField(max_length=200, db_index=True) value = models.TextField(blank=True, db_index=settings.DB_GIN_TRGM) def __str__(self): - return u"%s=%s" % (self.key, self.value) + return "%s=%s" % (self.key, self.value) ''' ItemSort table constructed based on info in settings.CONFIG['itemKeys'] ''' attrs = { '__module__': 'item.models', - 'item': models.OneToOneField('Item', related_name='sort', primary_key=True), + 'item': models.OneToOneField('Item', related_name='sort', primary_key=True, on_delete=models.CASCADE), 'duration': models.FloatField(null=True, blank=True, db_index=True), 'width': models.BigIntegerField(null=True, blank=True, db_index=True), 'height': models.BigIntegerField(null=True, blank=True, db_index=True), @@ -1826,14 +1822,13 @@ for key in list(filter(lambda k: k.get('sort', False) or k['type'] in ('integer' ItemSort = type('ItemSort', (models.Model,), attrs) ItemSort.fields = [f.name for f in ItemSort._meta.fields] -@python_2_unicode_compatible class Access(models.Model): class Meta: unique_together = ("item", "user") access = models.DateTimeField(auto_now=True) - item = models.ForeignKey(Item, related_name='accessed') - user = models.ForeignKey(User, null=True, related_name='accessed_items') + item = models.ForeignKey(Item, related_name='accessed', on_delete=models.CASCADE) + user = models.ForeignKey(User, null=True, related_name='accessed_items', on_delete=models.CASCADE) accessed = models.IntegerField(default=0) def save(self, *args, **kwargs): @@ -1846,10 +1841,9 @@ class Access(models.Model): def __str__(self): if self.user: - return u"%s/%s/%s" % (self.user, self.item, self.access) - return u"%s/%s" % (self.item, self.access) + return "%s/%s/%s" % (self.user, self.item, self.access) + return "%s/%s" % (self.item, self.access) -@python_2_unicode_compatible class Facet(models.Model): ''' used for keys that can have multiple values like people, languages etc. @@ -1860,13 +1854,13 @@ class Facet(models.Model): class Meta: unique_together = ("item", "key", "value") - item = models.ForeignKey('Item', related_name='facets') + item = models.ForeignKey('Item', related_name='facets', on_delete=models.CASCADE) key = models.CharField(max_length=200, db_index=True) value = models.CharField(max_length=1000, db_index=True) sortvalue = models.CharField(max_length=1000, db_index=True) def __str__(self): - return u"%s=%s" % (self.key, self.value) + return "%s=%s" % (self.key, self.value) def save(self, *args, **kwargs): if not self.sortvalue: @@ -1886,7 +1880,7 @@ class Description(models.Model): class AnnotationSequence(models.Model): - item = models.OneToOneField('Item', related_name='_annotation_sequence') + item = models.OneToOneField('Item', related_name='_annotation_sequence', on_delete=models.CASCADE) value = models.BigIntegerField(default=1) @classmethod diff --git a/pandora/item/site.py b/pandora/item/site.py new file mode 100644 index 00000000..cb186eab --- /dev/null +++ b/pandora/item/site.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +from django.urls import path, re_path + +from . import views + +urls = [ + [ + #frames + re_path(r'^(?P[A-Z0-9].*)/(?P\d+)p(?P[\d\.]*)\.jpg$', views.frame), + + #timelines + re_path(r'^(?P[A-Z0-9].*)/timeline(?P[a-z]*)(?P\d+)p(?P\d+)\.(?Ppng|jpg)$', views.timeline), + re_path(r'^(?P[A-Z0-9].*)/timeline(?P[a-z]*)(?P\d+)p\.(?Ppng|jpg)$', views.timeline), + + #download + re_path(r'^(?P[A-Z0-9].*)/download$', views.download), + re_path(r'^(?P[A-Z0-9].*)/download/$', views.download), + re_path(r'^(?P[A-Z0-9].*)/download/source/(?P\d+)?$', views.download_source), + re_path(r'^(?P[A-Z0-9].*)/download/(?P\d+)p(?P\d+)\.(?Pwebm|ogv|mp4)$', views.download), + re_path(r'^(?P[A-Z0-9].*)/download/(?P\d+)p\.(?Pwebm|ogv|mp4)$', views.download), + + #video + re_path(r'^(?P[A-Z0-9].*)/(?P\d+)p(?P\d*)\.(?Pwebm|ogv|mp4)$', views.video), + re_path(r'^(?P[A-Z0-9].*)/(?P\d+)p(?P\d*)\.(?P.+)\.(?Pwebm|ogv|mp4)$', views.video), + + #torrent + re_path(r'^(?P[A-Z0-9].*)/torrent$', views.torrent), + re_path(r'^(?P[A-Z0-9].*)/torrent/(?P.*?)$', views.torrent), + + #export + re_path(r'^(?P[A-Z0-9].*)/json$', views.item_json), + re_path(r'^(?P[A-Z0-9].*)/xml$', views.item_xml), + + #srt export + re_path(r'^(?P[A-Z0-9].*)/(?P.+)\.(?:(?P.{2})\.)?(?Psrt|vtt)$', views.srt), + + #icon + re_path(r'^(?P[A-Z0-9].*)/icon(?P\d*)\.jpg$', views.icon), + + #poster + re_path(r'^(?P[A-Z0-9].*)/posterframe(?P\d+).jpg$', views.poster_frame), + re_path(r'^(?P[A-Z0-9].*)/poster(?P\d+)\.jpg$', views.poster), + re_path(r'^(?P[A-Z0-9].*)/siteposter(?P\d*)\.jpg$', views.siteposter), + re_path(r'^(?P[A-Z0-9].*)/poster\.jpg$', views.siteposter), + + re_path(r'^random$', views.random_annotation), + ], + 'item', + 'item', +] diff --git a/pandora/item/tasks.py b/pandora/item/tasks.py index eca80083..667e2867 100644 --- a/pandora/item/tasks.py +++ b/pandora/item/tasks.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import os from datetime import timedelta, datetime diff --git a/pandora/item/timelines.py b/pandora/item/timelines.py index 87fe741c..f6ff9f72 100644 --- a/pandora/item/timelines.py +++ b/pandora/item/timelines.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, with_statement, print_function import math import os diff --git a/pandora/item/urls.py b/pandora/item/urls.py deleted file mode 100644 index 332da86d..00000000 --- a/pandora/item/urls.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- - -from django.conf.urls import url - -from . import views - -urlpatterns = [ - #frames - url(r'^(?P[A-Z0-9].*)/(?P\d+)p(?P[\d\.]*)\.jpg$', views.frame), - - #timelines - url(r'^(?P[A-Z0-9].*)/timeline(?P[a-z]*)(?P\d+)p(?P\d+)\.(?Ppng|jpg)$', views.timeline), - url(r'^(?P[A-Z0-9].*)/timeline(?P[a-z]*)(?P\d+)p\.(?Ppng|jpg)$', views.timeline), - - #download - url(r'^(?P[A-Z0-9].*)/download$', views.download), - url(r'^(?P[A-Z0-9].*)/download/$', views.download), - url(r'^(?P[A-Z0-9].*)/download/source/(?P\d+)?$', views.download_source), - url(r'^(?P[A-Z0-9].*)/download/(?P\d+)p(?P\d+)\.(?Pwebm|ogv|mp4)$', views.download), - url(r'^(?P[A-Z0-9].*)/download/(?P\d+)p\.(?Pwebm|ogv|mp4)$', views.download), - - #video - url(r'^(?P[A-Z0-9].*)/(?P\d+)p(?P\d*)\.(?Pwebm|ogv|mp4)$', views.video), - url(r'^(?P[A-Z0-9].*)/(?P\d+)p(?P\d*)\.(?P.+)\.(?Pwebm|ogv|mp4)$', views.video), - - #torrent - url(r'^(?P[A-Z0-9].*)/torrent$', views.torrent), - url(r'^(?P[A-Z0-9].*)/torrent/(?P.*?)$', views.torrent), - - #export - url(r'^(?P[A-Z0-9].*)/json$', views.item_json), - url(r'^(?P[A-Z0-9].*)/xml$', views.item_xml), - - #srt export - url(r'^(?P[A-Z0-9].*)/(?P.+)\.(?:(?P.{2})\.)?(?Psrt|vtt)$', views.srt), - - #icon - url(r'^(?P[A-Z0-9].*)/icon(?P\d*)\.jpg$', views.icon), - - #poster - url(r'^(?P[A-Z0-9].*)/posterframe(?P\d+).jpg$', views.poster_frame), - url(r'^(?P[A-Z0-9].*)/poster(?P\d+)\.jpg$', views.poster), - url(r'^(?P[A-Z0-9].*)/siteposter(?P\d*)\.jpg$', views.siteposter), - url(r'^(?P[A-Z0-9].*)/poster\.jpg$', views.siteposter), - - url(r'^random$', views.random_annotation), -] diff --git a/pandora/item/utils.py b/pandora/item/utils.py index be884bb9..24db30a3 100644 --- a/pandora/item/utils.py +++ b/pandora/item/utils.py @@ -55,14 +55,14 @@ def plural_key(term): def sort_title(title): - title = title.replace(u'Æ', 'Ae') + title = title.replace('Æ', 'Ae') if isinstance(title, bytes): title = title.decode('utf-8') title = ox.decode_html(title) title = sort_string(title) #title - title = re.sub(u'[\'!¿¡,\.;\-"\:\*\[\]]', '', title) + title = re.sub('[\'!¿¡,\.;\-"\:\*\[\]]', '', title) return title.strip() def get_positions(ids, pos, decode_id=False): diff --git a/pandora/item/views.py b/pandora/item/views.py index 97f8aa23..6092aff4 100644 --- a/pandora/item/views.py +++ b/pandora/item/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import os.path import mimetypes @@ -144,7 +143,7 @@ def get_positions(request, query): return utils.get_positions(ids, query['positions']) def is_editable(request, item): - if request.user.is_anonymous(): + if request.user.is_anonymous: return False if not hasattr(request, 'user_group_names'): request.user_group_names = {g.name for g in request.user.groups.all()} @@ -1097,7 +1096,7 @@ def video(request, id, resolution, format, index=None, track=None): ext = '.%s' % format duration = stream.info['duration'] - filename = u"Clip of %s - %s-%s - %s %s%s" % ( + filename = "Clip of %s - %s-%s - %s %s%s" % ( item.get('title'), ox.format_duration(t[0] * 1000).replace(':', '.')[:-4], ox.format_duration(t[1] * 1000).replace(':', '.')[:-4], @@ -1166,9 +1165,9 @@ def srt(request, id, layer, language=None, index=None, ext='srt'): content_type, encoder = _subtitle_formats[ext] response = HttpResponse() if language: - filename = u"%s.%s.%s" % (item.get('title'), language, ext) + filename = "%s.%s.%s" % (item.get('title'), language, ext) else: - filename = u"%s.%s" % (item.get('title'), ext) + filename = "%s.%s" % (item.get('title'), ext) response['Content-Disposition'] = "attachment; filename*=UTF-8''%s" % quote(filename.encode('utf-8')) response['Content-Type'] = content_type response.write(item.srt(layer, language, encoder=encoder)) @@ -1206,7 +1205,7 @@ def atom_xml(request): el.text = atom_link level = settings.CONFIG['capabilities']['canSeeItem']['guest'] - if not request.user.is_anonymous(): + if not request.user.is_anonymous: level = request.user.profile.level for item in models.Item.objects.filter(level__lte=level, rendered=True).order_by('-created')[:7]: if add_updated: @@ -1232,7 +1231,7 @@ def atom_xml(request): if item.get('director'): el = ET.SubElement(entry, "author") name = ET.SubElement(el, "name") - name.text = ox.decode_html(u', '.join(item.get('director'))) + name.text = ox.decode_html(', '.join(item.get('director'))) elif item.user: el = ET.SubElement(entry, "author") name = ET.SubElement(el, "name") @@ -1283,7 +1282,7 @@ def atom_xml(request): el = ET.SubElement(format, key) el.text = unicode(value) el = ET.SubElement(format, 'pixel_aspect_ratio') - el.text = u"1:1" + el.text = "1:1" if has_capability(request.user, 'canDownloadVideo'): if item.torrent: @@ -1386,7 +1385,7 @@ def sitemap_part_xml(request, part): def item_json(request, id): level = settings.CONFIG['capabilities']['canSeeItem']['guest'] - if not request.user.is_anonymous(): + if not request.user.is_anonymous: level = request.user.profile.level qs = models.Item.objects.filter(public_id=id, level__lte=level) if qs.count() == 0: @@ -1399,7 +1398,7 @@ def item_json(request, id): def item_xml(request, id): level = settings.CONFIG['capabilities']['canSeeItem']['guest'] - if not request.user.is_anonymous(): + if not request.user.is_anonymous: level = request.user.profile.level qs = models.Item.objects.filter(public_id=id, level__lte=level) if qs.count() == 0: @@ -1439,7 +1438,7 @@ def item(request, id): view = None template = 'index.html' level = settings.CONFIG['capabilities']['canSeeItem']['guest'] - if not request.user.is_anonymous(): + if not request.user.is_anonymous: level = request.user.profile.level qs = models.Item.objects.filter(public_id=id, level__lte=level) if qs.count() == 0: @@ -1484,7 +1483,7 @@ def item(request, id): else: title = key['title'] if key else k.capitalize() if isinstance(value, list): - value = value = u', '.join([unicode(v) for v in value]) + value = value = ', '.join([unicode(v) for v in value]) elif key and key.get('type') == 'float': value = '%0.3f' % value elif key and key.get('type') == 'time': @@ -1493,7 +1492,7 @@ def item(request, id): clips = [] clip = {'in': 0, 'annotations': []} # logged in users should have javascript. not adding annotations makes load faster - if not settings.USE_IMDB and request.user.is_anonymous(): + if not settings.USE_IMDB and request.user.is_anonymous: for a in item.annotations.exclude( layer='subtitles' ).exclude( @@ -1513,13 +1512,13 @@ def item(request, id): head_title = item.get('title', '') title = item.get('title', '') if item.get('director'): - head_title += u' (%s)' % u', '.join(item.get('director', [])) + head_title += ' (%s)' % ', '.join(item.get('director', [])) if item.get('year'): - head_title += u' %s' % item.get('year') - title += u' (%s)' % item.get('year') + head_title += ' %s' % item.get('year') + title += ' (%s)' % item.get('year') if view: - head_title += u' – %s' % view - head_title += u' – %s' % settings.SITENAME + head_title += ' – %s' % view + head_title += ' – %s' % settings.SITENAME head_title = ox.decode_html(head_title) title = ox.decode_html(title) ctx = { diff --git a/pandora/itemlist/managers.py b/pandora/itemlist/managers.py index 989c8704..e00c3a14 100644 --- a/pandora/itemlist/managers.py +++ b/pandora/itemlist/managers.py @@ -124,7 +124,7 @@ class ListManager(Manager): if conditions: qs = qs.filter(conditions) - if user.is_anonymous(): + if user.is_anonymous: qs = qs.filter(Q(status='public') | Q(status='featured')) else: qs = qs.filter(Q(status='public') | Q(status='featured') | Q(user=user)) diff --git a/pandora/itemlist/models.py b/pandora/itemlist/models.py index 7d696381..ad2cfb14 100644 --- a/pandora/itemlist/models.py +++ b/pandora/itemlist/models.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import os import re @@ -10,7 +9,6 @@ from django.db import models from django.db.models import Max from django.contrib.auth import get_user_model from django.conf import settings -from django.utils.encoding import python_2_unicode_compatible from oxdjango.fields import JSONField import ox @@ -28,7 +26,6 @@ def get_icon_path(f, x): return get_path(f, 'icon.jpg') def get_listview(): return settings.CONFIG['user']['ui']['listView'] def get_listsort(): return tuple(settings.CONFIG['user']['ui']['listSort']) -@python_2_unicode_compatible class List(models.Model): class Meta: @@ -36,7 +33,7 @@ class List(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) - user = models.ForeignKey(User, related_name='lists') + user = models.ForeignKey(User, related_name='lists', on_delete=models.CASCADE) groups = models.ManyToManyField(Group, blank=True, related_name='lists') name = models.CharField(max_length=255) status = models.CharField(max_length=20, default='private') @@ -50,7 +47,7 @@ class List(models.Model): view = models.TextField(default=get_listview) sort = JSONField(default=get_listsort, editable=False) - poster_frames = JSONField(default=[], editable=False) + poster_frames = JSONField(default=list, editable=False) #is through table still required? items = models.ManyToManyField('item.Item', related_name='lists', @@ -110,13 +107,13 @@ class List(models.Model): return self.get_id() def get_id(self): - return u'%s:%s' % (self.user.username, self.name) + return '%s:%s' % (self.user.username, self.name) def accessible(self, user): return self.user == user or self.status in ('public', 'featured') def editable(self, user): - if not user or user.is_anonymous(): + if not user or user.is_anonymous: return False if self.user == user or \ self.groups.filter(id__in=user.groups.all()).count() > 0 or \ @@ -244,7 +241,7 @@ class List(models.Model): elif key == 'subscribers': response[key] = self.subscribed_users.all().count() elif key == 'subscribed': - if user and not user.is_anonymous(): + if user and not user.is_anonymous: response[key] = self.subscribed_users.filter(id=user.id).exists() else: response[key] = getattr(self, { @@ -314,29 +311,27 @@ class List(models.Model): path = source return path -@python_2_unicode_compatible class ListItem(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) - list = models.ForeignKey(List) + list = models.ForeignKey(List, on_delete=models.CASCADE) index = models.IntegerField(default=0) - item = models.ForeignKey('item.Item') + item = models.ForeignKey('item.Item', on_delete=models.CASCADE) def __str__(self): - return u'%s in %s' % (self.item, self.list) + return '%s in %s' % (self.item, self.list) -@python_2_unicode_compatible class Position(models.Model): class Meta: unique_together = ("user", "list", "section") - list = models.ForeignKey(List, related_name='position') - user = models.ForeignKey(User) + list = models.ForeignKey(List, related_name='position', on_delete=models.CASCADE) + user = models.ForeignKey(User, on_delete=models.CASCADE) section = models.CharField(max_length=255) position = models.IntegerField(default=0) def __str__(self): - return u'%s/%s/%s' % (self.section, self.position, self.list) + return '%s/%s/%s' % (self.section, self.position, self.list) diff --git a/pandora/itemlist/views.py b/pandora/itemlist/views.py index 067b7d51..04edcc9d 100644 --- a/pandora/itemlist/views.py +++ b/pandora/itemlist/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import os import re @@ -75,7 +74,7 @@ def findLists(request, data): query = parse_query(data, request.user) #order - is_section_request = query['sort'] == [{u'operator': u'+', u'key': u'position'}] + is_section_request = query['sort'] == [{'operator': '+', 'key': 'position'}] def is_featured_condition(x): return x['key'] == 'status' and \ x['value'] == 'featured' and \ @@ -87,7 +86,7 @@ def findLists(request, data): if is_section_request: qs = query['qs'] - if not is_featured and not request.user.is_anonymous(): + if not is_featured and not request.user.is_anonymous: qs = qs.filter(position__in=models.Position.objects.filter(user=request.user)) qs = qs.order_by('position__position') else: diff --git a/pandora/log/models.py b/pandora/log/models.py index a4cde6c0..25e850c7 100644 --- a/pandora/log/models.py +++ b/pandora/log/models.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from django.db import models from django.contrib.auth import get_user_model -from django.utils.encoding import python_2_unicode_compatible import ox @@ -12,11 +10,10 @@ from . import managers User = get_user_model() -@python_2_unicode_compatible class Log(models.Model): created = models.DateTimeField(auto_now_add=True, db_index=True) modified = models.DateTimeField(auto_now=True) - user = models.ForeignKey(User, default=None, blank=True, null=True) + user = models.ForeignKey(User, default=None, blank=True, null=True, on_delete=models.CASCADE) url = models.CharField(max_length=1000, default='') line = models.IntegerField(default=0) text = models.TextField(blank=True) @@ -24,7 +21,7 @@ class Log(models.Model): objects = managers.LogManager() def __str__(self): - return u"%s" % self.id + return "%s" % self.id def json(self, keys=None): j = { diff --git a/pandora/log/tasks.py b/pandora/log/tasks.py index ffb9f567..768fd731 100644 --- a/pandora/log/tasks.py +++ b/pandora/log/tasks.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from datetime import timedelta, datetime diff --git a/pandora/log/utils.py b/pandora/log/utils.py index df115303..f2074567 100644 --- a/pandora/log/utils.py +++ b/pandora/log/utils.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import logging import sys @@ -34,7 +33,7 @@ class ErrorHandler(logging.Handler): request = record.request request_repr = repr(request) - if request.user.is_authenticated(): + if request.user.is_authenticated: user = request.user url = request.META.get('PATH_INFO', '') except: diff --git a/pandora/log/views.py b/pandora/log/views.py index 5016c406..d8dfa4f4 100644 --- a/pandora/log/views.py +++ b/pandora/log/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import ox from ox.utils import json @@ -25,7 +24,7 @@ def logError(request, data): returns {} see: findErrorLogs, removeErrorLogs ''' - if request.user.is_authenticated(): + if request.user.is_authenticated: user = request.user else: user = None diff --git a/pandora/news/admin.py b/pandora/news/admin.py index b7f636cb..4778b44b 100644 --- a/pandora/news/admin.py +++ b/pandora/news/admin.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from django.contrib import admin diff --git a/pandora/news/models.py b/pandora/news/models.py index 194a5ce7..98624ab6 100644 --- a/pandora/news/models.py +++ b/pandora/news/models.py @@ -1,14 +1,11 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from django.db import models -from django.utils.encoding import python_2_unicode_compatible import ox from . import managers -@python_2_unicode_compatible class News(models.Model): objects = managers.NewsManager() @@ -20,7 +17,7 @@ class News(models.Model): text = models.TextField() def editable(self, user): - return user.is_authenticated() and user.profile.capability("canEditSitePages") + return user.is_authenticated and user.profile.capability("canEditSitePages") def save(self, *args, **kwargs): super(News, self).save(*args, **kwargs) @@ -42,5 +39,5 @@ class News(models.Model): return j def __str__(self): - return u"%s/%s" % (self.date, self.title) + return "%s/%s" % (self.date, self.title) diff --git a/pandora/news/views.py b/pandora/news/views.py index 2a7fcf32..36e14590 100644 --- a/pandora/news/views.py +++ b/pandora/news/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import ox from ox.utils import json diff --git a/pandora/oxdjango/api/actions.py b/pandora/oxdjango/api/actions.py index f813e75a..95e1fe81 100644 --- a/pandora/oxdjango/api/actions.py +++ b/pandora/oxdjango/api/actions.py @@ -111,10 +111,10 @@ class ApiActions(dict): f = fc[len(fc)-1].cell_contents if PY2: info = f.func_code.co_filename[len(settings.PROJECT_ROOT)+1:] - info = u'%s:%s' % (info, f.func_code.co_firstlineno) + info = '%s:%s' % (info, f.func_code.co_firstlineno) else: info = f.__code__.co_filename[len(settings.PROJECT_ROOT)+1:] - info = u'%s:%s' % (info, f.__code__.co_firstlineno) + info = '%s:%s' % (info, f.__code__.co_firstlineno) return info, trim(inspect.getsource(f)) def register(self, method, action=None, cache=True, version=None): diff --git a/pandora/oxdjango/api/urls.py b/pandora/oxdjango/api/site.py similarity index 56% rename from pandora/oxdjango/api/urls.py rename to pandora/oxdjango/api/site.py index 159c4ff6..ddd5bc49 100644 --- a/pandora/oxdjango/api/urls.py +++ b/pandora/oxdjango/api/site.py @@ -1,13 +1,17 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -from django.conf.urls import url +from django.urls import path from . import views from . import actions actions.autodiscover() -urlpatterns = [ - url(r'^$', views.api), +urls = [ + [ + path(r'', views.api), + ], + 'api', + 'api' ] diff --git a/pandora/oxdjango/api/views.py b/pandora/oxdjango/api/views.py index 9e962346..1dfe7428 100644 --- a/pandora/oxdjango/api/views.py +++ b/pandora/oxdjango/api/views.py @@ -3,7 +3,7 @@ from __future__ import division, absolute_import import json -from django.shortcuts import render_to_response +from django.shortcuts import render from django.conf import settings from ..shortcuts import render_to_json_response, json_response, HttpErrorJson @@ -31,7 +31,7 @@ def api(request): 'settings': settings, 'sitename': settings.SITENAME } - response = render_to_response('api.html', context) + response = render(request, 'api.html', context) response['Access-Control-Allow-Origin'] = '*' return response if request.META.get('CONTENT_TYPE') == 'application/json': diff --git a/pandora/oxdjango/decorators.py b/pandora/oxdjango/decorators.py index ab4fbf2d..bd6942e5 100644 --- a/pandora/oxdjango/decorators.py +++ b/pandora/oxdjango/decorators.py @@ -12,7 +12,7 @@ def login_required_json(function=None): """ def _wrapped_view(request, *args, **kwargs): - if request.user.is_authenticated(): + if request.user.is_authenticated: return function(request, *args, **kwargs) return render_to_json_response({'status': {'code': 401, 'text': 'login required'}}) return wraps(function)(_wrapped_view) @@ -24,7 +24,7 @@ def admin_required_json(function=None): """ def _wrapped_view(request, *args, **kwargs): - if request.user.is_authenticated() and request.user.profile.get_level() == 'admin': + if request.user.is_authenticated and request.user.profile.get_level() == 'admin': return function(request, *args, **kwargs) return render_to_json_response({'status': {'code': 403, 'text': 'permission denied'}}) return wraps(function)(_wrapped_view) diff --git a/pandora/oxdjango/fields.py b/pandora/oxdjango/fields.py index b9b1e58b..f3b66321 100644 --- a/pandora/oxdjango/fields.py +++ b/pandora/oxdjango/fields.py @@ -64,7 +64,7 @@ class DictField(models.TextField): def dumps(cls, obj): return json.dumps(obj, default=to_json, ensure_ascii=False) - def from_db_value(self, value, expression, connection, context): + def from_db_value(self, value, expression, connection, context=None): if value is None: return value if isinstance(value, self._type): diff --git a/pandora/oxdjango/middleware.py b/pandora/oxdjango/middleware.py index 4883f23a..5b2ccaab 100644 --- a/pandora/oxdjango/middleware.py +++ b/pandora/oxdjango/middleware.py @@ -1,14 +1,17 @@ # -*- coding: utf-8 -*- +from django.utils.deprecation import MiddlewareMixin from .shortcuts import HttpErrorJson, render_to_json_response -class ExceptionMiddleware(object): +class ExceptionMiddleware(MiddlewareMixin): + def process_exception(self, request, exception): if isinstance(exception, HttpErrorJson): return render_to_json_response(exception.response) return None -class ChromeFrameMiddleware(object): +class ChromeFrameMiddleware(MiddlewareMixin): + def process_response(self, request, response): response['X-UA-Compatible'] = 'chrome=1' return response diff --git a/pandora/oxdjango/shortcuts.py b/pandora/oxdjango/shortcuts.py index 8246a98a..3baa1315 100644 --- a/pandora/oxdjango/shortcuts.py +++ b/pandora/oxdjango/shortcuts.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function import datetime from django.utils import datetime_safe from django.http import HttpResponse, Http404 @@ -23,7 +22,7 @@ def _to_json(python_object): return python_object.strftime('%Y-%m-%dT%H:%M:%SZ') if isinstance(python_object, datetime_safe.datetime): return python_object.strftime('%Y-%m-%dT%H:%M:%SZ') - raise TypeError(u'%s %s is not JSON serializable' % (repr(python_object), type(python_object))) + raise TypeError('%s %s is not JSON serializable' % (repr(python_object), type(python_object))) def json_dump(data, fp, indent=4): return json.dump(data, fp, indent=indent, default=_to_json, ensure_ascii=False) diff --git a/pandora/person/models.py b/pandora/person/models.py index 84fe86d6..54fae287 100644 --- a/pandora/person/models.py +++ b/pandora/person/models.py @@ -1,10 +1,8 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import unicodedata from django.db import models -from django.utils.encoding import python_2_unicode_compatible from oxdjango import fields import ox @@ -24,10 +22,9 @@ def get_name_sort(name, sortname=None): person.save() sortname = unicodedata.normalize('NFKD', person.sortname) else: - sortname = u'' + sortname = '' return sortname -@python_2_unicode_compatible class Person(models.Model): name = models.CharField(max_length=200, unique=True) sortname = models.CharField(max_length=200) diff --git a/pandora/person/tasks.py b/pandora/person/tasks.py index 2f360dae..a94ed273 100644 --- a/pandora/person/tasks.py +++ b/pandora/person/tasks.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from celery.task import task diff --git a/pandora/person/views.py b/pandora/person/views.py index aa6f6570..fd9258a1 100644 --- a/pandora/person/views.py +++ b/pandora/person/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import ox from ox.utils import json diff --git a/pandora/place/admin.py b/pandora/place/admin.py index 209244d3..9ee1aeae 100644 --- a/pandora/place/admin.py +++ b/pandora/place/admin.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from django.contrib import admin diff --git a/pandora/place/models.py b/pandora/place/models.py index 318a3d15..3c1acc6d 100644 --- a/pandora/place/models.py +++ b/pandora/place/models.py @@ -1,10 +1,8 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from django.db import models, transaction from django.contrib.auth import get_user_model -from django.utils.encoding import python_2_unicode_compatible import ox from oxdjango import fields @@ -15,7 +13,6 @@ from . import managers User = get_user_model() -@python_2_unicode_compatible class Place(models.Model): ''' Places are named locations, they should have geographical information attached to them. @@ -24,7 +21,7 @@ class Place(models.Model): modified = models.DateTimeField(auto_now=True) defined = models.BooleanField(default=True) - user = models.ForeignKey(User, null=True, related_name='places') + user = models.ForeignKey(User, null=True, related_name='places', on_delete=models.CASCADE) name = models.CharField(max_length=1024) alternativeNames = fields.TupleField(default=()) @@ -60,7 +57,7 @@ class Place(models.Model): @classmethod def get_or_create(model, name): - qs = model.objects.filter(name_find__contains=u'|%s|' % name.lower()) + qs = model.objects.filter(name_find__contains='|%s|' % name.lower()) if qs.count() == 0: instance = model(name=name) instance.save() @@ -69,7 +66,7 @@ class Place(models.Model): return instance def editable(self, user): - if user and not user.is_anonymous() \ + if user and not user.is_anonymous \ and (not self.user or \ self.user == user or \ user.profile.capability('canEditPlaces')): diff --git a/pandora/place/tasks.py b/pandora/place/tasks.py index ba6b8375..3feb88dd 100644 --- a/pandora/place/tasks.py +++ b/pandora/place/tasks.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from celery.task import task diff --git a/pandora/place/views.py b/pandora/place/views.py index 01533b52..662253c7 100644 --- a/pandora/place/views.py +++ b/pandora/place/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from django.db.models import Max, Min, Count from django.conf import settings @@ -50,7 +49,7 @@ def addPlace(request, data): n = 0 while _exists: _exists = models.Place.objects.filter(defined=True, - name_find__contains=u'|%s|' % name.lower()).count() > 0 + name_find__contains='|%s|' % name.lower()).count() > 0 if _exists: name = 'Untitled [%s]' %n n += 1 @@ -61,7 +60,7 @@ def addPlace(request, data): for n in names: n = ox.decode_html(name) if models.Place.objects.filter(defined=True, - name_find__contains=u'|%s|' % n.lower()).count() != 0: + name_find__contains='|%s|' % n.lower()).count() != 0: exists = True existing_names.append(n) ''' @@ -130,7 +129,7 @@ def editPlace(request, data): for name in names + alternative_names: name = ox.decode_html(name) if models.Place.objects.filter(defined=True, - name_find__contains=u'|%s|' % name.lower()).exclude(id=place.id).count() != 0: + name_find__contains='|%s|' % name.lower()).exclude(id=place.id).count() != 0: conflict = True conflict_names.append(name) ''' diff --git a/pandora/sequence/managers.py b/pandora/sequence/managers.py index 63bf693b..8f4c5d8d 100644 --- a/pandora/sequence/managers.py +++ b/pandora/sequence/managers.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from django.db.models import Q, Manager diff --git a/pandora/sequence/models.py b/pandora/sequence/models.py index 2970c3f2..b17c9277 100644 --- a/pandora/sequence/models.py +++ b/pandora/sequence/models.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from django.db import models -from django.utils.encoding import python_2_unicode_compatible from item.models import ItemSort @@ -15,7 +13,6 @@ def parse_hash(value): def format_hash(value): return hex(value + 9223372036854775808)[2:-1].upper() -@python_2_unicode_compatible class Sequence(models.Model): class Meta: unique_together = ("sort", "start", "end", "mode") @@ -25,7 +22,7 @@ class Sequence(models.Model): 'color': 1 } mode = models.IntegerField(choices=sorted(zip(MODE.values(), list(MODE)), key=lambda k: k[0]), default=0) - sort = models.ForeignKey(ItemSort, null=True, related_name='sequences') + sort = models.ForeignKey(ItemSort, null=True, related_name='sequences', on_delete=models.CASCADE) hash = models.BigIntegerField(db_index=True, default=-9223372036854775808) start = models.FloatField(default=-1) @@ -40,7 +37,7 @@ class Sequence(models.Model): @property def public_id(self): - return u"%s/%0.03f-%0.03f" % (self.sort.item.public_id, float(self.start), float(self.end)) + return "%s/%0.03f-%0.03f" % (self.sort.item.public_id, float(self.start), float(self.end)) def __str__(self): return self.public_id diff --git a/pandora/sequence/tasks.py b/pandora/sequence/tasks.py index 1282a66e..1d467f01 100644 --- a/pandora/sequence/tasks.py +++ b/pandora/sequence/tasks.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from six import string_types from django.db import connection, transaction diff --git a/pandora/sequence/views.py b/pandora/sequence/views.py index 89986129..08ee2543 100644 --- a/pandora/sequence/views.py +++ b/pandora/sequence/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from ox.utils import json from oxdjango.shortcuts import render_to_json_response, json_response diff --git a/pandora/settings.py b/pandora/settings.py index fd1cf7a0..aea1eb59 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -96,7 +96,7 @@ TEMPLATES = [ }, ] -MIDDLEWARE_CLASSES = ( +MIDDLEWARE = ( 'django.middleware.common.CommonMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', diff --git a/pandora/taskqueue/models.py b/pandora/taskqueue/models.py index 9dda9da1..214f9a41 100644 --- a/pandora/taskqueue/models.py +++ b/pandora/taskqueue/models.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from datetime import datetime, timedelta from time import time @@ -9,7 +8,6 @@ from django.contrib.auth import get_user_model from django.conf import settings from django.db import models from django.db.models import Q -from django.utils.encoding import python_2_unicode_compatible import celery.task.control import kombu.five import ox @@ -40,7 +38,6 @@ def get_tasks(username): tasks.append(task.json()) return tasks -@python_2_unicode_compatible class Task(models.Model): DONE = ['finished', 'failed', 'canceled'] @@ -51,8 +48,8 @@ class Task(models.Model): status = models.CharField(default='unknown', max_length=32) started = models.DateTimeField(null=True) ended = models.DateTimeField(null=True) - item = models.ForeignKey("item.Item", related_name='tasks') - user = models.ForeignKey(User, related_name='tasks', null=True) + item = models.ForeignKey("item.Item", related_name='tasks', on_delete=models.CASCADE) + user = models.ForeignKey(User, related_name='tasks', null=True, on_delete=models.CASCADE) def __str__(self): return "%s [%s]" % (self.item.public_id, self.status) diff --git a/pandora/taskqueue/views.py b/pandora/taskqueue/views.py index e4791768..aef64c99 100644 --- a/pandora/taskqueue/views.py +++ b/pandora/taskqueue/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import ox from oxdjango.decorators import login_required_json diff --git a/pandora/text/managers.py b/pandora/text/managers.py index 9de4ce65..e0eadc5a 100644 --- a/pandora/text/managers.py +++ b/pandora/text/managers.py @@ -126,7 +126,7 @@ class TextManager(Manager): if conditions: qs = qs.filter(conditions) - if user.is_anonymous(): + if user.is_anonymous: qs = qs.filter(Q(status='public') | Q(status='featured')) else: qs = qs.filter(Q(status='public') | Q(status='featured') | Q(user=user)) diff --git a/pandora/text/models.py b/pandora/text/models.py index fc9e7291..08bf6fa0 100644 --- a/pandora/text/models.py +++ b/pandora/text/models.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import os import re @@ -12,7 +11,6 @@ from django.db.models import Max from django.contrib.auth import get_user_model from django.conf import settings from django.db.models.signals import pre_delete -from django.utils.encoding import python_2_unicode_compatible import ox from oxdjango.fields import TupleField @@ -26,7 +24,6 @@ User = get_user_model() def get_path(i, x): return i.path(x) def get_icon_path(i, x): return get_path(i, 'icon.jpg') -@python_2_unicode_compatible class Text(models.Model): class Meta: @@ -34,7 +31,7 @@ class Text(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) - user = models.ForeignKey(User, related_name='texts') + user = models.ForeignKey(User, related_name='texts', on_delete=models.CASCADE) name = models.CharField(max_length=255) status = models.CharField(max_length=20, default='private') _status = ['private', 'public', 'featured'] @@ -77,13 +74,13 @@ class Text(models.Model): return '%s/text.pdf' % self.get_absolute_url() def get_id(self): - return u'%s:%s' % (self.user.username, self.name) + return '%s:%s' % (self.user.username, self.name) def accessible(self, user): return self.user == user or self.status in ('public', 'featured') def editable(self, user): - if not user or user.is_anonymous(): + if not user or user.is_anonymous: return False if self.user == user or \ user.is_staff or \ @@ -209,7 +206,7 @@ class Text(models.Model): elif key == 'subscribers': response[key] = self.subscribed_users.all().count() elif key == 'subscribed': - if user and not user.is_anonymous(): + if user and not user.is_anonymous: response[key] = self.subscribed_users.filter(id=user.id).exists() elif hasattr(self, _map.get(key, key)): response[key] = getattr(self, _map.get(key,key)) @@ -304,17 +301,16 @@ def delete_file(sender, **kwargs): t.file.delete(save=False) pre_delete.connect(delete_file, sender=Text) -@python_2_unicode_compatible class Position(models.Model): class Meta: unique_together = ("user", "text", "section") - text = models.ForeignKey(Text, related_name='position') - user = models.ForeignKey(User, related_name='text_position') + text = models.ForeignKey(Text, related_name='position', on_delete=models.CASCADE) + user = models.ForeignKey(User, related_name='text_position', on_delete=models.CASCADE) section = models.CharField(max_length=255) position = models.IntegerField(default=0) def __str__(self): - return u'%s/%s/%s' % (self.section, self.position, self.text) + return '%s/%s/%s' % (self.section, self.position, self.text) diff --git a/pandora/text/views.py b/pandora/text/views.py index ecea30c8..33338675 100644 --- a/pandora/text/views.py +++ b/pandora/text/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import os import re @@ -98,7 +97,7 @@ def getText(request, data): 'name': '', 'text': '', 'type': 'html', - 'editable': not request.user.is_anonymous() and request.user.profile.capability('canEditFeaturedTexts') + 'editable': not request.user.is_anonymous and request.user.profile.capability('canEditFeaturedTexts') } else: text = qs[0] @@ -206,7 +205,8 @@ def findTexts(request, data): query = parse_query(data, request.user) #order - is_section_request = query['sort'] == [{u'operator': u'+', u'key': u'position'}] + is_section_request = query['sort'] == [{'operator': '+', 'key': 'position'}] + def is_featured_condition(x): return x['key'] == 'status' and \ x['value'] == 'featured' and \ @@ -218,7 +218,7 @@ def findTexts(request, data): if is_section_request: qs = query['qs'] - if not is_featured and not request.user.is_anonymous(): + if not is_featured and not request.user.is_anonymous: qs = qs.filter(position__in=models.Position.objects.filter(user=request.user)) qs = qs.order_by('position__position') else: diff --git a/pandora/title/models.py b/pandora/title/models.py index fceb00c6..7e522c1c 100644 --- a/pandora/title/models.py +++ b/pandora/title/models.py @@ -1,10 +1,8 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import unicodedata from django.db import models -from django.utils.encoding import python_2_unicode_compatible import ox @@ -22,10 +20,9 @@ def get_title_sort(title): title.save() sorttitle = unicodedata.normalize('NFKD', title.sorttitle) else: - sorttitle = u'' + sorttitle = '' return sorttitle -@python_2_unicode_compatible class Title(models.Model): title = models.CharField(max_length=1000, unique=True) sorttitle = models.CharField(max_length=1000) diff --git a/pandora/title/views.py b/pandora/title/views.py index 178cb74d..aa48940c 100644 --- a/pandora/title/views.py +++ b/pandora/title/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import unicodedata diff --git a/pandora/translation/tasks.py b/pandora/translation/tasks.py index 09645963..8b0be30b 100644 --- a/pandora/translation/tasks.py +++ b/pandora/translation/tasks.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from datetime import timedelta, datetime diff --git a/pandora/tv/models.py b/pandora/tv/models.py index 6a68d214..e9778d70 100644 --- a/pandora/tv/models.py +++ b/pandora/tv/models.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from datetime import datetime, timedelta from random import randint @@ -7,22 +6,20 @@ from random import randint from django.db import models from django.db.models import Max from django.conf import settings -from django.utils.encoding import python_2_unicode_compatible from item.models import Item -@python_2_unicode_compatible class Channel(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) run = models.IntegerField(default=0) - list = models.OneToOneField('itemlist.List', related_name='channel', null=True, blank=True) - #list = models.ForeignKey('itemlist.List', related_name='channel', null=True, unique=True, blank=True) + list = models.OneToOneField('itemlist.List', related_name='channel', null=True, blank=True, on_delete=models.CASCADE) + #list = models.ForeignKey('itemlist.List', related_name='channel', null=True, unique=True, blank=True, on_delete=models.CASCADE) def __str__(self): - return u"%s %s" % (self.list or 'All', self.run) + return "%s %s" % (self.list or 'All', self.run) def update_program(self, now=None): if not now: @@ -73,18 +70,17 @@ class Channel(models.Model): else: return program.json(user, now) -@python_2_unicode_compatible class Program(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) run = models.IntegerField(default=0) start = models.DateTimeField() end = models.DateTimeField() - item = models.ForeignKey('item.Item', related_name='program') - channel = models.ForeignKey(Channel, related_name='program') + item = models.ForeignKey('item.Item', related_name='program', on_delete=models.CASCADE) + channel = models.ForeignKey(Channel, related_name='program', on_delete=models.CASCADE) def __str__(self): - return u"%s %s" % (self.item, self.start) + return "%s %s" % (self.item, self.start) def json(self, user, current=False): item_json = self.item.json() diff --git a/pandora/tv/tasks.py b/pandora/tv/tasks.py index 80d36fbc..20a9df94 100644 --- a/pandora/tv/tasks.py +++ b/pandora/tv/tasks.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from datetime import datetime, timedelta diff --git a/pandora/tv/views.py b/pandora/tv/views.py index fedf8c7b..89c0cae4 100644 --- a/pandora/tv/views.py +++ b/pandora/tv/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import from ox.utils import json from oxdjango.shortcuts import render_to_json_response, json_response diff --git a/pandora/urlalias/models.py b/pandora/urlalias/models.py index c42fa2e1..13fa2568 100644 --- a/pandora/urlalias/models.py +++ b/pandora/urlalias/models.py @@ -1,37 +1,32 @@ from django.db import models -from django.utils.encoding import python_2_unicode_compatible -@python_2_unicode_compatible class IDAlias(models.Model): old = models.CharField(max_length=255, unique=True) new = models.CharField(max_length=255) def __str__(self): - return u"%s=%s" % (self.old, self.new) + return "%s=%s" % (self.old, self.new) -@python_2_unicode_compatible class LayerAlias(models.Model): old = models.CharField(max_length=255, unique=True) new = models.CharField(max_length=255) def __str__(self): - return u"%s=%s" % (self.old, self.new) + return "%s=%s" % (self.old, self.new) -@python_2_unicode_compatible class ListAlias(models.Model): old = models.CharField(max_length=255, unique=True) new = models.CharField(max_length=255) def __str__(self): - return u"%s=%s" % (self.old, self.new) + return "%s=%s" % (self.old, self.new) -@python_2_unicode_compatible class Alias(models.Model): url = models.CharField(max_length=255, unique=True) target = models.CharField(max_length=255) def __str__(self): - return u"%s=%s" % (self.url, self.target) + return "%s=%s" % (self.url, self.target) diff --git a/pandora/urlalias/views.py b/pandora/urlalias/views.py index 8994837d..4e1d0989 100644 --- a/pandora/urlalias/views.py +++ b/pandora/urlalias/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import re diff --git a/pandora/urls.py b/pandora/urls.py index 12328f5d..cd241d66 100644 --- a/pandora/urls.py +++ b/pandora/urls.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import os -from django.conf.urls import url, include +from django.urls import path, re_path from oxdjango.http import HttpFileResponse from django.conf import settings @@ -13,7 +13,7 @@ admin.autodiscover() import app.monkey_patch -import oxdjango.api.urls +import oxdjango.api.site import app.views import archive.views @@ -24,7 +24,7 @@ import user.views import edit.views import itemlist.views import item.views -import item.urls +import item.site import translation.views import urlalias.views @@ -32,60 +32,59 @@ def serve_static_file(path, location, content_type): return HttpFileResponse(location, content_type=content_type) urlpatterns = [ - # Uncomment the admin/doc line below to enable admin documentation: - # urlurl(r'^admin/doc/', include('django.contrib.admindocs.urls')), - url(r'^admin/', include(admin.site.urls)), - url(r'^api/locale.(?P.*).json$', translation.views.locale_json), - url(r'^api/upload/text/?$', text.views.upload), - url(r'^api/upload/document/?$', document.views.upload), - url(r'^api/upload/direct/?$', archive.views.direct_upload), - url(r'^api/upload/?$', archive.views.firefogg_upload), - url(r'^url=(?P.*)$', app.views.redirect_url), - url(r'^file/(?P.*)$', archive.views.lookup_file), - url(r'^api/?', include(oxdjango.api.urls)), - url(r'^resetUI$', user.views.reset_ui), - url(r'^collection/(?P.*?)/icon(?P\d*).jpg$', documentcollection.views.icon), - url(r'^documents/(?P[A-Z0-9]+)/(?P\d*)p(?P[\d,]*).jpg$', document.views.thumbnail), - url(r'^documents/(?P[A-Z0-9]+)/(?P.*?\.[^\d]{3})$', document.views.file), - url(r'^edit/(?P.*?)/icon(?P\d*).jpg$', edit.views.icon), - url(r'^list/(?P.*?)/icon(?P\d*).jpg$', itemlist.views.icon), - url(r'^text/(?P.*?)/icon(?P\d*).jpg$', text.views.icon), - url(r'^texts/(?P.*?)/text.pdf$', text.views.pdf), - url(r'^texts/(?P.*?)/text.pdf.html$', text.views.pdf_viewer), - url(r'^texts/$', text.views.text), - url(r'^texts/(?P.*?)/\d+$', text.views.text), - url(r'^texts/(?P.*?)$', text.views.text), - url(r'^favicon.ico$', serve_static_file, { + #path('admin/', admin.site.urls), + + re_path(r'^api/locale.(?P.*).json$', translation.views.locale_json), + re_path(r'^api/upload/text/?$', text.views.upload), + re_path(r'^api/upload/document/?$', document.views.upload), + re_path(r'^api/upload/direct/?$', archive.views.direct_upload), + re_path(r'^api/upload/?$', archive.views.firefogg_upload), + re_path(r'^url=(?P.*)$', app.views.redirect_url), + re_path(r'^file/(?P.*)$', archive.views.lookup_file), + re_path(r'^api/?', oxdjango.api.site.urls), + re_path(r'^resetUI$', user.views.reset_ui), + re_path(r'^collection/(?P.*?)/icon(?P\d*).jpg$', documentcollection.views.icon), + re_path(r'^documents/(?P[A-Z0-9]+)/(?P\d*)p(?P[\d,]*).jpg$', document.views.thumbnail), + re_path(r'^documents/(?P[A-Z0-9]+)/(?P.*?\.[^\d]{3})$', document.views.file), + re_path(r'^edit/(?P.*?)/icon(?P\d*).jpg$', edit.views.icon), + re_path(r'^list/(?P.*?)/icon(?P\d*).jpg$', itemlist.views.icon), + re_path(r'^text/(?P.*?)/icon(?P\d*).jpg$', text.views.icon), + re_path(r'^texts/(?P.*?)/text.pdf$', text.views.pdf), + re_path(r'^texts/(?P.*?)/text.pdf.html$', text.views.pdf_viewer), + re_path(r'^texts/$', text.views.text), + re_path(r'^texts/(?P.*?)/\d+$', text.views.text), + re_path(r'^texts/(?P.*?)$', text.views.text), + re_path(r'^favicon.ico$', serve_static_file, { 'location': os.path.join(settings.STATIC_ROOT, 'png/icon.16.png'), 'content_type': 'image/x-icon' }), - url(r'^opensearch.xml$', app.views.opensearch_xml), - url(r'^oembed$', item.views.oembed), - url(r'^atom.xml$', item.views.atom_xml), - url(r'^robots.txt$', app.views.robots_txt), - url(r'^sitemap.xml$', item.views.sitemap_xml), - url(r'^sitemap(?P\d+).xml$', item.views.sitemap_part_xml), - url(r'', include(item.urls)), + re_path(r'^opensearch.xml$', app.views.opensearch_xml), + re_path(r'^oembed$', item.views.oembed), + re_path(r'^atom.xml$', item.views.atom_xml), + re_path(r'^robots.txt$', app.views.robots_txt), + re_path(r'^sitemap.xml$', item.views.sitemap_xml), + re_path(r'^sitemap(?P\d+).xml$', item.views.sitemap_part_xml), + path(r'', item.site.urls), ] #sould this not be enabled by default? nginx should handle those if settings.DEBUG: urlpatterns += [ - url(r'^data/(?P.*)$', django.views.static.serve, + re_path(r'^data/(?P.*)$', django.views.static.serve, {'document_root': settings.MEDIA_ROOT}), - url(r'^static/(?P.*)$', django.views.static.serve, + re_path(r'^static/(?P.*)$', django.views.static.serve, {'document_root': settings.STATIC_ROOT}), ] urlpatterns += [ - url(r'^(V[a-z0-9]+)$', urlalias.views.padma_video), - url(r'^(V[a-z0-9]+/.*)$', urlalias.views.padma_video), - url(r'^find$', urlalias.views.padma_find), + re_path(r'^(V[a-z0-9]+)$', urlalias.views.padma_video), + re_path(r'^(V[a-z0-9]+/.*)$', urlalias.views.padma_video), + re_path(r'^find$', urlalias.views.padma_find), ] urlpatterns += [ - url(r'^(?P[A-Z0-9x]+)/embed', app.views.embed), - url(r'^(?P[A-Z0-9x]+).*', item.views.item), - url(r'^[a-z0-9].+$', app.views.index), - url(r'^$', app.views.index), - url(r'^.*$', app.views.index), + re_path(r'^(?P[A-Z0-9x]+)/embed', app.views.embed), + re_path(r'^(?P[A-Z0-9x]+).*', item.views.item), + re_path(r'^[a-z0-9].+$', app.views.index), + re_path(r'^.*$', app.views.index), + path(r'', app.views.index), ] diff --git a/pandora/user/decorators.py b/pandora/user/decorators.py index f53a7b1f..1d816bce 100644 --- a/pandora/user/decorators.py +++ b/pandora/user/decorators.py @@ -8,7 +8,7 @@ from oxdjango.shortcuts import render_to_json_response, json_response def capability_required_json(capability): def capability_required(function=None): def _wrapped_view(request, *args, **kwargs): - if request.user.is_authenticated() and \ + if request.user.is_authenticated and \ request.user.profile.capability(capability): return function(request, *args, **kwargs) return render_to_json_response(json_response(status=403, text='permission denied')) diff --git a/pandora/user/middleware.py b/pandora/user/middleware.py index 8c1c35d6..36291497 100644 --- a/pandora/user/middleware.py +++ b/pandora/user/middleware.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- from django.conf import settings from django.contrib.sessions.models import Session +from django.utils.deprecation import MiddlewareMixin + +class UpdateSession(MiddlewareMixin): -class UpdateSession(object): def process_request(self, request): - if request.user.is_authenticated(): + if request.user.is_authenticated: expire_date = Session.objects.get(session_key=request.session.session_key).expire_date if (request.session.get_expiry_date() - expire_date).total_seconds() > settings.SESSION_UPDATE: request.session.modified = True diff --git a/pandora/user/models.py b/pandora/user/models.py index 0c94beb7..fba06561 100644 --- a/pandora/user/models.py +++ b/pandora/user/models.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import copy from datetime import datetime @@ -9,7 +8,6 @@ from django.contrib.auth import get_user_model from django.db import models from django.db.models import Max from django.conf import settings -from django.utils.encoding import python_2_unicode_compatible from oxdjango.fields import JSONField import ox @@ -21,10 +19,9 @@ from .utils import get_ip, get_location User = get_user_model() -@python_2_unicode_compatible class SessionData(models.Model): session_key = models.CharField(max_length=40, primary_key=True) - user = models.OneToOneField(User, null=True, blank=True, related_name='data') + user = models.OneToOneField(User, null=True, blank=True, related_name='data', on_delete=models.CASCADE) firstseen = models.DateTimeField(auto_now_add=True, db_index=True) lastseen = models.DateTimeField(default=datetime.now, db_index=True) username = models.CharField(max_length=255, null=True, db_index=True) @@ -50,7 +47,7 @@ class SessionData(models.Model): groupssort = models.CharField(default=None, blank=True, null=True, max_length=255) def __str__(self): - return u"%s" % self.session_key + return "%s" % self.session_key def parse_useragent(self): if self.useragent: @@ -69,8 +66,8 @@ class SessionData(models.Model): if self.ip: city, country = get_location(self.ip) if city: - location = u'%s, %s' % (city, country) - location_sort = u'%s, %s' % (country, city) + location = '%s, %s' % (city, country) + location_sort = '%s, %s' % (country, city) else: location = location_sort = country self.location = location @@ -98,10 +95,10 @@ class SessionData(models.Model): request.session.modified = True session_key = request.session.session_key assert session_key - if request.user.is_authenticated(): + if request.user.is_authenticated: cls.objects.filter(user=request.user).update(session_key=session_key) data, created = cls.objects.get_or_create(session_key=session_key) - if request.user.is_authenticated(): + if request.user.is_authenticated: data.user = request.user data.ip = get_ip(request) data.useragent = request.META.get('HTTP_USER_AGENT', '')[:4096] @@ -115,10 +112,10 @@ class SessionData(models.Model): ] screen = data.info.get('screen', {}) if screen and 'height' in screen and 'width' in screen: - data.screensize = u'%s\xd7%s' % (screen['width'], screen['height']) + data.screensize = '%s\xd7%s' % (screen['width'], screen['height']) window = data.info.get('window', {}) if window and 'outerHeight' in window and 'outerWidth' in window: - data.windowsize = u'%s\xd7%s' % (window['outerWidth'], window['outerHeight']) + data.windowsize = '%s\xd7%s' % (window['outerWidth'], window['outerHeight']) if not data.timesseen: data.timesseen = 0 data.timesseen += 1 @@ -172,10 +169,9 @@ class SessionData(models.Model): del j[key] return j -@python_2_unicode_compatible class UserProfile(models.Model): reset_code = models.CharField(max_length=255, blank=True, null=True, unique=True) - user = models.OneToOneField(User, related_name='profile') + user = models.OneToOneField(User, related_name='profile', on_delete=models.CASCADE) level = models.IntegerField(default=1) files_updated = models.DateTimeField(default=datetime.now) @@ -395,7 +391,7 @@ def get_ui(user_ui, user=None): def init_user(user, request=None): if request: SessionData.get_or_create(request) - if user.is_anonymous(): + if user.is_anonymous: result = settings.CONFIG['user'].copy() result['ui'] = get_ui(json.loads(request.session.get('ui', '{}'))) else: @@ -434,7 +430,7 @@ def user_json(user, keys=None): return j def has_capability(user, capability): - if user.is_anonymous(): + if user.is_anonymous: level = 'guest' else: level = user.profile.get_level() diff --git a/pandora/user/tasks.py b/pandora/user/tasks.py index f54eb190..b067d58f 100644 --- a/pandora/user/tasks.py +++ b/pandora/user/tasks.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import json from datetime import timedelta diff --git a/pandora/user/views.py b/pandora/user/views.py index 88dc2276..aa452cec 100644 --- a/pandora/user/views.py +++ b/pandora/user/views.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, print_function, absolute_import import random random.seed() @@ -118,7 +117,7 @@ def signout(request, data): see: signin, signup ''' response = json_response(text='ok') - if request.user.is_authenticated(): + if request.user.is_authenticated: uid = request.user.id profile = request.user.profile if profile.ui.get('page') == 'signout': @@ -608,7 +607,7 @@ def mail(request, data): 'url': request.build_absolute_uri('/'), } message = template.render(context, request) - subject = u'Fwd: %s' % subject + subject = 'Fwd: %s' % subject email_to = '"%s" <%s>' % (request.user.username, request.user.email) receipt = EmailMessage(subject, message, @@ -634,7 +633,7 @@ def contact(request, data): ''' name = data.get('name', '') email = data.get('email', '') - if request.user.is_authenticated(): + if request.user.is_authenticated: if not name: name = request.user.username if not email: @@ -657,10 +656,10 @@ def contact(request, data): message = ox.decode_html(template.render(context, request)) response = json_response(text='message sent') try: - send_mail(u'%s Contact - %s' % (settings.SITENAME, subject), message, email_from, email_to) + send_mail('%s Contact - %s' % (settings.SITENAME, subject), message, email_from, email_to) except BadHeaderError: response = json_response(status=400, text='invalid data') - if request.user.is_authenticated() \ + if request.user.is_authenticated \ and 'receipt' in data \ and data['receipt']: template = loader.get_template('contact_receipt.txt') @@ -735,7 +734,7 @@ actions.register(editPreferences, cache=False) def reset_ui(request): - if request.user.is_authenticated(): + if request.user.is_authenticated: profile = request.user.profile profile.ui = {} profile.save() @@ -752,7 +751,7 @@ def resetUI(request, data): see: setUI ''' response = json_response() - if request.user.is_authenticated(): + if request.user.is_authenticated: profile = request.user.profile profile.ui = {} profile.save() @@ -773,7 +772,7 @@ def setUI(request, data): notes: To set nested keys, use {'foo.bar.baz': value} see: resetUI ''' - if request.user.is_authenticated(): + if request.user.is_authenticated: profile = request.user.profile ui = profile.ui else: @@ -794,7 +793,7 @@ def setUI(request, data): del p[keys[0]] else: p[keys[0]] = value - if request.user.is_authenticated(): + if request.user.is_authenticated: profile.save() else: request.session['ui'] = json.dumps(ui) @@ -802,7 +801,7 @@ def setUI(request, data): if data.get('item'): item = get_object_or_404_json(Item, public_id=data['item']) with transaction.atomic(): - if request.user.is_authenticated(): + if request.user.is_authenticated: access, created = Access.objects.get_or_create(item=item, user=request.user) else: access, created = Access.objects.get_or_create(item=item, user=None) @@ -919,7 +918,7 @@ def addGroup(request, data): while not created: g, created = Group.objects.get_or_create(name=name) n += 1 - name = u'%s [%d]' % (_name, n) + name = '%s [%d]' % (_name, n) response['data'] = group_json(g) add_changelog(request, data, g.name) return render_to_json_response(response) @@ -950,7 +949,7 @@ def editGroup(request, data): _name = re.sub(' \[\d+\]$', '', name).strip() while Group.objects.filter(name=name).count(): n += 1 - name = u'%s [%d]' % (_name, n) + name = '%s [%d]' % (_name, n) g.name = name g.save() add_changelog(request, data, g.name) diff --git a/requirements.txt b/requirements.txt index 426726e2..436f7030 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,14 @@ -Django==1.11.28 +Django==3.0.6 simplejson chardet -celery>4 +celery>4.3.0 django-celery-results -django-extensions==2.0.7 -gunicorn==19.8.1 +django-extensions==2.2.9 +gunicorn==20.0.4 html5lib -requests==2.22.0 +requests==2.23.0 tornado<5 -geoip2==2.9.0 -youtube-dl>=2019.4.30 +geoip2==3.0.0 +youtube-dl>=2020.5.8 python-memcached elasticsearch diff --git a/scripts/poster.0xdb.py b/scripts/poster.0xdb.py index 928d5e0d..bd05982d 100755 --- a/scripts/poster.0xdb.py +++ b/scripts/poster.0xdb.py @@ -30,8 +30,8 @@ def get_frame(id, height, position): def render_poster(data, poster): title = ox.decode_html(data.get('title', '')).upper() - director = ox.decode_html(u', '.join(data.get('director', []))).upper() - for key, value in {u'\u03a0': 'PI', u'ß': u'SS'}.items(): + director = ox.decode_html(', '.join(data.get('director', []))).upper() + for key, value in {'\u03a0': 'PI', 'ß': 'SS'}.items(): title = title.replace(key, value) director = director.replace(key, value) year = str(data.get('year', '')) diff --git a/scripts/poster.indiancinema.py b/scripts/poster.indiancinema.py index 392fb50b..194700d5 100755 --- a/scripts/poster.indiancinema.py +++ b/scripts/poster.indiancinema.py @@ -17,7 +17,7 @@ static_root = os.path.join(os.path.dirname(__file__), 'data') def render_poster(data, poster): title = ox.decode_html(data.get('title', '')) - director = ox.decode_html(u', '.join(data.get('director', []))) + director = ox.decode_html(', '.join(data.get('director', []))) year = str(data.get('year', '')) frame = data.get('frame') timeline = data.get('timeline') diff --git a/scripts/poster.pandora.py b/scripts/poster.pandora.py index fa34e09e..6177efd8 100755 --- a/scripts/poster.pandora.py +++ b/scripts/poster.pandora.py @@ -17,7 +17,7 @@ static_root = os.path.join(os.path.dirname(__file__), 'data') def render_poster(data, poster): title = ox.decode_html(data.get('title', '')) - director = ox.decode_html(u', '.join(data.get('director', []))) + director = ox.decode_html(', '.join(data.get('director', []))) year = str(data.get('year', '')) series = data.get('isSeries', False) oxdb_id = data['oxdbId'] diff --git a/update.py b/update.py index 6b32fc67..ef802934 100755 --- a/update.py +++ b/update.py @@ -275,6 +275,9 @@ if __name__ == "__main__": ] with open('pandora/local_settings.py', 'w') as f: f.write('\n'.join(local_settings)) + if old <= 6313: + run('./bin/pip', 'uninstall', 'django-celery', '-y') + run('./bin/pip', 'install', '-r', 'requirements.txt') else: if len(sys.argv) == 1: branch = get_branch() From 548a73f121742795344d3c40b9ce885e3a795d44 Mon Sep 17 00:00:00 2001 From: j Date: Fri, 29 May 2020 12:17:18 +0200 Subject: [PATCH 13/37] python3 only: remove six.moves imports --- pandora/annotation/managers.py | 3 +-- pandora/app/config.py | 2 +- pandora/app/views.py | 3 +-- pandora/archive/extract.py | 3 +-- pandora/archive/models.py | 10 +++------- pandora/archive/tasks.py | 3 +-- pandora/archive/views.py | 3 +-- pandora/clip/managers.py | 5 ++--- pandora/document/managers.py | 3 +-- pandora/document/models.py | 22 +++++++++------------- pandora/document/views.py | 5 ++--- pandora/edit/models.py | 8 ++++---- pandora/entity/models.py | 9 ++++----- pandora/entity/views.py | 3 +-- pandora/event/managers.py | 3 +-- pandora/event/views.py | 5 ++--- pandora/home/models.py | 5 ++--- pandora/item/managers.py | 3 +-- pandora/item/models.py | 22 +++++++++------------- pandora/item/tasks.py | 4 ++-- pandora/item/utils.py | 7 +------ pandora/item/views.py | 17 +++++++---------- pandora/oxdjango/api/actions.py | 9 ++------- pandora/oxdjango/fields.py | 6 ++---- pandora/oxdjango/http.py | 6 +++--- pandora/person/managers.py | 3 +-- pandora/place/managers.py | 3 +-- pandora/place/views.py | 5 ++--- pandora/sequence/tasks.py | 3 +-- pandora/text/models.py | 4 ++-- pandora/title/managers.py | 3 +-- pandora/urlalias/views.py | 2 +- pandora/user/tasks.py | 4 ++-- 33 files changed, 75 insertions(+), 121 deletions(-) diff --git a/pandora/annotation/managers.py b/pandora/annotation/managers.py index 3b6ee2fe..e243cd16 100644 --- a/pandora/annotation/managers.py +++ b/pandora/annotation/managers.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import unicodedata -from six import string_types from django.db.models import Q, Manager from oxdjango.query import QuerySet @@ -68,7 +67,7 @@ def parseCondition(condition, user): else: key = k + get_operator(op, 'istr' if k in case_insensitive_keys else 'str') key = str(key) - if isinstance(v, string_types): + if isinstance(v, str): v = unicodedata.normalize('NFKD', v) if k not in case_sensitive_keys: v = v.lower() diff --git a/pandora/app/config.py b/pandora/app/config.py index 93714552..3bb7cc72 100644 --- a/pandora/app/config.py +++ b/pandora/app/config.py @@ -9,8 +9,8 @@ import sys import time from os.path import dirname, exists, join from glob import glob +import _thread as thread -from six.moves import _thread as thread from django.conf import settings from django.contrib.auth import get_user_model diff --git a/pandora/app/views.py b/pandora/app/views.py index 1ebc219c..5c76b8ec 100644 --- a/pandora/app/views.py +++ b/pandora/app/views.py @@ -3,7 +3,6 @@ import copy from datetime import datetime -from six import string_types from django.shortcuts import render, redirect from django.conf import settings from django.http import HttpResponse @@ -113,7 +112,7 @@ def getPage(request, data): } see: editPage ''' - if isinstance(data, string_types): + if isinstance(data, str): name = data else: name = data['name'] diff --git a/pandora/archive/extract.py b/pandora/archive/extract.py index 875759d4..dcb9bdcf 100644 --- a/pandora/archive/extract.py +++ b/pandora/archive/extract.py @@ -12,7 +12,6 @@ import shutil from distutils.spawn import find_executable from glob import glob -from six import string_types import numpy as np import ox import ox.image @@ -463,7 +462,7 @@ def timeline(video, prefix, modes=None, size=None): modes = ['antialias', 'slitscan', 'keyframes', 'audio', 'data'] if size is None: size = [64, 16] - if isinstance(video, string_types): + if isinstance(video, str): video = [video] cmd = ['../bin/oxtimelines', '-s', ','.join(map(str, reversed(sorted(size)))), diff --git a/pandora/archive/models.py b/pandora/archive/models.py index 391393ca..05882c40 100644 --- a/pandora/archive/models.py +++ b/pandora/archive/models.py @@ -6,7 +6,6 @@ import shutil import tempfile import time -from six import string_types, PY2 from django.conf import settings from django.contrib.auth import get_user_model from django.db import models @@ -28,9 +27,6 @@ from . import managers User = get_user_model() -if not PY2: - unicode = str - def data_path(f, x): return f.get_path('data.bin') @@ -165,7 +161,7 @@ class File(models.Model): if self.item: for key in self.ITEM_INFO: data[key] = self.item.get(key) - if isinstance(data[key], string_types): + if isinstance(data[key], str): data[key] = ox.decode_html(data[key]) elif isinstance(data[key], list): data[key] = [ox.decode_html(e) for e in data[key]] @@ -259,8 +255,8 @@ class File(models.Model): data = self.get_path_info() self.extension = data.get('extension') self.language = data.get('language') - self.part = ox.sort_string(unicode(data.get('part') or '')) - self.part_title = ox.sort_string(unicode(data.get('partTitle')) or '') + self.part = ox.sort_string(str(data.get('part') or '')) + self.part_title = ox.sort_string(str(data.get('partTitle')) or '') self.type = data.get('type') or 'unknown' self.version = data.get('version') diff --git a/pandora/archive/tasks.py b/pandora/archive/tasks.py index 8013d1ff..022cd377 100644 --- a/pandora/archive/tasks.py +++ b/pandora/archive/tasks.py @@ -2,7 +2,6 @@ from glob import glob -from six import string_types from celery.task import task from django.conf import settings from django.db.models import Q @@ -219,7 +218,7 @@ def move_media(data, user): data['public_id'] = data.pop('item').strip() if not is_imdb_id(data['public_id']): del data['public_id'] - if 'director' in data and isinstance(data['director'], string_types): + if 'director' in data and isinstance(data['director'], str): if data['director'] == '': data['director'] = [] else: diff --git a/pandora/archive/views.py b/pandora/archive/views.py index c7d80ff0..848bfc7e 100644 --- a/pandora/archive/views.py +++ b/pandora/archive/views.py @@ -7,7 +7,6 @@ from django.shortcuts import get_object_or_404, redirect, render from django.conf import settings from django.db.models import Count, Q -from six import string_types from celery.utils import get_full_cls_name from celery._state import current_app import ox @@ -555,7 +554,7 @@ def getPath(request, data): ''' response = json_response() ids = data['id'] - if isinstance(ids, string_types): + if isinstance(ids, str): ids = [ids] for f in models.File.objects.filter(oshash__in=ids).values('path', 'oshash').order_by('sort_path'): response['data'][f['oshash']] = f['path'] diff --git a/pandora/clip/managers.py b/pandora/clip/managers.py index 52f18e5d..eb7c6ca9 100644 --- a/pandora/clip/managers.py +++ b/pandora/clip/managers.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import unicodedata -from six import string_types from django.db.models import Q, Manager from django.conf import settings @@ -78,7 +77,7 @@ def parseCondition(condition, user): else: key = k + get_operator(op, 'istr' if k in case_insensitive_keys else 'str') key = str(key) - if isinstance(v, string_types) and op != '===': + if isinstance(v, str) and op != '===': v = unicodedata.normalize('NFKD', v).lower() if exclude: q = ~Q(**{key: v}) @@ -155,7 +154,7 @@ class ClipManager(Manager): def parse(condition): key = 'findvalue' + get_operator(condition.get('operator', '')) v = condition['value'] - if isinstance(v, string_types): + if isinstance(v, str): v = unicodedata.normalize('NFKD', v).lower() q = Q(**{key: v}) if condition['key'] in layer_ids: diff --git a/pandora/document/managers.py b/pandora/document/managers.py index fb862f79..4210cd62 100644 --- a/pandora/document/managers.py +++ b/pandora/document/managers.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import unicodedata -from six import string_types from django.db.models import Q, Manager from django.conf import settings @@ -152,7 +151,7 @@ def buildCondition(k, op, v, user, exclude=False, owner=None): value_key = 'find__value' else: value_key = k - if isinstance(v, string_types): + if isinstance(v, str): v = unicodedata.normalize('NFKD', v).lower() if k in facet_keys: in_find = False diff --git a/pandora/document/models.py b/pandora/document/models.py index a5a4f6b4..e2501244 100644 --- a/pandora/document/models.py +++ b/pandora/document/models.py @@ -1,12 +1,11 @@ # -*- coding: utf-8 -*- +from glob import glob +from urllib.parse import quote, unquote import os import re -from glob import glob import unicodedata -from six import PY2, string_types -from six.moves.urllib.parse import quote, unquote from django.db import models, transaction from django.db.models import Q, Sum, Max from django.contrib.auth import get_user_model @@ -34,9 +33,6 @@ from .fulltext import FulltextMixin User = get_user_model() -if not PY2: - unicode = str - def get_path(f, x): return f.path(x) @@ -88,7 +84,7 @@ class Document(models.Model, FulltextMixin): if not current_values: current_values = [] else: - current_values = [unicode(current_values)] + current_values = [str(current_values)] filter_map = utils.get_by_id(settings.CONFIG['documentKeys'], key).get('filterMap') if filter_map: @@ -139,7 +135,7 @@ class Document(models.Model, FulltextMixin): f, created = Find.objects.get_or_create(document=self, key=key) if isinstance(value, bool): value = value and 'true' or 'false' - if isinstance(value, string_types): + if isinstance(value, str): value = ox.decode_html(ox.strip_tags(value.strip())) value = unicodedata.normalize('NFKD', value).lower() f.value = value @@ -197,7 +193,7 @@ class Document(models.Model, FulltextMixin): return sort_value.lower() def set_value(s, name, value): - if isinstance(value, string_types): + if isinstance(value, str): value = ox.decode_html(value.lower()) if not value: value = None @@ -259,7 +255,7 @@ class Document(models.Model, FulltextMixin): set_value(s, name, value) elif sort_type == 'date': value = self.get_value(source) - if isinstance(value, string_types): + if isinstance(value, str): value = datetime_safe.datetime.strptime(value, '%Y-%m-%d') set_value(s, name, value) s.save() @@ -370,11 +366,11 @@ class Document(models.Model, FulltextMixin): self.data[key] = [ox.sanitize_html(t) for t in data[key]] elif ktype == '[string]': self.data[key] = [ox.escape_html(t) for t in data[key]] - elif isinstance(data[key], string_types): + elif isinstance(data[key], str): self.data[key] = ox.escape_html(data[key]) elif isinstance(data[key], list): def cleanup(i): - if isinstance(i, string_types): + if isinstance(i, str): i = ox.escape_html(i) return i self.data[key] = [cleanup(i) for i in data[key]] @@ -480,7 +476,7 @@ class Document(models.Model, FulltextMixin): if self.extension == 'html': response['text'] = self.data.get('text', '') if item: - if isinstance(item, string_types): + if isinstance(item, str): item = Item.objects.get(public_id=item) d = self.descriptions.filter(item=item) if d.exists(): diff --git a/pandora/document/views.py b/pandora/document/views.py index 0798913a..84279ac9 100644 --- a/pandora/document/views.py +++ b/pandora/document/views.py @@ -5,7 +5,6 @@ import re from glob import glob import unicodedata -from six import string_types import ox from ox.utils import json from oxdjango.api import actions @@ -70,7 +69,7 @@ def addDocument(request, data): else: ids = [data['id']] if 'item' in data: - if isinstance(data['item'], string_types): + if isinstance(data['item'], str): item = Item.objects.get(public_id=data['item']) if item.editable(request.user): for id in ids: @@ -87,7 +86,7 @@ def addDocument(request, data): document.add(item) add_changelog(request, data, data['item']) elif 'entity' in data: - if isinstance(data['entity'], string_types): + if isinstance(data['entity'], str): entity = Entity.get(data['entity']) if entity.editable(request.user): for id in ids: diff --git a/pandora/edit/models.py b/pandora/edit/models.py index 9d22247d..d71525a0 100644 --- a/pandora/edit/models.py +++ b/pandora/edit/models.py @@ -1,13 +1,13 @@ # -*- coding: utf-8 -*- -import re -import os -import shutil from glob import glob +from urllib.parse import quote +import os +import re +import shutil import subprocess import tempfile -from six.moves.urllib.parse import quote import ox from django.conf import settings from django.db import models, transaction diff --git a/pandora/entity/models.py b/pandora/entity/models.py index c39baccb..e97a6963 100644 --- a/pandora/entity/models.py +++ b/pandora/entity/models.py @@ -1,12 +1,11 @@ # -*- coding: utf-8 -*- +from glob import glob +from urllib.parse import quote, unquote import os import re -from glob import glob import unicodedata -from six import string_types -from six.moves.urllib.parse import quote, unquote from django.db import models, transaction from django.db.models import Max from django.contrib.auth import get_user_model @@ -188,7 +187,7 @@ class Entity(models.Model): .delete() else: #FIXME: more data validation - if isinstance(data[key], string_types): + if isinstance(data[key], str): self.data[key] = ox.sanitize_html(data[key]) else: self.data[key] = data[key] @@ -275,7 +274,7 @@ class Entity(models.Model): f, created = Find.objects.get_or_create(entity=self, key=key) if isinstance(value, bool): value = value and 'true' or 'false' - if isinstance(value, string_types): + if isinstance(value, str): value = ox.decode_html(ox.strip_tags(value.strip())) value = unicodedata.normalize('NFKD', value).lower() f.value = value diff --git a/pandora/entity/views.py b/pandora/entity/views.py index 483bf1a9..b1e2d7e2 100644 --- a/pandora/entity/views.py +++ b/pandora/entity/views.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from six import string_types import ox from ox.utils import json from oxdjango.api import actions @@ -69,7 +68,7 @@ def addEntity(request, data): for key in ('type', 'alternativeNames'): if key in data and data[key]: value = data[key] - if isinstance(value, string_types): + if isinstance(value, str): value = ox.escape_html(value) if key == 'alternativeNames': value = tuple([ox.escape_html(v) for v in value]) diff --git a/pandora/event/managers.py b/pandora/event/managers.py index 8dcec385..aed5533c 100644 --- a/pandora/event/managers.py +++ b/pandora/event/managers.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import unicodedata -from six import string_types from django.db.models import Q, Manager from oxdjango.query import QuerySet @@ -30,7 +29,7 @@ def parseCondition(condition, user): key = k + get_operator(op, 'istr') key = str(key) - if isinstance(v, string_types): + if isinstance(v, str): v = unicodedata.normalize('NFKD', v).lower() if exclude: q = ~Q(**{k: v}) diff --git a/pandora/event/views.py b/pandora/event/views.py index d4d63e6d..a535619f 100644 --- a/pandora/event/views.py +++ b/pandora/event/views.py @@ -3,7 +3,6 @@ from django.db.models import Count from django.conf import settings -from six import string_types import ox from ox.utils import json @@ -47,7 +46,7 @@ def addEvent(request, data): 'type', 'alternativeNames'): if key in data and data[key]: value = data[key] - if isinstance(value, string_types): + if isinstance(value, str): value = ox.escape_html(value) if key == 'alternativeNames': value = tuple([ox.escape_html(v) for v in value]) @@ -101,7 +100,7 @@ def editEvent(request, data): 'type', 'alternativeNames'): if key in data: value = data[key] - if isinstance(value, string_types): + if isinstance(value, str): value = ox.escape_html(value) if key == 'alternativeNames': value = tuple([ox.escape_html(v) for v in value]) diff --git a/pandora/home/models.py b/pandora/home/models.py index 49b08f73..72a06c03 100644 --- a/pandora/home/models.py +++ b/pandora/home/models.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- -from six import string_types -from six.moves.urllib.parse import quote +from urllib.parse import quote from django.db import models from django.db.models import Max @@ -43,7 +42,7 @@ class Item(models.Model): len([d for d in data[key] if isinstance(d, int)]) == 4): return False else: - if not isinstance(data[key], string_types): + if not isinstance(data[key], str): return False self.data[key] = data[key] if key == 'contentid' and self.data[key]: diff --git a/pandora/item/managers.py b/pandora/item/managers.py index bcb1d8cc..654f1dfe 100644 --- a/pandora/item/managers.py +++ b/pandora/item/managers.py @@ -3,7 +3,6 @@ from datetime import datetime import unicodedata -from six import string_types from django.db.models import Q, Manager from django.conf import settings @@ -123,7 +122,7 @@ def parseCondition(condition, user, owner=None): else: value_key = k if not k.startswith('public_id'): - if isinstance(v, string_types): + if isinstance(v, str): v = unicodedata.normalize('NFKD', v).lower() if k in facet_keys: in_find = False diff --git a/pandora/item/models.py b/pandora/item/models.py index 0519584d..1733e896 100644 --- a/pandora/item/models.py +++ b/pandora/item/models.py @@ -10,9 +10,7 @@ import unicodedata import uuid from datetime import datetime from glob import glob - -from six import PY2, string_types -from six.moves.urllib.parse import quote +from urllib.parse import quote from django.conf import settings from django.contrib.auth import get_user_model @@ -46,8 +44,6 @@ import archive.models User = get_user_model() -if not PY2: - unicode = str def get_id(info): q = Item.objects.all() @@ -284,11 +280,11 @@ class Item(models.Model): self.data[key] = [ox.escape_html(t) for t in data[key]] elif key in ('episodeTitle', 'seriesTitle', 'episodeDirector', 'seriesYear'): self.data[key] = ox.escape_html(data[key]) - elif isinstance(data[key], string_types): + elif isinstance(data[key], str): self.data[key] = ox.escape_html(data[key]) elif isinstance(data[key], list): def cleanup(i): - if isinstance(i, string_types): + if isinstance(i, str): i = ox.escape_html(i) return i self.data[key] = [cleanup(i) for i in data[key]] @@ -329,7 +325,7 @@ class Item(models.Model): if c: for t in list(c): if c[t]: - if isinstance(c[t][0], string_types): + if isinstance(c[t][0], str): c[t] = [{'id': i, 'title': None} for i in c[t]] ids = [i['id'] for i in c[t]] known = {} @@ -626,7 +622,7 @@ class Item(models.Model): if value: i[key] = value - if 'cast' in i and isinstance(i['cast'][0], string_types): + if 'cast' in i and isinstance(i['cast'][0], str): i['cast'] = [i['cast']] if 'cast' in i and isinstance(i['cast'][0], list): i['cast'] = [{'actor': x[0], 'character': x[1]} for x in i['cast']] @@ -797,7 +793,7 @@ class Item(models.Model): f, created = ItemFind.objects.get_or_create(item=self, key=key) if isinstance(value, bool): value = value and 'true' or 'false' - if isinstance(value, string_types): + if isinstance(value, str): value = ox.decode_html(ox.strip_tags(value.strip())) value = unicodedata.normalize('NFKD', value).lower() f.value = value @@ -916,7 +912,7 @@ class Item(models.Model): return sort_value.lower() def set_value(s, name, value): - if isinstance(value, string_types): + if isinstance(value, str): value = ox.decode_html(value.lower()) if not value: value = None @@ -1052,7 +1048,7 @@ class Item(models.Model): set_value(s, name, value) elif sort_type == 'date': value = value_ = self.get(source) - if isinstance(value, string_types): + if isinstance(value, str): value_ = None for fmt in ('%Y-%m-%d', '%Y-%m', '%Y'): try: @@ -1091,7 +1087,7 @@ class Item(models.Model): if not current_values: current_values = [] else: - current_values = [unicode(current_values)] + current_values = [str(current_values)] filter_map = utils.get_by_id(settings.CONFIG['itemKeys'], key).get('filterMap') if filter_map: diff --git a/pandora/item/tasks.py b/pandora/item/tasks.py index 667e2867..27511acc 100644 --- a/pandora/item/tasks.py +++ b/pandora/item/tasks.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- -import os from datetime import timedelta, datetime +from urllib.parse import quote import gzip +import os import random -from six.moves.urllib.parse import quote from celery.task import task, periodic_task from django.conf import settings from django.db import connection, transaction diff --git a/pandora/item/utils.py b/pandora/item/utils.py index 24db30a3..fce7d724 100644 --- a/pandora/item/utils.py +++ b/pandora/item/utils.py @@ -6,7 +6,6 @@ import unicodedata import ox from ox import sort_string -from six import PY2 def safe_filename(filename): @@ -92,11 +91,7 @@ def get_by_id(objects, id): return get_by_key(objects, 'id', id) def normalize_dict(encoding, data): - if PY2: - string_type = unicode - else: - string_type = str - if isinstance(data, string_type): + if isinstance(data, str): data = unicodedata.normalize(encoding, data) elif isinstance(data, dict): for key in data: diff --git a/pandora/item/views.py b/pandora/item/views.py index 6092aff4..23fadc29 100644 --- a/pandora/item/views.py +++ b/pandora/item/views.py @@ -1,13 +1,12 @@ # -*- coding: utf-8 -*- -import os.path +from datetime import datetime, timedelta +from urllib.parse import quote, urlparse import mimetypes +import os.path import random import time -from datetime import datetime, timedelta -from six import PY2 -from six.moves.urllib.parse import quote, urlparse from PIL import Image from django.db.models import Count, Sum from django.http import HttpResponse, HttpResponseForbidden, Http404 @@ -35,8 +34,6 @@ from changelog.models import add_changelog from oxdjango.api import actions -if not PY2: - unicode = str def _order_query(qs, sort, prefix='sort__'): order_by = [] @@ -1280,7 +1277,7 @@ def atom_xml(request): }.get(key, key)) if value and value != -1: el = ET.SubElement(format, key) - el.text = unicode(value) + el.text = str(value) el = ET.SubElement(format, 'pixel_aspect_ratio') el.text = "1:1" @@ -1357,7 +1354,7 @@ def oembed(request): oxml = ET.Element('oembed') for key in oembed: e = ET.SubElement(oxml, key) - e.text = unicode(oembed[key]) + e.text = str(oembed[key]) return HttpResponse( '\n' + ET.tostring(oxml).decode(), 'application/xml' @@ -1423,7 +1420,7 @@ def item_xml(request, id): xmltree(root, k, data[k]) else: e = ET.SubElement(root, key) - e.text = unicode(data) + e.text = str(data) oxml = ET.Element('item') xmltree(oxml, 'item', j) @@ -1483,7 +1480,7 @@ def item(request, id): else: title = key['title'] if key else k.capitalize() if isinstance(value, list): - value = value = ', '.join([unicode(v) for v in value]) + value = value = ', '.join([str(v) for v in value]) elif key and key.get('type') == 'float': value = '%0.3f' % value elif key and key.get('type') == 'time': diff --git a/pandora/oxdjango/api/actions.py b/pandora/oxdjango/api/actions.py index 95e1fe81..f8a91d3a 100644 --- a/pandora/oxdjango/api/actions.py +++ b/pandora/oxdjango/api/actions.py @@ -4,7 +4,6 @@ from __future__ import division, absolute_import import inspect import sys -from six import PY2 from django.conf import settings from ..shortcuts import render_to_json_response, json_response @@ -109,12 +108,8 @@ class ApiActions(dict): if name != 'api' and hasattr(f, 'func_closure') and f.func_closure: fc = list(filter(lambda c: hasattr(c.cell_contents, '__call__'), f.func_closure)) f = fc[len(fc)-1].cell_contents - if PY2: - info = f.func_code.co_filename[len(settings.PROJECT_ROOT)+1:] - info = '%s:%s' % (info, f.func_code.co_firstlineno) - else: - info = f.__code__.co_filename[len(settings.PROJECT_ROOT)+1:] - info = '%s:%s' % (info, f.__code__.co_firstlineno) + info = f.__code__.co_filename[len(settings.PROJECT_ROOT)+1:] + info = '%s:%s' % (info, f.__code__.co_firstlineno) return info, trim(inspect.getsource(f)) def register(self, method, action=None, cache=True, version=None): diff --git a/pandora/oxdjango/fields.py b/pandora/oxdjango/fields.py index f3b66321..daebd27c 100644 --- a/pandora/oxdjango/fields.py +++ b/pandora/oxdjango/fields.py @@ -8,8 +8,6 @@ from django.utils import datetime_safe import django.contrib.postgres.fields from django.core.serializers.json import DjangoJSONEncoder -from six import string_types - from ox.utils import json class JSONField(django.contrib.postgres.fields.JSONField): @@ -74,7 +72,7 @@ class DictField(models.TextField): except: raise Exception('failed to parse value: %s' % value) if value is not None: - if isinstance(value, string_types): + if isinstance(value, str): value = json.loads(value) assert isinstance(value, self._type) return value @@ -83,7 +81,7 @@ class DictField(models.TextField): if isinstance(value, self._type): value = self.dumps(value) if value is not None: - assert isinstance(value, string_types) + assert isinstance(value, str) value = models.TextField.get_prep_value(self, value) return value diff --git a/pandora/oxdjango/http.py b/pandora/oxdjango/http.py index 9bd252d2..38db62ac 100644 --- a/pandora/oxdjango/http.py +++ b/pandora/oxdjango/http.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -import os -import mimetypes from datetime import datetime, timedelta -from six.moves.urllib.parse import quote +from urllib.parse import quote +import mimetypes +import os from django.http import HttpResponse, Http404 from django.conf import settings diff --git a/pandora/person/managers.py b/pandora/person/managers.py index 4c718e6f..90418746 100644 --- a/pandora/person/managers.py +++ b/pandora/person/managers.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import unicodedata -from six import string_types from django.db.models import Q, Manager from item.utils import decode_id @@ -56,7 +55,7 @@ def parseCondition(condition, user): else: key = k + get_operator(op, 'istr') key = str(key) - if isinstance(v, string_types): + if isinstance(v, str): v = unicodedata.normalize('NFKD', v).lower() if exclude: q = ~Q(**{key: v}) diff --git a/pandora/place/managers.py b/pandora/place/managers.py index 75d59b1d..cea3777c 100644 --- a/pandora/place/managers.py +++ b/pandora/place/managers.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import unicodedata -from six import string_types from django.db.models import Q, Manager from item.utils import decode_id @@ -55,7 +54,7 @@ def parseCondition(condition, user): key = k + get_operator(op, 'istr') key = str(key) - if isinstance(v, string_types): + if isinstance(v, str): v = unicodedata.normalize('NFKD', v).lower() if exclude: diff --git a/pandora/place/views.py b/pandora/place/views.py index 662253c7..0c305d53 100644 --- a/pandora/place/views.py +++ b/pandora/place/views.py @@ -3,7 +3,6 @@ from django.db.models import Max, Min, Count from django.conf import settings -from six import string_types import ox from ox.utils import json @@ -115,7 +114,7 @@ def editPlace(request, data): ''' place = get_object_or_404_json(models.Place, pk=ox.fromAZ(data['id'])) names = data.get('name', []) - if isinstance(names, string_types): + if isinstance(names, str): names = [names] names = [ox.escape_html(n) for n in names] alternative_names = [ox.escape_html(n) for n in data.get('alternativeNames', [])] @@ -144,7 +143,7 @@ def editPlace(request, data): for key in data: if key != 'id': value = data[key] - if isinstance(value, string_types): + if isinstance(value, str): value = ox.escape_html(value) if isinstance(value, list): value = tuple(value) diff --git a/pandora/sequence/tasks.py b/pandora/sequence/tasks.py index 1d467f01..b90c0d5a 100644 --- a/pandora/sequence/tasks.py +++ b/pandora/sequence/tasks.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from six import string_types from django.db import connection, transaction from celery.task import task @@ -30,7 +29,7 @@ def get_sequences(public_id): sequence['duration'] = sequence['end'] - sequence['start'] if not keys: keys = ', '.join(['"%s"'%k for k in sequence.keys()]) - v = ', '.join([isinstance(v, string_types) and "'%s'"%v or str(v) + v = ', '.join([isinstance(v, str) and "'%s'"%v or str(v) for v in sequence.values()]) values.append('(%s)'%v) if values: diff --git a/pandora/text/models.py b/pandora/text/models.py index 08bf6fa0..2973ba6d 100644 --- a/pandora/text/models.py +++ b/pandora/text/models.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- +from glob import glob +from urllib.parse import quote import os import re import subprocess -from glob import glob -from six.moves.urllib.parse import quote from django.db import models from django.db.models import Max from django.contrib.auth import get_user_model diff --git a/pandora/title/managers.py b/pandora/title/managers.py index b70703a6..2c6ce6f7 100644 --- a/pandora/title/managers.py +++ b/pandora/title/managers.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import unicodedata -from six import string_types from django.db.models import Q, Manager from item.utils import decode_id @@ -47,7 +46,7 @@ def parseCondition(condition, user): return q if k == 'id': v = decode_id(v) - elif isinstance(v, string_types): + elif isinstance(v, str): v = unicodedata.normalize('NFKD', v).lower() if isinstance(v, bool): key = k diff --git a/pandora/urlalias/views.py b/pandora/urlalias/views.py index 4e1d0989..9ed1a36c 100644 --- a/pandora/urlalias/views.py +++ b/pandora/urlalias/views.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- +from urllib.parse import quote import re -from six.moves.urllib.parse import quote from django.shortcuts import get_object_or_404, redirect import app.views diff --git a/pandora/user/tasks.py b/pandora/user/tasks.py index b067d58f..c1da304c 100644 --- a/pandora/user/tasks.py +++ b/pandora/user/tasks.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- -import json from datetime import timedelta +from itertools import zip_longest +import json -from six.moves import zip_longest from celery.task import task, periodic_task from app.utils import limit_rate From 408788cac1b434f7cf852ec18b3295f4e8d23d5f Mon Sep 17 00:00:00 2001 From: j Date: Sat, 30 May 2020 00:05:50 +0200 Subject: [PATCH 14/37] use custom user table --- pandora/settings.py | 3 ++ pandora/system/__init__.py | 0 pandora/system/admin.py | 6 +++ pandora/system/apps.py | 5 +++ pandora/system/migrations/0001_initial.py | 42 +++++++++++++++++++ .../migrations/0002_rename_user_table.py | 21 ++++++++++ pandora/system/migrations/__init__.py | 0 pandora/system/models.py | 6 +++ pandora/system/tests.py | 3 ++ pandora/system/views.py | 3 ++ update.py | 14 +++++++ 11 files changed, 103 insertions(+) create mode 100644 pandora/system/__init__.py create mode 100644 pandora/system/admin.py create mode 100644 pandora/system/apps.py create mode 100644 pandora/system/migrations/0001_initial.py create mode 100644 pandora/system/migrations/0002_rename_user_table.py create mode 100644 pandora/system/migrations/__init__.py create mode 100644 pandora/system/models.py create mode 100644 pandora/system/tests.py create mode 100644 pandora/system/views.py diff --git a/pandora/settings.py b/pandora/settings.py index aea1eb59..6cb4f347 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -120,6 +120,7 @@ INSTALLED_APPS = ( # Uncomment the next line to enable admin documentation: # 'django.contrib.admindocs', 'django.contrib.humanize', + 'system', 'django_extensions', 'django_celery_results', @@ -151,6 +152,8 @@ INSTALLED_APPS = ( 'home', ) +AUTH_USER_MODEL = 'system.User' + # Log errors into db LOGGING = { 'version': 1, diff --git a/pandora/system/__init__.py b/pandora/system/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pandora/system/admin.py b/pandora/system/admin.py new file mode 100644 index 00000000..6d53b53a --- /dev/null +++ b/pandora/system/admin.py @@ -0,0 +1,6 @@ +from django.contrib import admin +from django.contrib.auth.admin import UserAdmin + +from .models import User + +admin.site.register(User, UserAdmin) diff --git a/pandora/system/apps.py b/pandora/system/apps.py new file mode 100644 index 00000000..5dc4d64b --- /dev/null +++ b/pandora/system/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class SystemConfig(AppConfig): + name = 'system' diff --git a/pandora/system/migrations/0001_initial.py b/pandora/system/migrations/0001_initial.py new file mode 100644 index 00000000..5c0a7b32 --- /dev/null +++ b/pandora/system/migrations/0001_initial.py @@ -0,0 +1,42 @@ +# Generated by Django 3.0.6 on 2020-05-29 21:41 + +import django.contrib.auth.models +import django.contrib.auth.validators +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('auth', '0008_alter_user_username_max_length'), + ] + + operations = [ + migrations.CreateModel( + name='User', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('password', models.CharField(max_length=255, verbose_name='password')), + ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')), + ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')), + ('username', models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.', max_length=255, unique=True, validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], verbose_name='username')), + ('first_name', models.CharField(blank=True, max_length=30, verbose_name='first name')), + ('last_name', models.CharField(blank=True, max_length=150, verbose_name='last name')), + ('email', models.EmailField(blank=True, max_length=255, verbose_name='email address')), + ('is_staff', models.BooleanField(default=False, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')), + ('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')), + ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')), + ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.Group', verbose_name='groups')), + ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.Permission', verbose_name='user permissions')), + ], + options={ + 'db_table': 'auth_user', + }, + managers=[ + ('objects', django.contrib.auth.models.UserManager()), + ], + ), + ] diff --git a/pandora/system/migrations/0002_rename_user_table.py b/pandora/system/migrations/0002_rename_user_table.py new file mode 100644 index 00000000..098c9b67 --- /dev/null +++ b/pandora/system/migrations/0002_rename_user_table.py @@ -0,0 +1,21 @@ +# Generated by Django 3.0.6 on 2020-05-29 21:54 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('system', '0001_initial'), + ] + + operations = [ + migrations.AlterModelOptions( + name='user', + options={'verbose_name': 'user', 'verbose_name_plural': 'users'}, + ), + migrations.AlterModelTable( + name='user', + table=None, + ), + ] diff --git a/pandora/system/migrations/__init__.py b/pandora/system/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pandora/system/models.py b/pandora/system/models.py new file mode 100644 index 00000000..696b770a --- /dev/null +++ b/pandora/system/models.py @@ -0,0 +1,6 @@ +from django.db import models +from django.contrib.auth.models import AbstractUser + + +class User(AbstractUser): + pass diff --git a/pandora/system/tests.py b/pandora/system/tests.py new file mode 100644 index 00000000..7ce503c2 --- /dev/null +++ b/pandora/system/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/pandora/system/views.py b/pandora/system/views.py new file mode 100644 index 00000000..91ea44a2 --- /dev/null +++ b/pandora/system/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/update.py b/update.py index ef802934..e1639f90 100755 --- a/update.py +++ b/update.py @@ -117,6 +117,12 @@ def run_git(path, *args): return subprocess.check_output(cmd, env=env).decode().strip() +def run_sql(sql): + cmd = [join(base, 'pandora/manage.py'), 'dbshell'] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + stdout, error = p.communicate(input=sql.encode()) + return stdout.decode() + def get_version(path): return run_git(path, 'rev-list', 'HEAD', '--count') @@ -278,6 +284,14 @@ if __name__ == "__main__": if old <= 6313: run('./bin/pip', 'uninstall', 'django-celery', '-y') run('./bin/pip', 'install', '-r', 'requirements.txt') + if old <= 6315: + for sql in [ + "INSERT INTO django_migrations (app, name, applied) VALUES ('system', '0001_initial', CURRENT_TIMESTAMP)", + "UPDATE django_content_type SET app_label = 'system' WHERE app_label = 'auth' and model = 'user'", + ]: + run_sql(sql) + run(join(base, 'pandora/manage.py'), 'migrate', 'system') + else: if len(sys.argv) == 1: branch = get_branch() From 361f27631535dae8778a8cbb6ff4f6d0cac30d7d Mon Sep 17 00:00:00 2001 From: j Date: Sat, 30 May 2020 01:50:32 +0200 Subject: [PATCH 15/37] fail with old python --- update.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/update.py b/update.py index e1639f90..669f0981 100755 --- a/update.py +++ b/update.py @@ -282,6 +282,12 @@ if __name__ == "__main__": with open('pandora/local_settings.py', 'w') as f: f.write('\n'.join(local_settings)) if old <= 6313: + if sys.version_info[0:2] < (3, 6): + print('Python 3.6 or late is required now, upgrade your system and run:') + print('') + print('./update.py postupdate %s %s' % (6313, new)) + print('') + sys.exit(1) run('./bin/pip', 'uninstall', 'django-celery', '-y') run('./bin/pip', 'install', '-r', 'requirements.txt') if old <= 6315: From fc18eecaf4893852c3142d1844728fa35db4286b Mon Sep 17 00:00:00 2001 From: j Date: Sat, 30 May 2020 01:50:53 +0200 Subject: [PATCH 16/37] don't overwrite /etc/nginx/sites-available/default --- vm/pandora_install.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vm/pandora_install.sh b/vm/pandora_install.sh index 5f957bb2..2d898c98 100755 --- a/vm/pandora_install.sh +++ b/vm/pandora_install.sh @@ -196,7 +196,10 @@ fi # configure nginx if [ "$NGINX" == "local" ]; then -cp "/srv/pandora/etc/nginx/pandora" "/etc/nginx/sites-available/default" +cp "/srv/pandora/etc/nginx/pandora" "/etc/nginx/sites-available/pandora" +rm /etc/nginx/sites-enabled/default +ln -s ../sites-available/pandora /etc/nginx/sites-enabled/pandora + read -r -d '' GZIP < Date: Sat, 30 May 2020 02:27:36 +0200 Subject: [PATCH 17/37] remove monkey patch --- pandora/app/monkey_patch.py | 15 --------------- pandora/item/management/commands/sqlfindindex.py | 2 -- 2 files changed, 17 deletions(-) diff --git a/pandora/app/monkey_patch.py b/pandora/app/monkey_patch.py index b578a01e..84b39001 100644 --- a/pandora/app/monkey_patch.py +++ b/pandora/app/monkey_patch.py @@ -31,19 +31,4 @@ def monkey_patch_username(): if isinstance(v, MaxLengthValidator): v.limit_value = 255 -def apply_patch(): - from django.db import connection, transaction - cursor = connection.cursor() - table = connection.introspection.get_table_description(cursor, User._meta.db_table) - sql = [] - for row in table: - if row.name in NEW_LENGTH and row.internal_size != NEW_LENGTH[row.name]: - sql.append('ALTER TABLE "%s" ALTER "%s" TYPE varchar(%d)' % (User._meta.db_table, row.name, NEW_LENGTH[row.name])) - - for q in sql: - cursor.execute(q) - if sql: - transaction.commit() - - monkey_patch_username() diff --git a/pandora/item/management/commands/sqlfindindex.py b/pandora/item/management/commands/sqlfindindex.py index 42031e67..21101418 100644 --- a/pandora/item/management/commands/sqlfindindex.py +++ b/pandora/item/management/commands/sqlfindindex.py @@ -30,8 +30,6 @@ class Command(BaseCommand): print(sql) cursor.execute(sql) - app.monkey_patch.apply_patch() - if settings.DB_GIN_TRGM: import entity.models import document.models From 5cde8977efc07c02029b01d71396d2d498efe997 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 30 May 2020 03:13:57 +0200 Subject: [PATCH 18/37] get city not country list --- pandora/app/config.py | 5 ++--- pandora/user/utils.py | 15 +++++++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/pandora/app/config.py b/pandora/app/config.py index 3bb7cc72..e950e5e9 100644 --- a/pandora/app/config.py +++ b/pandora/app/config.py @@ -394,8 +394,7 @@ def update_static(): def update_geoip(force=False): path = os.path.join(settings.GEOIP_PATH, 'GeoLite2-City.mmdb') if not os.path.exists(path) or force: - url = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz' - index = ox.net.read_url('https://db-ip.com/db/download/ip-to-country-lite').decode() + index = ox.net.read_url('https://db-ip.com/db/download/ip-to-city-lite').decode() match = re.compile('href=[\'"](http.*.mmdb.gz)').findall(index) if match: url = match[0] @@ -405,7 +404,7 @@ def update_geoip(force=False): os.unlink(path) os.system('gunzip "%s.gz"' % path) else: - print('failed to download dbip-country-lite-2020-03.mmdb.gz') + print('failed to download GeoLite2-City.mmdb') def init(): if not settings.RELOADER_RUNNING: diff --git a/pandora/user/utils.py b/pandora/user/utils.py index 2cc82f1c..808aaa69 100644 --- a/pandora/user/utils.py +++ b/pandora/user/utils.py @@ -17,15 +17,22 @@ def get_location(ip): country = city = None try: g = GeoIP2() - location = g.city(ip) + except: + country = city = None + else: + try: + location = g.city(ip) + except django.contrib.gis.geoip2.GeoIP2Exception: + try: + location = g.country(s.ip) + except: + location = None if location: country = ox.get_country_name(location['country_code']) - if location['city']: + if location.get('city'): city = location['city'] if isinstance(city, bytes): city = city.decode('latin-1') - except: - country = city = None return city, country From a0060d3e9135963748d06ab5380cee57cac53ef3 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 30 May 2020 03:17:12 +0200 Subject: [PATCH 19/37] drop get_indexes --- pandora/item/management/commands/sqlfindindex.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pandora/item/management/commands/sqlfindindex.py b/pandora/item/management/commands/sqlfindindex.py index 21101418..dfdf029e 100644 --- a/pandora/item/management/commands/sqlfindindex.py +++ b/pandora/item/management/commands/sqlfindindex.py @@ -47,7 +47,6 @@ class Command(BaseCommand): for k, c in contraints.items() if c['index'] or c['primary_key'] or c['unique'] } - #indexes = connection.introspection.get_indexes(cursor, table) drop = [] if column in indexes: for sql in ( @@ -66,7 +65,12 @@ class Command(BaseCommand): if options['debug']: print(sql) cursor.execute(sql) - indexes = connection.introspection.get_indexes(cursor, table) + contraints = connection.introspection.get_constraints(cursor, table) + indexes = { + ','.join(c['columns']): {'primary_key': c['primary_key'], 'unique': c['unique']} + for k, c in contraints.items() + if c['index'] or c['primary_key'] or c['unique'] + } if column not in indexes: create_index("%s_%s_idx" % (table, column), table, column) transaction.commit() From 38ce57bdae3696d160975f57b3880d86ea8be673 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 30 May 2020 03:27:51 +0200 Subject: [PATCH 20/37] 20.04 also works --- README.md | 6 +++--- vm/LXC_README.md | 2 +- vm/LXD_README.md | 8 +++----- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 71963b6a..e27645c9 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # pan.do/ra - open media archive - for more information on pan.do/ra visit our website at https://pan.do/ra + for more information about pan.do/ra visit our website at https://pan.do/ra ## Installing pan.do/ra - we recommend to run pan.do/ra inside of LXD or LXC or dedicated VM or server. + We recommend to run pan.do/ra inside of LXD or LXC or dedicated VM or server. You will need at least 2GB of free disk space - pan.do/ra is known to work with Ubuntu 18.04 and Debian/10 (buster), + pan.do/ra is known to work with Ubuntu 18.04, 20.04 and Debian/10 (buster), other distributions might also work, let us know if it works for you. Use the following commands as root to install pan.do/ra and all dependencies: diff --git a/vm/LXC_README.md b/vm/LXC_README.md index e8152239..8f91e41e 100644 --- a/vm/LXC_README.md +++ b/vm/LXC_README.md @@ -15,7 +15,7 @@ 2) Create a new container, use different names if installing multiple instances: - sudo lxc-create -n pandora -t ubuntu-cloud -- -r bionic + sudo lxc-create -n pandora -t ubuntu-cloud -- -r focal or diff --git a/vm/LXD_README.md b/vm/LXD_README.md index 3b003deb..9aa78a25 100644 --- a/vm/LXD_README.md +++ b/vm/LXD_README.md @@ -4,15 +4,13 @@ # Installing pan.do/ra inside LXD -1) Install lxd on the host (Ubuntu 16.04 or later): +1) Install lxd on the host (Ubuntu 18.04 or later, Debian/10): - sudo apt-get install lxd - -[on debian you can use snap install lxd] + sudo snap install lxd 2) Create a new container, use different names if installing multiple instances: - sudo lxc launch ubuntu:18.04 pandora + sudo lxc launch ubuntu:20.04 pandora or From 373d43351631c27d93a668314c0a7a0818063cf3 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 30 May 2020 12:19:02 +0200 Subject: [PATCH 21/37] add longer fields to custom user --- pandora/app/monkey_patch.py | 30 +++++++++++++++---- .../item/management/commands/sqlfindindex.py | 2 ++ .../system/migrations/0003_field_length.py | 24 +++++++++++++++ pandora/system/models.py | 15 +++++++++- 4 files changed, 64 insertions(+), 7 deletions(-) create mode 100644 pandora/system/migrations/0003_field_length.py diff --git a/pandora/app/monkey_patch.py b/pandora/app/monkey_patch.py index 84b39001..f4579d56 100644 --- a/pandora/app/monkey_patch.py +++ b/pandora/app/monkey_patch.py @@ -13,10 +13,17 @@ config.init() NEW_LENGTH = { 'username': 255, - 'email': 255, + 'email': 254, 'password': 255, } +def monkey_patch_groupname(): + f = Group._meta.get_field('name') + f.max_length = 255 + for v in f.validators: + if isinstance(v, MaxLengthValidator): + v.limit_value = 255 + def monkey_patch_username(): for field in NEW_LENGTH: f = User._meta.get_field(field) @@ -24,11 +31,22 @@ def monkey_patch_username(): for v in f.validators: if isinstance(v, MaxLengthValidator): v.limit_value = NEW_LENGTH[field] + monkey_patch_groupname() + +def apply_patch(): + from django.db import connection, transaction + cursor = connection.cursor() + table = connection.introspection.get_table_description(cursor, Group._meta.db_table) + sql = [] + for row in table: + if row.name == 'name' and row.internal_size != 255: + sql.append('ALTER TABLE "%s" ALTER "%s" TYPE varchar(%d)' % ( + Group._meta.db_table, row.name, 255) + ) + for q in sql: + cursor.execute(q) + if sql: + transaction.commit() - f = Group._meta.get_field('name') - f.max_length = 255 - for v in f.validators: - if isinstance(v, MaxLengthValidator): - v.limit_value = 255 monkey_patch_username() diff --git a/pandora/item/management/commands/sqlfindindex.py b/pandora/item/management/commands/sqlfindindex.py index dfdf029e..81c39488 100644 --- a/pandora/item/management/commands/sqlfindindex.py +++ b/pandora/item/management/commands/sqlfindindex.py @@ -30,6 +30,8 @@ class Command(BaseCommand): print(sql) cursor.execute(sql) + app.monkey_patch.apply_patch() + if settings.DB_GIN_TRGM: import entity.models import document.models diff --git a/pandora/system/migrations/0003_field_length.py b/pandora/system/migrations/0003_field_length.py new file mode 100644 index 00000000..2381ebe7 --- /dev/null +++ b/pandora/system/migrations/0003_field_length.py @@ -0,0 +1,24 @@ +# Generated by Django 3.0.6 on 2020-05-30 10:10 + +import django.contrib.auth.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('system', '0002_rename_user_table'), + ] + + operations = [ + migrations.AlterField( + model_name='user', + name='email', + field=models.EmailField(blank=True, max_length=254, verbose_name='email address'), + ), + migrations.AlterField( + model_name='user', + name='username', + field=models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 255 characters or fewer. Letters, digits and @/./+/-/_ only.', max_length=255, unique=True, validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], verbose_name='username'), + ), + ] diff --git a/pandora/system/models.py b/pandora/system/models.py index 696b770a..2f59135c 100644 --- a/pandora/system/models.py +++ b/pandora/system/models.py @@ -1,6 +1,19 @@ from django.db import models from django.contrib.auth.models import AbstractUser +from django.utils.translation import ugettext_lazy as _ class User(AbstractUser): - pass + + password = models.CharField(_('password'), max_length=255) + username = models.CharField( + _('username'), + max_length=255, + unique=True, + help_text=_('Required. 255 characters or fewer. Letters, digits and @/./+/-/_ only.'), + validators=[AbstractUser.username_validator], + error_messages={ + 'unique': _("A user with that username already exists."), + }, + ) + last_name = models.CharField(_('last name'), max_length=150, blank=True) From b889159b801140a228a1434e7f2cbaec6eacf1a4 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 30 May 2020 12:22:48 +0200 Subject: [PATCH 22/37] fix user migration --- pandora/system/migrations/0001_initial.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandora/system/migrations/0001_initial.py b/pandora/system/migrations/0001_initial.py index 5c0a7b32..b8f4f6d5 100644 --- a/pandora/system/migrations/0001_initial.py +++ b/pandora/system/migrations/0001_initial.py @@ -19,12 +19,12 @@ class Migration(migrations.Migration): name='User', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('password', models.CharField(max_length=255, verbose_name='password')), + ('password', models.CharField(max_length=128, verbose_name='password')), ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')), ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')), - ('username', models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.', max_length=255, unique=True, validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], verbose_name='username')), + ('username', models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.', max_length=150, unique=True, validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], verbose_name='username')), ('first_name', models.CharField(blank=True, max_length=30, verbose_name='first name')), - ('last_name', models.CharField(blank=True, max_length=150, verbose_name='last name')), + ('last_name', models.CharField(blank=True, max_length=30, verbose_name='last name')), ('email', models.EmailField(blank=True, max_length=255, verbose_name='email address')), ('is_staff', models.BooleanField(default=False, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')), ('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')), From 139511c3fe46ad26e9cc938e0dabfab0b346823a Mon Sep 17 00:00:00 2001 From: j Date: Sat, 30 May 2020 12:28:50 +0200 Subject: [PATCH 23/37] branch needs export --- README.md | 2 +- vm/LXC_README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e27645c9..044a3cb8 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ cd /root curl -sL https://pan.do/ra-install > pandora_install.sh chmod +x pandora_install.sh -BRANCH=stable # change to 'master' to get current developement version +export BRANCH=stable # change to 'master' to get current developement version ./pandora_install.sh 2>&1 | tee pandora_install.log ``` diff --git a/vm/LXC_README.md b/vm/LXC_README.md index 8f91e41e..440d412a 100644 --- a/vm/LXC_README.md +++ b/vm/LXC_README.md @@ -41,6 +41,6 @@ cd /root curl -sL https://pan.do/ra-install > pandora_install.sh chmod +x pandora_install.sh - BRANCH=stable # or master + export BRANCH=stable # or master ./pandora_install.sh 2>&1 | tee pandora_install.log From e99d59c0df97df453e849dea67c9ae9155d8dbfd Mon Sep 17 00:00:00 2001 From: j Date: Sat, 30 May 2020 12:37:08 +0200 Subject: [PATCH 24/37] install fixes --- pandora/system/migrations/0003_field_length.py | 10 ++++++++++ update.py | 2 +- vm/pandora_install.sh | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pandora/system/migrations/0003_field_length.py b/pandora/system/migrations/0003_field_length.py index 2381ebe7..a499f65c 100644 --- a/pandora/system/migrations/0003_field_length.py +++ b/pandora/system/migrations/0003_field_length.py @@ -11,6 +11,16 @@ class Migration(migrations.Migration): ] operations = [ + migrations.AlterField( + model_name='user', + name='password', + field=models.CharField(max_length=255, verbose_name='password'), + ), + migrations.AlterField( + model_name='user', + name='last_name', + fields=models.CharField(blank=True, max_length=30, verbose_name='last name'), + ), migrations.AlterField( model_name='user', name='email', diff --git a/update.py b/update.py index 669f0981..2c1de756 100755 --- a/update.py +++ b/update.py @@ -304,7 +304,7 @@ if __name__ == "__main__": development = branch == 'master' elif len(sys.argv) == 3 and sys.argv[1] == 'switch': branch = sys.argv[2] - development = False + development = branch == 'master' else: branch = 'master' development = True diff --git a/vm/pandora_install.sh b/vm/pandora_install.sh index 2d898c98..65d776c2 100755 --- a/vm/pandora_install.sh +++ b/vm/pandora_install.sh @@ -197,7 +197,7 @@ fi if [ "$NGINX" == "local" ]; then cp "/srv/pandora/etc/nginx/pandora" "/etc/nginx/sites-available/pandora" -rm /etc/nginx/sites-enabled/default +rm -f /etc/nginx/sites-enabled/default ln -s ../sites-available/pandora /etc/nginx/sites-enabled/pandora read -r -d '' GZIP < Date: Sat, 30 May 2020 12:39:03 +0200 Subject: [PATCH 25/37] fix migration --- pandora/system/migrations/0003_field_length.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandora/system/migrations/0003_field_length.py b/pandora/system/migrations/0003_field_length.py index a499f65c..6cb2938b 100644 --- a/pandora/system/migrations/0003_field_length.py +++ b/pandora/system/migrations/0003_field_length.py @@ -19,7 +19,7 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='user', name='last_name', - fields=models.CharField(blank=True, max_length=30, verbose_name='last name'), + field=models.CharField(blank=True, max_length=150, verbose_name='last name'), ), migrations.AlterField( model_name='user', From fa5cf7cf60d205be2849e35bf183f4b96f530574 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 30 May 2020 13:01:44 +0200 Subject: [PATCH 26/37] make manage.py accessible via ctl, install pandoractl --- ctl | 29 ++++++++++++++++++++++++++--- vm/pandora_install.sh | 5 +++-- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/ctl b/ctl index 1b93c214..bd789644 100755 --- a/ctl +++ b/ctl @@ -1,13 +1,17 @@ #!/bin/sh SERVICES="pandora pandora-tasks pandora-encoding pandora-cron pandora-websocketd" if [ -z "$1" ]; then - echo "Usage: $0 (start|stop|restart|reload)" + echo "Usage: $0 (start|stop|restart|reload|status)" exit 1 else action="$1" fi +self=`readlink "$0"` +if [ -z $self ]; then + self="$0" +fi if [ "$action" = "init" ]; then - cd "`dirname "$0"`" + cd "`dirname "$self"`" BASE=`pwd` SUDO="" PANDORA_USER=`ls -l update.py | cut -f3 -d" "` @@ -43,6 +47,18 @@ if [ "$action" = "init" ]; then fi exit 0 fi +if [ "$action" = "manage" ]; then + cd "`dirname "$self"`" + if [ `whoami` != 'root' ]; then + manage="./pandora/manage.py" + else + manage="sudo -u pandora ./pandora/manage.py" + fi + shift + $manage $@ + exit $? + +fi if [ `whoami` != 'root' ]; then echo you have to be root or run $0 with sudo exit 1 @@ -74,6 +90,13 @@ if [ "$action" = "install" ]; then fi exit 0 fi +if [ "status" = "$action" ]; then + export SYSTEMD_PAGER= +fi for service in $SERVICES; do - service $service $action + if [ -x /bin/systemctl ]; then + /bin/systemctl $action $service + else + service $service $action + fi done diff --git a/vm/pandora_install.sh b/vm/pandora_install.sh index 65d776c2..ec77d3b0 100755 --- a/vm/pandora_install.sh +++ b/vm/pandora_install.sh @@ -12,7 +12,7 @@ NGINX=${NGINX-local} BRANCH=${BRANCH-stable} # add a pandora user -echo Installing pandora with user: $PANDORA +echo Installing pandora with user: $PANDORA branch: $BRANCH getent passwd $PANDORA > /dev/null 2>&1 || adduser --disabled-password --gecos "" $PANDORA # @@ -197,8 +197,9 @@ fi if [ "$NGINX" == "local" ]; then cp "/srv/pandora/etc/nginx/pandora" "/etc/nginx/sites-available/pandora" -rm -f /etc/nginx/sites-enabled/default +rm -f /etc/nginx/sites-enabled/default /etc/nginx/sites-enabled/pandora ln -s ../sites-available/pandora /etc/nginx/sites-enabled/pandora +ln -s /srv/pandora/ctl /usr/local/bin/pandoractl read -r -d '' GZIP < Date: Sat, 30 May 2020 13:57:34 +0200 Subject: [PATCH 27/37] fix exception --- pandora/user/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandora/user/utils.py b/pandora/user/utils.py index 808aaa69..86a21ae1 100644 --- a/pandora/user/utils.py +++ b/pandora/user/utils.py @@ -1,4 +1,4 @@ -from django.contrib.gis.geoip2 import GeoIP2 +from django.contrib.gis.geoip2 import GeoIP, GeoIP2Exception2 import ox @@ -22,7 +22,7 @@ def get_location(ip): else: try: location = g.city(ip) - except django.contrib.gis.geoip2.GeoIP2Exception: + except GeoIP2Exception: try: location = g.country(s.ip) except: From d82cd9b6a7ffe40b9934c5031b0caae841afd7ca Mon Sep 17 00:00:00 2001 From: j Date: Sat, 30 May 2020 14:00:08 +0200 Subject: [PATCH 28/37] get current revno before switch --- update.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/update.py b/update.py index 2c1de756..bfa0a019 100755 --- a/update.py +++ b/update.py @@ -297,7 +297,7 @@ if __name__ == "__main__": ]: run_sql(sql) run(join(base, 'pandora/manage.py'), 'migrate', 'system') - + run(join(base, 'pandora/manage.py'), 'update_geoip') else: if len(sys.argv) == 1: branch = get_branch() @@ -316,13 +316,13 @@ if __name__ == "__main__": if exists(path): os.chdir(path) current_branch = get_branch(path) - if current_branch != branch: - run('git', 'remote', 'set-branches', 'origin', '*') - run('git', 'fetch', 'origin') - run('git', 'checkout', branch) - revno = get_version(path) if repo == 'pandora': pandora_old_revno = revno + if current_branch != branch: + run('git', 'remote', 'set-branches', 'origin', '*') + run('git', 'fetch', 'origin') + run('git', 'checkout', branch) + revno = get_version(path) current += revno url = repos[repo]['url'] print('Checking', repo) From 1cc18c52c35ce28907beb2d9fe7098364afbed68 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 30 May 2020 14:33:04 +0200 Subject: [PATCH 29/37] fix embed options --- static/js/embedDialog.js | 17 +++++++++-------- static/js/pandora.js | 3 +++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/static/js/embedDialog.js b/static/js/embedDialog.js index 1142cddb..60dba564 100644 --- a/static/js/embedDialog.js +++ b/static/js/embedDialog.js @@ -40,7 +40,7 @@ pandora.ui.embedDialog = function(/*[url, ]callback*/) { description: Ox._('Embed a Clip or a Full Video'), inputs: [ 'item', 'position', 'in', 'out', 'annotation', 'title', - 'showTimeline', 'showAnnotations', 'matchRatio' + 'showTimeline', 'showAnnotations', 'showLayers', 'matchRatio', 'timeline' ] }, { @@ -85,7 +85,7 @@ pandora.ui.embedDialog = function(/*[url, ]callback*/) { description: Ox._('Embed an Edited Video'), inputs: [ 'edit', 'editMode', 'position', - 'showTimeline', 'showAnnotations', 'matchRatio' + 'showTimeline', 'showAnnotations', 'showLayers', 'matchRatio', 'timeline' ] } ].map(function(item, index) { @@ -233,8 +233,7 @@ pandora.ui.embedDialog = function(/*[url, ]callback*/) { data = getData(); if (view == 'document') { var prefix = type == 'iframe' - ? (pandora.site.site.https ? 'https' : 'http') - + '://' + pandora.site.site.url + '/' + ? (document.location.protocol + '//' + document.location.host + '/') : '/', page = type == 'iframe' ? 'documents' : 'document', resolution = 480; @@ -260,9 +259,12 @@ pandora.ui.embedDialog = function(/*[url, ]callback*/) { options = Ox.serialize({ title: data.title || void 0, showTimeline: data.showTimeline || void 0, - timeline: data.timeline || void 0, + timeline: data.showTimeline && data.timeline ? data.timeline : void 0, showAnnotations: data.showAnnotations || void 0, - showLayers: data.showAnnotations && data.showLayers ? data.showLayers : void 0, + showLayers: data.showAnnotations && data.showLayers && + data.showLayers.length < pandora.site.layers.length + ? data.showLayers.join(',') + : void 0, matchRatio: Ox.contains(['video', 'edit'], view) ? data.matchRatio || void 0 : void 0, showInfo: Ox.contains(['list', 'text'], view) || (view == 'edit' && !!$input.editMode.value()) || void 0 }, true), @@ -275,8 +277,7 @@ pandora.ui.embedDialog = function(/*[url, ]callback*/) { return Ox.encodeHTMLEntities( ( type == 'iframe' - ? (pandora.site.site.https ? 'https' : 'http') - + '://' + pandora.site.site.url + '/' + ? (document.location.protocol + '//' + document.location.host + '/') : '/' ) + ( diff --git a/static/js/pandora.js b/static/js/pandora.js index 92f34709..8770be61 100644 --- a/static/js/pandora.js +++ b/static/js/pandora.js @@ -456,6 +456,9 @@ appPanel videoFormat: Ox.getVideoFormat(pandora.site.video.formats) }); + pandora.site.site.url = document.location.host + pandora.site.site.https = document.location.protocol == 'https:' + Ox.Map.GoogleApiKey = pandora.site.site.googleapikey; // set locale and initialize url controller From 88f0a11612e83a004706dfe69acedcdd91a0e056 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 30 May 2020 14:34:02 +0200 Subject: [PATCH 30/37] fix import --- pandora/user/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandora/user/utils.py b/pandora/user/utils.py index 86a21ae1..3648b78f 100644 --- a/pandora/user/utils.py +++ b/pandora/user/utils.py @@ -1,4 +1,4 @@ -from django.contrib.gis.geoip2 import GeoIP, GeoIP2Exception2 +from django.contrib.gis.geoip2 import GeoIP2, GeoIP2Exception2 import ox From 2e89299ecae084115107f0e03403c33b31bd23f3 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 30 May 2020 14:35:09 +0200 Subject: [PATCH 31/37] fix update --- update.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/update.py b/update.py index bfa0a019..566d23f9 100755 --- a/update.py +++ b/update.py @@ -316,14 +316,14 @@ if __name__ == "__main__": if exists(path): os.chdir(path) current_branch = get_branch(path) + revno = get_version(path) if repo == 'pandora': pandora_old_revno = revno + current += revno if current_branch != branch: run('git', 'remote', 'set-branches', 'origin', '*') run('git', 'fetch', 'origin') run('git', 'checkout', branch) - revno = get_version(path) - current += revno url = repos[repo]['url'] print('Checking', repo) run('git', 'pull') From 4e952c37461b3bf0e0f0951270596bcfea511fe7 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 30 May 2020 14:36:06 +0200 Subject: [PATCH 32/37] import again --- pandora/user/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandora/user/utils.py b/pandora/user/utils.py index 3648b78f..49fa7884 100644 --- a/pandora/user/utils.py +++ b/pandora/user/utils.py @@ -1,4 +1,4 @@ -from django.contrib.gis.geoip2 import GeoIP2, GeoIP2Exception2 +from django.contrib.gis.geoip2 import GeoIP2, GeoIP2Exception import ox From 5f9a2b987928311f58fa186028f5cec1f8c4dc1f Mon Sep 17 00:00:00 2001 From: j Date: Sat, 30 May 2020 17:44:49 +0200 Subject: [PATCH 33/37] reuse sudo code, install pandoractl --- ctl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ctl b/ctl index bd789644..b518cddd 100755 --- a/ctl +++ b/ctl @@ -49,15 +49,15 @@ if [ "$action" = "init" ]; then fi if [ "$action" = "manage" ]; then cd "`dirname "$self"`" - if [ `whoami` != 'root' ]; then - manage="./pandora/manage.py" - else - manage="sudo -u pandora ./pandora/manage.py" + BASE=`pwd` + SUDO="" + PANDORA_USER=`ls -l update.py | cut -f3 -d" "` + if [ `whoami` != $PANDORA_USER ]; then + SUDO="sudo -H -u $PANDORA_USER" fi shift - $manage $@ + $SUDO $BASE/pandora/manage.py $@ exit $? - fi if [ `whoami` != 'root' ]; then echo you have to be root or run $0 with sudo @@ -83,6 +83,7 @@ if [ "$action" = "install" ]; then systemctl enable ${service}.service done fi + test -e /usr/local/bin/pandoractl || ln -s /srv/pandora/ctl /usr/local/bin/pandoractl else if [ -d /etc/init ]; then cp $BASE/etc/init/* /etc/init/ From 830386214ccf1deca98fdaf0842512bc1af203a5 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 30 May 2020 21:23:04 +0200 Subject: [PATCH 34/37] log error --- pandora/websocket/worker.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pandora/websocket/worker.py b/pandora/websocket/worker.py index 72c35e5a..1a0e633d 100644 --- a/pandora/websocket/worker.py +++ b/pandora/websocket/worker.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import +import logging from django.conf import settings @@ -8,6 +8,7 @@ from kombu.mixins import ConsumerMixin from . import daemon, key +logger = logging.getLogger('pandora.websocket') queue = Queue('websocket', Exchange(key, type='direct'), routing_key=key) @@ -22,8 +23,11 @@ class Worker(ConsumerMixin): callbacks=[self.process_task])] def process_task(self, body, message): - if body['task'] == 'trigger_event': - daemon.trigger_event(*body['args']) + try: + if body['task'] == 'trigger_event': + daemon.trigger_event(*body['args']) + except: + logger.error('faild to trigger event %s', body, exc_info=True) message.ack() def run(): From a1e6ad7b3d6ddca62f543da10394662822532bc0 Mon Sep 17 00:00:00 2001 From: j Date: Sat, 30 May 2020 21:23:16 +0200 Subject: [PATCH 35/37] remove future imports --- pandora/app/utils.py | 2 -- pandora/archive/chunk.py | 2 -- pandora/home/apps.py | 2 -- pandora/item/data_api.py | 2 -- pandora/oxdjango/api/__init__.py | 2 -- pandora/oxdjango/api/actions.py | 2 -- pandora/oxdjango/api/site.py | 2 -- pandora/oxdjango/api/views.py | 2 -- pandora/oxdjango/decorators.py | 2 -- pandora/sequence/extract.py | 1 - pandora/settings.py | 1 - pandora/websocket/management/commands/websocketd.py | 2 -- scripts/poster.0xdb.py | 1 - scripts/poster.indiancinema.py | 1 - scripts/poster.padma.py | 1 - scripts/poster.pandora.py | 1 - update.py | 1 - 17 files changed, 27 deletions(-) diff --git a/pandora/app/utils.py b/pandora/app/utils.py index f272e6c8..ee2e9d08 100644 --- a/pandora/app/utils.py +++ b/pandora/app/utils.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, with_statement - import time from .models import Settings diff --git a/pandora/archive/chunk.py b/pandora/archive/chunk.py index 148ba24f..8422051e 100644 --- a/pandora/archive/chunk.py +++ b/pandora/archive/chunk.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, with_statement - import os import ox diff --git a/pandora/home/apps.py b/pandora/home/apps.py index 1d6cbbdf..90dc7137 100644 --- a/pandora/home/apps.py +++ b/pandora/home/apps.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - from django.apps import AppConfig diff --git a/pandora/item/data_api.py b/pandora/item/data_api.py index 81d9d72c..0c06015f 100644 --- a/pandora/item/data_api.py +++ b/pandora/item/data_api.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, with_statement - import ox from django.conf import settings diff --git a/pandora/oxdjango/api/__init__.py b/pandora/oxdjango/api/__init__.py index 4ecadf1d..72294dc6 100644 --- a/pandora/oxdjango/api/__init__.py +++ b/pandora/oxdjango/api/__init__.py @@ -1,3 +1 @@ -from __future__ import absolute_import - from .actions import actions diff --git a/pandora/oxdjango/api/actions.py b/pandora/oxdjango/api/actions.py index f8a91d3a..70249914 100644 --- a/pandora/oxdjango/api/actions.py +++ b/pandora/oxdjango/api/actions.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, absolute_import - import inspect import sys diff --git a/pandora/oxdjango/api/site.py b/pandora/oxdjango/api/site.py index ddd5bc49..cac1e021 100644 --- a/pandora/oxdjango/api/site.py +++ b/pandora/oxdjango/api/site.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import - from django.urls import path from . import views diff --git a/pandora/oxdjango/api/views.py b/pandora/oxdjango/api/views.py index 1dfe7428..3a4a2e89 100644 --- a/pandora/oxdjango/api/views.py +++ b/pandora/oxdjango/api/views.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division, absolute_import - import json from django.shortcuts import render diff --git a/pandora/oxdjango/decorators.py b/pandora/oxdjango/decorators.py index bd6942e5..e631daef 100644 --- a/pandora/oxdjango/decorators.py +++ b/pandora/oxdjango/decorators.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import - from functools import wraps from .shortcuts import render_to_json_response diff --git a/pandora/sequence/extract.py b/pandora/sequence/extract.py index 5cbcda50..930131cd 100644 --- a/pandora/sequence/extract.py +++ b/pandora/sequence/extract.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division import os from PIL import Image diff --git a/pandora/settings.py b/pandora/settings.py index 6cb4f347..99eda0ba 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -2,7 +2,6 @@ # Django settings for pan.do/ra project defaults, # create local_settings.py to overwrite # check pan.do/ra section below for relevant settings -from __future__ import absolute_import import os from os.path import join, normpath, dirname diff --git a/pandora/websocket/management/commands/websocketd.py b/pandora/websocket/management/commands/websocketd.py index 0eb9ed25..f93bc9d6 100644 --- a/pandora/websocket/management/commands/websocketd.py +++ b/pandora/websocket/management/commands/websocketd.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import - import os from django.core.management.base import BaseCommand diff --git a/scripts/poster.0xdb.py b/scripts/poster.0xdb.py index bd05982d..9b9f2470 100755 --- a/scripts/poster.0xdb.py +++ b/scripts/poster.0xdb.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from __future__ import division import os from PIL import Image diff --git a/scripts/poster.indiancinema.py b/scripts/poster.indiancinema.py index 194700d5..bc37fca5 100755 --- a/scripts/poster.indiancinema.py +++ b/scripts/poster.indiancinema.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from __future__ import division import os from PIL import Image diff --git a/scripts/poster.padma.py b/scripts/poster.padma.py index 81ec3dfe..015aaa9b 100755 --- a/scripts/poster.padma.py +++ b/scripts/poster.padma.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from __future__ import division import os from PIL import Image diff --git a/scripts/poster.pandora.py b/scripts/poster.pandora.py index 6177efd8..5f196562 100755 --- a/scripts/poster.pandora.py +++ b/scripts/poster.pandora.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from __future__ import division import os from PIL import Image diff --git a/update.py b/update.py index 566d23f9..3a2c7a40 100755 --- a/update.py +++ b/update.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -from __future__ import print_function import json import os import shutil From 3dc8a0aba0f74685283be80f29e2c9f99fcf7ba6 Mon Sep 17 00:00:00 2001 From: j Date: Sun, 31 May 2020 12:22:00 +0200 Subject: [PATCH 36/37] wrap update in ctl --- ctl | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ctl b/ctl index b518cddd..28c74e12 100755 --- a/ctl +++ b/ctl @@ -10,6 +10,7 @@ self=`readlink "$0"` if [ -z $self ]; then self="$0" fi + if [ "$action" = "init" ]; then cd "`dirname "$self"`" BASE=`pwd` @@ -47,7 +48,15 @@ if [ "$action" = "init" ]; then fi exit 0 fi + if [ "$action" = "manage" ]; then + cmd="pandora/manage.py" +fi +if [ "$action" = "update" ]; then + cmd="update.py" +fi + +if [ ! -z $cmd ]; then cd "`dirname "$self"`" BASE=`pwd` SUDO="" @@ -56,9 +65,10 @@ if [ "$action" = "manage" ]; then SUDO="sudo -H -u $PANDORA_USER" fi shift - $SUDO $BASE/pandora/manage.py $@ + $SUDO "$BASE/$cmd" $@ exit $? fi + if [ `whoami` != 'root' ]; then echo you have to be root or run $0 with sudo exit 1 From 769f0b7fd2dda5cdf0331a5f415c38ee329fd643 Mon Sep 17 00:00:00 2001 From: j Date: Sun, 31 May 2020 16:23:19 +0200 Subject: [PATCH 37/37] stay at last tile --- pandora/sequence/extract.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandora/sequence/extract.py b/pandora/sequence/extract.py index 930131cd..90db3b33 100644 --- a/pandora/sequence/extract.py +++ b/pandora/sequence/extract.py @@ -96,7 +96,9 @@ class DataTimeline(): def get_frame(self, pos): frame = int(pos * self.fps) tile = int(frame * 8 / self.timeline_width) - if self.current_tile != tile: + if len(self.file_names) <= tile: + tile = len(self.file_names) - 1 + if self.current_tile != tile and len(self.file_names): self.timeline_image = Image.open(self.file_names[tile]) self.current_tile = tile x = frame * 8 - tile * self.timeline_width