From 680efc967ca7918e2a3d0eb7ef8b0442383da98f Mon Sep 17 00:00:00 2001 From: j Date: Wed, 31 Aug 2016 12:23:41 +0200 Subject: [PATCH 1/5] processing trumps queued --- pandora/taskqueue/models.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pandora/taskqueue/models.py b/pandora/taskqueue/models.py index 546ccf5f..67298228 100644 --- a/pandora/taskqueue/models.py +++ b/pandora/taskqueue/models.py @@ -66,7 +66,8 @@ class Task(models.Model): task, created = cls.objects.get_or_create(item=item) if task.update(save=False) or created: task.user = user - task.started = datetime.now() + if not task.started: + task.started = datetime.now() task.ended = None task.save() @@ -83,10 +84,10 @@ class Task(models.Model): status = 'pending' elif self.item.files.filter(uploading=True).count(): status = 'uploading' - elif self.item.files.filter(queued=True).count(): - status = 'queued' elif self.item.files.filter(encoding=True).count(): status = 'processing' + elif self.item.files.filter(queued=True).count(): + status = 'queued' elif self.item.files.filter(failed=True).count(): status = 'failed' elif self.item.rendered: From 3913373c9b80e511b665ffb6e722b765a8fffaf7 Mon Sep 17 00:00:00 2001 From: j Date: Wed, 31 Aug 2016 17:38:40 +0200 Subject: [PATCH 2/5] update db if guest was reclassified as robot --- pandora/user/managers.py | 3 +-- pandora/user/models.py | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pandora/user/managers.py b/pandora/user/managers.py index d23007e5..3e7b9cae 100644 --- a/pandora/user/managers.py +++ b/pandora/user/managers.py @@ -37,7 +37,6 @@ def parseCondition(condition, user): else: key = k + get_operator(op, 'istr') key = str(key) - q = Q(**{key: v}) if exclude: q = ~q @@ -53,7 +52,7 @@ def parseConditions(conditions, operator, user): for condition in conditions: if 'conditions' in condition: q = parseConditions(condition['conditions'], - condition.get('operator', '&'), user) + condition.get('operator', '&'), user) if q: conn.append(q) pass diff --git a/pandora/user/models.py b/pandora/user/models.py index e810230e..3e3595af 100644 --- a/pandora/user/models.py +++ b/pandora/user/models.py @@ -141,6 +141,9 @@ class SessionData(models.Model): def json(self, keys=None, user=None): ua = ox.parse_useragent(self.useragent or '') + if ua['robot']['name'] and self.level != -1: + self.level = -1 + self.save() j = { 'browser': ua['browser']['string'], 'disabled': False, From 1c580a2df778508273128da3405156d902e39bc8 Mon Sep 17 00:00:00 2001 From: j Date: Wed, 31 Aug 2016 18:03:19 +0200 Subject: [PATCH 3/5] async update of annotation matches, fixes deadlock in load_subtitles --- pandora/annotation/models.py | 11 +++++++---- pandora/annotation/tasks.py | 22 ++++++++++++---------- pandora/item/models.py | 4 ++-- pandora/place/models.py | 4 ++-- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/pandora/annotation/models.py b/pandora/annotation/models.py index 19527afd..7b946b88 100644 --- a/pandora/annotation/models.py +++ b/pandora/annotation/models.py @@ -134,6 +134,7 @@ class Annotation(models.Model): def save(self, *args, **kwargs): from .tasks import update_matches + async = kwargs.pop('async', False) set_public_id = not self.id or not self.public_id layer = self.get_layer() @@ -177,14 +178,16 @@ class Annotation(models.Model): 'id': self.clip.id, self.layer: False }).update(**{self.layer: True}) - #update clip.findvalue + # update clip.findvalue self.clip.save() - #editAnnotations needs to be in snyc + # editAnnotations needs to be in snyc + # load_subtitles can not be in sync + fn = update_matches.delay if async else update_matches if layer.get('type') == 'place' or layer.get('hasPlaces'): - update_matches(self.id, 'place') + fn(self.id, 'place') if layer.get('type') == 'event' or layer.get('hasEvents'): - update_matches(self.id, 'event') + fn(self.id, 'event') def delete(self, *args, **kwargs): with transaction.atomic(): diff --git a/pandora/annotation/tasks.py b/pandora/annotation/tasks.py index 610555b0..969539d5 100644 --- a/pandora/annotation/tasks.py +++ b/pandora/annotation/tasks.py @@ -18,10 +18,13 @@ def update_matches(id, type): elif type == 'event': from event.models import Event as Model - a = Annotation.objects.get(pk=id) + try: + a = Annotation.objects.get(pk=id) + except Annotation.DoesNotExist: + return a_matches = getattr(a, type == 'place' and 'places' or 'events') - #remove undefined matches that only have this annotation + # remove undefined matches that only have this annotation for p in a_matches.filter(defined=False).exclude(name=a.value): if p.annotations.exclude(id=id).count() == 0: p.delete() @@ -33,8 +36,7 @@ def update_matches(id, type): if a.findvalue: names = {} for n in Model.objects.all().values('id', 'name', 'alternativeNames'): - names[n['id']] = [ox.decode_html(x) - for x in (n['name'],) + n['alternativeNames']] + names[n['id']] = [ox.decode_html(x) for x in (n['name'],) + n['alternativeNames']] value = a.findvalue.lower() current = [p.id for p in a_matches.all()] @@ -49,19 +51,19 @@ def update_matches(id, type): new = [] for i in matches: p = Model.objects.get(pk=i) - #only add places/events that did not get added as a super match - #i.e. only add The Paris Region and not Paris + # only add places/events that did not get added as a super match + # i.e. only add The Paris Region and not Paris if not filter(lambda n: n in name_matches, [n.lower() for n in p.get_super_matches()]): new.append(i) - removed = list(filter(lambda p: p not in new, current)) - added = list(filter(lambda p: p not in current, new)) - update = removed + added + removed = set(filter(lambda p: p not in new, current)) + added = set(filter(lambda p: p not in current, new)) + update = list(removed | added) if update: for e in Model.objects.filter(id__in=update): e.update_matches(Annotation.objects.filter(pk=a.id)) else: - #annotation has no value, remove all exisint matches + # annotation has no value, remove all exisint matches for e in a_matches.all(): e.update_matches(Annotation.objects.filter(pk=a.id)) diff --git a/pandora/item/models.py b/pandora/item/models.py index a1020dfe..6c22e8b6 100644 --- a/pandora/item/models.py +++ b/pandora/item/models.py @@ -1632,7 +1632,7 @@ class Item(models.Model): value=value, user=user ) - annotation.save() + annotation.save(async=True) # otherwise add empty 5 seconds annotation every minute if not subtitles_added: start = offset and int(offset / 60) * 60 + 60 or 0 @@ -1647,7 +1647,7 @@ class Item(models.Model): value='', user=user ) - annotation.save() + annotation.save(async=True) offset += f.duration # remove left over clips without annotations Clip.objects.filter(item=self, annotations__id=None).delete() diff --git a/pandora/place/models.py b/pandora/place/models.py index 71eac036..022778db 100644 --- a/pandora/place/models.py +++ b/pandora/place/models.py @@ -107,13 +107,13 @@ class Place(models.Model): numberofmatches = -1 for a in annotations.exclude(id__in=matches): self.annotations.remove(a) - #annotations of type place always need a place + # annotations of type place always need a place if a.get_layer().get('type') == 'place' and a.places.count() == 0: a.places.add(Place.get_or_create(a.value)) for p in a.places.exclude(id=self.id): p.update_matches() for a in matches.exclude(id__in=self.annotations.all()): - #need to check again since editEvent might have been called again + # need to check again since editEvent might have been called again if self.annotations.filter(id=a.id).count() == 0: self.annotations.add(a) ids = list(set([a['item_id'] for a in self.annotations.all().values('item_id')])) From 0f0f5f42d8c64b20605fc0c654d9f37d96e28bf3 Mon Sep 17 00:00:00 2001 From: j Date: Wed, 31 Aug 2016 19:07:44 +0200 Subject: [PATCH 4/5] space --- pandora/user/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandora/user/models.py b/pandora/user/models.py index 3e3595af..b183ee43 100644 --- a/pandora/user/models.py +++ b/pandora/user/models.py @@ -47,7 +47,7 @@ class SessionData(models.Model): objects = managers.SessionDataManager() - groupssort = models.CharField(default=None,blank=True,null=True, max_length=255) + groupssort = models.CharField(default=None, blank=True, null=True, max_length=255) def __unicode__(self): return u"%s" % self.session_key @@ -237,6 +237,7 @@ def get_ui(user_ui, user=None): ui = {} config = copy.deepcopy(settings.CONFIG) ui.update(config['user']['ui']) + def update_ui(ui, new): ''' only update set keys in dicts @@ -388,8 +389,7 @@ def has_capability(user, capability): else: level = user.profile.get_level() return level in settings.CONFIG['capabilities'][capability] \ - and settings.CONFIG['capabilities'][capability][level] - + and settings.CONFIG['capabilities'][capability][level] def merge_users(old, new): old.annotations.all().update(user=new) From 400162e4006f229cc393a12358bef6ce3b8571bf Mon Sep 17 00:00:00 2001 From: j Date: Mon, 5 Sep 2016 15:47:36 +0200 Subject: [PATCH 5/5] full_tile_widths can be 0 for items with many parts --- pandora/item/timelines.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/pandora/item/timelines.py b/pandora/item/timelines.py index 861ee1df..00c365c1 100644 --- a/pandora/item/timelines.py +++ b/pandora/item/timelines.py @@ -9,6 +9,16 @@ import re from PIL import Image from ox.utils import json +__all__ = ['join_tiles', 'split_tiles'] + +def divide(num, by): + # divide(100, 3) -> [33, 33, 34] + arr = [] + div = int(num / by) + mod = num % by + for i in range(int(by)): + arr.append(div + (i > by - 1 - mod)) + return arr def join_tiles(source_paths, durations, target_path): ''' @@ -17,15 +27,6 @@ def join_tiles(source_paths, durations, target_path): be written to target_path. ''' - def divide(num, by): - # divide(100, 3) -> [33, 33, 34] - arr = [] - div = int(num / by) - mod = num % by - for i in range(int(by)): - arr.append(div + (i > by - 1 - mod)) - return arr - def get_file_info(file_name): for mode in modes: if re.match('^timeline' + mode + '64p\d+\.jpg', file_name): @@ -84,11 +85,12 @@ def join_tiles(source_paths, durations, target_path): #print(image_file) if mode == full_tile_mode: # render full tile - resized = data['target_images']['large'].resize(( - data['full_tile_widths'][0], large_tile_h - ), Image.ANTIALIAS) - data['target_images']['full'].paste(resized, (data['full_tile_offset'], 0)) - data['full_tile_offset'] += data['full_tile_widths'][0] + if data['full_tile_widths'][0]: + resized = data['target_images']['large'].resize(( + data['full_tile_widths'][0], large_tile_h + ), Image.ANTIALIAS) + data['target_images']['full'].paste(resized, (data['full_tile_offset'], 0)) + data['full_tile_offset'] += data['full_tile_widths'][0] data['full_tile_widths'] = data['full_tile_widths'][1:] large_tile_i += 1 # open next large tile