From 1c580a2df778508273128da3405156d902e39bc8 Mon Sep 17 00:00:00 2001 From: j Date: Wed, 31 Aug 2016 18:03:19 +0200 Subject: [PATCH] async update of annotation matches, fixes deadlock in load_subtitles --- pandora/annotation/models.py | 11 +++++++---- pandora/annotation/tasks.py | 22 ++++++++++++---------- pandora/item/models.py | 4 ++-- pandora/place/models.py | 4 ++-- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/pandora/annotation/models.py b/pandora/annotation/models.py index 19527afd2..7b946b883 100644 --- a/pandora/annotation/models.py +++ b/pandora/annotation/models.py @@ -134,6 +134,7 @@ class Annotation(models.Model): def save(self, *args, **kwargs): from .tasks import update_matches + async = kwargs.pop('async', False) set_public_id = not self.id or not self.public_id layer = self.get_layer() @@ -177,14 +178,16 @@ class Annotation(models.Model): 'id': self.clip.id, self.layer: False }).update(**{self.layer: True}) - #update clip.findvalue + # update clip.findvalue self.clip.save() - #editAnnotations needs to be in snyc + # editAnnotations needs to be in snyc + # load_subtitles can not be in sync + fn = update_matches.delay if async else update_matches if layer.get('type') == 'place' or layer.get('hasPlaces'): - update_matches(self.id, 'place') + fn(self.id, 'place') if layer.get('type') == 'event' or layer.get('hasEvents'): - update_matches(self.id, 'event') + fn(self.id, 'event') def delete(self, *args, **kwargs): with transaction.atomic(): diff --git a/pandora/annotation/tasks.py b/pandora/annotation/tasks.py index 610555b0c..969539d5d 100644 --- a/pandora/annotation/tasks.py +++ b/pandora/annotation/tasks.py @@ -18,10 +18,13 @@ def update_matches(id, type): elif type == 'event': from event.models import Event as Model - a = Annotation.objects.get(pk=id) + try: + a = Annotation.objects.get(pk=id) + except Annotation.DoesNotExist: + return a_matches = getattr(a, type == 'place' and 'places' or 'events') - #remove undefined matches that only have this annotation + # remove undefined matches that only have this annotation for p in a_matches.filter(defined=False).exclude(name=a.value): if p.annotations.exclude(id=id).count() == 0: p.delete() @@ -33,8 +36,7 @@ def update_matches(id, type): if a.findvalue: names = {} for n in Model.objects.all().values('id', 'name', 'alternativeNames'): - names[n['id']] = [ox.decode_html(x) - for x in (n['name'],) + n['alternativeNames']] + names[n['id']] = [ox.decode_html(x) for x in (n['name'],) + n['alternativeNames']] value = a.findvalue.lower() current = [p.id for p in a_matches.all()] @@ -49,19 +51,19 @@ def update_matches(id, type): new = [] for i in matches: p = Model.objects.get(pk=i) - #only add places/events that did not get added as a super match - #i.e. only add The Paris Region and not Paris + # only add places/events that did not get added as a super match + # i.e. only add The Paris Region and not Paris if not filter(lambda n: n in name_matches, [n.lower() for n in p.get_super_matches()]): new.append(i) - removed = list(filter(lambda p: p not in new, current)) - added = list(filter(lambda p: p not in current, new)) - update = removed + added + removed = set(filter(lambda p: p not in new, current)) + added = set(filter(lambda p: p not in current, new)) + update = list(removed | added) if update: for e in Model.objects.filter(id__in=update): e.update_matches(Annotation.objects.filter(pk=a.id)) else: - #annotation has no value, remove all exisint matches + # annotation has no value, remove all exisint matches for e in a_matches.all(): e.update_matches(Annotation.objects.filter(pk=a.id)) diff --git a/pandora/item/models.py b/pandora/item/models.py index a1020dfe1..6c22e8b67 100644 --- a/pandora/item/models.py +++ b/pandora/item/models.py @@ -1632,7 +1632,7 @@ class Item(models.Model): value=value, user=user ) - annotation.save() + annotation.save(async=True) # otherwise add empty 5 seconds annotation every minute if not subtitles_added: start = offset and int(offset / 60) * 60 + 60 or 0 @@ -1647,7 +1647,7 @@ class Item(models.Model): value='', user=user ) - annotation.save() + annotation.save(async=True) offset += f.duration # remove left over clips without annotations Clip.objects.filter(item=self, annotations__id=None).delete() diff --git a/pandora/place/models.py b/pandora/place/models.py index 71eac036b..022778db5 100644 --- a/pandora/place/models.py +++ b/pandora/place/models.py @@ -107,13 +107,13 @@ class Place(models.Model): numberofmatches = -1 for a in annotations.exclude(id__in=matches): self.annotations.remove(a) - #annotations of type place always need a place + # annotations of type place always need a place if a.get_layer().get('type') == 'place' and a.places.count() == 0: a.places.add(Place.get_or_create(a.value)) for p in a.places.exclude(id=self.id): p.update_matches() for a in matches.exclude(id__in=self.annotations.all()): - #need to check again since editEvent might have been called again + # need to check again since editEvent might have been called again if self.annotations.filter(id=a.id).count() == 0: self.annotations.add(a) ids = list(set([a['item_id'] for a in self.annotations.all().values('item_id')]))