From 39f42aa1dd8d8d3292c55ed9d980c0813c4d10f5 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Mon, 22 Aug 2011 09:05:14 +0200 Subject: [PATCH 1/3] subs are not video or audio --- pandora/archive/models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandora/archive/models.py b/pandora/archive/models.py index 9cfed69f..bfbaf3bb 100644 --- a/pandora/archive/models.py +++ b/pandora/archive/models.py @@ -127,7 +127,7 @@ class File(models.Model): self.display_aspect_ratio = "4:3" self.width = '320' self.height = '240' - if 'audio' in self.info and self.info['audio']: + if 'audio' in self.info and self.info['audio'] and self.duration > 0: audio = self.info['audio'][0] self.audio_codec = audio['codec'] self.samplerate = audio.get('samplerate', 0) @@ -137,6 +137,9 @@ class File(models.Model): self.is_audio = True else: self.is_audio = False + self.audio_codec = '' + self.sampleate = 0 + self.channels = 0 if self.framerate: self.pixels = int(self.width * self.height * float(utils.parse_decimal(self.framerate)) * self.duration) From a6e120322fb93f34be148b40625133e4644a8785 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Tue, 23 Aug 2011 12:47:59 +0200 Subject: [PATCH 2/3] speedup annotation looksup --- pandora/annotation/models.py | 14 ++++++++------ pandora/annotation/views.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/pandora/annotation/models.py b/pandora/annotation/models.py index 5bc06919..d2d127d3 100644 --- a/pandora/annotation/models.py +++ b/pandora/annotation/models.py @@ -79,8 +79,9 @@ class Annotation(models.Model): user = models.ForeignKey(User) item = models.ForeignKey('item.Item', related_name='annotations') + public_id = models.CharField(max_length=128, unique=True) #seconds - start = models.FloatField(default=-1) + start = models.FloatField(default=-1, db_index=True) end = models.FloatField(default=-1) layer = models.ForeignKey(Layer) @@ -100,12 +101,15 @@ class Annotation(models.Model): else: return self.value - def get_id(self): - return '%s/%s' % (self.item.itemId, ox.to32(self.id)) + def save(self, *args, **kwargs): + if not self.id: + super(Annotation, self).save(*args, **kwargs) + self.public_id = '%s/%s' % (self.item.itemId, ox.to32(self.id)) + super(Annotation, self).save(*args, **kwargs) def json(self, layer=False, keys=None): j = { - 'id': self.get_id(), + 'id': self.public_id, 'user': self.user.username, 'in': self.start, 'out': self.end, @@ -125,8 +129,6 @@ class Annotation(models.Model): streams = self.item.streams() if streams: j['videoRatio'] = streams[0].aspect_ratio - if 'item' in keys: - j['item'] = self.item.itemId return j def __unicode__(self): diff --git a/pandora/annotation/views.py b/pandora/annotation/views.py index e37aa6f0..11056fba 100644 --- a/pandora/annotation/views.py +++ b/pandora/annotation/views.py @@ -58,7 +58,7 @@ def findAnnotations(request): qs = qs[query['range'][0]:query['range'][1]] response['data']['items'] = [p.json(keys=data['keys']) for p in qs] elif 'position' in query: - ids = [i.get_id() for i in qs] + ids = [i.public_id for i in qs] data['conditions'] = data['conditions'] + { 'value': data['position'], 'key': query['sort'][0]['key'], From 50adcb66fc18ba0ee06e242f91bd7d27c15ccac7 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Tue, 23 Aug 2011 19:39:34 +0200 Subject: [PATCH 3/3] rework files --- pandora/archive/admin.py | 2 +- pandora/archive/models.py | 40 ++++-------- pandora/archive/tasks.py | 3 + pandora/archive/views.py | 9 ++- pandora/item/models.py | 131 +++++++++++++++++++++++--------------- pandora/item/urls.py | 3 +- pandora/item/views.py | 16 ++++- pandora/user/models.py | 14 ++-- pandora/user/views.py | 18 +++++- 9 files changed, 138 insertions(+), 98 deletions(-) diff --git a/pandora/archive/admin.py b/pandora/archive/admin.py index e14ecddb..2a0fa6fb 100644 --- a/pandora/archive/admin.py +++ b/pandora/archive/admin.py @@ -9,7 +9,7 @@ import models class FileAdmin(admin.ModelAdmin): search_fields = ['name', 'folder','oshash', 'video_codec'] - list_display = ['available', 'is_main', '__unicode__', 'itemId'] + list_display = ['available', 'wanted', 'active', '__unicode__', 'itemId'] list_display_links = ('__unicode__', ) def itemId(self, obj): diff --git a/pandora/archive/models.py b/pandora/archive/models.py index bfbaf3bb..f1750016 100644 --- a/pandora/archive/models.py +++ b/pandora/archive/models.py @@ -30,7 +30,7 @@ class File(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) - verified = models.BooleanField(default=False) + active = models.BooleanField(default=False) auto = models.BooleanField(default=True) oshash = models.CharField(max_length=16, unique=True) @@ -69,34 +69,17 @@ class File(models.Model): #This is true if derivative is available or subtitles where uploaded available = models.BooleanField(default = False) + wanted = models.BooleanField(default = False) + uploading = models.BooleanField(default = False) is_audio = models.BooleanField(default=False) is_video = models.BooleanField(default=False) - is_extra = models.BooleanField(default=False) - is_main = models.BooleanField(default=False) is_subtitle = models.BooleanField(default=False) - is_version = models.BooleanField(default=False) def __unicode__(self): return self.name def set_state(self): - instance = self.get_instance() - if instance: - if instance.name.lower().startswith('extras/'): - self.is_extra = True - self.is_main = False - elif instance.name.lower().startswith('versions/'): - self.is_version = True - self.is_main = False - else: - self.is_extra = False - self.is_main = True - else: - self.is_main = False - self.is_extra = False - self.is_version = False - self.name = self.get_name() self.folder = self.get_folder() self.sort_name = utils.sort_string(canonicalTitle(self.name)) @@ -284,8 +267,8 @@ class File(models.Model): 'available': self.available, 'duration': duration, 'framerate': self.framerate, - 'height': self.height, - 'width': self.width, + #'height': self.height, + #'width': self.width, 'resolution': resolution, 'id': self.oshash, 'samplerate': self.samplerate, @@ -293,12 +276,12 @@ class File(models.Model): 'audio_codec': self.audio_codec, 'name': self.name, 'size': self.size, - 'info': self.info, - 'users': list(set([i.volume.user.username for i in self.instances.all()])), + #'info': self.info, + 'users': list(set([u.username + for u in User.objects.filter(volumes__files__in=self.instances.all())])), 'instances': [i.json() for i in self.instances.all()], 'folder': self.get_folder(), 'type': self.get_type(), - 'is_main': self.is_main, 'part': self.get_part() } if keys: @@ -314,12 +297,12 @@ class File(models.Model): if self.language: name = name[-(len(self.language)+1)] qs = self.item.files.filter(Q(is_video=True)|Q(is_audio=True), - is_main=True, name__startswith=name) + active=True, name__startswith=name) if qs.count()>0: return qs[0].part - if not self.is_extra: + if self.active: files = list(self.item.files.filter(type=self.type, language=self.language, - is_main=self.is_main).order_by('sort_name')) + active=self.active).order_by('sort_name')) if self in files: return files.index(self) + 1 return None @@ -412,6 +395,7 @@ class Instance(models.Model): name = models.CharField(max_length=2048) folder = models.CharField(max_length=2048) + extra = models.BooleanField(default=False) file = models.ForeignKey(File, related_name='instances') volume = models.ForeignKey(Volume, related_name='files') diff --git a/pandora/archive/tasks.py b/pandora/archive/tasks.py index 0b621c62..cce44282 100644 --- a/pandora/archive/tasks.py +++ b/pandora/archive/tasks.py @@ -66,6 +66,9 @@ def update_or_create_instance(volume, f): instance.file = get_or_create_file(volume, f, volume.user, item) for key in _INSTANCE_KEYS: setattr(instance, key, f[key]) + if instance.name.lower().startswith('extras/') or \ + instance.name.lower().startswith('versions/'): + instance.extra = True instance.save() instance.file.save() return instance diff --git a/pandora/archive/views.py b/pandora/archive/views.py index ce202ab5..e179b755 100644 --- a/pandora/archive/views.py +++ b/pandora/archive/views.py @@ -98,11 +98,12 @@ def update(request): if volume: files = files.filter(volume=volume) response['data']['info'] = [f.file.oshash for f in files.filter(file__info='{}')] - #needs some flag to find those that are actually used main is to generic response['data']['data'] = [f.file.oshash for f in files.filter(file__is_video=True, - file__is_main=True)] + file__available=False, + file__wanted=True)] response['data']['data'] += [f.file.oshash for f in files.filter(file__is_audio=True, - file__is_main=True)] + file__available=False, + file__wanted=True)] response['data']['file'] = [f.file.oshash for f in files.filter(file__is_subtitle=True, name__endswith='.srt')] @@ -188,6 +189,7 @@ def firefogg_upload(request): response['result'] = -1 elif form.cleaned_data['done']: f.available = True + f.uploading = False f.save() #FIXME: this fails badly if rabbitmq goes down try: @@ -205,6 +207,7 @@ def firefogg_upload(request): if f.editable(request.user): f.streams.all().delete() f.available = False + f.uploading = True f.save() response = { #is it possible to no hardcode url here? diff --git a/pandora/item/models.py b/pandora/item/models.py index 1ecc18b1..3dcbc705 100644 --- a/pandora/item/models.py +++ b/pandora/item/models.py @@ -34,6 +34,8 @@ from .timelines import join_timelines from archive import extract from annotation.models import Annotation, Layer +import archive.models + from person.models import get_name_sort from app.models import site_config @@ -340,11 +342,11 @@ class Item(models.Model): posters = [] - poster = self.path('poster.local.jpg') + poster = self.path('siteposter.jpg') poster = os.path.abspath(os.path.join(settings.MEDIA_ROOT, poster)) if os.path.exists(poster): posters.append({ - 'url': '/%s/poster.jpg' % self.itemId, + 'url': '/%s/siteposter.jpg' % self.itemId, 'width': 640, 'height': 1024, 'source': settings.URL, @@ -614,18 +616,17 @@ class Item(models.Model): s.wordsperminute = 0 s.clips = 0 #FIXME: get clips from all layers or something s.popularity = 0 #FIXME: get popularity from somewhere - videos = self.main_videos() - if len(videos) > 0: + videos = self.files.filter(active=True, is_video=True) + if videos.count() > 0: s.duration = sum([v.duration for v in videos]) - s.resolution = videos[0].width * videos[0].height + v = videos[0] + s.resolution = v.width * v.height s.aspectratio = float(utils.parse_decimal(v.display_aspect_ratio)) - #FIXME: should be average over all files - if 'bitrate' in videos[0].info: - s.bitrate = videos[0].info['bitrate'] s.pixels = sum([v.pixels for v in videos]) s.numberoffiles = self.files.all().count() - s.parts = len(videos) + s.parts = videos.count() s.size = sum([v.size for v in videos]) #FIXME: only size of movies? + s.bitrate = s.size * 8 / s.duration s.volume = 0 else: s.duration = None @@ -707,38 +708,62 @@ class Item(models.Model): @property def timeline_prefix(self): - videos = self.main_videos() + videos = self.streams() if len(videos) == 1: return os.path.join(settings.MEDIA_ROOT, videos[0].path('timeline')) return os.path.join(settings.MEDIA_ROOT, self.path(), 'timeline') - def main_videos(self): - #FIXME: needs to check if more than one user has main files and only - # take from "higher" user - videos = self.files.filter(is_main=True, is_video=True, available=True, instances__gt=0).order_by('part') - if videos.count()>0: - first = videos[0] - user = first.instances.all()[0].volume.user - #only take videos from same user and with same width/height - def check(v): - if v.instances.filter(volume__user=user).count()>0 and \ - first.width == v.width and first.height == v.height: - return True - return False - videos = filter(check, videos) + def get_files(self, user): + #FIXME: limit by user + return [f.json() for f in self.files.all()] + + def users_with_files(self): + return User.objects.filter(volumes__files__file__item=self).distinct() + + def update_wanted(self): + users = self.users_with_files() + if users.filter(is_superuser=True).count()>0: + files = self.files.filter(instances__volume__user__is_superuser=True) + users = User.objects.filter(volumes__files__file__item__in=files, + is_superuser=True).distinct() + elif users.filter(is_staff=True).count()>0: + files = self.files.filter(instances__volume__user__is_staff=True) + users = User.objects.filter(volumes__files__file__item__in=files, + is_staff=True).distinct() else: - audio = self.files.filter(is_main=True, is_audio=True, available=True) - if audio.count()>0: - first = audio[0] - user = first.instances.all()[0].volume.user - #only take videos from same user and with same width/height - def check(v): - if v.instances.filter(volume__user=user).count()>0: - return True - return False - videos = filter(check, audio) - - return videos + files = self.files.all() + files = files.filter(is_video=True, instances__extra=False, instances__gt=0).order_by('part') + folders = list(set([f.folder for f in files])) + if len(folders) > 1: + files = files.filter(folder=folders[0]) + files.update(wanted=True) + + def update_selected(self): + files = archive.models.File.objects.filter(item=self, + streams__available=True, + streams__source=None) + def get_level(users): + if users.filter(is_superuser=True).count() > 0: level = 0 + elif users.filter(is_staff=True).count() > 0: level = 1 + else: level = 2 + return level + + users = User.objects.filter(volumes__files__file__in=self.files.filter(active=True)).distinct() + current_level = get_level(users) + + users = User.objects.filter(volumes__files__file__in=files).distinct() + possible_level = get_level(users) + + if possible_level < current_level: + files = self.files.filter(instances__volume__user__in=users).order_by('part') + #FIXME: this should be instance folders + folders = list(set([f.folder + for f in files.filter(is_video=True, instances__extra=False)])) + files = files.filter(folder__startswith=folders[0]) + files.update(active=True) + self.rendered = False + self.save() + self.update_timeline() def make_torrent(self): base = self.path('torrent') @@ -751,26 +776,25 @@ class Item(models.Model): base = os.path.abspath(os.path.join(settings.MEDIA_ROOT, base)) size = 0 duration = 0.0 - if len(self.main_videos()) == 1: + streams = self.streams() + if streams.count() == 1: url = "%s/torrent/%s.webm" % (self.get_absolute_url(), quote(self.get('title').encode('utf-8'))) video = "%s.webm" % base - v = self.main_videos()[0] + v = streams[0] os.symlink(v.video.path, video) - info = ox.avinfo(video) - size = info.get('size', 0) - duration = info.get('duration', 0.0) + size = v.video.size + duration = v.duration else: url = "%s/torrent/" % self.get_absolute_url() part = 1 os.makedirs(base) - for v in self.main_videos(): + for v in streams: video = "%s/%s.Part %d.webm" % (base, self.get('title'), part) part += 1 os.symlink(v.video.path, video) - info = ox.avinfo(video) - size += info.get('size', 0) - duration += info.get('duration', 0.0) + size += v.video.size + duration += v.duration video = base torrent = '%s.torrent' % base @@ -794,7 +818,7 @@ class Item(models.Model): def streams(self): return [video.streams.filter(source=None, available=True)[0] - for video in self.main_videos()] + for video in self.files.filter(is_video=True, active=True)] def update_timeline(self, force=False): config = site_config() @@ -846,8 +870,7 @@ class Item(models.Model): else: poster= self.path('poster.jpg') path = os.path.abspath(os.path.join(settings.MEDIA_ROOT, poster)) - posters = glob(path.replace('.jpg', '*.jpg')) - for f in filter(lambda p: not p.endswith('poster.local.jpg'), posters): + for f in glob(path.replace('.jpg', '*.jpg')): os.unlink(f) def prefered_poster_url(self): @@ -883,7 +906,7 @@ class Item(models.Model): self.poster.save('poster.jpg', ContentFile(f.read())) def make_local_poster(self): - poster = self.path('poster.local.jpg') + poster = self.path('siteposter.jpg') poster = os.path.abspath(os.path.join(settings.MEDIA_ROOT, poster)) frame = self.get_poster_frame_path() @@ -914,12 +937,14 @@ class Item(models.Model): ox.makedirs(os.path.join(settings.MEDIA_ROOT,self.path())) p = subprocess.Popen(cmd) p.wait() + for f in glob(poster.replace('.jpg', '*.jpg')): + os.unlink(f) return poster def poster_frames(self): frames = [] offset = 0 - for f in self.main_videos(): + for f in self.files(active=True, is_video=True): for ff in f.frames.all(): frames.append({ 'position': offset + ff.position, @@ -976,7 +1001,7 @@ class Item(models.Model): Annotation.objects.filter(layer=layer,item=self).delete() offset = 0 language = '' - languages = [f.language for f in self.files.filter(is_main=True, is_subtitle=True, + languages = [f.language for f in self.files.filter(active=True, is_subtitle=True, available=True)] if languages: if 'en' in languages: @@ -985,7 +1010,7 @@ class Item(models.Model): language = '' else: language = languages[0] - for f in self.files.filter(is_main=True, is_subtitle=True, + for f in self.files.filter(active=True, is_subtitle=True, available=True, language=language).order_by('part'): user = f.instances.all()[0].volume.user for data in f.srt(offset): @@ -999,7 +1024,7 @@ class Item(models.Model): ) annotation.save() duration = self.files.filter(Q(is_audio=True)|Q(is_video=True)) \ - .filter(is_main=True, available=True, part=f.part) + .filter(active=True, available=True, part=f.part) if duration: duration = duration[0].duration else: diff --git a/pandora/item/urls.py b/pandora/item/urls.py index 1ee7264b..436212ef 100644 --- a/pandora/item/urls.py +++ b/pandora/item/urls.py @@ -23,7 +23,8 @@ urlpatterns = patterns("item.views", #poster (r'^(?P[A-Z0-9].+)/poster(?P\d+)\.jpg$', 'poster'), - (r'^(?P[A-Z0-9].+)/poster\.jpg$', 'poster_local'), + (r'^(?P[A-Z0-9].+)/siteposter(?P\d+)\.jpg$', 'siteposter'), + (r'^(?P[A-Z0-9].+)/poster\.jpg$', 'siteposter'), (r'^(?P[A-Z0-9].+)/frameposter(?P\d+).jpg$', 'poster_frame'), diff --git a/pandora/item/views.py b/pandora/item/views.py index ac1cc8ca..61ff36a3 100644 --- a/pandora/item/views.py +++ b/pandora/item/views.py @@ -5,6 +5,7 @@ import os.path from datetime import datetime, timedelta import mimetypes +import Image from django.db.models import Count, Sum, Max from django.http import HttpResponse, Http404 from django.shortcuts import get_object_or_404, redirect @@ -343,6 +344,8 @@ def get(request): info['stream'] = item.get_stream() if not data['keys'] or 'layers' in data['keys']: info['layers'] = item.get_layers(request.user) + if data['keys'] and 'files' in data['keys']: + info['files'] = item.get_files(request.user) response['data'] = info else: response = json_response(status=403, text='permission denied') @@ -582,7 +585,7 @@ def poster_frame(request, id, position): raise Http404 -def image_to_response(item, image, size=None): +def image_to_response(image, size=None): if size: size = int(size) path = image.path.replace('.jpg', '.%d.jpg'%size) @@ -596,10 +599,17 @@ def image_to_response(item, image, size=None): path = image.path return HttpFileResponse(path, content_type='image/jpeg') -def poster_local(request, id): +def siteposter(request, id, size=None): item = get_object_or_404(models.Item, itemId=id) - poster = item.path('poster.local.jpg') + poster = item.path('siteposter.jpg') poster = os.path.abspath(os.path.join(settings.MEDIA_ROOT, poster)) + if size: + image = Image.open(poster) + image_size = max(image.size) + if size < image_size: + path = poster.replace('.jpg', '.%d.jpg'%size) + extract.resize_image(poster, path, size=size) + poster = path return HttpFileResponse(poster, content_type='image/jpeg') def poster(request, id, size=None): diff --git a/pandora/user/models.py b/pandora/user/models.py index e1caf575..5fbe434d 100644 --- a/pandora/user/models.py +++ b/pandora/user/models.py @@ -81,6 +81,13 @@ class UserProfile(models.Model): del ui['lists'][i] return ui + def get_level(self): + if self.user.is_superuser: + return 'admin' + elif self.user.is_staff: + return 'staff' + return 'member' + def user_post_save(sender, instance, **kwargs): profile, new = UserProfile.objects.get_or_create(user=instance) @@ -92,12 +99,7 @@ def get_user_json(user): result = {} for key in ('username', ): result[key] = getattr(user, key) - if user.is_superuser: - result['level'] = 'admin' - elif user.is_staff: - result['level'] = 'staff' - else: - result['level'] = 'member' + result['level'] = profile.get_level() result['groups'] = [g.name for g in user.groups.all()] result['preferences'] = profile.get_preferences() result['ui'] = profile.get_ui() diff --git a/pandora/user/views.py b/pandora/user/views.py index 2fa5bb53..f8fbd294 100644 --- a/pandora/user/views.py +++ b/pandora/user/views.py @@ -309,22 +309,34 @@ def findUser(request): param data { key: "username", value: "foo", operator: "=" + keys: [] } return { 'status': {'code': int, 'text': string} 'data': { - users = ['user1', 'user2'] + users = [{username: 'user1', level: ...}, {username: 'user2', ..}] } } ''' #FIXME: support other operators and keys data = json.loads(request.POST['data']) response = json_response(status=200, text='ok') + keys = data.get('keys') + if not keys: + keys = ['username', 'level'] + def user_json(user, keys): + return { + 'usernname': user.username, + 'level': user.get_profile().get_level() + } + if data['key'] == 'email': - response['data']['users'] = [u.username for u in User.objects.filter(email__iexact=data['value'])] + response['data']['users'] = [user_json(u, keys) + for u in User.objects.filter(email__iexact=data['value'])] else: - response['data']['users'] = [u.username for u in User.objects.filter(username__iexact=data['value'])] + response['data']['users'] = [user_json(u, keys) + for u in User.objects.filter(username__iexact=data['value'])] return render_to_json_response(response) actions.register(findUser)