From 4328d54450c5759e2dbdae642e002c7a71e8d77c Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Wed, 6 Apr 2011 20:57:25 +0200 Subject: [PATCH] server side cleanups and fixed for pandora_client sync --- pandora/archive/admin.py | 4 +-- pandora/archive/extract.py | 67 ++++++++++++++++++++------------------ pandora/archive/models.py | 39 ++++++++++++++-------- pandora/archive/tasks.py | 2 ++ pandora/archive/views.py | 22 ++++++++----- pandora/item/models.py | 8 ++--- pandora/item/utils.py | 6 +++- pandora/settings.py | 1 + 8 files changed, 88 insertions(+), 61 deletions(-) diff --git a/pandora/archive/admin.py b/pandora/archive/admin.py index a8057b7b7..e14ecddb4 100644 --- a/pandora/archive/admin.py +++ b/pandora/archive/admin.py @@ -8,7 +8,7 @@ import models class FileAdmin(admin.ModelAdmin): - search_fields = ['name', 'video_codec'] + search_fields = ['name', 'folder','oshash', 'video_codec'] list_display = ['available', 'is_main', '__unicode__', 'itemId'] list_display_links = ('__unicode__', ) @@ -21,7 +21,7 @@ admin.site.register(models.File, FileAdmin) class InstanceAdmin(admin.ModelAdmin): - search_fields = ['name', 'folder', 'volume__name'] + search_fields = ['name', 'folder', 'volume__name', 'file__oshash'] form = InstanceAdminForm admin.site.register(models.Instance, InstanceAdmin) diff --git a/pandora/archive/extract.py b/pandora/archive/extract.py index a2a20702e..638aae557 100644 --- a/pandora/archive/extract.py +++ b/pandora/archive/extract.py @@ -47,11 +47,8 @@ class AspectRatio(fractions.Fraction): def stream(video, target, profile, info): if not os.path.exists(target): - fdir = os.path.dirname(target) - if not os.path.exists(fdir): - os.makedirs(fdir) + ox.makedirs(os.path.dirname(target)) - dar = AspectRatio(info['video'][0]['display_aspect_ratio']) ''' WebM look into lag @@ -109,36 +106,27 @@ def stream(video, target, profile, info): audiochannels = 1 bpp = 0.17 - fps = AspectRatio(info['video'][0]['framerate']) + + if info['video'] and 'display_aspect_ratio' in info['video'][0]: + fps = AspectRatio(info['video'][0]['framerate']) - width = int(dar * height) - width += width % 2 + dar = AspectRatio(info['video'][0]['display_aspect_ratio']) + width = int(dar * height) + width += width % 2 - bitrate = height*width*fps*bpp/1000 - aspect = dar.ratio - #use 1:1 pixel aspect ratio if dar is close to that - if abs(width/height - dar) < 0.02: - aspect = '%s:%s' % (width, height) + bitrate = height*width*fps*bpp/1000 + aspect = dar.ratio + #use 1:1 pixel aspect ratio if dar is close to that + if abs(width/height - dar) < 0.02: + aspect = '%s:%s' % (width, height) - if info['audio']: - audio_settings = ['-ar', str(audiorate), '-aq', str(audioquality)] - if audiochannels and 'channels' in info['audio'][0] and info['audio'][0]['channels'] > audiochannels: - audio_settings += ['-ac', str(audiochannels)] - if audiobitrate: - audio_settings += ['-ab', audiobitrate] - if format == 'mp4': - audio_settings += ['-acodec', 'libfaac'] - else: - audio_settings += ['-acodec', 'libvorbis'] - else: - audio_settings = ['-an'] - - if info['video']: video_settings = [ '-vb', '%dk'%bitrate, '-g', '%d' % int(fps*2), '-s', '%dx%d'%(width, height), '-aspect', aspect, + #'-vf', 'yadif', ] + if format == 'mp4': #quicktime does not support bpyramid ''' @@ -185,6 +173,19 @@ def stream(video, target, profile, info): else: video_settings = ['-vn'] + if info['audio']: + audio_settings = ['-ar', str(audiorate), '-aq', str(audioquality)] + if audiochannels and 'channels' in info['audio'][0] and info['audio'][0]['channels'] > audiochannels: + audio_settings += ['-ac', str(audiochannels)] + if audiobitrate: + audio_settings += ['-ab', audiobitrate] + if format == 'mp4': + audio_settings += ['-acodec', 'libfaac'] + else: + audio_settings += ['-acodec', 'libvorbis'] + else: + audio_settings = ['-an'] + ffmpeg = FFMPEG2THEORA.replace('2theora', '') cmd = [ffmpeg, '-y', '-i', video] \ + audio_settings \ @@ -199,12 +200,16 @@ def stream(video, target, profile, info): cmd += [target] print cmd - p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=open('/dev/null', 'w'), stderr=subprocess.STDOUT) + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, + stdout=open('/dev/null', 'w'), + stderr=subprocess.STDOUT) p.communicate() if format == 'mp4': cmd = ['qt-faststart', "%s.mp4"%target, target] print cmd - p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=open('/dev/null', 'w'), stderr=subprocess.STDOUT) + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, + stdout=open('/dev/null', 'w'), + stderr=subprocess.STDOUT) p.communicate() os.unlink("%s.mp4"%target) return True @@ -212,7 +217,8 @@ def stream(video, target, profile, info): def run_command(cmd, timeout=10): #print cmd - p = subprocess.Popen(cmd, stdout=open('/dev/null', 'w'), stderr=subprocess.STDOUT) + p = subprocess.Popen(cmd, stdout=open('/dev/null', 'w'), + stderr=subprocess.STDOUT) while timeout > 0: time.sleep(0.2) timeout -= 0.2 @@ -236,8 +242,7 @@ def frame(videoFile, frame, position, width=128, redo=False): if exists(videoFile): frameFolder = os.path.dirname(frame) if redo or not exists(frame): - if not exists(frameFolder): - os.makedirs(frameFolder) + ox.makedirs(frameFolder) cmd = ['oxframe', '-i', videoFile, '-o', frame, '-p', str(position), '-x', str(width)] run_command(cmd) diff --git a/pandora/archive/models.py b/pandora/archive/models.py index f8981679a..99c3ccd61 100644 --- a/pandora/archive/models.py +++ b/pandora/archive/models.py @@ -74,10 +74,26 @@ class File(models.Model): return self.name def save(self, *args, **kwargs): - self.name= self.get_name() + instance = self.get_instance() + if instance: + if instance.name.lower().startswith('extras/'): + self.is_extra = True + self.is_main = False + elif instance.name.lower().startswith('versions/'): + self.is_version = True + self.is_main = False + else: + self.is_extra = False + self.is_main = True + else: + self.is_main = False + self.is_extra = False + self.is_version = False + + self.name = self.get_name() self.folder = self.get_folder() - if self.name and not self.sort_name: - self.sort_name = utils.sort_string(canonicalTitle(self.name)) + self.sort_name = utils.sort_string(canonicalTitle(self.name)) + if self.info: for key in ('duration', 'size'): setattr(self, key, self.info.get(key, 0)) @@ -126,16 +142,6 @@ class File(models.Model): if not self.is_audio and not self.is_video and self.name.endswith('.srt'): self.is_subtitle = True - if self.name and self.name.lower().startswith('extras/'): - self.is_extra = True - self.is_main = False - elif self.name and self.name.lower().startswith('versions/'): - self.is_version = True - self.is_main = False - else: - self.is_extra = False - self.is_main = True - self.part = self.get_part() self.type = self.get_type() @@ -290,6 +296,11 @@ class File(models.Model): return 'subtitle' return 'unknown' + def get_instance(self): + if self.instances.all().count() > 0: + return self.instances.all()[0] + return None + def get_folder(self): name = os.path.splitext(self.get_name())[0] if self.item: @@ -307,6 +318,8 @@ class File(models.Model): return u'' def get_name(self): + if self.is_extra: + return self.get_instance().name if self.item: name = self.item.get('title', 'Untitled') name = re.sub(r'[:\\/]', '_', name) diff --git a/pandora/archive/tasks.py b/pandora/archive/tasks.py index e3f136810..a1d16abc3 100644 --- a/pandora/archive/tasks.py +++ b/pandora/archive/tasks.py @@ -47,6 +47,7 @@ def update_or_create_instance(volume, f): updated=True if updated: instance.save() + instance.file.save() else: instance = models.Instance() instance.volume = volume @@ -54,6 +55,7 @@ def update_or_create_instance(volume, f): for key in _INSTANCE_KEYS: setattr(instance, key, f[key]) instance.save() + instance.file.save() return instance @task(ignore_resulsts=True, queue='default') diff --git a/pandora/archive/views.py b/pandora/archive/views.py index 7b8259971..3c4d8de67 100644 --- a/pandora/archive/views.py +++ b/pandora/archive/views.py @@ -66,6 +66,7 @@ def update(request): ''' data = json.loads(request.POST['data']) user = request.user + upload_only = data.get('upload', False) response = json_response({'info': [], 'data': [], 'file': []}) volume = None @@ -91,15 +92,18 @@ def update(request): del info[key] instance.file.info = info instance.file.save() - - files = models.Instance.objects.filter(volume__user=user, file__available=False) - if volume: - files = files.filter(volume=volume) - response['data']['info'] = [f.file.oshash for f in files.filter(file__info='{}')] - #needs some flag to find those that are actually used main is to generic - response['data']['data'] = [f.file.oshash for f in files.filter(file__is_video=True, file__is_main=True)] - response['data']['data'] += [f.file.oshash for f in files.filter(file__is_audio=True, file__is_main=True)] - response['data']['file'] = [f.file.oshash for f in files.filter(file__is_subtitle=True)] + if not upload_only: + files = models.Instance.objects.filter(volume__user=user, file__available=False) + if volume: + files = files.filter(volume=volume) + response['data']['info'] = [f.file.oshash for f in files.filter(file__info='{}')] + #needs some flag to find those that are actually used main is to generic + response['data']['data'] = [f.file.oshash for f in files.filter(file__is_video=True, + file__is_main=True)] + response['data']['data'] += [f.file.oshash for f in files.filter(file__is_audio=True, + file__is_main=True)] + response['data']['file'] = [f.file.oshash for f in files.filter(file__is_subtitle=True, + name__endswith='.srt')] return render_to_json_response(response) actions.register(update, cache=False) diff --git a/pandora/item/models.py b/pandora/item/models.py index 91cd62d31..9501c3ced 100644 --- a/pandora/item/models.py +++ b/pandora/item/models.py @@ -550,7 +550,7 @@ class Item(models.Model): for key in self.facet_keys: current_values = self.get(key, []) if not isinstance(current_values, list): - current_values = [current_values] + current_values = [unicode(current_values)] current_values = list(set(current_values)) saved_values = [i.value for i in Facet.objects.filter(item=self, key=key)] removed_values = filter(lambda i: i not in current_values, saved_values) @@ -632,8 +632,7 @@ class Item(models.Model): cmd = [] if os.path.exists(stream.video.path): os.unlink(stream.video.path) - elif not os.path.exists(os.path.dirname(stream.video.path)): - os.makedirs(os.path.dirname(stream.video.path)) + ox.makedirs(os.path.dirname(stream.video.path)) if len(files.values()) > 1: if len(files.values()) > 4: print "FIXME: to many files for this item, not merging entire tv shows" @@ -753,8 +752,7 @@ class Item(models.Model): posters = self.local_posters() timeline = self.path('timeline.64.png') timeline = os.path.abspath(os.path.join(settings.MEDIA_ROOT, timeline)) - if not os.path.exists(os.path.join(settings.MEDIA_ROOT,self.path())): - os.makedirs(os.path.join(settings.MEDIA_ROOT,self.path())) + ox.makedirs(os.path.join(settings.MEDIA_ROOT,self.path())) for poster in posters: frame = posters[poster] cmd = [settings.ITEM_POSTER, diff --git a/pandora/item/utils.py b/pandora/item/utils.py index f781ad06f..9f29fc2cd 100644 --- a/pandora/item/utils.py +++ b/pandora/item/utils.py @@ -102,6 +102,8 @@ def parse_title(_title, searchTitle = False): if year and title.endswith(year): title = title[:-len(year)].strip() title = normalizeTitle(title) + if searchTitle and year: + title = u"%s %s" % (title, year) return title @@ -180,7 +182,9 @@ def parse_path(path): r['season'], r['episode'] = parse_season_episode(path) r['series_title'] = parse_series_title(path) - r['imdbId'] = ox.web.imdb.guess(search_title, ', '.join(r['director']), timeout=-1) + #FIXME: use oxdata/id/?title=title&director=director&year=year + #r['imdbId'] = ox.web.imdb.guess(search_title, ', '.join(r['director']), timeout=-1) + r['imdbId'] = ox.web.imdb.guess(search_title, timeout=-1) r['oxdbId'] = oxdb_id(r['title'], r['director'], r.get('year', ''), r.get('season', ''), r.get('episode', ''), episode_title=r['episode_title'], diff --git a/pandora/settings.py b/pandora/settings.py index f727c4942..e2e27ed38 100644 --- a/pandora/settings.py +++ b/pandora/settings.py @@ -27,6 +27,7 @@ ADMINS = ( DEFAULT_FROM_EMAIL='admin@' + URL.split('/')[0] #DEFAULT_FROM_EMAIL='admin@example.com' +SERVER_EMAIL=DEFAULT_FROM_EMAIL MANAGERS = ADMINS