From 2dd8f949b6a63abe6bac35e7f44dc9bdfc27cb65 Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Thu, 19 Jun 2008 12:14:38 +0200 Subject: [PATCH] more vi:si:et:sw=4:sts=4:ts=4 --- oxdbarchive/backup.py | 113 +++++++++++++++++++------------------ oxdbarchive/cache.py | 111 ++++++++++++++++++------------------ oxdbarchive/controllers.py | 80 +++++++++++++------------- oxdbarchive/cron.py | 59 ++++++++++--------- oxdbarchive/extract.py | 2 - oxdbarchive/midentify.py | 2 - oxdbarchive/model.py | 88 +++++++++++++++++++++++++---- oxdbarchive/oxdb_import.py | 2 - oxdbarchive/oxdb_utils.py | 2 - oxdbarchive/timeline.py | 2 - 10 files changed, 258 insertions(+), 203 deletions(-) diff --git a/oxdbarchive/backup.py b/oxdbarchive/backup.py index 72147d1..acded75 100644 --- a/oxdbarchive/backup.py +++ b/oxdbarchive/backup.py @@ -1,65 +1,68 @@ +# -*- coding: utf-8 -*- +# vi:si:et:sw=4:sts=4:ts=4 from oxdb_utils import oxdb_makedir from os.path import * -import cache +import cache import extract + class BackupMovieFile: - def __init__(self, parent): - self.parent = parent - - def extractFrame(self, position, img_folder): - return self.parent.extractFrame(position) - + def __init__(self, parent): + self.parent = parent + + def extractFrame(self, position, img_folder): + return self.parent.extractFrame(position) + class Backup: - def __init__(self, md5sum): - self.md5sum = md5sum - self.timelineFile = join(cache.cache_root, 'timeline', self.md5sum[:4], "%s.png" % self.md5sum) - self.posterStill = join(cache.cache_root, 'posterStill', self.md5sum[:4], "%s.png" % self.md5sum) - self.mini_movie_file = join(cache.cache_root, 'mini', self.md5sum[:4], "%s.avi" % self.md5sum) - self.movieFile = BackupMovieFile(self) - self.frameFolder = join(cache.cache_root, 'frame', self.md5sum[:4], self.md5sum) - - def exists(self): - return exists(self.mini_movie_file) + def __init__(self, md5sum): + self.md5sum = md5sum + self.timelineFile = join(cache.cache_root, 'timeline', self.md5sum[:4], "%s.png" % self.md5sum) + self.posterStill = join(cache.cache_root, 'posterStill', self.md5sum[:4], "%s.png" % self.md5sum) + self.mini_movie_file = join(cache.cache_root, 'mini', self.md5sum[:4], "%s.avi" % self.md5sum) + self.movieFile = BackupMovieFile(self) + self.frameFolder = join(cache.cache_root, 'frame', self.md5sum[:4], self.md5sum) - def timeline(self): - return cache.loadTimeline(self) - - def frame(self, pos): - return cache.loadFrame(self, pos) - - def posterStill(self, pos): - return cache.loadPosterStill(self, pos) + def exists(self): + return exists(self.mini_movie_file) - def flvClip(self, position): - return cache.loadFlvClip(self, position) + def timeline(self): + return cache.loadTimeline(self) - def extractFlvClip(self, inpoint, outpoint=-1): - flash_folder = self.frameFolder - movie_file = self.mini_movie_file - position = inpoint.replace(':', '.') - flash_movie = join(self.frameFolder, '%s.%s' % (position, 'flv')) - width = 128 - height = int(width / self.frameAspect) - height = height - height % 2 - inpoint = inpoint.replace('.', ':') - if outpoint == -1: - outpoint = shift_time(5000, inpoint) - else: - outpoint = outpoint.replace('.', ':') - extract.extract_flash(movie_file, flash_movie, inpoint, outpoint, width, height, offset = 0) - - def extractFrame(self, position, img_folder=-1): - img_folder = self.frameFolder - movie_file = self.mini_movie_file - oxdb_makedir(img_folder) - position = position.replace('.', ':') - extract.extract_frame(movie_file, position, img_folder, offset = 0, redo = False) - - def extractPosterStill(self, position): - oxdb_makedir(dirname(self.posterStillFile)) - mFile = self.absolutePath - if os.path.splitext(mFile)[-1] in ('.mov', '.mpg', '.mpeg'): - mFile = self.mini_movie_file - extract.extract_poster_still(mFile, self.posterStillFile, position) + def frame(self, pos): + return cache.loadFrame(self, pos) + + def posterStill(self, pos): + return cache.loadPosterStill(self, pos) + + def flvClip(self, position): + return cache.loadFlvClip(self, position) + + def extractFlvClip(self, inpoint, outpoint=-1): + flash_folder = self.frameFolder + movie_file = self.mini_movie_file + position = inpoint.replace(':', '.') + flash_movie = join(self.frameFolder, '%s.%s' % (position, 'flv')) + width = 128 + height = int(width / self.frameAspect) + height = height - height % 2 + inpoint = inpoint.replace('.', ':') + if outpoint == -1: + outpoint = shift_time(5000, inpoint) + else: + outpoint = outpoint.replace('.', ':') + extract.extract_flash(movie_file, flash_movie, inpoint, outpoint, width, height, offset = 0) + + def extractFrame(self, position, img_folder=-1): + img_folder = self.frameFolder + movie_file = self.mini_movie_file + oxdb_makedir(img_folder) + position = position.replace('.', ':') + extract.extract_frame(movie_file, position, img_folder, offset = 0, redo = False) + + def extractPosterStill(self, position): + oxdb_makedir(dirname(self.posterStillFile)) + mFile = self.absolutePath + if os.path.splitext(mFile)[-1] in ('.mov', '.mpg', '.mpeg'): + mFile = self.mini_movie_file + extract.extract_poster_still(mFile, self.posterStillFile, position) diff --git a/oxdbarchive/cache.py b/oxdbarchive/cache.py index 16e120f..6afb372 100644 --- a/oxdbarchive/cache.py +++ b/oxdbarchive/cache.py @@ -1,7 +1,5 @@ -# -*- Mode: Python; -*- # -*- coding: utf-8 -*- -# vi:si:et:sw=2:sts=2:ts=2 - +# vi:si:et:sw=4:sts=4:ts=4 import os from os.path import abspath, exists, join, dirname, basename import shutil @@ -18,70 +16,71 @@ img_extension = "jpg" frame_cache_root = join(cache_root, 'frame') def loadFile(f_name): - f = open(f_name) - data = f.read() - f.close() - return data + f = open(f_name) + data = f.read() + f.close() + return data def saveFile(f_name, data): - f = open(f_name, 'w') - f.write(data) - f.close() + f = open(f_name, 'w') + f.write(data) + f.close() def loadStaticFile(fname): - return loadFile(join(dirname(abspath(__file__)), "static", fname)) + return loadFile(join(dirname(abspath(__file__)), "static", fname)) def loadFrame(afile, position): - position = basename(position).replace(':', '.') - frame = join(afile.frameFolder, '%s.%s' % (position, img_extension)) - if not exists(frame): - afile.extractFrame(position) - if exists(frame): - return loadFile(frame) - return '' + position = basename(position).replace(':', '.') + frame = join(afile.frameFolder, '%s.%s' % (position, img_extension)) + if not exists(frame): + afile.extractFrame(position) + if exists(frame): + return loadFile(frame) + return '' def loadFlvClip(afile, position): - position = basename(position).replace(':', '.') - positions = position.split(';') - if len(positions) > 1: - position = positions[0] - outpoint = positions[1] - else: - outpoint = -1 - flash = join(afile.frameFolder, '%s.%s' % (position, 'flv')) - if not exists(flash): - afile.extractFlvClip(position, outpoint) - if exists(flash): - return loadFile(flash) - return '' + position = basename(position).replace(':', '.') + positions = position.split(';') + if len(positions) > 1: + position = positions[0] + outpoint = positions[1] + else: + outpoint = -1 + flash = join(afile.frameFolder, '%s.%s' % (position, 'flv')) + if not exists(flash): + afile.extractFlvClip(position, outpoint) + if exists(flash): + return loadFile(flash) + return '' def loadOggClip(afile, position): - position = basename(position).replace(':', '.') - positions = position.split(';') - if len(positions) > 1: - position = positions[0] - outpoint = positions[1] - else: - outpoint = -1 - clip = join(afile.frameFolder, '%s.%s' % (position, 'ogv')) - if not exists(clip): - afile.extractOggClip(position, outpoint) - if exists(clip): - return loadFile(clip) - return '' + position = basename(position).replace(':', '.') + positions = position.split(';') + if len(positions) > 1: + position = positions[0] + outpoint = positions[1] + else: + outpoint = -1 + clip = join(afile.frameFolder, '%s.%s' % (position, 'ogv')) + if not exists(clip): + afile.extractOggClip(position, outpoint) + if exists(clip): + return loadFile(clip) + return '' def loadTimeline(afile): - timeline = afile.timelineFile - if not exists(timeline): - afile.extractTimeline() - if exists(timeline): - return loadFile(timeline) - return '' + timeline = afile.timelineFile + if not exists(timeline): + afile.extractTimeline() + if exists(timeline): + return loadFile(timeline) + return '' def loadPosterStill(afile, position): - still = afile.posterStillFile - if not exists(still): - afile.extractPosterStill(position) - if exists(still): - return loadFile(still) - return '' + still = afile.posterStillFile + if not exists(still): + afile.extractPosterStill(position) + if exists(still): + return loadFile(still) + return '' + diff --git a/oxdbarchive/controllers.py b/oxdbarchive/controllers.py index a360714..c5248d5 100644 --- a/oxdbarchive/controllers.py +++ b/oxdbarchive/controllers.py @@ -1,7 +1,5 @@ -# -*- Mode: Python; -*- # -*- coding: utf-8 -*- -# vi:si:et:sw=2:sts=2:ts=2 - +# vi:si:et:sw=4:sts=4:ts=4 from turbogears import controllers, expose, flash, redirect from model import * from json import * @@ -18,45 +16,43 @@ import backup ''' def httpExpires(sec): - exp_date = cherrypy.lib.httptools.HTTPDate(time.gmtime(time.mktime(time.gmtime()) + sec)) - return exp_date + exp_date = cherrypy.lib.httptools.HTTPDate(time.gmtime(time.mktime(time.gmtime()) + sec)) + return exp_date class Root(controllers.RootController): - @expose() - def default(self, md5Hash = None, action = None, position = None): - if not md5Hash: - return dict() - try: - f = ArchiveFile.byMd5sum(md5Hash) - except: - #return dict() - f = backup.Backup(md5Hash) - if not f.exists(): + @expose() + def default(self, md5Hash = None, action = None, position = None): + if not md5Hash: + return dict() + try: + f = ArchiveFile.byMd5sum(md5Hash) + except: + #return dict() + f = backup.Backup(md5Hash) + if not f.exists(): + return dict() + if action == 'metadata': + return dict(metadata = f) + elif action in ('timeline', 'timeline.png'): + cherrypy.response.headerMap['Content-Type'] = "image/png" + cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15) + return f.timeline() + elif position: #clip / frame + position = position.replace('.png', '').replace('.jpg', '') + position = position.replace('.flv', '').replace('.ogv', '') + position = position.replace('-', ':').replace('.',':') + cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15) + if action == 'clip': + cherrypy.response.headerMap['Content-Type'] = "video/x-flv" + return f.flvClip(position) + if action == 'ogv': + cherrypy.response.headerMap['Content-Type'] = "video/ogg" + return f.oggClip(position) + elif action == 'frame': + cherrypy.response.headerMap['Content-Type'] = "image/jpeg" + return f.frame(position) + elif action == 'posterStill': + cherrypy.response.headerMap['Content-Type'] = "image/png" + return f.posterStill(position) return dict() - if action == 'metadata': - return dict(metadata = f) - elif action in ('timeline', 'timeline.png'): - cherrypy.response.headerMap['Content-Type'] = "image/png" - cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15) - return f.timeline() - elif position: #clip / frame - position = position.replace('.png', '').replace('.jpg', '') - position = position.replace('.flv', '').replace('.ogv', '') - position = position.replace('-', ':').replace('.',':') - if action == 'clip': - cherrypy.response.headerMap['Content-Type'] = "video/x-flv" - cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15) - return f.flvClip(position) - if action == 'ogv': - cherrypy.response.headerMap['Content-Type'] = "video/ogg" - cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15) - return f.oggClip(position) - elif action == 'frame': - cherrypy.response.headerMap['Content-Type'] = "image/jpeg" - cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15) - return f.frame(position) - elif action == 'posterStill': - cherrypy.response.headerMap['Content-Type'] = "image/png" - cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15) - return f.posterStill(position) - return dict() + diff --git a/oxdbarchive/cron.py b/oxdbarchive/cron.py index 7584e4d..f703cbd 100644 --- a/oxdbarchive/cron.py +++ b/oxdbarchive/cron.py @@ -1,48 +1,47 @@ -# -*- Mode: Python; -*- # -*- coding: utf-8 -*- -# vi:si:et:sw=2:sts=2:ts=2 - +# vi:si:et:sw=4:sts=4:ts=4 from model import * import cache def cronDaily(): - findNew() - extractNew() + findNew() + extractNew() def findNew(): - archive = Archive.get(1) - archive.importFiles() + archive = Archive.get(1) + archive.importFiles() def extractFrames(): - for f in ArchiveFile.select(ArchiveFile.q.extracted == False): - f.extractFrames() + for f in ArchiveFile.select(ArchiveFile.q.extracted == False): + f.extractFrames() def extractTimelines(): - for f in ArchiveFile.select(ArchiveFile.q.extracted == False): - f.extractTimeline() + for f in ArchiveFile.select(ArchiveFile.q.extracted == False): + f.extractTimeline() def extractNew(): - for f in ArchiveFile.select(ArchiveFile.q.extracted == False): - f.extractAll() + for f in ArchiveFile.select(ArchiveFile.q.extracted == False): + f.extractAll() def cleanClipMovieCache(): - cache = os.path.abspath('oxdb/cache/mini/') - for d, p, files in os.walk(cache): - for f in files: - md5sum = f.replace('.avi', '') - try: - fm = ArchiveFile.byMd5sum(md5sum) - except: - os.remove(os.path.join(cache, d, f)) + cache = os.path.abspath('oxdb/cache/mini/') + for d, p, files in os.walk(cache): + for f in files: + md5sum = f.replace('.avi', '') + try: + fm = ArchiveFile.byMd5sum(md5sum) + except: + os.remove(os.path.join(cache, d, f)) #possible fuctions: def extractSubtitles(): - extractSubtitlesScript = abspath(join(dirname(cache.cache_root), "tools/subtitles.py")) - for f in ArchiveFile.select(LIKE(ArchiveFile.q.path, '%' + '.idx')): - base = f.absolutePath.replace('.idx', '') - srtFile = f.absolutePath.replace('.idx', '.srt') - if not os.path.exists(srtFile): - cmd = 'python "%s" "%s"' % (extractSubtitlesScript, base) - os.system(cmd) - #only one subtitle at a time - return + extractSubtitlesScript = abspath(join(dirname(cache.cache_root), "tools/subtitles.py")) + for f in ArchiveFile.select(LIKE(ArchiveFile.q.path, '%' + '.idx')): + base = f.absolutePath.replace('.idx', '') + srtFile = f.absolutePath.replace('.idx', '.srt') + if not os.path.exists(srtFile): + cmd = 'python "%s" "%s"' % (extractSubtitlesScript, base) + os.system(cmd) + #only one subtitle at a time + return + diff --git a/oxdbarchive/extract.py b/oxdbarchive/extract.py index dc5cbd5..98a26e7 100644 --- a/oxdbarchive/extract.py +++ b/oxdbarchive/extract.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -# -*- Mode: Python; -*- # vi:si:et:sw=2:sts=2:ts=2 - import re import os from os.path import abspath, join, dirname diff --git a/oxdbarchive/midentify.py b/oxdbarchive/midentify.py index fc78663..1a298c8 100644 --- a/oxdbarchive/midentify.py +++ b/oxdbarchive/midentify.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -# -*- Mode: Python; -*- # vi:si:et:sw=2:sts=2:ts=2 - import os import sys diff --git a/oxdbarchive/model.py b/oxdbarchive/model.py index 7ecd017..d4e6dfc 100644 --- a/oxdbarchive/model.py +++ b/oxdbarchive/model.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -# -*- Mode: Python; -*- # vi:si:et:sw=2:sts=2:ts=2 import gzip import StringIO @@ -353,10 +352,13 @@ class ArchiveFile(SQLObject): self.frameAspect = "%0.6f" % aspect return aspect + def _get_sceneWidth(self): + return 128 + def _get_sceneHeight(self): default = 80 if not self.subtitle: - h = int(128 / self.frameAspect) + h = int(self.sceneWidth / self.frameAspect) h = h + h % 2 return h return default @@ -450,9 +452,8 @@ class ArchiveFile(SQLObject): movie_file = self.mini_movie_file position = inpoint.replace(':', '.') flash_movie = join(self.frameFolder, '%s.%s' % (position, 'flv')) - width = 128 - height = int(width / self.frameAspect) - height = height - height % 2 + width = self.sceneWidth + height = self.sceneHeight inpoint = inpoint.replace('.', ':') if outpoint == -1: s = self._findSubtitleByInPoint(inpoint) @@ -471,9 +472,8 @@ class ArchiveFile(SQLObject): movie_file = self.mini_movie_file position = inpoint.replace(':', '.') clip_movie = join(self.frameFolder, '%s.%s' % (position, 'ogv')) - width = 128 - height = int(width / self.frameAspect) - height = height - height % 2 + width = self.sceneWidth + height = self.sceneHeight inpoint = inpoint.replace('.', ':') if outpoint == -1: s = self._findSubtitleByInPoint(inpoint) @@ -534,7 +534,7 @@ class ArchiveFile(SQLObject): options = '' options += " -ovc lavc -lavcopts vcodec=mjpeg" options += " -af volnorm=1 -oac mp3lame -lameopts br=64:mode=3 -af resample=44100" - options += " -vf scale -zoom -xy 128" + options += " -vf scale -zoom -xy %s" % self.sceneWidth options += ' "%s"' % movie_file.replace('"', '\\"') options += ' -o "%s"' % mini_movie_file cmd = "mencoder %s >/dev/null 2>&1" % options @@ -542,7 +542,75 @@ class ArchiveFile(SQLObject): r = os.system(cmd.encode('utf-8')) if r == 0: self.extracted = True - + + def extractOggMovie(self, force = False): + if self.broken: + return + if not self.height: + #only if midentify works we should try to extract the clip + return + if self.path.split('.')[-1] in ('mp3', 'wav', 'srt', 'sub', 'idx', 'rar','jpg', 'png'): + #ignore files known to not be + return + + oggFile = self.mini_movie_file.replace('.avi', '.ogg') + movieFile = self.absolutePath + if not movieFile or not exists(movieFile): + return + if exists(oggFile) and not force: + debug("clip exists, skipping extraction %s" % oggFile) + return + self.extractedOgg = False + oxdb_makedir(dirname(oggFile)) + options = '' + options += " --no-skeleton -K 16 -V 180 -a -1 -H 44100 -S 1 --speedlevel 0 -c 2 " + options += " -x %s -y %s" % (self.sceneWidth, self.sceneHeight) + options += ' "%s"' % movieFile.replace('"', '\\"') + options += ' -o "%s"' % oggFile + cmd = "ffmpeg2theora %s >/dev/null 2>&1" % options + r = os.system(cmd.encode('utf-8')) + if r == 0: + self.extractedOgg = True + + def extractH264Movie(self, force = False): + if self.broken: + return + if not self.height: + #only if midentify works we should try to extract the clip + return + if self.path.split('.')[-1] in ('mp3', 'wav', 'srt', 'sub', 'idx', 'rar','jpg', 'png'): + #ignore files known to not be + return + + h264File = self.mini_movie_file.replace('.avi', '.mp4') + h264FileTmp = h264File + ".tmp.mp4" + movieFile = self.absolutePath + if not movieFile or not exists(movieFile): + return + if exists(h264File) and not force: + debug("clip exists, skipping extraction %s" % h264File) + return + self.extractedH264 = False + height = self.sceneHeight + oxdb_makedir(dirname(h264File)) + options = '' + options += " -vcodec libx264 -b 112k -bf 3 -subq 6 -cmp 256 -refs 5 -qmin 10 " + options += " -qmax 51 -qdiff 4 -coder 1 -loop 1 -me hex -me_range 16 -trellis 1 " + options += " -flags +mv4 -flags2 +bpyramid+wpred+mixed_refs+brdo+8x8dct " + options += " -partitions parti4x4+parti8x8+partp4x4+partp8x8+partb8x8 -g 250 " + options += " -keyint_min 16 -sc_threshold 40 -i_qfactor 0.71" + options += " -s %sx%s" % (self.sceneWidth, height) + + cmd = '''ffmpeg -y -i "%s" -an -pass 1 -threads 2 %s "%s" >/dev/null 2>&1''' % (movieFile, options, h264FileTmp) + print cmd + r = os.system(cmd.encode('utf-8')) + cmd = '''ffmpeg -y -i "%s" -acodec libfaac -ac 2 -ar 44100 -ab 48k -pass 2 -threads 2 %s "%s" >/dev/null 2>&1''' % (movieFile, options, h264FileTmp) + r = os.system(cmd.encode('utf-8')) + cmd = '''qt-faststart "%s" "%s" >/dev/null 2>&1 && rm "%s"''' % (h264FileTmp, h264File, h264FileTmp) + r = os.system(cmd.encode('utf-8')) + if r == 0: + self.extractedH264 = True + def removeTimeline(self): if exists(self.timelineFile): os.unlink(self.timelineFile) diff --git a/oxdbarchive/oxdb_import.py b/oxdbarchive/oxdb_import.py index 539b7ab..46f4f65 100644 --- a/oxdbarchive/oxdb_import.py +++ b/oxdbarchive/oxdb_import.py @@ -1,11 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# -*- Mode: Python; -*- # vi:si:et:sw=2:sts=2:ts=2 # OXDb Import client, crawls the filesystem and gathers information about # movies # - import md5 import os import sys diff --git a/oxdbarchive/oxdb_utils.py b/oxdbarchive/oxdb_utils.py index aaca210..8ebc008 100644 --- a/oxdbarchive/oxdb_utils.py +++ b/oxdbarchive/oxdb_utils.py @@ -1,11 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# -*- Mode: Python; -*- # vi:si:et:sw=2:sts=2:ts=2 # OXDb Import client, crawls the filesystem and gathers information about # movies # - import md5 import os import sys diff --git a/oxdbarchive/timeline.py b/oxdbarchive/timeline.py index ce97076..8a8cbcb 100644 --- a/oxdbarchive/timeline.py +++ b/oxdbarchive/timeline.py @@ -1,7 +1,5 @@ -# -*- Mode: Python; -*- # -*- coding: utf-8 -*- # vi:si:et:sw=2:sts=2:ts=2 - import Image import math from StringIO import StringIO