more vi:si:et:sw=4:sts=4:ts=4

This commit is contained in:
j 2008-06-19 12:14:38 +02:00
parent ecfb2ff28c
commit 2dd8f949b6
10 changed files with 258 additions and 203 deletions

View file

@ -1,65 +1,68 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from oxdb_utils import oxdb_makedir from oxdb_utils import oxdb_makedir
from os.path import * from os.path import *
import cache import cache
import extract import extract
class BackupMovieFile:
def __init__(self, parent):
self.parent = parent
def extractFrame(self, position, img_folder): class BackupMovieFile:
return self.parent.extractFrame(position) def __init__(self, parent):
self.parent = parent
def extractFrame(self, position, img_folder):
return self.parent.extractFrame(position)
class Backup: class Backup:
def __init__(self, md5sum): def __init__(self, md5sum):
self.md5sum = md5sum self.md5sum = md5sum
self.timelineFile = join(cache.cache_root, 'timeline', self.md5sum[:4], "%s.png" % self.md5sum) self.timelineFile = join(cache.cache_root, 'timeline', self.md5sum[:4], "%s.png" % self.md5sum)
self.posterStill = join(cache.cache_root, 'posterStill', self.md5sum[:4], "%s.png" % self.md5sum) self.posterStill = join(cache.cache_root, 'posterStill', self.md5sum[:4], "%s.png" % self.md5sum)
self.mini_movie_file = join(cache.cache_root, 'mini', self.md5sum[:4], "%s.avi" % self.md5sum) self.mini_movie_file = join(cache.cache_root, 'mini', self.md5sum[:4], "%s.avi" % self.md5sum)
self.movieFile = BackupMovieFile(self) self.movieFile = BackupMovieFile(self)
self.frameFolder = join(cache.cache_root, 'frame', self.md5sum[:4], self.md5sum) self.frameFolder = join(cache.cache_root, 'frame', self.md5sum[:4], self.md5sum)
def exists(self): def exists(self):
return exists(self.mini_movie_file) return exists(self.mini_movie_file)
def timeline(self): def timeline(self):
return cache.loadTimeline(self) return cache.loadTimeline(self)
def frame(self, pos): def frame(self, pos):
return cache.loadFrame(self, pos) return cache.loadFrame(self, pos)
def posterStill(self, pos): def posterStill(self, pos):
return cache.loadPosterStill(self, pos) return cache.loadPosterStill(self, pos)
def flvClip(self, position): def flvClip(self, position):
return cache.loadFlvClip(self, position) return cache.loadFlvClip(self, position)
def extractFlvClip(self, inpoint, outpoint=-1): def extractFlvClip(self, inpoint, outpoint=-1):
flash_folder = self.frameFolder flash_folder = self.frameFolder
movie_file = self.mini_movie_file movie_file = self.mini_movie_file
position = inpoint.replace(':', '.') position = inpoint.replace(':', '.')
flash_movie = join(self.frameFolder, '%s.%s' % (position, 'flv')) flash_movie = join(self.frameFolder, '%s.%s' % (position, 'flv'))
width = 128 width = 128
height = int(width / self.frameAspect) height = int(width / self.frameAspect)
height = height - height % 2 height = height - height % 2
inpoint = inpoint.replace('.', ':') inpoint = inpoint.replace('.', ':')
if outpoint == -1: if outpoint == -1:
outpoint = shift_time(5000, inpoint) outpoint = shift_time(5000, inpoint)
else: else:
outpoint = outpoint.replace('.', ':') outpoint = outpoint.replace('.', ':')
extract.extract_flash(movie_file, flash_movie, inpoint, outpoint, width, height, offset = 0) extract.extract_flash(movie_file, flash_movie, inpoint, outpoint, width, height, offset = 0)
def extractFrame(self, position, img_folder=-1): def extractFrame(self, position, img_folder=-1):
img_folder = self.frameFolder img_folder = self.frameFolder
movie_file = self.mini_movie_file movie_file = self.mini_movie_file
oxdb_makedir(img_folder) oxdb_makedir(img_folder)
position = position.replace('.', ':') position = position.replace('.', ':')
extract.extract_frame(movie_file, position, img_folder, offset = 0, redo = False) extract.extract_frame(movie_file, position, img_folder, offset = 0, redo = False)
def extractPosterStill(self, position): def extractPosterStill(self, position):
oxdb_makedir(dirname(self.posterStillFile)) oxdb_makedir(dirname(self.posterStillFile))
mFile = self.absolutePath mFile = self.absolutePath
if os.path.splitext(mFile)[-1] in ('.mov', '.mpg', '.mpeg'): if os.path.splitext(mFile)[-1] in ('.mov', '.mpg', '.mpeg'):
mFile = self.mini_movie_file mFile = self.mini_movie_file
extract.extract_poster_still(mFile, self.posterStillFile, position) extract.extract_poster_still(mFile, self.posterStillFile, position)

View file

@ -1,7 +1,5 @@
# -*- Mode: Python; -*-
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# vi:si:et:sw=2:sts=2:ts=2 # vi:si:et:sw=4:sts=4:ts=4
import os import os
from os.path import abspath, exists, join, dirname, basename from os.path import abspath, exists, join, dirname, basename
import shutil import shutil
@ -18,70 +16,71 @@ img_extension = "jpg"
frame_cache_root = join(cache_root, 'frame') frame_cache_root = join(cache_root, 'frame')
def loadFile(f_name): def loadFile(f_name):
f = open(f_name) f = open(f_name)
data = f.read() data = f.read()
f.close() f.close()
return data return data
def saveFile(f_name, data): def saveFile(f_name, data):
f = open(f_name, 'w') f = open(f_name, 'w')
f.write(data) f.write(data)
f.close() f.close()
def loadStaticFile(fname): def loadStaticFile(fname):
return loadFile(join(dirname(abspath(__file__)), "static", fname)) return loadFile(join(dirname(abspath(__file__)), "static", fname))
def loadFrame(afile, position): def loadFrame(afile, position):
position = basename(position).replace(':', '.') position = basename(position).replace(':', '.')
frame = join(afile.frameFolder, '%s.%s' % (position, img_extension)) frame = join(afile.frameFolder, '%s.%s' % (position, img_extension))
if not exists(frame): if not exists(frame):
afile.extractFrame(position) afile.extractFrame(position)
if exists(frame): if exists(frame):
return loadFile(frame) return loadFile(frame)
return '' return ''
def loadFlvClip(afile, position): def loadFlvClip(afile, position):
position = basename(position).replace(':', '.') position = basename(position).replace(':', '.')
positions = position.split(';') positions = position.split(';')
if len(positions) > 1: if len(positions) > 1:
position = positions[0] position = positions[0]
outpoint = positions[1] outpoint = positions[1]
else: else:
outpoint = -1 outpoint = -1
flash = join(afile.frameFolder, '%s.%s' % (position, 'flv')) flash = join(afile.frameFolder, '%s.%s' % (position, 'flv'))
if not exists(flash): if not exists(flash):
afile.extractFlvClip(position, outpoint) afile.extractFlvClip(position, outpoint)
if exists(flash): if exists(flash):
return loadFile(flash) return loadFile(flash)
return '' return ''
def loadOggClip(afile, position): def loadOggClip(afile, position):
position = basename(position).replace(':', '.') position = basename(position).replace(':', '.')
positions = position.split(';') positions = position.split(';')
if len(positions) > 1: if len(positions) > 1:
position = positions[0] position = positions[0]
outpoint = positions[1] outpoint = positions[1]
else: else:
outpoint = -1 outpoint = -1
clip = join(afile.frameFolder, '%s.%s' % (position, 'ogv')) clip = join(afile.frameFolder, '%s.%s' % (position, 'ogv'))
if not exists(clip): if not exists(clip):
afile.extractOggClip(position, outpoint) afile.extractOggClip(position, outpoint)
if exists(clip): if exists(clip):
return loadFile(clip) return loadFile(clip)
return '' return ''
def loadTimeline(afile): def loadTimeline(afile):
timeline = afile.timelineFile timeline = afile.timelineFile
if not exists(timeline): if not exists(timeline):
afile.extractTimeline() afile.extractTimeline()
if exists(timeline): if exists(timeline):
return loadFile(timeline) return loadFile(timeline)
return '' return ''
def loadPosterStill(afile, position): def loadPosterStill(afile, position):
still = afile.posterStillFile still = afile.posterStillFile
if not exists(still): if not exists(still):
afile.extractPosterStill(position) afile.extractPosterStill(position)
if exists(still): if exists(still):
return loadFile(still) return loadFile(still)
return '' return ''

View file

@ -1,7 +1,5 @@
# -*- Mode: Python; -*-
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# vi:si:et:sw=2:sts=2:ts=2 # vi:si:et:sw=4:sts=4:ts=4
from turbogears import controllers, expose, flash, redirect from turbogears import controllers, expose, flash, redirect
from model import * from model import *
from json import * from json import *
@ -18,45 +16,43 @@ import backup
''' '''
def httpExpires(sec): def httpExpires(sec):
exp_date = cherrypy.lib.httptools.HTTPDate(time.gmtime(time.mktime(time.gmtime()) + sec)) exp_date = cherrypy.lib.httptools.HTTPDate(time.gmtime(time.mktime(time.gmtime()) + sec))
return exp_date return exp_date
class Root(controllers.RootController): class Root(controllers.RootController):
@expose() @expose()
def default(self, md5Hash = None, action = None, position = None): def default(self, md5Hash = None, action = None, position = None):
if not md5Hash: if not md5Hash:
return dict() return dict()
try: try:
f = ArchiveFile.byMd5sum(md5Hash) f = ArchiveFile.byMd5sum(md5Hash)
except: except:
#return dict() #return dict()
f = backup.Backup(md5Hash) f = backup.Backup(md5Hash)
if not f.exists(): if not f.exists():
return dict()
if action == 'metadata':
return dict(metadata = f)
elif action in ('timeline', 'timeline.png'):
cherrypy.response.headerMap['Content-Type'] = "image/png"
cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
return f.timeline()
elif position: #clip / frame
position = position.replace('.png', '').replace('.jpg', '')
position = position.replace('.flv', '').replace('.ogv', '')
position = position.replace('-', ':').replace('.',':')
cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
if action == 'clip':
cherrypy.response.headerMap['Content-Type'] = "video/x-flv"
return f.flvClip(position)
if action == 'ogv':
cherrypy.response.headerMap['Content-Type'] = "video/ogg"
return f.oggClip(position)
elif action == 'frame':
cherrypy.response.headerMap['Content-Type'] = "image/jpeg"
return f.frame(position)
elif action == 'posterStill':
cherrypy.response.headerMap['Content-Type'] = "image/png"
return f.posterStill(position)
return dict() return dict()
if action == 'metadata':
return dict(metadata = f)
elif action in ('timeline', 'timeline.png'):
cherrypy.response.headerMap['Content-Type'] = "image/png"
cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
return f.timeline()
elif position: #clip / frame
position = position.replace('.png', '').replace('.jpg', '')
position = position.replace('.flv', '').replace('.ogv', '')
position = position.replace('-', ':').replace('.',':')
if action == 'clip':
cherrypy.response.headerMap['Content-Type'] = "video/x-flv"
cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
return f.flvClip(position)
if action == 'ogv':
cherrypy.response.headerMap['Content-Type'] = "video/ogg"
cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
return f.oggClip(position)
elif action == 'frame':
cherrypy.response.headerMap['Content-Type'] = "image/jpeg"
cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
return f.frame(position)
elif action == 'posterStill':
cherrypy.response.headerMap['Content-Type'] = "image/png"
cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
return f.posterStill(position)
return dict()

View file

@ -1,48 +1,47 @@
# -*- Mode: Python; -*-
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# vi:si:et:sw=2:sts=2:ts=2 # vi:si:et:sw=4:sts=4:ts=4
from model import * from model import *
import cache import cache
def cronDaily(): def cronDaily():
findNew() findNew()
extractNew() extractNew()
def findNew(): def findNew():
archive = Archive.get(1) archive = Archive.get(1)
archive.importFiles() archive.importFiles()
def extractFrames(): def extractFrames():
for f in ArchiveFile.select(ArchiveFile.q.extracted == False): for f in ArchiveFile.select(ArchiveFile.q.extracted == False):
f.extractFrames() f.extractFrames()
def extractTimelines(): def extractTimelines():
for f in ArchiveFile.select(ArchiveFile.q.extracted == False): for f in ArchiveFile.select(ArchiveFile.q.extracted == False):
f.extractTimeline() f.extractTimeline()
def extractNew(): def extractNew():
for f in ArchiveFile.select(ArchiveFile.q.extracted == False): for f in ArchiveFile.select(ArchiveFile.q.extracted == False):
f.extractAll() f.extractAll()
def cleanClipMovieCache(): def cleanClipMovieCache():
cache = os.path.abspath('oxdb/cache/mini/') cache = os.path.abspath('oxdb/cache/mini/')
for d, p, files in os.walk(cache): for d, p, files in os.walk(cache):
for f in files: for f in files:
md5sum = f.replace('.avi', '') md5sum = f.replace('.avi', '')
try: try:
fm = ArchiveFile.byMd5sum(md5sum) fm = ArchiveFile.byMd5sum(md5sum)
except: except:
os.remove(os.path.join(cache, d, f)) os.remove(os.path.join(cache, d, f))
#possible fuctions: #possible fuctions:
def extractSubtitles(): def extractSubtitles():
extractSubtitlesScript = abspath(join(dirname(cache.cache_root), "tools/subtitles.py")) extractSubtitlesScript = abspath(join(dirname(cache.cache_root), "tools/subtitles.py"))
for f in ArchiveFile.select(LIKE(ArchiveFile.q.path, '%' + '.idx')): for f in ArchiveFile.select(LIKE(ArchiveFile.q.path, '%' + '.idx')):
base = f.absolutePath.replace('.idx', '') base = f.absolutePath.replace('.idx', '')
srtFile = f.absolutePath.replace('.idx', '.srt') srtFile = f.absolutePath.replace('.idx', '.srt')
if not os.path.exists(srtFile): if not os.path.exists(srtFile):
cmd = 'python "%s" "%s"' % (extractSubtitlesScript, base) cmd = 'python "%s" "%s"' % (extractSubtitlesScript, base)
os.system(cmd) os.system(cmd)
#only one subtitle at a time #only one subtitle at a time
return return

View file

@ -1,7 +1,5 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# -*- Mode: Python; -*-
# vi:si:et:sw=2:sts=2:ts=2 # vi:si:et:sw=2:sts=2:ts=2
import re import re
import os import os
from os.path import abspath, join, dirname from os.path import abspath, join, dirname

View file

@ -1,7 +1,5 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# -*- Mode: Python; -*-
# vi:si:et:sw=2:sts=2:ts=2 # vi:si:et:sw=2:sts=2:ts=2
import os import os
import sys import sys

View file

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# -*- Mode: Python; -*-
# vi:si:et:sw=2:sts=2:ts=2 # vi:si:et:sw=2:sts=2:ts=2
import gzip import gzip
import StringIO import StringIO
@ -353,10 +352,13 @@ class ArchiveFile(SQLObject):
self.frameAspect = "%0.6f" % aspect self.frameAspect = "%0.6f" % aspect
return aspect return aspect
def _get_sceneWidth(self):
return 128
def _get_sceneHeight(self): def _get_sceneHeight(self):
default = 80 default = 80
if not self.subtitle: if not self.subtitle:
h = int(128 / self.frameAspect) h = int(self.sceneWidth / self.frameAspect)
h = h + h % 2 h = h + h % 2
return h return h
return default return default
@ -450,9 +452,8 @@ class ArchiveFile(SQLObject):
movie_file = self.mini_movie_file movie_file = self.mini_movie_file
position = inpoint.replace(':', '.') position = inpoint.replace(':', '.')
flash_movie = join(self.frameFolder, '%s.%s' % (position, 'flv')) flash_movie = join(self.frameFolder, '%s.%s' % (position, 'flv'))
width = 128 width = self.sceneWidth
height = int(width / self.frameAspect) height = self.sceneHeight
height = height - height % 2
inpoint = inpoint.replace('.', ':') inpoint = inpoint.replace('.', ':')
if outpoint == -1: if outpoint == -1:
s = self._findSubtitleByInPoint(inpoint) s = self._findSubtitleByInPoint(inpoint)
@ -471,9 +472,8 @@ class ArchiveFile(SQLObject):
movie_file = self.mini_movie_file movie_file = self.mini_movie_file
position = inpoint.replace(':', '.') position = inpoint.replace(':', '.')
clip_movie = join(self.frameFolder, '%s.%s' % (position, 'ogv')) clip_movie = join(self.frameFolder, '%s.%s' % (position, 'ogv'))
width = 128 width = self.sceneWidth
height = int(width / self.frameAspect) height = self.sceneHeight
height = height - height % 2
inpoint = inpoint.replace('.', ':') inpoint = inpoint.replace('.', ':')
if outpoint == -1: if outpoint == -1:
s = self._findSubtitleByInPoint(inpoint) s = self._findSubtitleByInPoint(inpoint)
@ -534,7 +534,7 @@ class ArchiveFile(SQLObject):
options = '' options = ''
options += " -ovc lavc -lavcopts vcodec=mjpeg" options += " -ovc lavc -lavcopts vcodec=mjpeg"
options += " -af volnorm=1 -oac mp3lame -lameopts br=64:mode=3 -af resample=44100" options += " -af volnorm=1 -oac mp3lame -lameopts br=64:mode=3 -af resample=44100"
options += " -vf scale -zoom -xy 128" options += " -vf scale -zoom -xy %s" % self.sceneWidth
options += ' "%s"' % movie_file.replace('"', '\\"') options += ' "%s"' % movie_file.replace('"', '\\"')
options += ' -o "%s"' % mini_movie_file options += ' -o "%s"' % mini_movie_file
cmd = "mencoder %s >/dev/null 2>&1" % options cmd = "mencoder %s >/dev/null 2>&1" % options
@ -543,6 +543,74 @@ class ArchiveFile(SQLObject):
if r == 0: if r == 0:
self.extracted = True self.extracted = True
def extractOggMovie(self, force = False):
if self.broken:
return
if not self.height:
#only if midentify works we should try to extract the clip
return
if self.path.split('.')[-1] in ('mp3', 'wav', 'srt', 'sub', 'idx', 'rar','jpg', 'png'):
#ignore files known to not be
return
oggFile = self.mini_movie_file.replace('.avi', '.ogg')
movieFile = self.absolutePath
if not movieFile or not exists(movieFile):
return
if exists(oggFile) and not force:
debug("clip exists, skipping extraction %s" % oggFile)
return
self.extractedOgg = False
oxdb_makedir(dirname(oggFile))
options = ''
options += " --no-skeleton -K 16 -V 180 -a -1 -H 44100 -S 1 --speedlevel 0 -c 2 "
options += " -x %s -y %s" % (self.sceneWidth, self.sceneHeight)
options += ' "%s"' % movieFile.replace('"', '\\"')
options += ' -o "%s"' % oggFile
cmd = "ffmpeg2theora %s >/dev/null 2>&1" % options
r = os.system(cmd.encode('utf-8'))
if r == 0:
self.extractedOgg = True
def extractH264Movie(self, force = False):
if self.broken:
return
if not self.height:
#only if midentify works we should try to extract the clip
return
if self.path.split('.')[-1] in ('mp3', 'wav', 'srt', 'sub', 'idx', 'rar','jpg', 'png'):
#ignore files known to not be
return
h264File = self.mini_movie_file.replace('.avi', '.mp4')
h264FileTmp = h264File + ".tmp.mp4"
movieFile = self.absolutePath
if not movieFile or not exists(movieFile):
return
if exists(h264File) and not force:
debug("clip exists, skipping extraction %s" % h264File)
return
self.extractedH264 = False
height = self.sceneHeight
oxdb_makedir(dirname(h264File))
options = ''
options += " -vcodec libx264 -b 112k -bf 3 -subq 6 -cmp 256 -refs 5 -qmin 10 "
options += " -qmax 51 -qdiff 4 -coder 1 -loop 1 -me hex -me_range 16 -trellis 1 "
options += " -flags +mv4 -flags2 +bpyramid+wpred+mixed_refs+brdo+8x8dct "
options += " -partitions parti4x4+parti8x8+partp4x4+partp8x8+partb8x8 -g 250 "
options += " -keyint_min 16 -sc_threshold 40 -i_qfactor 0.71"
options += " -s %sx%s" % (self.sceneWidth, height)
cmd = '''ffmpeg -y -i "%s" -an -pass 1 -threads 2 %s "%s" >/dev/null 2>&1''' % (movieFile, options, h264FileTmp)
print cmd
r = os.system(cmd.encode('utf-8'))
cmd = '''ffmpeg -y -i "%s" -acodec libfaac -ac 2 -ar 44100 -ab 48k -pass 2 -threads 2 %s "%s" >/dev/null 2>&1''' % (movieFile, options, h264FileTmp)
r = os.system(cmd.encode('utf-8'))
cmd = '''qt-faststart "%s" "%s" >/dev/null 2>&1 && rm "%s"''' % (h264FileTmp, h264File, h264FileTmp)
r = os.system(cmd.encode('utf-8'))
if r == 0:
self.extractedH264 = True
def removeTimeline(self): def removeTimeline(self):
if exists(self.timelineFile): if exists(self.timelineFile):
os.unlink(self.timelineFile) os.unlink(self.timelineFile)

View file

@ -1,11 +1,9 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# -*- Mode: Python; -*-
# vi:si:et:sw=2:sts=2:ts=2 # vi:si:et:sw=2:sts=2:ts=2
# OXDb Import client, crawls the filesystem and gathers information about # OXDb Import client, crawls the filesystem and gathers information about
# movies # movies
# #
import md5 import md5
import os import os
import sys import sys

View file

@ -1,11 +1,9 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# -*- Mode: Python; -*-
# vi:si:et:sw=2:sts=2:ts=2 # vi:si:et:sw=2:sts=2:ts=2
# OXDb Import client, crawls the filesystem and gathers information about # OXDb Import client, crawls the filesystem and gathers information about
# movies # movies
# #
import md5 import md5
import os import os
import sys import sys

View file

@ -1,7 +1,5 @@
# -*- Mode: Python; -*-
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# vi:si:et:sw=2:sts=2:ts=2 # vi:si:et:sw=2:sts=2:ts=2
import Image import Image
import math import math
from StringIO import StringIO from StringIO import StringIO