more vi:si:et:sw=4:sts=4:ts=4

This commit is contained in:
j 2008-06-19 12:14:38 +02:00
parent ecfb2ff28c
commit 2dd8f949b6
10 changed files with 258 additions and 203 deletions

View File

@ -1,65 +1,68 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from oxdb_utils import oxdb_makedir
from os.path import *
import cache
import cache
import extract
class BackupMovieFile:
def __init__(self, parent):
self.parent = parent
def extractFrame(self, position, img_folder):
return self.parent.extractFrame(position)
def __init__(self, parent):
self.parent = parent
def extractFrame(self, position, img_folder):
return self.parent.extractFrame(position)
class Backup:
def __init__(self, md5sum):
self.md5sum = md5sum
self.timelineFile = join(cache.cache_root, 'timeline', self.md5sum[:4], "%s.png" % self.md5sum)
self.posterStill = join(cache.cache_root, 'posterStill', self.md5sum[:4], "%s.png" % self.md5sum)
self.mini_movie_file = join(cache.cache_root, 'mini', self.md5sum[:4], "%s.avi" % self.md5sum)
self.movieFile = BackupMovieFile(self)
self.frameFolder = join(cache.cache_root, 'frame', self.md5sum[:4], self.md5sum)
def exists(self):
return exists(self.mini_movie_file)
def __init__(self, md5sum):
self.md5sum = md5sum
self.timelineFile = join(cache.cache_root, 'timeline', self.md5sum[:4], "%s.png" % self.md5sum)
self.posterStill = join(cache.cache_root, 'posterStill', self.md5sum[:4], "%s.png" % self.md5sum)
self.mini_movie_file = join(cache.cache_root, 'mini', self.md5sum[:4], "%s.avi" % self.md5sum)
self.movieFile = BackupMovieFile(self)
self.frameFolder = join(cache.cache_root, 'frame', self.md5sum[:4], self.md5sum)
def timeline(self):
return cache.loadTimeline(self)
def frame(self, pos):
return cache.loadFrame(self, pos)
def posterStill(self, pos):
return cache.loadPosterStill(self, pos)
def exists(self):
return exists(self.mini_movie_file)
def flvClip(self, position):
return cache.loadFlvClip(self, position)
def timeline(self):
return cache.loadTimeline(self)
def extractFlvClip(self, inpoint, outpoint=-1):
flash_folder = self.frameFolder
movie_file = self.mini_movie_file
position = inpoint.replace(':', '.')
flash_movie = join(self.frameFolder, '%s.%s' % (position, 'flv'))
width = 128
height = int(width / self.frameAspect)
height = height - height % 2
inpoint = inpoint.replace('.', ':')
if outpoint == -1:
outpoint = shift_time(5000, inpoint)
else:
outpoint = outpoint.replace('.', ':')
extract.extract_flash(movie_file, flash_movie, inpoint, outpoint, width, height, offset = 0)
def extractFrame(self, position, img_folder=-1):
img_folder = self.frameFolder
movie_file = self.mini_movie_file
oxdb_makedir(img_folder)
position = position.replace('.', ':')
extract.extract_frame(movie_file, position, img_folder, offset = 0, redo = False)
def extractPosterStill(self, position):
oxdb_makedir(dirname(self.posterStillFile))
mFile = self.absolutePath
if os.path.splitext(mFile)[-1] in ('.mov', '.mpg', '.mpeg'):
mFile = self.mini_movie_file
extract.extract_poster_still(mFile, self.posterStillFile, position)
def frame(self, pos):
return cache.loadFrame(self, pos)
def posterStill(self, pos):
return cache.loadPosterStill(self, pos)
def flvClip(self, position):
return cache.loadFlvClip(self, position)
def extractFlvClip(self, inpoint, outpoint=-1):
flash_folder = self.frameFolder
movie_file = self.mini_movie_file
position = inpoint.replace(':', '.')
flash_movie = join(self.frameFolder, '%s.%s' % (position, 'flv'))
width = 128
height = int(width / self.frameAspect)
height = height - height % 2
inpoint = inpoint.replace('.', ':')
if outpoint == -1:
outpoint = shift_time(5000, inpoint)
else:
outpoint = outpoint.replace('.', ':')
extract.extract_flash(movie_file, flash_movie, inpoint, outpoint, width, height, offset = 0)
def extractFrame(self, position, img_folder=-1):
img_folder = self.frameFolder
movie_file = self.mini_movie_file
oxdb_makedir(img_folder)
position = position.replace('.', ':')
extract.extract_frame(movie_file, position, img_folder, offset = 0, redo = False)
def extractPosterStill(self, position):
oxdb_makedir(dirname(self.posterStillFile))
mFile = self.absolutePath
if os.path.splitext(mFile)[-1] in ('.mov', '.mpg', '.mpeg'):
mFile = self.mini_movie_file
extract.extract_poster_still(mFile, self.posterStillFile, position)

View File

@ -1,7 +1,5 @@
# -*- Mode: Python; -*-
# -*- coding: utf-8 -*-
# vi:si:et:sw=2:sts=2:ts=2
# vi:si:et:sw=4:sts=4:ts=4
import os
from os.path import abspath, exists, join, dirname, basename
import shutil
@ -18,70 +16,71 @@ img_extension = "jpg"
frame_cache_root = join(cache_root, 'frame')
def loadFile(f_name):
f = open(f_name)
data = f.read()
f.close()
return data
f = open(f_name)
data = f.read()
f.close()
return data
def saveFile(f_name, data):
f = open(f_name, 'w')
f.write(data)
f.close()
f = open(f_name, 'w')
f.write(data)
f.close()
def loadStaticFile(fname):
return loadFile(join(dirname(abspath(__file__)), "static", fname))
return loadFile(join(dirname(abspath(__file__)), "static", fname))
def loadFrame(afile, position):
position = basename(position).replace(':', '.')
frame = join(afile.frameFolder, '%s.%s' % (position, img_extension))
if not exists(frame):
afile.extractFrame(position)
if exists(frame):
return loadFile(frame)
return ''
position = basename(position).replace(':', '.')
frame = join(afile.frameFolder, '%s.%s' % (position, img_extension))
if not exists(frame):
afile.extractFrame(position)
if exists(frame):
return loadFile(frame)
return ''
def loadFlvClip(afile, position):
position = basename(position).replace(':', '.')
positions = position.split(';')
if len(positions) > 1:
position = positions[0]
outpoint = positions[1]
else:
outpoint = -1
flash = join(afile.frameFolder, '%s.%s' % (position, 'flv'))
if not exists(flash):
afile.extractFlvClip(position, outpoint)
if exists(flash):
return loadFile(flash)
return ''
position = basename(position).replace(':', '.')
positions = position.split(';')
if len(positions) > 1:
position = positions[0]
outpoint = positions[1]
else:
outpoint = -1
flash = join(afile.frameFolder, '%s.%s' % (position, 'flv'))
if not exists(flash):
afile.extractFlvClip(position, outpoint)
if exists(flash):
return loadFile(flash)
return ''
def loadOggClip(afile, position):
position = basename(position).replace(':', '.')
positions = position.split(';')
if len(positions) > 1:
position = positions[0]
outpoint = positions[1]
else:
outpoint = -1
clip = join(afile.frameFolder, '%s.%s' % (position, 'ogv'))
if not exists(clip):
afile.extractOggClip(position, outpoint)
if exists(clip):
return loadFile(clip)
return ''
position = basename(position).replace(':', '.')
positions = position.split(';')
if len(positions) > 1:
position = positions[0]
outpoint = positions[1]
else:
outpoint = -1
clip = join(afile.frameFolder, '%s.%s' % (position, 'ogv'))
if not exists(clip):
afile.extractOggClip(position, outpoint)
if exists(clip):
return loadFile(clip)
return ''
def loadTimeline(afile):
timeline = afile.timelineFile
if not exists(timeline):
afile.extractTimeline()
if exists(timeline):
return loadFile(timeline)
return ''
timeline = afile.timelineFile
if not exists(timeline):
afile.extractTimeline()
if exists(timeline):
return loadFile(timeline)
return ''
def loadPosterStill(afile, position):
still = afile.posterStillFile
if not exists(still):
afile.extractPosterStill(position)
if exists(still):
return loadFile(still)
return ''
still = afile.posterStillFile
if not exists(still):
afile.extractPosterStill(position)
if exists(still):
return loadFile(still)
return ''

View File

@ -1,7 +1,5 @@
# -*- Mode: Python; -*-
# -*- coding: utf-8 -*-
# vi:si:et:sw=2:sts=2:ts=2
# vi:si:et:sw=4:sts=4:ts=4
from turbogears import controllers, expose, flash, redirect
from model import *
from json import *
@ -18,45 +16,43 @@ import backup
'''
def httpExpires(sec):
exp_date = cherrypy.lib.httptools.HTTPDate(time.gmtime(time.mktime(time.gmtime()) + sec))
return exp_date
exp_date = cherrypy.lib.httptools.HTTPDate(time.gmtime(time.mktime(time.gmtime()) + sec))
return exp_date
class Root(controllers.RootController):
@expose()
def default(self, md5Hash = None, action = None, position = None):
if not md5Hash:
return dict()
try:
f = ArchiveFile.byMd5sum(md5Hash)
except:
#return dict()
f = backup.Backup(md5Hash)
if not f.exists():
@expose()
def default(self, md5Hash = None, action = None, position = None):
if not md5Hash:
return dict()
try:
f = ArchiveFile.byMd5sum(md5Hash)
except:
#return dict()
f = backup.Backup(md5Hash)
if not f.exists():
return dict()
if action == 'metadata':
return dict(metadata = f)
elif action in ('timeline', 'timeline.png'):
cherrypy.response.headerMap['Content-Type'] = "image/png"
cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
return f.timeline()
elif position: #clip / frame
position = position.replace('.png', '').replace('.jpg', '')
position = position.replace('.flv', '').replace('.ogv', '')
position = position.replace('-', ':').replace('.',':')
cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
if action == 'clip':
cherrypy.response.headerMap['Content-Type'] = "video/x-flv"
return f.flvClip(position)
if action == 'ogv':
cherrypy.response.headerMap['Content-Type'] = "video/ogg"
return f.oggClip(position)
elif action == 'frame':
cherrypy.response.headerMap['Content-Type'] = "image/jpeg"
return f.frame(position)
elif action == 'posterStill':
cherrypy.response.headerMap['Content-Type'] = "image/png"
return f.posterStill(position)
return dict()
if action == 'metadata':
return dict(metadata = f)
elif action in ('timeline', 'timeline.png'):
cherrypy.response.headerMap['Content-Type'] = "image/png"
cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
return f.timeline()
elif position: #clip / frame
position = position.replace('.png', '').replace('.jpg', '')
position = position.replace('.flv', '').replace('.ogv', '')
position = position.replace('-', ':').replace('.',':')
if action == 'clip':
cherrypy.response.headerMap['Content-Type'] = "video/x-flv"
cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
return f.flvClip(position)
if action == 'ogv':
cherrypy.response.headerMap['Content-Type'] = "video/ogg"
cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
return f.oggClip(position)
elif action == 'frame':
cherrypy.response.headerMap['Content-Type'] = "image/jpeg"
cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
return f.frame(position)
elif action == 'posterStill':
cherrypy.response.headerMap['Content-Type'] = "image/png"
cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
return f.posterStill(position)
return dict()

View File

@ -1,48 +1,47 @@
# -*- Mode: Python; -*-
# -*- coding: utf-8 -*-
# vi:si:et:sw=2:sts=2:ts=2
# vi:si:et:sw=4:sts=4:ts=4
from model import *
import cache
def cronDaily():
findNew()
extractNew()
findNew()
extractNew()
def findNew():
archive = Archive.get(1)
archive.importFiles()
archive = Archive.get(1)
archive.importFiles()
def extractFrames():
for f in ArchiveFile.select(ArchiveFile.q.extracted == False):
f.extractFrames()
for f in ArchiveFile.select(ArchiveFile.q.extracted == False):
f.extractFrames()
def extractTimelines():
for f in ArchiveFile.select(ArchiveFile.q.extracted == False):
f.extractTimeline()
for f in ArchiveFile.select(ArchiveFile.q.extracted == False):
f.extractTimeline()
def extractNew():
for f in ArchiveFile.select(ArchiveFile.q.extracted == False):
f.extractAll()
for f in ArchiveFile.select(ArchiveFile.q.extracted == False):
f.extractAll()
def cleanClipMovieCache():
cache = os.path.abspath('oxdb/cache/mini/')
for d, p, files in os.walk(cache):
for f in files:
md5sum = f.replace('.avi', '')
try:
fm = ArchiveFile.byMd5sum(md5sum)
except:
os.remove(os.path.join(cache, d, f))
cache = os.path.abspath('oxdb/cache/mini/')
for d, p, files in os.walk(cache):
for f in files:
md5sum = f.replace('.avi', '')
try:
fm = ArchiveFile.byMd5sum(md5sum)
except:
os.remove(os.path.join(cache, d, f))
#possible fuctions:
def extractSubtitles():
extractSubtitlesScript = abspath(join(dirname(cache.cache_root), "tools/subtitles.py"))
for f in ArchiveFile.select(LIKE(ArchiveFile.q.path, '%' + '.idx')):
base = f.absolutePath.replace('.idx', '')
srtFile = f.absolutePath.replace('.idx', '.srt')
if not os.path.exists(srtFile):
cmd = 'python "%s" "%s"' % (extractSubtitlesScript, base)
os.system(cmd)
#only one subtitle at a time
return
extractSubtitlesScript = abspath(join(dirname(cache.cache_root), "tools/subtitles.py"))
for f in ArchiveFile.select(LIKE(ArchiveFile.q.path, '%' + '.idx')):
base = f.absolutePath.replace('.idx', '')
srtFile = f.absolutePath.replace('.idx', '.srt')
if not os.path.exists(srtFile):
cmd = 'python "%s" "%s"' % (extractSubtitlesScript, base)
os.system(cmd)
#only one subtitle at a time
return

View File

@ -1,7 +1,5 @@
# -*- coding: utf-8 -*-
# -*- Mode: Python; -*-
# vi:si:et:sw=2:sts=2:ts=2
import re
import os
from os.path import abspath, join, dirname

View File

@ -1,7 +1,5 @@
# -*- coding: utf-8 -*-
# -*- Mode: Python; -*-
# vi:si:et:sw=2:sts=2:ts=2
import os
import sys

View File

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
# -*- Mode: Python; -*-
# vi:si:et:sw=2:sts=2:ts=2
import gzip
import StringIO
@ -353,10 +352,13 @@ class ArchiveFile(SQLObject):
self.frameAspect = "%0.6f" % aspect
return aspect
def _get_sceneWidth(self):
return 128
def _get_sceneHeight(self):
default = 80
if not self.subtitle:
h = int(128 / self.frameAspect)
h = int(self.sceneWidth / self.frameAspect)
h = h + h % 2
return h
return default
@ -450,9 +452,8 @@ class ArchiveFile(SQLObject):
movie_file = self.mini_movie_file
position = inpoint.replace(':', '.')
flash_movie = join(self.frameFolder, '%s.%s' % (position, 'flv'))
width = 128
height = int(width / self.frameAspect)
height = height - height % 2
width = self.sceneWidth
height = self.sceneHeight
inpoint = inpoint.replace('.', ':')
if outpoint == -1:
s = self._findSubtitleByInPoint(inpoint)
@ -471,9 +472,8 @@ class ArchiveFile(SQLObject):
movie_file = self.mini_movie_file
position = inpoint.replace(':', '.')
clip_movie = join(self.frameFolder, '%s.%s' % (position, 'ogv'))
width = 128
height = int(width / self.frameAspect)
height = height - height % 2
width = self.sceneWidth
height = self.sceneHeight
inpoint = inpoint.replace('.', ':')
if outpoint == -1:
s = self._findSubtitleByInPoint(inpoint)
@ -534,7 +534,7 @@ class ArchiveFile(SQLObject):
options = ''
options += " -ovc lavc -lavcopts vcodec=mjpeg"
options += " -af volnorm=1 -oac mp3lame -lameopts br=64:mode=3 -af resample=44100"
options += " -vf scale -zoom -xy 128"
options += " -vf scale -zoom -xy %s" % self.sceneWidth
options += ' "%s"' % movie_file.replace('"', '\\"')
options += ' -o "%s"' % mini_movie_file
cmd = "mencoder %s >/dev/null 2>&1" % options
@ -542,7 +542,75 @@ class ArchiveFile(SQLObject):
r = os.system(cmd.encode('utf-8'))
if r == 0:
self.extracted = True
def extractOggMovie(self, force = False):
if self.broken:
return
if not self.height:
#only if midentify works we should try to extract the clip
return
if self.path.split('.')[-1] in ('mp3', 'wav', 'srt', 'sub', 'idx', 'rar','jpg', 'png'):
#ignore files known to not be
return
oggFile = self.mini_movie_file.replace('.avi', '.ogg')
movieFile = self.absolutePath
if not movieFile or not exists(movieFile):
return
if exists(oggFile) and not force:
debug("clip exists, skipping extraction %s" % oggFile)
return
self.extractedOgg = False
oxdb_makedir(dirname(oggFile))
options = ''
options += " --no-skeleton -K 16 -V 180 -a -1 -H 44100 -S 1 --speedlevel 0 -c 2 "
options += " -x %s -y %s" % (self.sceneWidth, self.sceneHeight)
options += ' "%s"' % movieFile.replace('"', '\\"')
options += ' -o "%s"' % oggFile
cmd = "ffmpeg2theora %s >/dev/null 2>&1" % options
r = os.system(cmd.encode('utf-8'))
if r == 0:
self.extractedOgg = True
def extractH264Movie(self, force = False):
if self.broken:
return
if not self.height:
#only if midentify works we should try to extract the clip
return
if self.path.split('.')[-1] in ('mp3', 'wav', 'srt', 'sub', 'idx', 'rar','jpg', 'png'):
#ignore files known to not be
return
h264File = self.mini_movie_file.replace('.avi', '.mp4')
h264FileTmp = h264File + ".tmp.mp4"
movieFile = self.absolutePath
if not movieFile or not exists(movieFile):
return
if exists(h264File) and not force:
debug("clip exists, skipping extraction %s" % h264File)
return
self.extractedH264 = False
height = self.sceneHeight
oxdb_makedir(dirname(h264File))
options = ''
options += " -vcodec libx264 -b 112k -bf 3 -subq 6 -cmp 256 -refs 5 -qmin 10 "
options += " -qmax 51 -qdiff 4 -coder 1 -loop 1 -me hex -me_range 16 -trellis 1 "
options += " -flags +mv4 -flags2 +bpyramid+wpred+mixed_refs+brdo+8x8dct "
options += " -partitions parti4x4+parti8x8+partp4x4+partp8x8+partb8x8 -g 250 "
options += " -keyint_min 16 -sc_threshold 40 -i_qfactor 0.71"
options += " -s %sx%s" % (self.sceneWidth, height)
cmd = '''ffmpeg -y -i "%s" -an -pass 1 -threads 2 %s "%s" >/dev/null 2>&1''' % (movieFile, options, h264FileTmp)
print cmd
r = os.system(cmd.encode('utf-8'))
cmd = '''ffmpeg -y -i "%s" -acodec libfaac -ac 2 -ar 44100 -ab 48k -pass 2 -threads 2 %s "%s" >/dev/null 2>&1''' % (movieFile, options, h264FileTmp)
r = os.system(cmd.encode('utf-8'))
cmd = '''qt-faststart "%s" "%s" >/dev/null 2>&1 && rm "%s"''' % (h264FileTmp, h264File, h264FileTmp)
r = os.system(cmd.encode('utf-8'))
if r == 0:
self.extractedH264 = True
def removeTimeline(self):
if exists(self.timelineFile):
os.unlink(self.timelineFile)

View File

@ -1,11 +1,9 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# -*- Mode: Python; -*-
# vi:si:et:sw=2:sts=2:ts=2
# OXDb Import client, crawls the filesystem and gathers information about
# movies
#
import md5
import os
import sys

View File

@ -1,11 +1,9 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# -*- Mode: Python; -*-
# vi:si:et:sw=2:sts=2:ts=2
# OXDb Import client, crawls the filesystem and gathers information about
# movies
#
import md5
import os
import sys

View File

@ -1,7 +1,5 @@
# -*- Mode: Python; -*-
# -*- coding: utf-8 -*-
# vi:si:et:sw=2:sts=2:ts=2
import Image
import math
from StringIO import StringIO