0x2620/oxdbarchive
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

* cleanup oxdbarchive, consolidated cache code

Browse source 
* no tg_flash
* fix extract_timeline
This commit is contained in:
j 2007-07-10 13:32:21 +00:00
parent 40d1554124
commit 9779cfc86a
7 changed files with 45 additions and 321 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
oxdbarchive/cache.py
View file

@ -17,8 +17,6 @@ img_extension = "jpg"
frame_cache_root = join(cache_root, 'frame')
mini_movie_folder = '/mnt/storage/oil/oxdb/mini'
def loadFile(f_name):
f = open(f_name)
data = f.read()
@ -33,16 +31,9 @@ def saveFile(f_name, data):
def loadStaticFile(fname):
return loadFile(join(dirname(abspath(__file__)), "static", fname))
def framePath(frameType, movieID, position):
position = position.replace(':', '.')
frame_root = join(cache_root, frameType)
frame = join(frame_root, imgName(join(movieID, position)))
if not exists(dirname(frame)):
os.makedirs(dirname(frame))
return frame
def loadDefaultFrame(afile):
frame = framePath('frame', afile, 'default').replace('jpg', 'png')
frame = join(afile.frameFolder, 'default.png')
if not exists(frame):
data = loadStaticFile('images/stillDark.png')
imageIO = StringIO(data)
@ -56,7 +47,7 @@ def loadDefaultFrame(afile):
def loadFrame(afile, position):
position = basename(position)
frame = framePath('frame', afile.oxdb, position)
frame = join(afile.frameFolder, '%s.%s' % (position, img_extension))
if not exists(frame):
afile.extractFrame(position)
if exists(frame):
@ -65,7 +56,7 @@ def loadFrame(afile, position):
def loadClip(afile, position):
position = basename(position)
flash = framePath('frame', afile.oxdb, position).replace(img_extension, 'flv')
flash = join(afile.frameFolder, '%s.%s' % (position, 'flv'))
if not exists(flash):
afile.extractClip(position)
if exists(flash):

1
oxdbarchive/config/app.cfg
View file

@ -25,6 +25,7 @@
# Allow every exposed function to be called as json,
# tg.allow_json = False
tg.no_empty_flash = True
# List of Widgets to include on every page.
# for exemple ['turbogears.mochikit']

19
oxdbarchive/controllers.py
View file

@ -1,3 +1,7 @@
# -*- Mode: Python; -*-
# -*- coding: utf-8 -*-
# vi:si:et:sw=2:sts=2:ts=2
from turbogears import controllers, expose, flash, redirect
from model import *
import cherrypy
@ -20,20 +24,19 @@ class Root(controllers.RootController):
def default(self, md5Hash, action, position = None):
f = ArchiveFile.byMd5sum(md5Hash)
if action == 'metadata':
print f
return dict(meta = f)
elif position or action == 'timeline': #clip / frame / timeline
elif action in ('timeline', 'timeline.png'):
cherrypy.response.headerMap['Content-Type'] = "image/jpeg"
cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
return f.timeline()
elif position: #clip / frame
cherrypy.response.headerMap['Content-Type'] = "image/jpeg"
cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
position = position.replace('.png', '').replace('.jpg', '')
position = position.replace('-', ':').replace('.',':')
if action == 'timeline':
return f.timeline()
if action == 'clip':
return f.clip(position)
elif action == 'frame':
return f.frame(position)
redirect('http://0xdb.oil21.org/')
@expose()
def index(self):
redirect('http://0xdb.oil21.org/')
redirect('http://0xdb.oil21.org/')

55
oxdbarchive/model.py
View file

@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
# -*- Mode: Python; -*-
# vi:si:et:sw=2:sts=2:ts=2
from sqlobject import *
from turbogears.database import PackageHub
import turbogears
@ -15,7 +16,6 @@ from glob import glob
import shutil
import oxdb_cache
import cache
import oxdb_import
from oxdb_utils import oxdb_title, oxdb_director, oxdb_id
@ -63,7 +63,8 @@ class Archive(SQLObject):
params is a dict with at least md5sum, path, date but also needs
audio, video, length, size, bpp for new files
'''
params['path'] = params['path'].replace(self.basePath, '')
params['path'] = params['path'].replace(self.basePath, u'')
q = ArchiveFile.select(AND(
ArchiveFile.q.archiveID == self.id,
ArchiveFile.q.md5sum == params['md5sum'],
@ -140,8 +141,6 @@ class Archive(SQLObject):
md5sum_on_disk.append(oxdb_files[f]['md5sum'])
else:
meta = oxdb_import.oxdb_file_metadata(meta)
#remove base
meta['path'] = f.encode('utf-8')
#FIXME: check input
for key in ['bpp', 'size', 'length', 'date']:
meta[key] = int(float(meta[key]))
@ -160,9 +159,9 @@ class Archive(SQLObject):
class ArchiveFile(SQLObject):
'''
ALTER TABLE file_meta CHANGE size size bigint;
ALTER TABLE file_meta CHANGE pixels pixels bigint;
ALTER TABLE file_meta CHANGE srt srt LONGTEXT;
ALTER TABLE archive_file CHANGE size size bigint;
ALTER TABLE archive_file CHANGE pixels pixels bigint;
ALTER TABLE archive_file CHANGE srt srt LONGTEXT;
'''
md5sum = UnicodeCol(length=128, alternateID=True)
oxdb = UnicodeCol(length=128)
@ -328,10 +327,18 @@ class ArchiveFile(SQLObject):
if f.nameExtra == self.nameExtra or f.nameExtra == 'en':
self.subtitle_meta_id = f.id
def _get_mini_movie_file(self):
return join(oxdb_cache.mini_movie_folder, self.md5sum[:4], "%s.avi" % self.md5sum)
return join(cache.cache_root, 'mini', self.md5sum[:4], "%s.avi" % self.md5sum)
def _get_frameFolder(self):
f = join(cache.cache_root, 'mini', self.md5sum[:4], self.md5sum)
if not exists(f):
os.makedirs(f)
return f
def _get_timelineFile(self):
return join(cache.cache_root, 'timeline', self.md5sum[:4], "%s.png" % self.md5sum)
def removeMiniMovie(self):
if os.path.exists(self.mini_movie_file):
os.remove(self.mini_movie_file)
@ -345,16 +352,16 @@ class ArchiveFile(SQLObject):
return None
def extractAll(self, force = False):
self.updateMeta()
self.extractClipMovie()
self.extractTimeline()
def extractClip(self, inpoint, outpoint=-1, flash_folder=oxdb_cache.frame_cache_root):
if not self.extracted or force:
self.updateMeta()
self.extractClipMovie()
self.extractTimeline()
self.extracted = True
def extractClip(self, inpoint, outpoint=-1, flash_folder=cache.frame_cache_root):
movie_file = self.mini_movie_file
flash_folder = join(flash_folder, self.oxdb)
flash_movie = join(flash_folder, "%s.flv" % inpoint.replace(':', '.'))
if not os.path.exists(flash_folder):
os.makedirs(flash_folder)
position = inpoint.replace(':', '.')
flash_movie = join(self.frameFolder, '%s.%s' % (position, 'flv'))
width = 128
height = int(width / (self.width / self.height))
height = height - height % 2
@ -364,12 +371,10 @@ class ArchiveFile(SQLObject):
outpoint = s['stop']
else:
outpoint = shift_time(2000, inpoint)
if self.part > 1:
offset = self.offset
extract_flash(movie_file, flash_movie, inpoint, outpoint, width, height, offset = 0)
#extract_flash_ng(self.absolutePath, flash_movie, inpoint, outpoint, width, height, offset)
def extractFrame(self, position, img_folder=oxdb_cache.frame_cache_root):
def extractFrame(self, position, img_folder=cache.frame_cache_root):
if self.movieFile:
return self.movieFile.extractFrame(position, img_folder)
movie_file = self.mini_movie_file
@ -378,7 +383,7 @@ class ArchiveFile(SQLObject):
os.makedirs(img_folder)
extract_frame(movie_file, position, img_folder, offset = 0, redo = False)
def extractFrames(self, img_folder=oxdb_cache.frame_cache_root):
def extractFrames(self, img_folder=cache.frame_cache_root):
if self.movieFile:
return self.movieFile.extractFrames(img_folder)
movie_file = self.absolutePath
@ -394,7 +399,7 @@ class ArchiveFile(SQLObject):
movie_file = self.absolutePath
if not movie_file or not os.path.exists(movie_file):
return
if os.path.exists(mini_movie_file):
if os.path.exists(mini_movie_file) and not force:
print "clip exists, skipping extraction", mini_movie_file
return
if not os.path.exists(dirname(mini_movie_file)):
@ -409,8 +414,6 @@ class ArchiveFile(SQLObject):
print cmd.encode('utf-8')
os.system(cmd.encode('utf-8'))
def _get_timelineFile(self):
return join(oxdb_cache.cache_root, 'timeline', self.md5sum[:4], "%s.png" % self.md5sum)
def removeTimeline(self):
if os.path.exists(self.timelineFile):
@ -428,7 +431,7 @@ class ArchiveFile(SQLObject):
t = self.timelineFile
if os.path.exists(self.mini_movie_file):
if not os.path.exists(t):
if not os.path.exists(os.path.dirname(t)):
os.makedirs(os.path.dirname(t))
#lets only extract the timeline if it does not exist yet
if os.path.exists(t):

273
oxdbarchive/oxdb_cache.py
View file

@ -1,273 +0,0 @@
# -*- Mode: Python; -*-
# -*- coding: utf-8 -*-
# vi:si:et:sw=2:sts=2:ts=2
import os
from os.path import abspath, exists, join, dirname, basename
import shutil
from glob import glob
import Image
from StringIO import StringIO
from scrapeit.utils import read_url
cache_root = join(dirname(abspath(__file__)), 'cache')
img_extension = "jpg"
frame_cache_root = join(cache_root, 'frame')
mini_movie_folder = '/mnt/storage/oil/oxdb/mini'
def loadFile(f_name):
f = open(f_name)
data = f.read()
f.close()
return data
def saveFile(f_name, data):
f = open(f_name, 'w')
f.write(data)
f.close()
def loadStaticFile(fname):
return loadFile(join(dirname(abspath(__file__)), "static", fname))
'''
returns name including a possible directory level for a given hash
'''
def imgName(imdb):
return "%s.%s" % (imdb, img_extension)
'''
returns path to an icon from iconType for given icon in the cache
'''
def iconPath(iconType, movie):
icon_root = join(cache_root, iconType)
icon = join(icon_root, imgName(movie.imdb))
if not exists(dirname(icon)):
os.makedirs(dirname(icon))
return icon
'''
render reflection of sourceFile on targetFile,
uses alpha, target files needs to support RGBA, i.e. png
'''
def _writeReflection(sourceFile, targetFile, height = 0.5, opacity = 0.25):
sourceImage = Image.open(sourceFile).convert('RGB')
sourceSource = sourceImage.size[0]
sourceHeight = sourceImage.size[1]
targetWidth = sourceImage.size[0]
targetHeight = int(round(sourceHeight * height))
targetImage = Image.new('RGBA', (targetWidth, targetHeight))
for y in range(0, targetHeight):
brightness = int(255 * (targetHeight - y) * opacity / targetHeight)
for x in range(0, targetWidth):
targetColor = sourceImage.getpixel((x, sourceHeight - 1 - y))
targetColor += (brightness, )
targetImage.putpixel((x, y), targetColor)
targetImage.save(targetFile, optimized = True)
def resizePoster(data, max_resolution, format = 'JPEG'):
posterIO = StringIO(data)
sourceImage = Image.open(posterIO)
sourceWidth = sourceImage.size[0]
sourceHeight = sourceImage.size[1]
if int(round((float(max_resolution[1]) * sourceWidth) / sourceHeight)) < max_resolution[0]:
max_resolution[0] = int(round((float(max_resolution[1]) * sourceWidth) / sourceHeight))
if int(round((float(max_resolution[0]) / sourceWidth) * sourceHeight)) < max_resolution[1]:
max_resolution[1] = int(round((float(max_resolution[0]) / sourceWidth) * sourceHeight))
if sourceWidth >= sourceHeight:
targetWidth = max_resolution[0]
targetHeight = int(round((float(max_resolution[0]) / sourceWidth) * sourceHeight))
else:
targetWidth = int(round((float(max_resolution[1]) * sourceWidth) / sourceHeight))
targetHeight = max_resolution[1]
if targetWidth < sourceWidth:
resizeMethod = Image.ANTIALIAS
else:
resizeMethod = Image.BICUBIC
targetSize = (targetWidth, targetHeight)
targetImage = sourceImage.resize(targetSize, resizeMethod)
f = StringIO()
if format == 'JPEG':
targetImage.save(f, 'JPEG', quality=90)
else:
targetImage.save(f, 'PNG')
return f.getvalue()
'''
download poster from imdb and resize it before saving into cache
returns poster data
'''
def downloadPoster(movie):
icon = iconPath('poster', movie)
if movie.posterFile:
data = loadFile(movie.posterFile)
else:
data = read_url(movie.poster)
posterIO = StringIO(data)
sourceImage = Image.open(posterIO)
sourceWidth = sourceImage.size[0]
sourceHeight = sourceImage.size[1]
if sourceWidth >= sourceHeight:
targetWidth = 128
targetHeight = int(round((128.0 / sourceWidth) * sourceHeight))
else:
targetWidth = int(round((128.0 * sourceWidth) / sourceHeight))
targetHeight = 128
if targetWidth < sourceWidth:
resizeMethod = Image.ANTIALIAS
else:
resizeMethod = Image.BICUBIC
targetSize = (targetWidth, targetHeight)
targetImage = sourceImage.resize(targetSize, resizeMethod).convert('RGB')
targetImage.save(icon)
return loadFile(icon)
'''
return icon data, reads from remote url if not cached
'''
def loadPoster(movie):
if not movie.hasPoster():
return ''
#return loadStaticFile('images/posterDark.png')
#return loadStaticFile('images/posterBlack.png')
icon = iconPath('poster', movie)
if exists(icon):
data = loadFile(icon)
else:
data = downloadPoster(movie)
return data
'''
return icon reflection data, renders reflection if it does not exists
'''
def loadPosterReflection(movie):
icon = iconPath('poster', movie)
iconReflection = iconPath('posterReflection', movie).replace('jpg', 'png')
if not exists(iconReflection):
if not exists(icon):
loadPoster(movie)
if exists(icon):
_writeReflection(icon, iconReflection)
else:
return loadStaticFile('images/posterDark.reflection.png')
return loadFile(iconReflection)
'''
returns path to a frame from type for given movie in the cache
'''
def framePath(frameType, movie, position):
position = position.replace(':', '.')
frame_root = join(cache_root, frameType)
frame = join(frame_root, imgName(join(movie.imdb, position)))
if not exists(dirname(frame)):
os.makedirs(dirname(frame))
return frame
def loadClip(movie, position):
position = basename(position)
flash = framePath('frame', movie, position).replace(img_extension, 'flv')
if not exists(flash):
movie.extractClip(position)
if exists(flash):
return loadFile(flash)
return ''
'''
returns png frame of the given position.
'''
def loadFrame(movie, position):
position = basename(position)
frame = framePath('frame', movie, position)
if not exists(frame):
#movie.extractClip(position)
movie.extractFrame(position)
if exists(frame):
return loadFile(frame)
return loadDefaultFrame(movie)
def loadDefaultFrameReflection(movie):
frame = framePath('frame', movie, 'default').replace('jpg', 'png')
frameReflection = framePath('frameReflection', movie, 'default').replace('jpg', 'png')
if not exists(frameReflection):
if not exists(frame):
loadDefaultFrame(movie)
if exists(frame):
_writeReflection(frame, frameReflection)
else:
return loadStaticFile('images/stillDark.reflection.png')
return loadFile(frameReflection)
def loadDefaultFrame(movie):
frame = framePath('frame', movie, 'default').replace('jpg', 'png')
if not exists(frame):
data = loadStaticFile('images/stillDark.png')
imageIO = StringIO(data)
sourceImage = Image.open(imageIO)
sourceWidth = sourceImage.size[0]
sourceHeight = sourceImage.size[1]
top = (sourceHeight - movie.sceneHeight) / 2
targetImage = sourceImage.crop((0, top, sourceWidth, top + movie.sceneHeight))
targetImage.save(frame, 'PNG')
return loadFile(frame)
'''
returns png frame reflection of the given position.
'''
def loadFrameReflection(movie, position):
position = basename(position)
frame = framePath('frame', movie, position)
frameReflection = framePath('frameReflection', movie, position).replace('jpg', 'png')
if not exists(frameReflection):
if not exists(frame):
loadFrame(movie, position)
if exists(frame):
_writeReflection(frame, frameReflection)
else:
return loadDefaultFrameReflection(movie)
return loadFile(frameReflection)
def loadTimeline(movie, position):
bar = framePath('timeline', movie, position).replace('jpg', 'png')
if exists(bar):
return loadFile(bar)
print bar
return ''
#FIXME load and return bar hre
'''
move cache files to new imdb
'''
def moveCache(old_imdb, new_imdb):
old = join(cache_root, 'frame', old_imdb)
new = join(cache_root, 'frame', new_imdb)
if exists(old) and not exists(new):
shutil.move(old, new)
if exists(old):
shutil.rmtree(old)
old = join(cache_root, 'frameReflection', old_imdb)
new = join(cache_root, 'frameReflection', new_imdb)
if exists(old) and not exists(new):
shutil.move(old, new)
if exists(old):
shutil.rmtree(old)
old = join(cache_root, 'timeline', old_imdb)
new = join(cache_root, 'timeline', new_imdb)
if exists(old) and not exists(new):
shutil.move(old, new)
if exists(old):
shutil.rmtree(old)

BIN
oxdbarchive/static/images/stillDark.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 3.2 KiB

3
oxdbarchive/tools/extract_timeline.py
View file

@ -52,7 +52,6 @@ class GstTimeline:
format, self.length = q.parse_duration()
seconds = int(math.ceil(float(self.length) / gst.SECOND))
print "seconds", seconds
self.timelineImage = Image.new("RGB", (seconds, bar_height))
self.timelineImagePos = 0
@ -112,7 +111,7 @@ def usage():
if __name__ == "__main__":
if len(sys.argv) < 2:
usage()
if not os.path.exists(sys.argv[1]):
if not os.path.exists(os.path.dirname(sys.argv[1])):
print "target does not exist"
sys.exit(1)