more vi:si:et:sw=4:sts=4:ts=4

oxdbarchive/backup.py

@@ -1,65 +1,68 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
 from oxdb_utils import oxdb_makedir
 from os.path import *
-import cache 
+import cache
 import extract
 
+
 class BackupMovieFile:
-  def __init__(self, parent):
+    def __init__(self, parent):
-    self.parent = parent
+        self.parent = parent
-    
+
-  def extractFrame(self, position, img_folder):
+    def extractFrame(self, position, img_folder):
-    return self.parent.extractFrame(position)
+        return self.parent.extractFrame(position)
-  
+
 class Backup:
-  def __init__(self, md5sum):
+    def __init__(self, md5sum):
-    self.md5sum = md5sum
+        self.md5sum = md5sum
-    self.timelineFile = join(cache.cache_root, 'timeline', self.md5sum[:4], "%s.png" % self.md5sum)
+        self.timelineFile = join(cache.cache_root, 'timeline', self.md5sum[:4], "%s.png" % self.md5sum)
-    self.posterStill = join(cache.cache_root, 'posterStill', self.md5sum[:4], "%s.png" % self.md5sum)
+        self.posterStill = join(cache.cache_root, 'posterStill', self.md5sum[:4], "%s.png" % self.md5sum)
-    self.mini_movie_file = join(cache.cache_root, 'mini', self.md5sum[:4], "%s.avi" % self.md5sum)
+        self.mini_movie_file = join(cache.cache_root, 'mini', self.md5sum[:4], "%s.avi" % self.md5sum)
-    self.movieFile = BackupMovieFile(self)
+        self.movieFile = BackupMovieFile(self)
-    self.frameFolder = join(cache.cache_root, 'frame', self.md5sum[:4], self.md5sum)
+        self.frameFolder = join(cache.cache_root, 'frame', self.md5sum[:4], self.md5sum)
-  
-  def exists(self):
-    return exists(self.mini_movie_file)
 
-  def timeline(self):
+    def exists(self):
-    return cache.loadTimeline(self)
+        return exists(self.mini_movie_file)
-  
-  def frame(self, pos):
-    return cache.loadFrame(self, pos)
-    
-  def posterStill(self, pos):
-    return cache.loadPosterStill(self, pos)
 
-  def flvClip(self, position):
+    def timeline(self):
-    return cache.loadFlvClip(self, position)
+        return cache.loadTimeline(self)
 
-  def extractFlvClip(self, inpoint, outpoint=-1):
+    def frame(self, pos):
-    flash_folder = self.frameFolder
+        return cache.loadFrame(self, pos)
-    movie_file = self.mini_movie_file
+
-    position = inpoint.replace(':', '.')
+    def posterStill(self, pos):
-    flash_movie = join(self.frameFolder, '%s.%s' % (position, 'flv'))
+        return cache.loadPosterStill(self, pos)
-    width = 128
+
-    height = int(width / self.frameAspect)
+    def flvClip(self, position):
-    height = height - height % 2
+        return cache.loadFlvClip(self, position)
-    inpoint = inpoint.replace('.', ':')
+
-    if outpoint == -1:
+    def extractFlvClip(self, inpoint, outpoint=-1):
-      outpoint = shift_time(5000, inpoint)
+        flash_folder = self.frameFolder
-    else:
+        movie_file = self.mini_movie_file
-      outpoint = outpoint.replace('.', ':')
+        position = inpoint.replace(':', '.')
-    extract.extract_flash(movie_file, flash_movie, inpoint, outpoint, width, height, offset = 0)
+        flash_movie = join(self.frameFolder, '%s.%s' % (position, 'flv'))
-  
+        width = 128
-  def extractFrame(self, position, img_folder=-1): 
+        height = int(width / self.frameAspect)
-    img_folder = self.frameFolder 
+        height = height - height % 2
-    movie_file = self.mini_movie_file
+        inpoint = inpoint.replace('.', ':')
-    oxdb_makedir(img_folder)
+        if outpoint == -1:
-    position = position.replace('.', ':')
+            outpoint = shift_time(5000, inpoint)
-    extract.extract_frame(movie_file, position, img_folder, offset = 0, redo = False)
+        else:
-  
+            outpoint = outpoint.replace('.', ':')
-  def extractPosterStill(self, position):
+        extract.extract_flash(movie_file, flash_movie, inpoint, outpoint, width, height, offset = 0)
-    oxdb_makedir(dirname(self.posterStillFile))
+
-    mFile = self.absolutePath
+    def extractFrame(self, position, img_folder=-1):
-    if os.path.splitext(mFile)[-1] in ('.mov', '.mpg', '.mpeg'):
+        img_folder = self.frameFolder
-      mFile = self.mini_movie_file
+        movie_file = self.mini_movie_file
-    extract.extract_poster_still(mFile, self.posterStillFile, position)
+        oxdb_makedir(img_folder)
+        position = position.replace('.', ':')
+        extract.extract_frame(movie_file, position, img_folder, offset = 0, redo = False)
+
+    def extractPosterStill(self, position):
+        oxdb_makedir(dirname(self.posterStillFile))
+        mFile = self.absolutePath
+        if os.path.splitext(mFile)[-1] in ('.mov', '.mpg', '.mpeg'):
+            mFile = self.mini_movie_file
+        extract.extract_poster_still(mFile, self.posterStillFile, position)
 

oxdbarchive/cache.py

@@ -1,7 +1,5 @@
-# -*- Mode: Python; -*-
 # -*- coding: utf-8 -*-
-# vi:si:et:sw=2:sts=2:ts=2
+# vi:si:et:sw=4:sts=4:ts=4
-
 import os
 from os.path import abspath, exists, join, dirname, basename
 import shutil
@@ -18,70 +16,71 @@ img_extension = "jpg"
 frame_cache_root = join(cache_root, 'frame')
 
 def loadFile(f_name):
-  f = open(f_name)
+    f = open(f_name)
-  data = f.read()
+    data = f.read()
-  f.close()
+    f.close()
-  return data
+    return data
 
 def saveFile(f_name, data):
-  f = open(f_name, 'w')
+    f = open(f_name, 'w')
-  f.write(data)
+    f.write(data)
-  f.close()
+    f.close()
 
 def loadStaticFile(fname):
-  return loadFile(join(dirname(abspath(__file__)), "static", fname))
+    return loadFile(join(dirname(abspath(__file__)), "static", fname))
 
 def loadFrame(afile, position):
-  position = basename(position).replace(':', '.')
+    position = basename(position).replace(':', '.')
-  frame = join(afile.frameFolder, '%s.%s' % (position, img_extension))
+    frame = join(afile.frameFolder, '%s.%s' % (position, img_extension))
-  if not exists(frame):
+    if not exists(frame):
-    afile.extractFrame(position)
+        afile.extractFrame(position)
-  if exists(frame):
+    if exists(frame):
-    return loadFile(frame)
+        return loadFile(frame)
-  return ''
+    return ''
 
 def loadFlvClip(afile, position):
-  position = basename(position).replace(':', '.')
+    position = basename(position).replace(':', '.')
-  positions = position.split(';')
+    positions = position.split(';')
-  if len(positions) > 1:
+    if len(positions) > 1:
-    position = positions[0]
+        position = positions[0]
-    outpoint = positions[1]
+        outpoint = positions[1]
-  else:
+    else:
-    outpoint = -1
+        outpoint = -1
-  flash = join(afile.frameFolder, '%s.%s' % (position, 'flv'))
+    flash = join(afile.frameFolder, '%s.%s' % (position, 'flv'))
-  if not exists(flash):
+    if not exists(flash):
-    afile.extractFlvClip(position, outpoint)
+        afile.extractFlvClip(position, outpoint)
-  if exists(flash):
+    if exists(flash):
-    return loadFile(flash)
+        return loadFile(flash)
-  return ''
+    return ''
 
 def loadOggClip(afile, position):
-  position = basename(position).replace(':', '.')
+    position = basename(position).replace(':', '.')
-  positions = position.split(';')
+    positions = position.split(';')
-  if len(positions) > 1:
+    if len(positions) > 1:
-    position = positions[0]
+        position = positions[0]
-    outpoint = positions[1]
+        outpoint = positions[1]
-  else:
+    else:
-    outpoint = -1
+        outpoint = -1
-  clip = join(afile.frameFolder, '%s.%s' % (position, 'ogv'))
+    clip = join(afile.frameFolder, '%s.%s' % (position, 'ogv'))
-  if not exists(clip):
+    if not exists(clip):
-    afile.extractOggClip(position, outpoint)
+        afile.extractOggClip(position, outpoint)
-  if exists(clip):
+    if exists(clip):
-    return loadFile(clip)
+        return loadFile(clip)
-  return ''
+    return ''
 
 def loadTimeline(afile):
-  timeline = afile.timelineFile
+    timeline = afile.timelineFile
-  if not exists(timeline):
+    if not exists(timeline):
-    afile.extractTimeline()
+        afile.extractTimeline()
-  if exists(timeline):
+    if exists(timeline):
-    return loadFile(timeline)
+        return loadFile(timeline)
-  return ''
+    return ''
 
 def loadPosterStill(afile, position):
-  still = afile.posterStillFile
+    still = afile.posterStillFile
-  if not exists(still):
+    if not exists(still):
-    afile.extractPosterStill(position)
+        afile.extractPosterStill(position)
-  if exists(still):
+    if exists(still):
-    return loadFile(still)
+        return loadFile(still)
-  return ''
+    return ''
+

oxdbarchive/controllers.py

@@ -1,7 +1,5 @@
-# -*- Mode: Python; -*-
 # -*- coding: utf-8 -*-
-# vi:si:et:sw=2:sts=2:ts=2
+# vi:si:et:sw=4:sts=4:ts=4
-
 from turbogears import controllers, expose, flash, redirect
 from model import *
 from json import *
@@ -18,45 +16,43 @@ import backup
 '''
 
 def httpExpires(sec):
-  exp_date = cherrypy.lib.httptools.HTTPDate(time.gmtime(time.mktime(time.gmtime()) + sec))
+    exp_date = cherrypy.lib.httptools.HTTPDate(time.gmtime(time.mktime(time.gmtime()) + sec))
-  return exp_date
+    return exp_date
 
 class Root(controllers.RootController):
-  @expose()
+    @expose()
-  def default(self, md5Hash = None, action = None, position = None):
+    def default(self, md5Hash = None, action = None, position = None):
-    if not md5Hash:
+        if not md5Hash:
-      return dict()
+            return dict()
-    try:
+        try:
-      f = ArchiveFile.byMd5sum(md5Hash)
+            f = ArchiveFile.byMd5sum(md5Hash)
-    except:
+        except:
-      #return dict()
+            #return dict()
-      f = backup.Backup(md5Hash)
+            f = backup.Backup(md5Hash)
-      if not f.exists():
+            if not f.exists():
+                return dict()
+        if action == 'metadata':
+            return dict(metadata = f)
+        elif action in ('timeline', 'timeline.png'):
+            cherrypy.response.headerMap['Content-Type'] = "image/png"
+            cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
+            return f.timeline()
+        elif position: #clip / frame
+            position = position.replace('.png', '').replace('.jpg', '')
+            position = position.replace('.flv', '').replace('.ogv', '')
+            position = position.replace('-', ':').replace('.',':')
+            cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
+            if action == 'clip':
+                cherrypy.response.headerMap['Content-Type'] = "video/x-flv"
+                return f.flvClip(position)
+            if action == 'ogv':
+                cherrypy.response.headerMap['Content-Type'] = "video/ogg"
+                return f.oggClip(position)
+            elif action == 'frame':
+                cherrypy.response.headerMap['Content-Type'] = "image/jpeg"
+                return f.frame(position)
+            elif action == 'posterStill':
+                cherrypy.response.headerMap['Content-Type'] = "image/png"
+                return f.posterStill(position)
         return dict()
-    if action == 'metadata':
+
-      return dict(metadata = f)
-    elif action in ('timeline', 'timeline.png'):
-      cherrypy.response.headerMap['Content-Type'] = "image/png"
-      cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
-      return f.timeline()
-    elif position: #clip / frame
-      position = position.replace('.png', '').replace('.jpg', '')
-      position = position.replace('.flv', '').replace('.ogv', '')
-      position = position.replace('-', ':').replace('.',':')
-      if action == 'clip':
-        cherrypy.response.headerMap['Content-Type'] = "video/x-flv"
-        cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
-        return f.flvClip(position)
-      if action == 'ogv':
-        cherrypy.response.headerMap['Content-Type'] = "video/ogg"
-        cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
-        return f.oggClip(position)
-      elif action == 'frame':
-        cherrypy.response.headerMap['Content-Type'] = "image/jpeg"
-        cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
-        return f.frame(position)
-      elif action == 'posterStill':
-        cherrypy.response.headerMap['Content-Type'] = "image/png"
-        cherrypy.response.headerMap["Expires"] = httpExpires(60*60*24*15)
-        return f.posterStill(position)
-    return dict()

oxdbarchive/cron.py

@@ -1,48 +1,47 @@
-# -*- Mode: Python; -*-
 # -*- coding: utf-8 -*-
-# vi:si:et:sw=2:sts=2:ts=2
+# vi:si:et:sw=4:sts=4:ts=4
-
 from model import *
 import cache
 
 def cronDaily():
-  findNew()
+    findNew()
-  extractNew()
+    extractNew()
 
 def findNew():
-  archive = Archive.get(1)
+    archive = Archive.get(1)
-  archive.importFiles()
+    archive.importFiles()
 
 def extractFrames():
-  for f in ArchiveFile.select(ArchiveFile.q.extracted == False):
+    for f in ArchiveFile.select(ArchiveFile.q.extracted == False):
-    f.extractFrames()
+        f.extractFrames()
 
 def extractTimelines():
-  for f in ArchiveFile.select(ArchiveFile.q.extracted == False):
+    for f in ArchiveFile.select(ArchiveFile.q.extracted == False):
-    f.extractTimeline()
+        f.extractTimeline()
 
 def extractNew():
-  for f in ArchiveFile.select(ArchiveFile.q.extracted == False):
+    for f in ArchiveFile.select(ArchiveFile.q.extracted == False):
-    f.extractAll()
+        f.extractAll()
 
 def cleanClipMovieCache():
-  cache = os.path.abspath('oxdb/cache/mini/')
+    cache = os.path.abspath('oxdb/cache/mini/')
-  for d, p, files in os.walk(cache):
+    for d, p, files in os.walk(cache):
-    for f in files:
+        for f in files:
-      md5sum = f.replace('.avi', '')
+            md5sum = f.replace('.avi', '')
-      try:
+            try:
-        fm = ArchiveFile.byMd5sum(md5sum)
+                fm = ArchiveFile.byMd5sum(md5sum)
-      except:
+            except:
-        os.remove(os.path.join(cache, d, f))
+                os.remove(os.path.join(cache, d, f))
 
 #possible fuctions:
 def extractSubtitles():
-  extractSubtitlesScript = abspath(join(dirname(cache.cache_root), "tools/subtitles.py"))
+    extractSubtitlesScript = abspath(join(dirname(cache.cache_root), "tools/subtitles.py"))
-  for f in ArchiveFile.select(LIKE(ArchiveFile.q.path, '%' + '.idx')):
+    for f in ArchiveFile.select(LIKE(ArchiveFile.q.path, '%' + '.idx')):
-    base = f.absolutePath.replace('.idx', '')
+        base = f.absolutePath.replace('.idx', '')
-    srtFile = f.absolutePath.replace('.idx', '.srt')
+        srtFile = f.absolutePath.replace('.idx', '.srt')
-    if not os.path.exists(srtFile):
+        if not os.path.exists(srtFile):
-      cmd = 'python "%s" "%s"' % (extractSubtitlesScript, base)
+            cmd = 'python "%s" "%s"' % (extractSubtitlesScript, base)
-      os.system(cmd)
+            os.system(cmd)
-      #only one subtitle at a time
+            #only one subtitle at a time
-      return
+            return
+

oxdbarchive/extract.py

@@ -1,7 +1,5 @@
 # -*- coding: utf-8 -*-
-# -*- Mode: Python; -*-
 # vi:si:et:sw=2:sts=2:ts=2
-
 import re
 import os
 from os.path import abspath, join, dirname

oxdbarchive/midentify.py

@@ -1,7 +1,5 @@
 # -*- coding: utf-8 -*-
-# -*- Mode: Python; -*-
 # vi:si:et:sw=2:sts=2:ts=2
-
 import os
 import sys
 

oxdbarchive/model.py

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-# -*- Mode: Python; -*-
 # vi:si:et:sw=2:sts=2:ts=2
 import gzip
 import StringIO
@@ -353,10 +352,13 @@ class ArchiveFile(SQLObject):
       self.frameAspect = "%0.6f" % aspect
     return aspect
   
+  def _get_sceneWidth(self):
+    return 128
+
   def _get_sceneHeight(self):
     default = 80
     if not self.subtitle:
-      h = int(128 / self.frameAspect)
+      h = int(self.sceneWidth / self.frameAspect)
       h = h + h % 2
       return h
     return default
@@ -450,9 +452,8 @@ class ArchiveFile(SQLObject):
     movie_file = self.mini_movie_file
     position = inpoint.replace(':', '.')
     flash_movie = join(self.frameFolder, '%s.%s' % (position, 'flv'))
-    width = 128
+    width = self.sceneWidth
-    height = int(width / self.frameAspect)
+    height = self.sceneHeight
-    height = height - height % 2
     inpoint = inpoint.replace('.', ':')
     if outpoint == -1:
       s = self._findSubtitleByInPoint(inpoint)
@@ -471,9 +472,8 @@ class ArchiveFile(SQLObject):
     movie_file = self.mini_movie_file
     position = inpoint.replace(':', '.')
     clip_movie = join(self.frameFolder, '%s.%s' % (position, 'ogv'))
-    width = 128
+    width = self.sceneWidth
-    height = int(width / self.frameAspect)
+    height = self.sceneHeight
-    height = height - height % 2
     inpoint = inpoint.replace('.', ':')
     if outpoint == -1:
       s = self._findSubtitleByInPoint(inpoint)
@@ -534,7 +534,7 @@ class ArchiveFile(SQLObject):
     options = ''
     options += " -ovc lavc -lavcopts vcodec=mjpeg"
     options += " -af volnorm=1 -oac mp3lame -lameopts br=64:mode=3 -af resample=44100"
-    options += " -vf scale -zoom -xy 128"
+    options += " -vf scale -zoom -xy %s" % self.sceneWidth
     options += ' "%s"' % movie_file.replace('"', '\\"')
     options += ' -o "%s"' % mini_movie_file
     cmd = "mencoder %s >/dev/null 2>&1" % options
@@ -542,7 +542,75 @@ class ArchiveFile(SQLObject):
     r = os.system(cmd.encode('utf-8'))
     if r == 0:
       self.extracted = True
-  
+
+  def extractOggMovie(self, force = False): 
+    if self.broken:
+      return
+    if not self.height:
+      #only if midentify works we should try to extract the clip
+      return
+    if self.path.split('.')[-1] in ('mp3', 'wav', 'srt', 'sub', 'idx', 'rar','jpg', 'png'):
+      #ignore files known to not be 
+      return
+
+    oggFile = self.mini_movie_file.replace('.avi', '.ogg')
+    movieFile = self.absolutePath
+    if not movieFile or not exists(movieFile):
+      return
+    if exists(oggFile) and not force:
+      debug("clip exists, skipping extraction %s" % oggFile)
+      return
+    self.extractedOgg = False
+    oxdb_makedir(dirname(oggFile))
+    options = ''
+    options += " --no-skeleton -K 16 -V 180 -a -1 -H 44100 -S 1 --speedlevel 0 -c 2 "
+    options += " -x %s -y %s" % (self.sceneWidth, self.sceneHeight)
+    options += ' "%s"' % movieFile.replace('"', '\\"')
+    options += ' -o "%s"' % oggFile
+    cmd = "ffmpeg2theora %s >/dev/null 2>&1" % options
+    r = os.system(cmd.encode('utf-8'))
+    if r == 0:
+      self.extractedOgg = True
+
+  def extractH264Movie(self, force = False): 
+    if self.broken:
+      return
+    if not self.height:
+      #only if midentify works we should try to extract the clip
+      return
+    if self.path.split('.')[-1] in ('mp3', 'wav', 'srt', 'sub', 'idx', 'rar','jpg', 'png'):
+      #ignore files known to not be 
+      return
+
+    h264File = self.mini_movie_file.replace('.avi', '.mp4')
+    h264FileTmp = h264File + ".tmp.mp4"
+    movieFile = self.absolutePath
+    if not movieFile or not exists(movieFile):
+      return
+    if exists(h264File) and not force:
+      debug("clip exists, skipping extraction %s" % h264File)
+      return
+    self.extractedH264 = False
+    height = self.sceneHeight
+    oxdb_makedir(dirname(h264File))
+    options = ''
+    options += " -vcodec libx264 -b 112k -bf 3 -subq 6 -cmp 256 -refs 5 -qmin 10 "
+    options += " -qmax 51 -qdiff 4 -coder 1 -loop 1 -me hex -me_range 16 -trellis 1 "
+    options += " -flags +mv4 -flags2 +bpyramid+wpred+mixed_refs+brdo+8x8dct "
+    options += " -partitions parti4x4+parti8x8+partp4x4+partp8x8+partb8x8 -g 250 "
+    options += " -keyint_min 16 -sc_threshold 40 -i_qfactor 0.71"
+    options += " -s %sx%s" % (self.sceneWidth, height)
+
+    cmd = '''ffmpeg -y -i "%s" -an -pass 1 -threads 2 %s "%s" >/dev/null 2>&1''' % (movieFile, options, h264FileTmp)
+    print cmd
+    r = os.system(cmd.encode('utf-8'))
+    cmd = '''ffmpeg -y -i "%s" -acodec libfaac -ac 2 -ar 44100 -ab 48k -pass 2 -threads 2 %s "%s" >/dev/null 2>&1''' % (movieFile, options, h264FileTmp)
+    r = os.system(cmd.encode('utf-8'))
+    cmd = '''qt-faststart "%s" "%s" >/dev/null 2>&1 && rm "%s"''' % (h264FileTmp, h264File, h264FileTmp)
+    r = os.system(cmd.encode('utf-8'))
+    if r == 0:
+      self.extractedH264 = True
+
   def removeTimeline(self):
     if exists(self.timelineFile):
       os.unlink(self.timelineFile)

oxdbarchive/oxdb_import.py

@@ -1,11 +1,9 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# -*- Mode: Python; -*-
 # vi:si:et:sw=2:sts=2:ts=2
 # OXDb Import client, crawls the filesystem and gathers information about 
 # movies
 #
-
 import md5
 import os
 import sys

oxdbarchive/oxdb_utils.py

@@ -1,11 +1,9 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# -*- Mode: Python; -*-
 # vi:si:et:sw=2:sts=2:ts=2
 # OXDb Import client, crawls the filesystem and gathers information about
 # movies
 #
-
 import md5
 import os
 import sys

oxdbarchive/timeline.py

@@ -1,7 +1,5 @@
-# -*- Mode: Python; -*-
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=2:sts=2:ts=2
-
 import Image
 import math
 from StringIO import StringIO