From a55164012259cece36a2763feb9416dbb687db6a Mon Sep 17 00:00:00 2001
From: j <0x006A@0x2620.org>
Date: Thu, 3 Jul 2008 18:24:54 +0200
Subject: [PATCH] extract still, check if archive is online and nothing changed

---
 cron-oxdbarchive.py                |  2 +-
 oxdbarchive/cache.py               |  7 +--
 oxdbarchive/config/app.cfg         |  2 +
 oxdbarchive/controllers.py         |  3 ++
 oxdbarchive/cron.py                |  2 +-
 oxdbarchive/extract.py             | 10 +++++
 oxdbarchive/model.py               | 72 +++++++++++++++++++++++-------
 oxdbarchive/tools/extract_frame.py | 12 +++--
 8 files changed, 85 insertions(+), 25 deletions(-)

diff --git a/cron-oxdbarchive.py b/cron-oxdbarchive.py
index c9cf598..8493096 100755
--- a/cron-oxdbarchive.py
+++ b/cron-oxdbarchive.py
@@ -23,4 +23,4 @@ else:
 from oxdbarchive.controllers import Root
 from oxdbarchive import cron
 
-cron.cronDaily()
\ No newline at end of file
+cron.cronDaily()
diff --git a/oxdbarchive/cache.py b/oxdbarchive/cache.py
index 6afb372..52f7fd9 100644
--- a/oxdbarchive/cache.py
+++ b/oxdbarchive/cache.py
@@ -76,10 +76,11 @@ def loadTimeline(afile):
         return loadFile(timeline)
     return ''
 
-def loadPosterStill(afile, position):
-    still = afile.posterStillFile
+def loadStil(afile, position):
+    still = afile.stillFile
+    still.replace('.png', '%s.png' % position)
     if not exists(still):
-        afile.extractPosterStill(position)
+        afile.extractStill(position)
     if exists(still):
         return loadFile(still)
     return ''
diff --git a/oxdbarchive/config/app.cfg b/oxdbarchive/config/app.cfg
index 6e06b13..1c54bdb 100644
--- a/oxdbarchive/config/app.cfg
+++ b/oxdbarchive/config/app.cfg
@@ -12,6 +12,8 @@
 # template name
 # tg.defaultview = "kid"
 
+tg.defaultview = "genshi"
+
 # The following kid settings determine the settings used by the kid serializer.
 
 # One of (html|html-strict|xhtml|xhtml-strict|xml|json)
diff --git a/oxdbarchive/controllers.py b/oxdbarchive/controllers.py
index c5248d5..43bd8ff 100644
--- a/oxdbarchive/controllers.py
+++ b/oxdbarchive/controllers.py
@@ -51,6 +51,9 @@ class Root(controllers.RootController):
             elif action == 'frame':
                 cherrypy.response.headerMap['Content-Type'] = "image/jpeg"
                 return f.frame(position)
+            elif action == 'still':
+                cherrypy.response.headerMap['Content-Type'] = "image/png"
+                return f.still(position)
             elif action == 'posterStill':
                 cherrypy.response.headerMap['Content-Type'] = "image/png"
                 return f.posterStill(position)
diff --git a/oxdbarchive/cron.py b/oxdbarchive/cron.py
index f703cbd..2b9415b 100644
--- a/oxdbarchive/cron.py
+++ b/oxdbarchive/cron.py
@@ -9,7 +9,7 @@ def cronDaily():
 
 def findNew():
     archive = Archive.get(1)
-    archive.importFiles()
+    archive.findNew()
 
 def extractFrames():
     for f in ArchiveFile.select(ArchiveFile.q.extracted == False):
diff --git a/oxdbarchive/extract.py b/oxdbarchive/extract.py
index fd8c895..f65313e 100644
--- a/oxdbarchive/extract.py
+++ b/oxdbarchive/extract.py
@@ -160,6 +160,16 @@ def extract_frame(movie_file, timestamp, img_folder, width=128, offset = 0, redo
     print "update the cache %s missing" % movie_file
   shutil.rmtree(framedir)
 
+def extract_still(movie_file, png_file, inpoint):
+  ext = movie_file.split('.')[-1]
+  if ext in ('sub', 'srt'):
+    print "this is not a movie file, will not try to extract frames"
+    return
+  inpoint = time2ms(inpoint)
+  extractClipScript = abspath(join(dirname(__file__), "tools/extract_frame.py"))
+  cmd = '''%s "%s" "%s" %s 0 -1''' % (extractClipScript, movie_file, png_file, inpoint)
+  run_command(cmd.encode('utf-8'))
+
 def extract_poster_still(movie_file, png_file, inpoint):
   ext = movie_file.split('.')[-1]
   if ext in ('sub', 'srt'):
diff --git a/oxdbarchive/model.py b/oxdbarchive/model.py
index d4e6dfc..044eea5 100644
--- a/oxdbarchive/model.py
+++ b/oxdbarchive/model.py
@@ -2,6 +2,8 @@
 # vi:si:et:sw=2:sts=2:ts=2
 import gzip
 import StringIO
+from glob import glob
+import os
 
 from sqlobject import *
 from turbogears.database import PackageHub
@@ -19,7 +21,7 @@ import shutil
 import socket
 
 import simplejson
-from oxutils.net import getUrl
+from oxlib.net import getUrl
 import subtitles
 
 import cache
@@ -50,6 +52,28 @@ class Archive(SQLObject):
       self.basePath = basePath
     return basePath
 
+  def findNew(self):
+    #only update archive if not modified for more than 2 hours
+    if archive.isOnline():
+      if archive.sinceLastModification() > 60*60*2:
+        archive.importFiles()
+      else:
+        print "ignoring %s, was changed withing last 2 hours" % archive.basePath
+    else:
+      print "WARNING %s, is not online, check power and disk" % archive.basePath
+
+  def isOnline(self):
+      r = os.system('find "%s" >/dev/null 2>&1' % self.basePath)
+      if r:
+          return False
+      r = glob("%s*" % self.basePath)
+      if not r:
+          return False
+      return True
+
+  def sinceLastModification(self):
+    return time.time() - max([os.stat(f).st_mtime for f in glob('%s/*'% self.basePath)])
+
   def notifyFrontend(self, action, md5sum):
     if self.baseUrlFrontend:
       dto = socket.getdefaulttimeout()
@@ -412,7 +436,11 @@ class ArchiveFile(SQLObject):
   
   def _get_posterStillFile(self):
     return join(cache.cache_root, 'posterStill', self.md5sum[:4], "%s.png" % self.md5sum)
-  
+
+  def stillFile(self, position):
+    position = oxlib.getValidFilename(position)
+    return join(cache.cache_root, 'still', self.md5sum[:4], "%s_%s.png" % (self.md5sum, position))
+
   def removeMiniMovie(self): 
     if exists(self.mini_movie_file):
       os.remove(self.mini_movie_file)
@@ -504,7 +532,7 @@ class ArchiveFile(SQLObject):
   def extractFrames(self, img_folder=cache.frame_cache_root):
     for p in self._startPoints():
       self.frame(p)
-  
+
   def extractPosterStill(self, position):
     oxdb_makedir(dirname(self.posterStillFile))
     mFile = self.absolutePath
@@ -515,22 +543,22 @@ class ArchiveFile(SQLObject):
   def extractClipMovie(self, force = False): 
     if self.broken:
       return
-    if not self.height:
-      #only if midentify works we should try to extract the clip
-      return
-    if self.path.split('.')[-1] in ('mp3', 'wav', 'srt', 'sub', 'idx', 'rar','jpg', 'png'):
-      #ignore files known to not be 
+    #do not try to extract non movie files, or files that could not be identified
+    if not self.height or self.path.split('.')[-1] in ('mp3', 'wav', 'srt', 'sub', 'idx', 'rar','jpg', 'png'):
+      self.extracted = True
       return
     mini_movie_file = self.mini_movie_file
     movie_file = self.absolutePath
     if not movie_file or not exists(movie_file):
       return
-    if exists(mini_movie_file) and not force:
+    if not exists(mini_movie_file):
+      self.extracted = False
+    if self.extracted and not force:
       debug("clip exists, skipping extraction %s" % mini_movie_file)
       return
     self.extracted = False
     oxdb_makedir(dirname(mini_movie_file))
-    
+
     options = ''
     options += " -ovc lavc -lavcopts vcodec=mjpeg"
     options += " -af volnorm=1 -oac mp3lame -lameopts br=64:mode=3 -af resample=44100"
@@ -624,7 +652,7 @@ class ArchiveFile(SQLObject):
     #return if its not a video
     if self.height <= 0:
       return
-    
+
     if not (self.extracted and exists(self.mini_movie_file)):
       debug("mini movie missing, skipping %s" % self.path)
       return 
@@ -637,13 +665,10 @@ class ArchiveFile(SQLObject):
     oxdb_makedir(dirname(t))
 
     #this fails in tg-admin shell
-    #extractTimelineScript = abspath(join(dirname(__file__), "tools/extract_timeline.py"))
     extractTimelineScript = abspath(join(dirname(cache.cache_root), "tools/extract_timeline.py"))
-    #this fails is called inside server
-    #extractTimelineScript = "oxdbarchive/tools/extract_timeline.py"
     cmd = "python %s %s %s" %(extractTimelineScript, t, self.mini_movie_file)
     os.system(cmd)
-  
+
   def loadSubtitleFromFile(self):
     if self.movieFile:
       movieFile = self.movieFile
@@ -706,9 +731,22 @@ class ArchiveFile(SQLObject):
   
   def frame(self, position):
     return cache.loadFrame(self, position)
-  
+
   def timeline(self):
     return cache.loadTimeline(self)
-  
+
   def posterStill(self, position):
     return cache.loadPosterStill(self, position)
+
+  def still(self, position):
+    still = self.stillFile(position)
+    if not exists(still):
+      oxdb_makedir(dirname(still))
+      movieFile = self.absolutePath
+      if os.path.splitext(movieFile)[-1] in ('.mov', '.mpg', '.mpeg'):
+        movieFile = self.mini_movie_file
+      extract_still(movieFile, still, position)
+    if exists(still):
+        return cache.loadFile(still)
+    return ''
+
diff --git a/oxdbarchive/tools/extract_frame.py b/oxdbarchive/tools/extract_frame.py
index 77b23f4..5b1310c 100755
--- a/oxdbarchive/tools/extract_frame.py
+++ b/oxdbarchive/tools/extract_frame.py
@@ -98,7 +98,10 @@ class ExtractFrame:
     caps = sink.sink_pads().next().get_negotiated_caps()
     for s in caps:
       input_size = (s['width'], s['height'])
-      self.frame_size = self.scaleto(s['width'], s['height'])
+      if self.width > 0:
+        self.frame_size = self.scaleto(s['width'], s['height'])
+      else:
+        self.frame_size = None
       #Why are the last frames broken, aka have green pixels
       save_last_frame = (4*gst.SECOND/float(s['framerate']))
       if (self.duration-self.frame_pos) < save_last_frame:
@@ -166,8 +169,11 @@ class ExtractFrame:
       self.quit()
 
   def quit(self):
-    if self.frame_img and self.frame_size:
-      img = self.frame_img.resize(self.frame_size, Image.ANTIALIAS)
+    if self.frame_img:
+      if self.frame_size:
+        img = self.frame_img.resize(self.frame_size, Image.ANTIALIAS)
+      else:
+        img = self.frame_img
       img.save(self.frame_file)
       self.debug('frame saved at %s' % self.frame_file)
     self.pipeline.set_state(gst.STATE_NULL)