* use extract_frame.py from pad.ma

* update json request to only get md5sum from frontend
 * cronjob now imports all new files to db, extracts them
   and only after that send notification to frontend

oxdbarchive/extract.py

@@ -154,9 +154,9 @@ def extract_poster_still(movie_file, png_file, inpoint):
     print "this is not a movie file, will not try to extract frames"
     return
   inpoint = time_str2msec(inpoint)
-  extractClipScript = abspath(join(dirname(__file__), "tools/extract_poster_still.py"))
+  extractClipScript = abspath(join(dirname(__file__), "tools/extract_frame.py"))
 
-  cmd = '''%s "%s" "%s" %s''' % (extractClipScript, movie_file, png_file, inpoint)
+  cmd = '''%s "%s" "%s" %s 0 128''' % (extractClipScript, movie_file, png_file, inpoint)
   os.system(cmd.encode('utf-8'))
 
 def extract_subtitles(movie_file, srt, img_folder, width=128, offset = 0, redo = False):

oxdbarchive/model.py

@@ -40,14 +40,16 @@ class Archive(SQLObject):
   name = UnicodeCol(length=255, alternateID=True)
   basePath = UnicodeCol()
   baseUrlFrontend = UnicodeCol(default = '')
-  
+  published = DateTimeCol(defalut=datetime.now)
+  modified = DateTimeCol(defalut=datetime.now)
+
   def _get_basePath(self):
     basePath = self._SO_get_basePath()
     if not basePath.endswith('/'):
       basePath = basePath + "/"
       self.basePath = basePath
     return basePath
-  
+
   def notifyFrontend(self, action, md5sum):
     if self.baseUrlFrontend:
       dto = socket.getdefaulttimeout()
@@ -141,9 +143,9 @@ class Archive(SQLObject):
         ret = "added entry"
     f.updateMeta()
     f.extractAll()
-    self.notifyFrontend('add', f.md5sum)
+    f.modified = datetime.now()
     return ret.encode('utf-8')
-  
+
   def removeFile(self, md5sum):
     '''
       remove file based on md5sum from archive
@@ -158,35 +160,7 @@ class Archive(SQLObject):
         self.notifyFrontend('remove', md5sum)
       return dict(result="file removed")
     return dict(result="not in archive")
-  
+
-  #FIXME this fails for old frontends because it notifies the frontend
-  def importFrontend(self):
-    if self.baseUrlFrontend:
-      dto = socket.getdefaulttimeout()
-      socket.setdefaulttimeout(256)
-      url = "%s/%s" % (self.baseUrlFrontend, 'list')
-      result = read_url(url)
-      files = simplejson.loads(result)['files']
-      socket.setdefaulttimeout(dto)
-      for f in files:
-        meta = files[f]
-        fname = join(self.basePath, f)
-        if exists(fname):
-          stats = oxdb_import.oxdb_file_stats(fname)
-          meta['date'] = stats['date']
-          meta['path'] = f
-          meta['video'] = ''
-          meta['audio'] = ''
-          meta['length'] = 0
-          meta['bpp'] = 0
-          for key in ('bpp', 'size', 'length', 'date'):
-            meta[key] = int(float(meta[key]))
-          meta['date'] = datetime.fromtimestamp(meta['date'])
-          print self.addFile(meta), f
-        else:
-          print "remove", f
-          self.removeFile(meta['md5sum'])
-  
   def importFiles(self):
     stats = {'skipped': 0, 'added': 0, 'remove':0}
     print self.basePath
@@ -215,38 +189,38 @@ class Archive(SQLObject):
         self.removeFile(oxdb_files[f]['md5sum'])
         stats['remove'] += 1
     print stats
+    print "updating information on frontend"
+    self.updateFrontend()
     return stats
   
   '''
-    notify frontend about each file we have.
+    Interaction with frontend, 
+      - update send modified files since last sync.
+      - bootstrap send all files
+      - sync get list from frontend and remove/add those that are not in sync
   '''
+  def updateFrontned(self):
+    for f in ArchiveFile.select(ArchiveFile.modified >= self.published):
+       self.notifyFrontend('add', f.md5sum)
+    self.published=datetime.now()
+
   def bootstrapFrontend(self):
     for f in self.files:
       self.notifyFrontend('add', f.md5sum)
   
-  def cleanupFrontend(self):
+  def syncFrontend(self):
     dto = socket.getdefaulttimeout()
     socket.setdefaulttimeout(256)
     data = read_url("%s/list" % self.baseUrlFrontend)
-    files = simplejson.loads(data)['files']
+    md5sums = simplejson.loads(data)['files']
     socket.setdefaulttimeout(dto)
-    md5sums = [str(f['md5sum']) for f in files.values()]
     for md5sum in md5sums:
       try:
         f = ArchiveFile.byMd5sum(md5sum)
       except SQLObjectNotFound:
         self.notifyFrontend('remove', md5sum)
-  
-  def fillFrontend(self):
-    dto = socket.getdefaulttimeout()
-    socket.setdefaulttimeout(256)
-    data = read_url("%s/list" % self.baseUrlFrontend)
-    files = simplejson.loads(data)['files']
-    socket.setdefaulttimeout(dto)
-    md5sums = [str(f['md5sum']) for f in files.values()]
     for f in ArchiveFile.select(NOT(IN(ArchiveFile.q.md5sum, md5sums))):
       self.notifyFrontend('add', f.md5sum)
-  
 
 class ArchiveFile(SQLObject):
   '''
@@ -267,11 +241,11 @@ class ArchiveFile(SQLObject):
   size = IntCol()
   bpp = FloatCol(default = -1)
   pixels = IntCol(default = 0)
-  
+
-  date_added = DateTimeCol(default = datetime.now)
+  date_added = DateTimeCol(default=datetime.now)
-  pubDate = DateTimeCol(default = datetime.now)
+  published = DateTimeCol(defalut=datetime.now)
-  modDate = DateTimeCol(default = datetime.now)
+  modified = DateTimeCol(defalut=datetime.now)
-  
+
   height = IntCol(default = -1)
   width = IntCol(default = -1)
   frameAspect = UnicodeCol(default = "1.6", length = 100)

oxdbarchive/oxdb_utils.py

@@ -46,9 +46,9 @@ def oxdb_filenameUmlaute(string):
   
 def oxdb_director(director):
   director = os.path.basename(os.path.dirname(director))
-  director.replace(' & ', ', ')
+  director = director.replace('&', ', ').replace(' ,  ', ', ')
   return director
-  
+
 def oxdb_title(title):
   '''
     normalize filename to get movie title
@@ -215,4 +215,4 @@ def oxdb_makedir(folder):
       os.makedirs(folder)
     except os.error, e:
       if e.errno != errno.EEXIST or not os.path.isdir(folder):
-        raise
+        raise

oxdbarchive/tools/extract_frame.py

@@ -0,0 +1,194 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=2:sts=2:ts=2
+# GPL written 2008 by j@pad.ma
+import gobject
+gobject.threads_init()
+
+import pygst
+pygst.require("0.10")
+import gst
+import Image
+
+
+DEBUG=0
+
+class ExtractFrame:
+  video = None
+  audio = None
+  info = {}
+  frame_img = None
+  frame_size = None
+  duration = 0
+  height = 0
+  width = 128
+  def __init__(self, videofile, frame, frame_pos, width=128, height=0):
+    self.width = width
+    self.height = height
+    self.frame_file = frame
+    self.frame_pos = frame_pos
+    self.mainloop = gobject.MainLoop()
+    self.pipeline = gst.parse_launch('filesrc name=input ! decodebin name=dbin')
+    self.input  = self.pipeline.get_by_name('input')
+    self.input.props.location = videofile
+    self.dbin  = self.pipeline.get_by_name('dbin')
+
+    self.bus = self.pipeline.get_bus()
+    self.dbin.connect('new-decoded-pad', self.demux_pad_added)
+
+    self.bus.add_signal_watch()
+    self.watch_id = self.bus.connect("message", self.onBusMessage)
+
+    self.pipeline.set_state(gst.STATE_PAUSED)
+    self.pipeline.get_state()
+
+  def run(self):
+    self.pipeline.set_state(gst.STATE_PLAYING)
+    self.pipeline.get_state()
+
+    #duration
+    pads = None
+    if self.video:
+      pads = self.video.sink_pads()
+    elif self.audio:
+      pads = self.audio.sink_pads()
+    if pads:
+      q = gst.query_new_duration(gst.FORMAT_TIME)
+      for pad in pads:
+        if pad.get_peer() and pad.get_peer().query(q):
+          format, self.duration = q.parse_duration()
+          self.info["duration"] = self.duration/gst.MSECOND
+
+    #seek
+    if self.frame_pos > self.duration:
+      self.debug('seek point greater than file duration %s' % (self.duration/gst.MSECOND))
+      return
+
+    if (self.duration - self.frame_pos) < gst.SECOND:
+      seek_pos = self.duration- 10 * gst.SECOND
+    else:
+      seek_pos = self.frame_pos - 10 * gst.SECOND
+    seek_pos = max(0, seek_pos)
+    
+    #extract
+    self.debug('seek tp %s'%seek_pos)
+    self.seek(seek_pos)
+    self.debug('get frame tp %s'%seek_pos)
+    self.frame_ho = self.video.connect("handoff", self.get_frame_cb)
+    self.pipeline.set_state(gst.STATE_PLAYING)
+    self.pipeline.get_state()
+
+
+    self.mainloop.run()
+    return self.info
+
+  def seek(self, seek_pos):
+    event = gst.event_new_seek(1.0, gst.FORMAT_TIME,
+            gst.SEEK_FLAG_FLUSH | gst.SEEK_FLAG_ACCURATE,
+            gst.SEEK_TYPE_SET, seek_pos,
+            gst.SEEK_TYPE_NONE, 0)
+
+    res = self.video.send_event(event)
+    if res:
+      self.pipeline.set_new_stream_time(0L)
+    else:
+      gst.error("seek to %r failed" % frame_pos)
+
+  def get_frame_cb(self, sink, frame_buffer, pad):
+    caps = sink.sink_pads().next().get_negotiated_caps()
+    for s in caps:
+      input_size = (s['width'], s['height'])
+      self.frame_size = self.scaleto(s['width'], s['height'])
+      #Why are the last frames broken, aka have green pixels
+      save_last_frame = (4*gst.SECOND/float(s['framerate']))
+      if (self.duration-self.frame_pos) < save_last_frame:
+        self.frame_pos = self.duration-save_last_frame
+    position, format = sink.query_position(gst.FORMAT_TIME)
+    if not self.frame_img or position <= self.frame_pos:
+      self.frame_img = Image.fromstring('RGB', input_size, frame_buffer)
+    else:
+      self.video.disconnect(self.frame_ho)
+      self.bus.post(gst.message_new_eos(self.pipeline))
+
+  def scaleto(self, width, height):
+    if self.width:
+      height = int(self.width / (float(width) / height))
+      height = height - height % 2
+      return (self.width, height)
+    else:
+      width = int(self.height * (float(width) / height))
+      width = width - width % 2
+      return (width, self.height)
+
+
+  def get_audio_info_cb(self, sink, buffer, pad):
+    caps = sink.sink_pads().next().get_negotiated_caps()
+    for s in caps:
+      self.info["channels"] = s['channels']
+      self.info["samplerate"] = s['rate']
+    self.audio.disconnect(self.audio_cb)
+
+  def get_frame_info_cb(self, sink, buffer, pad):
+    caps = sink.sink_pads().next().get_negotiated_caps()
+    for s in caps:
+      self.info["width"] = s['width']
+      self.info["height"] = s['height']
+      self.info["framerate"] = float(s['framerate'])
+      self.info["pixel-aspect-ratio"] = "%d:%d" % (s['pixel-aspect-ratio'].num, s['pixel-aspect-ratio'].denom)
+    self.video.disconnect(self.video_cb)
+
+  def demux_pad_added(self, element, pad, bool):
+    caps = pad.get_caps()
+    structure = caps[0]
+    stream_type = structure.get_name()
+    if stream_type.startswith('video'):
+      colorspace = gst.element_factory_make("ffmpegcolorspace");
+      self.pipeline.add (colorspace);
+      colorspace.set_state (gst.STATE_PLAYING);
+      pad.link (colorspace.get_pad("sink"));
+
+      self.video = gst.element_factory_make("fakesink")
+      self.video.props.signal_handoffs = True
+      self.pipeline.add(self.video)
+      self.video.set_state (gst.STATE_PLAYING);
+      colorspace.link (self.video, gst.caps_from_string('video/x-raw-rgb'));
+      self.video_cb = self.video.connect("handoff", self.get_frame_info_cb)
+    elif stream_type.startswith('audio'):
+      self.audio = gst.element_factory_make("fakesink")
+      self.audio.props.signal_handoffs = True
+      self.pipeline.add(self.audio)
+      self.audio.set_state (gst.STATE_PLAYING);
+      pad.link(self.audio.get_pad('sink'))
+      self.audio_cb = self.audio.connect("handoff", self.get_audio_info_cb)
+
+  def onBusMessage(self, bus, message):
+    if message.src == self.pipeline and message.type == gst.MESSAGE_EOS:
+      self.quit()
+
+  def quit(self):
+    if self.frame_img and self.frame_size:
+      img = self.frame_img.resize(self.frame_size, Image.ANTIALIAS)
+      img.save(self.frame_file)
+      self.debug('frame saved at %s' % self.frame_file)
+    self.pipeline.set_state(gst.STATE_NULL)
+    self.pipeline.get_state()
+    self.mainloop.quit()
+
+  def debug(self, msg):
+    if DEBUG:
+      print msg
+
+if __name__ == "__main__":
+  import sys
+  width = 128
+  height = 0
+  inputFile = sys.argv[1]
+  outputFile = sys.argv[2]
+  offset = int(float(sys.argv[3]) * gst.MSECOND)
+  if len(sys.argv) > 4:
+    width = int(sys.argv[4])
+  if len(sys.argv) > 5:
+    height = int(sys.argv[5])
+  f = ExtractFrame(inputFile, outputFile, offset, width, height)
+  f.run()
+