* use extract_frame.py from pad.ma
* update json request to only get md5sum from frontend * cronjob now imports all new files to db, extracts them and only after that send notification to frontend
This commit is contained in:
j
parent
33518260ee
commit
802a274aba
4 changed files with 224 additions and 56 deletions
|
|
@ -154,9 +154,9 @@ def extract_poster_still(movie_file, png_file, inpoint):
|
|||
print "this is not a movie file, will not try to extract frames"
|
||||
return
|
||||
inpoint = time_str2msec(inpoint)
|
||||
extractClipScript = abspath(join(dirname(__file__), "tools/extract_poster_still.py"))
|
||||
extractClipScript = abspath(join(dirname(__file__), "tools/extract_frame.py"))
|
||||
|
||||
cmd = '''%s "%s" "%s" %s''' % (extractClipScript, movie_file, png_file, inpoint)
|
||||
cmd = '''%s "%s" "%s" %s 0 128''' % (extractClipScript, movie_file, png_file, inpoint)
|
||||
os.system(cmd.encode('utf-8'))
|
||||
|
||||
def extract_subtitles(movie_file, srt, img_folder, width=128, offset = 0, redo = False):
|
||||
|
|
|
|||
|
|
@ -40,14 +40,16 @@ class Archive(SQLObject):
|
|||
name = UnicodeCol(length=255, alternateID=True)
|
||||
basePath = UnicodeCol()
|
||||
baseUrlFrontend = UnicodeCol(default = '')
|
||||
|
||||
published = DateTimeCol(defalut=datetime.now)
|
||||
modified = DateTimeCol(defalut=datetime.now)
|
||||
|
||||
def _get_basePath(self):
|
||||
basePath = self._SO_get_basePath()
|
||||
if not basePath.endswith('/'):
|
||||
basePath = basePath + "/"
|
||||
self.basePath = basePath
|
||||
return basePath
|
||||
|
||||
|
||||
def notifyFrontend(self, action, md5sum):
|
||||
if self.baseUrlFrontend:
|
||||
dto = socket.getdefaulttimeout()
|
||||
|
|
@ -141,9 +143,9 @@ class Archive(SQLObject):
|
|||
ret = "added entry"
|
||||
f.updateMeta()
|
||||
f.extractAll()
|
||||
self.notifyFrontend('add', f.md5sum)
|
||||
f.modified = datetime.now()
|
||||
return ret.encode('utf-8')
|
||||
|
||||
|
||||
def removeFile(self, md5sum):
|
||||
'''
|
||||
remove file based on md5sum from archive
|
||||
|
|
@ -158,35 +160,7 @@ class Archive(SQLObject):
|
|||
self.notifyFrontend('remove', md5sum)
|
||||
return dict(result="file removed")
|
||||
return dict(result="not in archive")
|
||||
|
||||
#FIXME this fails for old frontends because it notifies the frontend
|
||||
def importFrontend(self):
|
||||
if self.baseUrlFrontend:
|
||||
dto = socket.getdefaulttimeout()
|
||||
socket.setdefaulttimeout(256)
|
||||
url = "%s/%s" % (self.baseUrlFrontend, 'list')
|
||||
result = read_url(url)
|
||||
files = simplejson.loads(result)['files']
|
||||
socket.setdefaulttimeout(dto)
|
||||
for f in files:
|
||||
meta = files[f]
|
||||
fname = join(self.basePath, f)
|
||||
if exists(fname):
|
||||
stats = oxdb_import.oxdb_file_stats(fname)
|
||||
meta['date'] = stats['date']
|
||||
meta['path'] = f
|
||||
meta['video'] = ''
|
||||
meta['audio'] = ''
|
||||
meta['length'] = 0
|
||||
meta['bpp'] = 0
|
||||
for key in ('bpp', 'size', 'length', 'date'):
|
||||
meta[key] = int(float(meta[key]))
|
||||
meta['date'] = datetime.fromtimestamp(meta['date'])
|
||||
print self.addFile(meta), f
|
||||
else:
|
||||
print "remove", f
|
||||
self.removeFile(meta['md5sum'])
|
||||
|
||||
|
||||
def importFiles(self):
|
||||
stats = {'skipped': 0, 'added': 0, 'remove':0}
|
||||
print self.basePath
|
||||
|
|
@ -215,38 +189,38 @@ class Archive(SQLObject):
|
|||
self.removeFile(oxdb_files[f]['md5sum'])
|
||||
stats['remove'] += 1
|
||||
print stats
|
||||
print "updating information on frontend"
|
||||
self.updateFrontend()
|
||||
return stats
|
||||
|
||||
'''
|
||||
notify frontend about each file we have.
|
||||
Interaction with frontend,
|
||||
- update send modified files since last sync.
|
||||
- bootstrap send all files
|
||||
- sync get list from frontend and remove/add those that are not in sync
|
||||
'''
|
||||
def updateFrontned(self):
|
||||
for f in ArchiveFile.select(ArchiveFile.modified >= self.published):
|
||||
self.notifyFrontend('add', f.md5sum)
|
||||
self.published=datetime.now()
|
||||
|
||||
def bootstrapFrontend(self):
|
||||
for f in self.files:
|
||||
self.notifyFrontend('add', f.md5sum)
|
||||
|
||||
def cleanupFrontend(self):
|
||||
def syncFrontend(self):
|
||||
dto = socket.getdefaulttimeout()
|
||||
socket.setdefaulttimeout(256)
|
||||
data = read_url("%s/list" % self.baseUrlFrontend)
|
||||
files = simplejson.loads(data)['files']
|
||||
md5sums = simplejson.loads(data)['files']
|
||||
socket.setdefaulttimeout(dto)
|
||||
md5sums = [str(f['md5sum']) for f in files.values()]
|
||||
for md5sum in md5sums:
|
||||
try:
|
||||
f = ArchiveFile.byMd5sum(md5sum)
|
||||
except SQLObjectNotFound:
|
||||
self.notifyFrontend('remove', md5sum)
|
||||
|
||||
def fillFrontend(self):
|
||||
dto = socket.getdefaulttimeout()
|
||||
socket.setdefaulttimeout(256)
|
||||
data = read_url("%s/list" % self.baseUrlFrontend)
|
||||
files = simplejson.loads(data)['files']
|
||||
socket.setdefaulttimeout(dto)
|
||||
md5sums = [str(f['md5sum']) for f in files.values()]
|
||||
for f in ArchiveFile.select(NOT(IN(ArchiveFile.q.md5sum, md5sums))):
|
||||
self.notifyFrontend('add', f.md5sum)
|
||||
|
||||
|
||||
class ArchiveFile(SQLObject):
|
||||
'''
|
||||
|
|
@ -267,11 +241,11 @@ class ArchiveFile(SQLObject):
|
|||
size = IntCol()
|
||||
bpp = FloatCol(default = -1)
|
||||
pixels = IntCol(default = 0)
|
||||
|
||||
date_added = DateTimeCol(default = datetime.now)
|
||||
pubDate = DateTimeCol(default = datetime.now)
|
||||
modDate = DateTimeCol(default = datetime.now)
|
||||
|
||||
|
||||
date_added = DateTimeCol(default=datetime.now)
|
||||
published = DateTimeCol(defalut=datetime.now)
|
||||
modified = DateTimeCol(defalut=datetime.now)
|
||||
|
||||
height = IntCol(default = -1)
|
||||
width = IntCol(default = -1)
|
||||
frameAspect = UnicodeCol(default = "1.6", length = 100)
|
||||
|
|
|
|||
|
|
@ -46,9 +46,9 @@ def oxdb_filenameUmlaute(string):
|
|||
|
||||
def oxdb_director(director):
|
||||
director = os.path.basename(os.path.dirname(director))
|
||||
director.replace(' & ', ', ')
|
||||
director = director.replace('&', ', ').replace(' , ', ', ')
|
||||
return director
|
||||
|
||||
|
||||
def oxdb_title(title):
|
||||
'''
|
||||
normalize filename to get movie title
|
||||
|
|
@ -215,4 +215,4 @@ def oxdb_makedir(folder):
|
|||
os.makedirs(folder)
|
||||
except os.error, e:
|
||||
if e.errno != errno.EEXIST or not os.path.isdir(folder):
|
||||
raise
|
||||
raise
|
||||
|
|
|
|||
194
oxdbarchive/tools/extract_frame.py
Executable file
194
oxdbarchive/tools/extract_frame.py
Executable file
|
|
@ -0,0 +1,194 @@
|
|||
#!/usr/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
# vi:si:et:sw=2:sts=2:ts=2
|
||||
# GPL written 2008 by j@pad.ma
|
||||
import gobject
|
||||
gobject.threads_init()
|
||||
|
||||
import pygst
|
||||
pygst.require("0.10")
|
||||
import gst
|
||||
import Image
|
||||
|
||||
|
||||
DEBUG=0
|
||||
|
||||
class ExtractFrame:
|
||||
video = None
|
||||
audio = None
|
||||
info = {}
|
||||
frame_img = None
|
||||
frame_size = None
|
||||
duration = 0
|
||||
height = 0
|
||||
width = 128
|
||||
def __init__(self, videofile, frame, frame_pos, width=128, height=0):
|
||||
self.width = width
|
||||
self.height = height
|
||||
self.frame_file = frame
|
||||
self.frame_pos = frame_pos
|
||||
self.mainloop = gobject.MainLoop()
|
||||
self.pipeline = gst.parse_launch('filesrc name=input ! decodebin name=dbin')
|
||||
self.input = self.pipeline.get_by_name('input')
|
||||
self.input.props.location = videofile
|
||||
self.dbin = self.pipeline.get_by_name('dbin')
|
||||
|
||||
self.bus = self.pipeline.get_bus()
|
||||
self.dbin.connect('new-decoded-pad', self.demux_pad_added)
|
||||
|
||||
self.bus.add_signal_watch()
|
||||
self.watch_id = self.bus.connect("message", self.onBusMessage)
|
||||
|
||||
self.pipeline.set_state(gst.STATE_PAUSED)
|
||||
self.pipeline.get_state()
|
||||
|
||||
def run(self):
|
||||
self.pipeline.set_state(gst.STATE_PLAYING)
|
||||
self.pipeline.get_state()
|
||||
|
||||
#duration
|
||||
pads = None
|
||||
if self.video:
|
||||
pads = self.video.sink_pads()
|
||||
elif self.audio:
|
||||
pads = self.audio.sink_pads()
|
||||
if pads:
|
||||
q = gst.query_new_duration(gst.FORMAT_TIME)
|
||||
for pad in pads:
|
||||
if pad.get_peer() and pad.get_peer().query(q):
|
||||
format, self.duration = q.parse_duration()
|
||||
self.info["duration"] = self.duration/gst.MSECOND
|
||||
|
||||
#seek
|
||||
if self.frame_pos > self.duration:
|
||||
self.debug('seek point greater than file duration %s' % (self.duration/gst.MSECOND))
|
||||
return
|
||||
|
||||
if (self.duration - self.frame_pos) < gst.SECOND:
|
||||
seek_pos = self.duration- 10 * gst.SECOND
|
||||
else:
|
||||
seek_pos = self.frame_pos - 10 * gst.SECOND
|
||||
seek_pos = max(0, seek_pos)
|
||||
|
||||
#extract
|
||||
self.debug('seek tp %s'%seek_pos)
|
||||
self.seek(seek_pos)
|
||||
self.debug('get frame tp %s'%seek_pos)
|
||||
self.frame_ho = self.video.connect("handoff", self.get_frame_cb)
|
||||
self.pipeline.set_state(gst.STATE_PLAYING)
|
||||
self.pipeline.get_state()
|
||||
|
||||
|
||||
self.mainloop.run()
|
||||
return self.info
|
||||
|
||||
def seek(self, seek_pos):
|
||||
event = gst.event_new_seek(1.0, gst.FORMAT_TIME,
|
||||
gst.SEEK_FLAG_FLUSH | gst.SEEK_FLAG_ACCURATE,
|
||||
gst.SEEK_TYPE_SET, seek_pos,
|
||||
gst.SEEK_TYPE_NONE, 0)
|
||||
|
||||
res = self.video.send_event(event)
|
||||
if res:
|
||||
self.pipeline.set_new_stream_time(0L)
|
||||
else:
|
||||
gst.error("seek to %r failed" % frame_pos)
|
||||
|
||||
def get_frame_cb(self, sink, frame_buffer, pad):
|
||||
caps = sink.sink_pads().next().get_negotiated_caps()
|
||||
for s in caps:
|
||||
input_size = (s['width'], s['height'])
|
||||
self.frame_size = self.scaleto(s['width'], s['height'])
|
||||
#Why are the last frames broken, aka have green pixels
|
||||
save_last_frame = (4*gst.SECOND/float(s['framerate']))
|
||||
if (self.duration-self.frame_pos) < save_last_frame:
|
||||
self.frame_pos = self.duration-save_last_frame
|
||||
position, format = sink.query_position(gst.FORMAT_TIME)
|
||||
if not self.frame_img or position <= self.frame_pos:
|
||||
self.frame_img = Image.fromstring('RGB', input_size, frame_buffer)
|
||||
else:
|
||||
self.video.disconnect(self.frame_ho)
|
||||
self.bus.post(gst.message_new_eos(self.pipeline))
|
||||
|
||||
def scaleto(self, width, height):
|
||||
if self.width:
|
||||
height = int(self.width / (float(width) / height))
|
||||
height = height - height % 2
|
||||
return (self.width, height)
|
||||
else:
|
||||
width = int(self.height * (float(width) / height))
|
||||
width = width - width % 2
|
||||
return (width, self.height)
|
||||
|
||||
|
||||
def get_audio_info_cb(self, sink, buffer, pad):
|
||||
caps = sink.sink_pads().next().get_negotiated_caps()
|
||||
for s in caps:
|
||||
self.info["channels"] = s['channels']
|
||||
self.info["samplerate"] = s['rate']
|
||||
self.audio.disconnect(self.audio_cb)
|
||||
|
||||
def get_frame_info_cb(self, sink, buffer, pad):
|
||||
caps = sink.sink_pads().next().get_negotiated_caps()
|
||||
for s in caps:
|
||||
self.info["width"] = s['width']
|
||||
self.info["height"] = s['height']
|
||||
self.info["framerate"] = float(s['framerate'])
|
||||
self.info["pixel-aspect-ratio"] = "%d:%d" % (s['pixel-aspect-ratio'].num, s['pixel-aspect-ratio'].denom)
|
||||
self.video.disconnect(self.video_cb)
|
||||
|
||||
def demux_pad_added(self, element, pad, bool):
|
||||
caps = pad.get_caps()
|
||||
structure = caps[0]
|
||||
stream_type = structure.get_name()
|
||||
if stream_type.startswith('video'):
|
||||
colorspace = gst.element_factory_make("ffmpegcolorspace");
|
||||
self.pipeline.add (colorspace);
|
||||
colorspace.set_state (gst.STATE_PLAYING);
|
||||
pad.link (colorspace.get_pad("sink"));
|
||||
|
||||
self.video = gst.element_factory_make("fakesink")
|
||||
self.video.props.signal_handoffs = True
|
||||
self.pipeline.add(self.video)
|
||||
self.video.set_state (gst.STATE_PLAYING);
|
||||
colorspace.link (self.video, gst.caps_from_string('video/x-raw-rgb'));
|
||||
self.video_cb = self.video.connect("handoff", self.get_frame_info_cb)
|
||||
elif stream_type.startswith('audio'):
|
||||
self.audio = gst.element_factory_make("fakesink")
|
||||
self.audio.props.signal_handoffs = True
|
||||
self.pipeline.add(self.audio)
|
||||
self.audio.set_state (gst.STATE_PLAYING);
|
||||
pad.link(self.audio.get_pad('sink'))
|
||||
self.audio_cb = self.audio.connect("handoff", self.get_audio_info_cb)
|
||||
|
||||
def onBusMessage(self, bus, message):
|
||||
if message.src == self.pipeline and message.type == gst.MESSAGE_EOS:
|
||||
self.quit()
|
||||
|
||||
def quit(self):
|
||||
if self.frame_img and self.frame_size:
|
||||
img = self.frame_img.resize(self.frame_size, Image.ANTIALIAS)
|
||||
img.save(self.frame_file)
|
||||
self.debug('frame saved at %s' % self.frame_file)
|
||||
self.pipeline.set_state(gst.STATE_NULL)
|
||||
self.pipeline.get_state()
|
||||
self.mainloop.quit()
|
||||
|
||||
def debug(self, msg):
|
||||
if DEBUG:
|
||||
print msg
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
width = 128
|
||||
height = 0
|
||||
inputFile = sys.argv[1]
|
||||
outputFile = sys.argv[2]
|
||||
offset = int(float(sys.argv[3]) * gst.MSECOND)
|
||||
if len(sys.argv) > 4:
|
||||
width = int(sys.argv[4])
|
||||
if len(sys.argv) > 5:
|
||||
height = int(sys.argv[5])
|
||||
f = ExtractFrame(inputFile, outputFile, offset, width, height)
|
||||
f.run()
|
||||
|
||||
Loading…
Reference in a new issue