diff --git a/oxdbarchive/extract.py b/oxdbarchive/extract.py index 23a2a2c..1c524b1 100644 --- a/oxdbarchive/extract.py +++ b/oxdbarchive/extract.py @@ -154,9 +154,9 @@ def extract_poster_still(movie_file, png_file, inpoint): print "this is not a movie file, will not try to extract frames" return inpoint = time_str2msec(inpoint) - extractClipScript = abspath(join(dirname(__file__), "tools/extract_poster_still.py")) + extractClipScript = abspath(join(dirname(__file__), "tools/extract_frame.py")) - cmd = '''%s "%s" "%s" %s''' % (extractClipScript, movie_file, png_file, inpoint) + cmd = '''%s "%s" "%s" %s 0 128''' % (extractClipScript, movie_file, png_file, inpoint) os.system(cmd.encode('utf-8')) def extract_subtitles(movie_file, srt, img_folder, width=128, offset = 0, redo = False): diff --git a/oxdbarchive/model.py b/oxdbarchive/model.py index 3f2911c..607f34d 100644 --- a/oxdbarchive/model.py +++ b/oxdbarchive/model.py @@ -40,14 +40,16 @@ class Archive(SQLObject): name = UnicodeCol(length=255, alternateID=True) basePath = UnicodeCol() baseUrlFrontend = UnicodeCol(default = '') - + published = DateTimeCol(defalut=datetime.now) + modified = DateTimeCol(defalut=datetime.now) + def _get_basePath(self): basePath = self._SO_get_basePath() if not basePath.endswith('/'): basePath = basePath + "/" self.basePath = basePath return basePath - + def notifyFrontend(self, action, md5sum): if self.baseUrlFrontend: dto = socket.getdefaulttimeout() @@ -141,9 +143,9 @@ class Archive(SQLObject): ret = "added entry" f.updateMeta() f.extractAll() - self.notifyFrontend('add', f.md5sum) + f.modified = datetime.now() return ret.encode('utf-8') - + def removeFile(self, md5sum): ''' remove file based on md5sum from archive @@ -158,35 +160,7 @@ class Archive(SQLObject): self.notifyFrontend('remove', md5sum) return dict(result="file removed") return dict(result="not in archive") - - #FIXME this fails for old frontends because it notifies the frontend - def importFrontend(self): - if self.baseUrlFrontend: - dto = socket.getdefaulttimeout() - socket.setdefaulttimeout(256) - url = "%s/%s" % (self.baseUrlFrontend, 'list') - result = read_url(url) - files = simplejson.loads(result)['files'] - socket.setdefaulttimeout(dto) - for f in files: - meta = files[f] - fname = join(self.basePath, f) - if exists(fname): - stats = oxdb_import.oxdb_file_stats(fname) - meta['date'] = stats['date'] - meta['path'] = f - meta['video'] = '' - meta['audio'] = '' - meta['length'] = 0 - meta['bpp'] = 0 - for key in ('bpp', 'size', 'length', 'date'): - meta[key] = int(float(meta[key])) - meta['date'] = datetime.fromtimestamp(meta['date']) - print self.addFile(meta), f - else: - print "remove", f - self.removeFile(meta['md5sum']) - + def importFiles(self): stats = {'skipped': 0, 'added': 0, 'remove':0} print self.basePath @@ -215,38 +189,38 @@ class Archive(SQLObject): self.removeFile(oxdb_files[f]['md5sum']) stats['remove'] += 1 print stats + print "updating information on frontend" + self.updateFrontend() return stats ''' - notify frontend about each file we have. + Interaction with frontend, + - update send modified files since last sync. + - bootstrap send all files + - sync get list from frontend and remove/add those that are not in sync ''' + def updateFrontned(self): + for f in ArchiveFile.select(ArchiveFile.modified >= self.published): + self.notifyFrontend('add', f.md5sum) + self.published=datetime.now() + def bootstrapFrontend(self): for f in self.files: self.notifyFrontend('add', f.md5sum) - def cleanupFrontend(self): + def syncFrontend(self): dto = socket.getdefaulttimeout() socket.setdefaulttimeout(256) data = read_url("%s/list" % self.baseUrlFrontend) - files = simplejson.loads(data)['files'] + md5sums = simplejson.loads(data)['files'] socket.setdefaulttimeout(dto) - md5sums = [str(f['md5sum']) for f in files.values()] for md5sum in md5sums: try: f = ArchiveFile.byMd5sum(md5sum) except SQLObjectNotFound: self.notifyFrontend('remove', md5sum) - - def fillFrontend(self): - dto = socket.getdefaulttimeout() - socket.setdefaulttimeout(256) - data = read_url("%s/list" % self.baseUrlFrontend) - files = simplejson.loads(data)['files'] - socket.setdefaulttimeout(dto) - md5sums = [str(f['md5sum']) for f in files.values()] for f in ArchiveFile.select(NOT(IN(ArchiveFile.q.md5sum, md5sums))): self.notifyFrontend('add', f.md5sum) - class ArchiveFile(SQLObject): ''' @@ -267,11 +241,11 @@ class ArchiveFile(SQLObject): size = IntCol() bpp = FloatCol(default = -1) pixels = IntCol(default = 0) - - date_added = DateTimeCol(default = datetime.now) - pubDate = DateTimeCol(default = datetime.now) - modDate = DateTimeCol(default = datetime.now) - + + date_added = DateTimeCol(default=datetime.now) + published = DateTimeCol(defalut=datetime.now) + modified = DateTimeCol(defalut=datetime.now) + height = IntCol(default = -1) width = IntCol(default = -1) frameAspect = UnicodeCol(default = "1.6", length = 100) diff --git a/oxdbarchive/oxdb_utils.py b/oxdbarchive/oxdb_utils.py index adcdb60..a5720a6 100644 --- a/oxdbarchive/oxdb_utils.py +++ b/oxdbarchive/oxdb_utils.py @@ -46,9 +46,9 @@ def oxdb_filenameUmlaute(string): def oxdb_director(director): director = os.path.basename(os.path.dirname(director)) - director.replace(' & ', ', ') + director = director.replace('&', ', ').replace(' , ', ', ') return director - + def oxdb_title(title): ''' normalize filename to get movie title @@ -215,4 +215,4 @@ def oxdb_makedir(folder): os.makedirs(folder) except os.error, e: if e.errno != errno.EEXIST or not os.path.isdir(folder): - raise \ No newline at end of file + raise diff --git a/oxdbarchive/tools/extract_frame.py b/oxdbarchive/tools/extract_frame.py new file mode 100755 index 0000000..77b23f4 --- /dev/null +++ b/oxdbarchive/tools/extract_frame.py @@ -0,0 +1,194 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# vi:si:et:sw=2:sts=2:ts=2 +# GPL written 2008 by j@pad.ma +import gobject +gobject.threads_init() + +import pygst +pygst.require("0.10") +import gst +import Image + + +DEBUG=0 + +class ExtractFrame: + video = None + audio = None + info = {} + frame_img = None + frame_size = None + duration = 0 + height = 0 + width = 128 + def __init__(self, videofile, frame, frame_pos, width=128, height=0): + self.width = width + self.height = height + self.frame_file = frame + self.frame_pos = frame_pos + self.mainloop = gobject.MainLoop() + self.pipeline = gst.parse_launch('filesrc name=input ! decodebin name=dbin') + self.input = self.pipeline.get_by_name('input') + self.input.props.location = videofile + self.dbin = self.pipeline.get_by_name('dbin') + + self.bus = self.pipeline.get_bus() + self.dbin.connect('new-decoded-pad', self.demux_pad_added) + + self.bus.add_signal_watch() + self.watch_id = self.bus.connect("message", self.onBusMessage) + + self.pipeline.set_state(gst.STATE_PAUSED) + self.pipeline.get_state() + + def run(self): + self.pipeline.set_state(gst.STATE_PLAYING) + self.pipeline.get_state() + + #duration + pads = None + if self.video: + pads = self.video.sink_pads() + elif self.audio: + pads = self.audio.sink_pads() + if pads: + q = gst.query_new_duration(gst.FORMAT_TIME) + for pad in pads: + if pad.get_peer() and pad.get_peer().query(q): + format, self.duration = q.parse_duration() + self.info["duration"] = self.duration/gst.MSECOND + + #seek + if self.frame_pos > self.duration: + self.debug('seek point greater than file duration %s' % (self.duration/gst.MSECOND)) + return + + if (self.duration - self.frame_pos) < gst.SECOND: + seek_pos = self.duration- 10 * gst.SECOND + else: + seek_pos = self.frame_pos - 10 * gst.SECOND + seek_pos = max(0, seek_pos) + + #extract + self.debug('seek tp %s'%seek_pos) + self.seek(seek_pos) + self.debug('get frame tp %s'%seek_pos) + self.frame_ho = self.video.connect("handoff", self.get_frame_cb) + self.pipeline.set_state(gst.STATE_PLAYING) + self.pipeline.get_state() + + + self.mainloop.run() + return self.info + + def seek(self, seek_pos): + event = gst.event_new_seek(1.0, gst.FORMAT_TIME, + gst.SEEK_FLAG_FLUSH | gst.SEEK_FLAG_ACCURATE, + gst.SEEK_TYPE_SET, seek_pos, + gst.SEEK_TYPE_NONE, 0) + + res = self.video.send_event(event) + if res: + self.pipeline.set_new_stream_time(0L) + else: + gst.error("seek to %r failed" % frame_pos) + + def get_frame_cb(self, sink, frame_buffer, pad): + caps = sink.sink_pads().next().get_negotiated_caps() + for s in caps: + input_size = (s['width'], s['height']) + self.frame_size = self.scaleto(s['width'], s['height']) + #Why are the last frames broken, aka have green pixels + save_last_frame = (4*gst.SECOND/float(s['framerate'])) + if (self.duration-self.frame_pos) < save_last_frame: + self.frame_pos = self.duration-save_last_frame + position, format = sink.query_position(gst.FORMAT_TIME) + if not self.frame_img or position <= self.frame_pos: + self.frame_img = Image.fromstring('RGB', input_size, frame_buffer) + else: + self.video.disconnect(self.frame_ho) + self.bus.post(gst.message_new_eos(self.pipeline)) + + def scaleto(self, width, height): + if self.width: + height = int(self.width / (float(width) / height)) + height = height - height % 2 + return (self.width, height) + else: + width = int(self.height * (float(width) / height)) + width = width - width % 2 + return (width, self.height) + + + def get_audio_info_cb(self, sink, buffer, pad): + caps = sink.sink_pads().next().get_negotiated_caps() + for s in caps: + self.info["channels"] = s['channels'] + self.info["samplerate"] = s['rate'] + self.audio.disconnect(self.audio_cb) + + def get_frame_info_cb(self, sink, buffer, pad): + caps = sink.sink_pads().next().get_negotiated_caps() + for s in caps: + self.info["width"] = s['width'] + self.info["height"] = s['height'] + self.info["framerate"] = float(s['framerate']) + self.info["pixel-aspect-ratio"] = "%d:%d" % (s['pixel-aspect-ratio'].num, s['pixel-aspect-ratio'].denom) + self.video.disconnect(self.video_cb) + + def demux_pad_added(self, element, pad, bool): + caps = pad.get_caps() + structure = caps[0] + stream_type = structure.get_name() + if stream_type.startswith('video'): + colorspace = gst.element_factory_make("ffmpegcolorspace"); + self.pipeline.add (colorspace); + colorspace.set_state (gst.STATE_PLAYING); + pad.link (colorspace.get_pad("sink")); + + self.video = gst.element_factory_make("fakesink") + self.video.props.signal_handoffs = True + self.pipeline.add(self.video) + self.video.set_state (gst.STATE_PLAYING); + colorspace.link (self.video, gst.caps_from_string('video/x-raw-rgb')); + self.video_cb = self.video.connect("handoff", self.get_frame_info_cb) + elif stream_type.startswith('audio'): + self.audio = gst.element_factory_make("fakesink") + self.audio.props.signal_handoffs = True + self.pipeline.add(self.audio) + self.audio.set_state (gst.STATE_PLAYING); + pad.link(self.audio.get_pad('sink')) + self.audio_cb = self.audio.connect("handoff", self.get_audio_info_cb) + + def onBusMessage(self, bus, message): + if message.src == self.pipeline and message.type == gst.MESSAGE_EOS: + self.quit() + + def quit(self): + if self.frame_img and self.frame_size: + img = self.frame_img.resize(self.frame_size, Image.ANTIALIAS) + img.save(self.frame_file) + self.debug('frame saved at %s' % self.frame_file) + self.pipeline.set_state(gst.STATE_NULL) + self.pipeline.get_state() + self.mainloop.quit() + + def debug(self, msg): + if DEBUG: + print msg + +if __name__ == "__main__": + import sys + width = 128 + height = 0 + inputFile = sys.argv[1] + outputFile = sys.argv[2] + offset = int(float(sys.argv[3]) * gst.MSECOND) + if len(sys.argv) > 4: + width = int(sys.argv[4]) + if len(sys.argv) > 5: + height = int(sys.argv[5]) + f = ExtractFrame(inputFile, outputFile, offset, width, height) + f.run() +