oxdb archive, backend 219

oxdbarchive/model.py

@@ -0,0 +1,499 @@
+# -*- coding: utf-8 -*-
+# -*- Mode: Python; -*-
+# vi:si:et:sw=2:sts=2:ts=2
+from sqlobject import *
+from turbogears.database import PackageHub
+import turbogears
+import re
+from urllib import quote, quote_plus
+import os
+from os.path import abspath, join, dirname
+from datetime import datetime
+import time
+import math
+from glob import glob
+import shutil
+
+
+import oxdb_cache
+import cache
+import oxdb_import
+from oxdb_utils import oxdb_title, oxdb_director, oxdb_id
+from subtitles import *
+import midentify
+
+
+hub = PackageHub('oxdbarchive')
+__connection__ = hub
+
+class Archive(SQLObject):
+  name = UnicodeCol(length=255, alternateID=True)
+  basePath = UnicodeCol()
+  
+  def _get_basePath(self):
+    basePath = self._SO_get_basePath()
+    if not basePath.endswith('/'):
+      basePath = basePath + "/"
+      self.basePath = basePath
+    return basePath
+  
+  def _get_files(self):
+    q = ArchiveFile.select(ArchiveFile.q.archiveID == self.id)
+    return [f for f in q]
+  
+  def _get_file_list(self):
+    files = {}
+    for f in self.files:
+      try:
+        d = dict(md5sum = f.md5sum, size = f.size)
+        files[f.path] = d
+      except SQLObjectNotFound:
+        f.destroySelf()
+    return files
+  
+  def addLocalFile(self, fname, movie = None): 
+    params = oxdb_import.oxdb_file_stats(fname)
+    params = oxdb_import.oxdb_file_metadata(params)
+    params['date'] = datetime.fromtimestamp(params['date'])
+    return self.addFile(params, movie)
+  
+  def addFile(self, params, movie = None):
+    '''
+      updates or adds new file to database, 
+      params is a dict with at least md5sum, path, date but also needs
+      audio, video, length, size, bpp for new files
+    '''
+    params['path'] = params['path'].replace(self.basePath, '')
+    q = ArchiveFile.select(AND(
+        ArchiveFile.q.archiveID == self.id,
+        ArchiveFile.q.md5sum == params['md5sum'],
+      ))
+    if q.count() > 0:
+      '''update existing entry'''
+      f = q[0]
+      #FIXME: deal with double files here. right now they are changed
+      if f.path != params['path']:
+        ret =  "this file is already in the database, first time at:\n\t%s\n\t" % f.path
+      else:
+        ret = "updated entry"
+      for field in ('path', 'date'):
+        setattr(f, field, params[field])
+    else:
+      #just a new md5? happens for srt files quite often
+      qq = ArchiveFile.select(AND(
+        ArchiveFile.q.archiveID == self.id,
+        ArchiveFile.q.path == params['path'],
+      ))
+      f = None
+      if qq.count() == 1:
+        f = qq[0]
+        ret = "updated entry"
+      else:
+        ''' add new file to database '''
+        title = oxdb_title(params['path'])
+        director = oxdb_director(params['path'])
+        oxdb = oxdb_id(title, director)
+        f = ArchiveFile(
+          archive = self,
+          path = params['path'], 
+          date = params['date'],
+          oxdb = oxdb,
+          md5sum = params['md5sum'],
+          audio = params['audio'],
+          video = params['video'],
+          length = params['length'],
+          size = params['size'],
+          bpp = params['bpp'],
+          date_added = datetime.now(),
+          subtitle = params['path'].endswith('.srt'),
+        )
+        ret = "added entry"
+    f.updateMeta()
+    return ret
+  
+  def removeFile(self, md5sum):
+    '''
+      remove file based on md5sum from archive
+    '''
+    q = ArchiveFile.select(AND(
+        ArchiveFile.q.archiveID == self.id,
+        ArchiveFile.q.md5sum == md5sum,
+      ))
+    if q.count() == 1:
+      for i in q: 
+        ArchiveFile.delete(i.id)
+      return dict(result="file removed")
+    return dict(result="not in archive")
+  
+  def importFiles(self):
+    stats = {'skipped': 0, 'added': 0, 'remove':0}
+    print self.basePath
+    files = oxdb_import.oxdb_spider(self.basePath)
+  
+    oxdb_files = self.file_list()
+    md5sum_on_disk = []
+    for f in files:
+      meta = oxdb_import.oxdb_file_stats(f)
+      f = f.replace(base, '')
+      if oxdb_files.has_key(f) and oxdb_files[f]['size'] == meta['size']:
+        stats['skipped'] += 1
+        md5sum_on_disk.append(oxdb_files[f]['md5sum'])
+      else:
+        meta = oxdb_import.oxdb_file_metadata(meta)
+        #remove base
+        meta['path'] = f.encode('utf-8')
+        #ignore files changed in the last 5 minutes
+        print self.addFile(meta), f
+        stats['added'] += 1
+        md5sum_on_disk.append(meta['md5sum'])
+    for f in oxdb_files:
+      if oxdb_files[f]['md5sum'] not in md5sum_on_disk:
+        print "remove", f
+        self.removeFile({'md5sum':oxdb_files[f]['md5sum']})
+        stats['remove'] += 1
+    print stats
+    return stats
+
+
+class ArchiveFile(SQLObject):
+  '''
+  ALTER TABLE file_meta CHANGE size size bigint;
+  ALTER TABLE file_meta CHANGE pixels pixels bigint;
+  ALTER TABLE file_meta CHANGE srt srt LONGTEXT;
+  '''
+  md5sum = UnicodeCol(length=128, alternateID=True)
+  oxdb = UnicodeCol(length=128)
+  path = UnicodeCol()
+  date = DateTimeCol()
+  
+  archive = ForeignKey('Archive')
+  
+  audio = UnicodeCol()
+  video = UnicodeCol()
+  length = IntCol()
+  size = IntCol()
+  bpp = IntCol(default = 0)
+  pixels = IntCol(default = 0)
+  
+  date_added = DateTimeCol(default = datetime.now)
+  pubDate = DateTimeCol(default = datetime.now)
+  modDate = DateTimeCol(default = datetime.now)
+  
+  height = IntCol(default = -1)
+  width = IntCol(default = -1)
+  frameAspect = UnicodeCol(default = "1.6", length = 100)
+  
+  bitrate = IntCol(default = -1)
+  fps = IntCol(default = -1)
+  srt = UnicodeCol(default = '')
+  subtitle_meta_id = IntCol(default = -1)
+  subtitle = BoolCol(default = False)
+  part = IntCol(default = 1)
+  
+  broken = BoolCol(default = False)
+  
+  extracted = BoolCol(default = False)
+  
+  filename = UnicodeCol(default = '')
+  
+  def _get_part(self):
+    part = 1
+    parts = re.compile('Part (\d)').findall(self.path)
+    if not parts:
+      parts = re.compile('CD (\d)').findall(self.path)
+    if parts:
+      part = int(parts[-1])
+    self.part = part
+    return part
+  
+  def _get_offset(self):
+    if not self.part:
+      self.part = 1
+    if self.part == 1:
+      return 0
+    length = 0
+    q = ArchiveFile.select(AND(
+      ArchiveFile.q.oxdb == self.oxdb,
+      ArchiveFile.q.part < self.part,
+      ArchiveFile.q.subtitle == False,
+      ))
+    for f in q:
+      length += f.length
+    return length
+  
+  def _get_ext(self):
+    return self.path.split('.')[-1]
+  
+  def _get_preferredVersion(self):
+    e = self.nameExtra.lower()
+    for pref in ('directors cut', 'long version'):
+      if pref in e:
+        return True
+    return False
+  
+  def _get_nameExtra(self):
+    path = os.path.basename(self.path)
+    parts = path.replace(self.title, '').split('.')[:-1]
+    parts = filter(lambda x: not x.startswith('Part'), parts)
+    parts = filter(lambda x: not x.startswith('Season'), parts)
+    parts = filter(lambda x: not x.startswith('Episode'), parts)
+    parts = filter(lambda x: not x.startswith('vo'), parts)
+    extra = '.'.join(parts)
+    if extra.startswith('.'):
+      extra = extra[1:]
+    return extra
+  
+  def _get_title(self):
+    return oxdb_title(self.path)
+  
+  def _get_director(self):
+    return oxdb_director(self.path)
+  
+  def _get_absolutePath(self):
+    return join(self.archive.basePath, self.path)
+  
+  def updateMeta(self):
+    self.findSubtitleLink()
+    if os.path.exists(self.absolutePath):
+      info = midentify.identify(self.absolutePath)
+      self.length = info['length']
+      self.width = info['width']
+      self.frameAspect = "%0.6f" % info['aspect']
+      self.height = info['height']      
+      self.bitrate = info['video_bitrate']
+      self.fps = info['fps']
+      self.audio = info['audio_codec']
+      self.video = info['video_codec']
+      self.updatePixels()
+      self.updateBpp()
+      self.loadSubtitleFromFile()
+      self.oxdb = oxdb_id(self.title, self.director)
+  
+  def _get_frameAspect(self):
+    aspect =  float(self._SO_get_frameAspect())
+    if self.subtitle:
+      return 1
+    if aspect == -1:
+      if self.height:
+        aspect = float(self.width) / self.height
+      else:
+        aspect = 16.0 / 10
+      self.frameAspect = "%0.6f" % aspect
+    return aspect
+  
+  def _get_sceneHeight(self):
+    default = 80
+    if not self.subtitle:
+      h = int(128 / self.frameAspect)
+      h = h + h % 2
+      return h
+    return default
+
+  def _get_movieFile(self):
+    if self.subtitle and self.subtitle_meta_id>0:
+      try:
+        m = ArchiveFile.get(self.subtitle_meta_id)
+      except:
+        m = None
+        self.subtitle_meta_id = -1
+        self.srt = ''
+      return m
+    return None
+  
+  def _get_subtitleFile(self):
+    if not self.subtitle and self.subtitle_meta_id>0:
+      try:
+        s = ArchiveFile.get(self.subtitle_meta_id)
+      except:
+        s = None
+        self.subtitle_meta_id = -1
+        self.srt = ''
+      return s
+    return None
+  
+  def findSubtitleLink(self):
+    subtitle = not self.subtitle
+    q = ArchiveFile.select(AND(
+      ArchiveFile.q.oxdb == self.oxdb,
+      ArchiveFile.q.part == self.part,
+      ArchiveFile.q.subtitle == subtitle,
+      ))
+    self.subtitle_meta_id = -1
+    if q.count():
+      for f in q:
+        if not f.path.endswith('.sub'):
+          if f.nameExtra == self.nameExtra or f.nameExtra == 'en':
+            self.subtitle_meta_id = f.id
+  
+  
+  def _get_mini_movie_file(self):
+    return join(oxdb_cache.mini_movie_folder, self.md5sum[:4], "%s.avi" % self.md5sum)
+  
+  def removeMiniMovie(self): 
+    if os.path.exists(self.mini_movie_file):
+      os.remove(self.mini_movie_file)
+  
+  def _findSubtitleByStart(self, start):
+    if self.srt:
+      d = srt2dict(self.srt)
+      for s in d.values():
+        if s['start'] == start:
+          return s
+    return None
+    
+  def extractAll(self, force = False):
+    self.updateMeta()
+    self.extractClipMovie()
+    self.extractTimeline()
+  
+  def extractClip(self, inpoint, outpoint=-1, flash_folder=oxdb_cache.frame_cache_root): 
+    movie_file = self.mini_movie_file
+    flash_folder = join(flash_folder, self.oxdb)
+    flash_movie = join(flash_folder, "%s.flv" % inpoint.replace(':', '.'))
+    if not os.path.exists(flash_folder):
+      os.makedirs(flash_folder)
+    width = 128
+    height = int(width / (self.width / self.height))
+    height = height - height % 2
+    if outpoint == -1:
+      s = self._findSubtitleByStart(inpoint)
+      if s:
+        outpoint = s['stop']
+      else:
+        outpoint = shift_time(2000, inpoint)
+    if self.part > 1:
+      offset = self.offset
+    extract_flash(movie_file, flash_movie, inpoint, outpoint, width, height, offset = 0)
+    #extract_flash_ng(self.absolutePath, flash_movie, inpoint, outpoint, width, height, offset)
+  
+  def extractFrame(self, position, img_folder=oxdb_cache.frame_cache_root): 
+    if self.movieFile:
+      return self.movieFile.extractFrame(position, img_folder)
+    movie_file = self.mini_movie_file
+    img_folder = join(img_folder, self.oxdb)
+    if not os.path.exists(img_folder):
+      os.makedirs(img_folder)
+    extract_frame(movie_file, position, img_folder, offset = 0, redo = False)
+  
+  def extractFrames(self, img_folder=oxdb_cache.frame_cache_root): 
+    if self.movieFile:
+      return self.movieFile.extractFrames(img_folder)
+    movie_file = self.absolutePath
+    img_folder = join(img_folder, self.oxdb)
+    if not os.path.exists(img_folder):
+      os.makedirs(img_folder)    
+    extract_subtitles(movie_file, self.srt.encode('utf-8'), img_folder, width=100, offset=self.offset)
+  
+  def extractClipMovie(self, force = False): 
+    if self.broken:
+      return
+    mini_movie_file = self.mini_movie_file
+    movie_file = self.absolutePath
+    if not movie_file or not os.path.exists(movie_file):
+      return
+    if os.path.exists(mini_movie_file):
+      print "clip exists, skipping extraction", mini_movie_file
+      return
+    if not os.path.exists(dirname(mini_movie_file)):
+      os.makedirs(dirname(mini_movie_file))
+    options = ''
+    options += " -ovc lavc -lavcopts vcodec=mjpeg"
+    options += " -af volnorm=1 -oac mp3lame -lameopts br=64:mode=3 -af resample=44100"
+    options += " -vf scale -zoom -xy 128"
+    options += ' "%s"' % movie_file
+    options += ' -o "%s"' % mini_movie_file
+    cmd = "mencoder %s >/dev/null 2>&1" % options
+    print cmd.encode('utf-8')
+    os.system(cmd.encode('utf-8'))
+
+  def _get_timelineFile(self):
+    return join(oxdb_cache.cache_root, 'timeline', self.md5sum[:4], "%s.png" % self.md5sum)
+    
+  def removeTimeline(self):
+    if os.path.exists(self.timelineFile):
+      os.unlink(self.timelineFile)
+      
+  def extractTimeline(self, force = False):
+    if self.broken:
+      return
+    if force:
+      self.removeTimeline()
+
+    #return if its not a video
+    if self.height <= 0:
+      return
+    
+    t = self.timelineFile
+    if os.path.exists(self.mini_movie_file):
+      if not os.path.exists(t):
+        os.makedirs(os.path.dirname(t))
+    #lets only extract the timeline if it does not exist yet
+    if os.path.exists(t):
+      print "skipping, ", self.path
+      return
+    extractTimelineScript = abspath(join(dirname(__file__), "tools/extract_timeline.py"))
+    cmd = "python %s %s %s" %(extractTimelineScript, t, self.mini_movie_file)
+    os.system(cmd)
+  
+  def loadSubtitleFromFile(self): 
+    if self.movieFile:
+      movieFile = self.movieFile
+      subtitle = self
+    else:
+      movieFile = self
+      subtitle = self.subtitleFile
+    if movieFile:
+      movieFile.srt = ''
+    if subtitle and movieFile:
+      if not subtitle.absolutePath or not os.path.exists(subtitle.absolutePath):
+        return
+      if not subtitle.absolutePath.endswith('.srt'):
+        print "this is not a subtitle", subtitle.absolutePath
+        return
+      movieFile.srt  = loadSrt(subtitle.absolutePath)
+  
+  def _set_fps(self, fps):
+    fps = int(fps * 10000)
+    self._SO_set_fps(fps)
+  
+  def _get_fps(self):
+    fps = self._SO_get_fps()
+    if fps:
+      return float(fps) / 10000
+    return 0.0
+  
+  def _get_resolution(self):
+    if self.subtitle or (not self.width or not self.height):
+      return u''
+    return "%sx%s" % (self.width, self.height)
+  
+  def updateBpp(self):
+    if self.height and self.width and self.fps and self.bitrate:
+      bpp = int(self.height * self.width * self.fps / self.bitrate)
+      self.bpp = bpp
+    else:
+      bpp = 0
+    return bpp
+  
+  def updatePixels(self):
+    if self.length and self.fps and self.width and self.height:
+      pixels = int((self.length / 1000) * self.fps * self.width * self.height)
+      self.pixels = pixels
+    else:
+      pixels = 0
+    return pixels
+  
+  def _get_pixels(self):
+    pixels = self._SO_get_pixels()
+    if not pixels:
+      pixels = self.updatePixels()
+    return pixels
+  
+  def clip(self, position):
+    return cache.loadClip(self, position)
+  
+  def frame(self, position):
+    return cache.loadFrame(self, position)
+  
+  def timeline(self):
+    return cache.loadTimeline(self)