foundations to directly use upoaded videos if format is usable

pandora/archive/external.py

@@ -97,6 +97,17 @@ def download(item_id, url):
         tmp = tmp.decode('utf-8')
     os.chdir(tmp)
     cmd = ['youtube-dl', '-q', media['url']]
+    if settings.CONFIG['video'].get('reuseUload', False):
+        max_resolution = max(settings.CONFIG['video']['resolutions'])
+        format = settings.CONFIG['video']['formats'][0]
+        if format == 'mp4':
+            cmd += [
+                '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/bestvideo+bestaudio',
+                '--merge-output-format', 'mp4'
+            ]
+        elif format == 'webm':
+            cmd += ['--merge-output-format', 'webm']
+
     p = subprocess.Popen(cmd,
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE, close_fds=True)

pandora/archive/extract.py

@@ -698,3 +698,99 @@ def chop(video, start, end, subtitles=None, dest=None, encode=False):
         return f
     else:
         return None
+
+def has_faststart(path):
+    cmd = [settings.FFPROBE, '-v', 'trace', '-i', path]
+    p = subprocess.Popen(cmd, stdin=subprocess.PIPE,
+                         stdout=subprocess.PIPE,
+                         stderr=subprocess.STDOUT,
+                         close_fds=True)
+    stdout, stderr = p.communicate()
+    moov = "type:'moov'"
+    mdat = "type:'mdat'"
+    blocks = [b for b in stdout.decode().split('\n') if moov in b or mdat in b]
+    if blocks and moov in blocks[0]:
+        return True
+    return False
+
+def remux_stream(src, dst):
+    info = ox.avinfo(src)
+    if info.get('audio'):
+        audio = ['-c:a', 'copy']
+    else:
+        audio = []
+    if info.get('video'):
+        video = ['-c:v', 'copy']
+    else:
+        video = []
+    cmd = [
+        settings.FFMPEG,
+        '-nostats', '-loglevel', 'error',
+        '-map_metadata', '-1', '-sn',
+        '-i', src,
+    ] + video + [
+    ] + audio + [
+        '-movflags', '+faststart',
+        dst
+    ]
+    p = subprocess.Popen(cmd, stdin=subprocess.PIPE,
+                         stdout=open('/dev/null', 'w'),
+                         stderr=open('/dev/null', 'w'),
+                         close_fds=True)
+    p.wait()
+    return True, None
+
+
+def ffprobe(path, *args):
+    cmd = [settings.FFPROBE, '-loglevel', 'error', '-print_format', 'json', '-i', path] + list(args)
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
+    stdout, stderr = p.communicate()
+    return json.loads(stdout.decode())
+
+
+def get_chapters(path):
+    info = ffprobe(path, '-show_chapters')
+    chapters = []
+    n = 0
+    for chapter in info.get('chapters', []):
+        n += 1
+        chapters.append({
+            'in': chapter['start_time'],
+            'out': chapter['end_time'],
+            'value': chapter.get('tags', {}).get('title', 'Chapter %s' % n)
+        })
+    return chapters
+
+def get_text_subtitles(path):
+    subtitles = []
+    for stream in ffprobe(path, '-show_streams')['streams']:
+        if stream.get('codec_name') in ('subrip', 'aas', 'text'):
+            subtitles.append({
+                'index': stream['index'],
+                'language': stream['tags']['language'],
+            })
+    return subtitles
+
+def has_img_subtitles(path):
+    subtitles = []
+    for stream in ffprobe(path, '-show_streams')['streams']:
+        if stream.get('codec_type') == 'subtitle' and stream.get('codec_name') in ('dvbsub', 'pgssub'):
+            subtitles.append({
+                'index': stream['index'],
+                'language': stream['tags']['language'],
+            })
+    return subtitles
+
+def extract_subtitles(path, language=None):
+    extra = []
+    if language:
+        tracks = get_text_subtitles(path)
+        track = [t for t in tracks if t['language'] == language]
+        if track:
+            extra = ['-map', '0:%s' % track[0]['index']]
+        else:
+            raise Exception("unknown language: %s" % language)
+    cmd = ['ffmpeg', '-loglevel', 'error', '-i', path] + extra + ['-f', 'srt', '-']
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
+    stdout, stderr = p.communicate()
+    return ox.srt.loads(stdout.decode())

pandora/archive/models.py

@@ -376,6 +376,50 @@ class File(models.Model):
             return save_chunk(stream, stream.media, chunk, offset, name, done_cb)
         return False, 0
 
+    def extract_text_data(self):
+        if self.data:
+            for sub in extract.get_text_subtitles(self.data.path):
+                srt = extract.extract_subtitles(self.data.path, sub['language'])
+                # fixme add subtitles, possibly with language!
+            chapters = extract.get_chapters(self.data.path)
+            if chapters:
+                # fixme add chapters as notes
+                pass
+
+    def get_codec(self, type):
+        track = self.info.get(type)
+        if track:
+            return track[0].get('codec')
+
+    MP4_VCODECS = ['h264']
+    MP4_ACODECS = ['aac', None]
+    WEBM_VCODECS = ['vp8', 'vp9']
+    WEBM_ACODECS = ['vorbis', 'opus', None]
+
+    def can_remux(self):
+        config = settings.CONFIG['video']
+        height = self.info['video'][0]['height'] if self.info.get('video') else None
+        max_resolution = max(config['resolutions'])
+        if height <= max_resolution and self.extension in ('mov', 'mkv', 'mp4', 'm4v'):
+            vcodec = self.get_codec('video')
+            acodec = self.get_codec('audio')
+            if vcodec in self.MP4_VCODECS and acodec in self.MP4_ACODECS:
+                return True
+        return False
+
+    def can_stream(self):
+        config = settings.CONFIG['video']
+        height = self.info['video'][0]['height'] if self.info.get('video') else None
+        max_resolution = max(config['resolutions'])
+        if height <= max_resolution and config['formats'][0] == self.extension:
+            vcodec = self.get_codec('video')
+            acodec = self.get_codec('audio')
+            if self.extension in ['mp4', 'm4v'] and vcodec in self.MP4_VCODECS and acodec in self.MP4_ACODECS:
+                return extract.has_faststart(self.data.path)
+            elif self.extension == 'webm' and vcodec in self.WEBM_VCODECS and acodec in self.WEBM_ACODECS:
+                return True
+        return False
+
     def stream_resolution(self):
         config = settings.CONFIG['video']
         height = self.info['video'][0]['height'] if self.info.get('video') else None
@@ -747,13 +791,27 @@ class Stream(models.Model):
                         derivative.encode()
 
     def encode(self):
+        reuse = settings.CONFIG['video'].get('reuseUpload', False)
         media = self.source.media.path if self.source else self.file.data.path
-
         if not self.media:
             self.media.name = self.path(self.name())
         target = self.media.path
         info = ox.avinfo(media)
-        ok, error = extract.stream(media, target, self.name(), info, flags=self.flags)
+
+        done = False
+        if reuse and not self.source:
+            if self.file.can_stream():
+                ok, error = True, None
+                ox.makedirs(os.path.dirname(target))
+                shutil.move(self.file.data.path, target)
+                self.file.data.name = ''
+                self.file.save()
+            elif self.file.can_remux():
+                ok, error = extract.remux_stream(media, target)
+                done = True
+        if not done:
+            ok, error = extract.stream(media, target, self.name(), info, flags=self.flags)
+
         # file could have been moved while encoding
         # get current version from db and update
         self.refresh_from_db()

pandora/settings.py

@@ -178,6 +178,7 @@ CACHES = {
 AUTH_PROFILE_MODULE = 'user.UserProfile'
 AUTH_CHECK_USERNAME = True
 FFMPEG = 'ffmpeg'
+FFPROBE = 'ffprobe'
 FFMPEG_SUPPORTS_VP9 = True
 FFMPEG_DEBUG = False