make private items private

optional fulltext search for documents using elasticsearch
text is extracted from pdfs and via ocr from images

ctl

@@ -9,33 +9,37 @@ fi
 if [ "$action" = "init" ]; then
     cd "`dirname "$0"`"
     BASE=`pwd`
-    python3 -m venv --system-site-packages .
-
+    SUDO=""
+    PANDORA_USER=`ls -l update.py | cut -f3 -d" "`
+    if [ `whoami` != $PANDORA_USER ]; then
+        SUDO="sudo -H -u $PANDORA_USER"
+    fi
+    $SUDO python3 -m venv --system-site-packages .
     branch=`cat .git/HEAD  | sed 's@/@\n@g' | tail -n1`
 
     # Work around broken venv module in Ubuntu 16.04 / Debian 9
     if [ ! -e bin/pip ]; then
-        bin/python3 -m pip install -U --ignore-installed "pip<9"
+        $SUDO bin/python3 -m pip install -U --ignore-installed "pip<9"
     fi
     if [ ! -d static/oxjs ]; then
-        git clone --depth 1 -b $branch https://git.0x2620.org/oxjs.git static/oxjs
+        $SUDO git clone --depth 1 -b $branch https://git.0x2620.org/oxjs.git static/oxjs
     fi
-    mkdir -p src
+    $SUDO mkdir -p src
     if [ ! -d src/oxtimelines ]; then
-        git clone --depth 1 -b $branch https://git.0x2620.org/oxtimelines.git src/oxtimelines
+        $SUDO git clone --depth 1 -b $branch https://git.0x2620.org/oxtimelines.git src/oxtimelines
     fi
     for package in oxtimelines python-ox; do
         cd ${BASE}
         if [ ! -d src/${package} ]; then
-            git clone --depth 1 -b $branch https://git.0x2620.org/${package}.git src/${package}
+            $SUDO git clone --depth 1 -b $branch https://git.0x2620.org/${package}.git src/${package}
         fi
         cd ${BASE}/src/${package}
-        ${BASE}/bin/python setup.py develop
+        $SUDO ${BASE}/bin/python setup.py develop
     done
     cd ${BASE}
-    ./bin/pip install -r requirements.txt
+    $SUDO ./bin/pip install -r requirements.txt
     if [ ! -e pandora/gunicorn_config.py ]; then
-        cp pandora/gunicorn_config.py.in pandora/gunicorn_config.py
+        $SUDO cp pandora/gunicorn_config.py.in pandora/gunicorn_config.py
     fi
     exit 0
 fi

etc/nginx/pandora

@@ -42,7 +42,7 @@ server {
             proxy_set_header Proxy "";
             proxy_redirect off;
             proxy_buffering off;
-            proxy_read_timeout 999999999;
+            proxy_read_timeout 99999;
             proxy_pass http://127.0.0.1:2622/;
     }
 
@@ -55,11 +55,11 @@ server {
             proxy_buffering off;
             proxy_read_timeout 90;    #should be in sync with gunicorn timeout
             proxy_connect_timeout 90; #should be in sync with gunicorn timeout
+            client_max_body_size 32m;
             if (!-f $request_filename) {
                 proxy_pass http://127.0.0.1:2620;
                 break;
             }
-            client_max_body_size 32m;
     }
 
     error_page  400  /;

pandora/annotation/models.py

@@ -378,6 +378,8 @@ class Annotation(models.Model):
                 streams = self.item.streams()
                 if streams:
                     j['videoRatio'] = streams[0].aspect_ratio
+            if 'clip' in keys:
+                j[key] = self.clip.public_id
             for key in keys:
                 if key not in j:
                     if key in self._clip_keys:

pandora/app/config.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 from __future__ import division, print_function, absolute_import
 
+import codecs
 import os
-import sys
+import re
 import shutil
 import subprocess
+import sys
 import time
-import codecs
 from os.path import dirname, exists, join
 from glob import glob
 
@@ -71,7 +72,7 @@ def load_config(init=False):
         if getattr(settings, 'SITEURL', False):
             config['site']['url'] = settings.SITEURL
         settings.URL = config['site']['url']
-        settings.EMAIL_SUBJECT_PREFIX = '[%s]'%settings.SITENAME
+        settings.EMAIL_SUBJECT_PREFIX = '[%s]' % settings.SITENAME
         settings.DEFAULT_FROM_EMAIL = config['site']['email']['system']
         settings.SERVER_EMAIL = config['site']['email']['system']
         config['site']['videoprefix'] = settings.VIDEO_PREFIX
@@ -79,18 +80,32 @@ def load_config(init=False):
         config['site']['googleapikey'] = getattr(settings, 'GOOGLE_API_KEY')
         config['site']['version'] = get_version()
         config['site']['dontValidateUser'] = not settings.AUTH_CHECK_USERNAME
-        if not 'folderdepth' in config['site']:
+        if 'folderdepth' not in config['site']:
             config['site']['folderdepth'] = settings.USE_IMDB and 4 or 3
-        if 'sendReferrer' in config and not 'sendReferrer' in config['site']:
+        if 'sendReferrer' in config and 'sendReferrer' not in config['site']:
             config['site']['sendReferrer'] = config.pop('sendReferrer')
 
         # enable default filters if needed
         default_filters = [f['id'] for f in config['user']['ui']['filters']]
+        available_filters = [key['id'] for key in config['itemKeys'] if key.get('filter')]
+        unknown_ids = set(default_filters) - set(available_filters)
+        if unknown_ids:
+            sys.stderr.write('WARNING: unknown item keys in default filters: %s.\n' % list(unknown_ids))
+            unused_filters = [key for key in available_filters if key not in default_filters]
+            if len(unused_filters) < len(unknown_ids):
+                sys.stderr.write('you need at least 5 item filters')
+            else:
+                auto_filters = unused_filters[:len(unknown_ids)]
+                default_filters += auto_filters
+                for key in auto_filters:
+                    config['user']['ui']['filters'].append({
+                        "id": key, "sort": [{"key": "items", "operator": "-"}]
+                    })
+                sys.stderr.write('         using the following document filters instead: %s.\n' % auto_filters)
         for key in config['itemKeys']:
             if key['id'] in default_filters and not key.get('filter'):
                 key['filter'] = True
                 sys.stderr.write('enabled filter for "%s" since its used as default filter.\n' % (key['id']))
-
         config['keys'] = {}
         for key in config['itemKeys']:
             config['keys'][key['id']] = key
@@ -148,6 +163,17 @@ def load_config(init=False):
             if level not in config[key]:
                 config[key] = default.get(key, 0)
 
+        config['user']['ui']['documentsSort'] = [
+            s for s in config['user']['ui']['documentsSort']
+            if get_by_id(config['documentKeys'], s['key'])
+        ]
+        if not config['user']['ui']['documentsSort']:
+            sort_key = [k for k in config['documentKeys'] if k['id'] != '*'][0]
+            config['user']['ui']['documentsSort'] = [{
+                "key": sort_key['id'],
+                "operator": sort_key.get('operator', '+')
+            }]
+
         for key in ('language', 'importMetadata'):
             if key not in config:
                 sys.stderr.write("adding default value:\n\t\"%s\": %s,\n\n" % (key, json.dumps(default[key])))
@@ -161,6 +187,32 @@ def load_config(init=False):
         if 'downloadFormat' not in config['video']:
             config['video']['downloadFormat'] = default['video']['downloadFormat']
 
+
+        # enable default document filters if needed
+        default_filters = [f['id'] for f in config['user']['ui']['documentFilters']]
+        available_filters = [key['id'] for key in config['documentKeys'] if key.get('filter')]
+        unknown_ids = set(default_filters) - set(available_filters)
+        if unknown_ids:
+            sys.stderr.write('WARNING: unknown document keys in default filters: %s.\n' % list(unknown_ids))
+            unused_filters = [key for key in available_filters if key not in default_filters]
+            if len(unused_filters) < len(unknown_ids):
+                sys.stderr.write('you need at least 5 item filters')
+            else:
+                auto_filters = unused_filters[:len(unknown_ids)]
+                default_filters += auto_filters
+                for key in auto_filters:
+                    config['user']['ui']['documentFilters'].append({
+                        "id": key, "sort": [{"key": "items", "operator": "-"}]
+                    })
+                sys.stderr.write('         using the following document filters instead: %s.\n' % auto_filters)
+
+        for key in config['documentKeys']:
+            if key['id'] in default_filters and not key.get('filter'):
+                key['filter'] = True
+                sys.stderr.write('enabled filter for document key "%s" since its used as default filter.\n' % (key['id']))
+
+
+
         old_formats = getattr(settings, 'CONFIG', {}).get('video', {}).get('formats', [])
         formats = config.get('video', {}).get('formats')
         if set(old_formats) != set(formats):
@@ -348,11 +400,17 @@ def update_geoip(force=False):
     path = os.path.join(settings.GEOIP_PATH, 'GeoLite2-City.mmdb')
     if not os.path.exists(path) or force:
         url = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz'
+        index = ox.net.read_url('https://db-ip.com/db/download/ip-to-country-lite').decode()
+        match = re.compile('href=[\'"](http.*.mmdb.gz)').findall(index)
+        if match:
+            url = match[0]
             print('download', url)
-        ox.net.save_url(url, "%s.gz"%path)
+            ox.net.save_url(url, "%s.gz" % path)
             if os.path.exists(path):
                 os.unlink(path)
             os.system('gunzip "%s.gz"' % path)
+        else:
+            print('failed to download dbip-country-lite-2020-03.mmdb.gz')
 
 def init():
     if not settings.RELOADER_RUNNING:

pandora/archive/external.py

@@ -97,6 +97,17 @@ def download(item_id, url):
         tmp = tmp.decode('utf-8')
     os.chdir(tmp)
     cmd = ['youtube-dl', '-q', media['url']]
+    if settings.CONFIG['video'].get('reuseUload', False):
+        max_resolution = max(settings.CONFIG['video']['resolutions'])
+        format = settings.CONFIG['video']['formats'][0]
+        if format == 'mp4':
+            cmd += [
+                '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/bestvideo+bestaudio',
+                '--merge-output-format', 'mp4'
+            ]
+        elif format == 'webm':
+            cmd += ['--merge-output-format', 'webm']
+
     p = subprocess.Popen(cmd,
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE, close_fds=True)

pandora/archive/extract.py

@@ -191,7 +191,7 @@ def stream(video, target, profile, info, audio_track=0, flags={}):
             w = info['video'][0]['width'] - flags['crop']['left'] - flags['crop']['right']
             x = flags['crop']['left']
             y = flags['crop']['top']
-            crop = ',crop=w=%s:h=%s:x=%s:y=%s' (w, h, x, y)
+            crop = ',crop=w=%s:h=%s:x=%s:y=%s' % (w, h, x, y)
             aspect = dar * (info['video'][0]['width'] / info['video'][0]['height']) * (w/h)
             if abs(w/h - aspect) < 0.02:
                 aspect = '%s:%s' % (w, h)
@@ -216,6 +216,7 @@ def stream(video, target, profile, info, audio_track=0, flags={}):
             '-vb', '%dk' % bitrate,
             '-aspect', aspect,
             # '-vf', 'yadif',
+            '-max_muxing_queue_size', '512',
             '-vf', 'hqdn3d%s,scale=%s:%s' % (crop, width, height),
             '-g', '%d' % int(fps*5),
         ]
@@ -238,6 +239,7 @@ def stream(video, target, profile, info, audio_track=0, flags={}):
                 '-preset:v', 'medium',
                 '-profile:v', 'high',
                 '-level', '4.0',
+                '-pix_fmt', 'yuv420p',
             ]
         video_settings += ['-map', '0:%s,0:0' % info['video'][0]['id']]
         audio_only = False
@@ -609,11 +611,14 @@ def timeline_strip(item, cuts, info, prefix):
             timeline_image.save(timeline_file)
 
 
-def chop(video, start, end, subtitles=None):
+def chop(video, start, end, subtitles=None, dest=None, encode=False):
     t = end - start
-    tmp = tempfile.mkdtemp()
     ext = os.path.splitext(video)[1]
+    if dest is None:
+        tmp = tempfile.mkdtemp()
         choped_video = '%s/tmp%s' % (tmp, ext)
+    else:
+        choped_video = dest
     if subtitles and ext == '.mp4':
         subtitles_f = choped_video + '.full.srt'
         with open(subtitles_f, 'wb') as fd:
@@ -625,25 +630,167 @@ def chop(video, start, end, subtitles=None):
         if subtitles_f:
             os.unlink(subtitles_f)
     else:
+        if encode:
+            bpp = 0.17
+            if ext == '.mp4':
+                vcodec = [
+                    '-c:v', 'libx264',
+                    '-preset:v', 'medium',
+                    '-profile:v', 'high',
+                    '-level', '4.0',
+                ]
+                acodec = [
+                    '-c:a', 'aac',
+                    '-aq', '6',
+                    '-strict', '-2'
+                ]
+            else:
+                vcodec = [
+                    '-c:v', 'libvpx',
+                    '-deadline', 'good',
+                    '-cpu-used', '0',
+                    '-lag-in-frames', '25',
+                    '-auto-alt-ref', '1',
+                ]
+                acodec = [
+                    '-c:a', 'libvorbis',
+                    '-aq', '6',
+                ]
+            info = ox.avinfo(video)
+            if not info['audio']:
+                acodec = []
+            if not info['video']:
+                vcodec = []
+            else:
+                height = info['video'][0]['height']
+                width = info['video'][0]['width']
+                fps = 30
+                bitrate = height*width*fps*bpp/1000
+                vcodec += ['-vb', '%dk' % bitrate]
+            encoding = vcodec + acodec
+        else:
+            encoding = [
+                '-c:v', 'copy',
+                '-c:a', 'copy',
+            ]
         cmd = [
             settings.FFMPEG,
             '-y',
             '-i', video,
             '-ss', '%.3f' % start,
             '-t', '%.3f' % t,
-            '-c:v', 'copy',
-            '-c:a', 'copy',
+        ] + encoding + [
             '-f', ext[1:],
             choped_video
         ]
+        print(cmd)
+        p = subprocess.Popen(cmd, stdin=subprocess.PIPE,
+                             stdout=open('/dev/null', 'w'),
+                             stderr=open('/dev/null', 'w'),
+                             close_fds=True)
+        p.wait()
+    if subtitles_f and os.path.exists(subtitles_f):
+        os.unlink(subtitles_f)
+    if dest is None:
+        f = open(choped_video, 'rb')
+        os.unlink(choped_video)
+        os.rmdir(tmp)
+        return f
+    else:
+        return None
+
+def has_faststart(path):
+    cmd = [settings.FFPROBE, '-v', 'trace', '-i', path]
+    p = subprocess.Popen(cmd, stdin=subprocess.PIPE,
+                         stdout=subprocess.PIPE,
+                         stderr=subprocess.STDOUT,
+                         close_fds=True)
+    stdout, stderr = p.communicate()
+    moov = "type:'moov'"
+    mdat = "type:'mdat'"
+    blocks = [b for b in stdout.decode().split('\n') if moov in b or mdat in b]
+    if blocks and moov in blocks[0]:
+        return True
+    return False
+
+def remux_stream(src, dst):
+    info = ox.avinfo(src)
+    if info.get('audio'):
+        audio = ['-c:a', 'copy']
+    else:
+        audio = []
+    if info.get('video'):
+        video = ['-c:v', 'copy']
+    else:
+        video = []
+    cmd = [
+        settings.FFMPEG,
+        '-nostats', '-loglevel', 'error',
+        '-map_metadata', '-1', '-sn',
+        '-i', src,
+    ] + video + [
+    ] + audio + [
+        '-movflags', '+faststart',
+        dst
+    ]
     p = subprocess.Popen(cmd, stdin=subprocess.PIPE,
                          stdout=open('/dev/null', 'w'),
                          stderr=open('/dev/null', 'w'),
                          close_fds=True)
     p.wait()
-    f = open(choped_video, 'rb')
-    os.unlink(choped_video)
-    if subtitles_f and os.path.exists(subtitles_f):
-        os.unlink(subtitles_f)
-    os.rmdir(tmp)
-    return f
+    return True, None
+
+
+def ffprobe(path, *args):
+    cmd = [settings.FFPROBE, '-loglevel', 'error', '-print_format', 'json', '-i', path] + list(args)
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
+    stdout, stderr = p.communicate()
+    return json.loads(stdout.decode())
+
+
+def get_chapters(path):
+    info = ffprobe(path, '-show_chapters')
+    chapters = []
+    n = 0
+    for chapter in info.get('chapters', []):
+        n += 1
+        chapters.append({
+            'in': chapter['start_time'],
+            'out': chapter['end_time'],
+            'value': chapter.get('tags', {}).get('title', 'Chapter %s' % n)
+        })
+    return chapters
+
+def get_text_subtitles(path):
+    subtitles = []
+    for stream in ffprobe(path, '-show_streams')['streams']:
+        if stream.get('codec_name') in ('subrip', 'aas', 'text'):
+            subtitles.append({
+                'index': stream['index'],
+                'language': stream['tags']['language'],
+            })
+    return subtitles
+
+def has_img_subtitles(path):
+    subtitles = []
+    for stream in ffprobe(path, '-show_streams')['streams']:
+        if stream.get('codec_type') == 'subtitle' and stream.get('codec_name') in ('dvbsub', 'pgssub'):
+            subtitles.append({
+                'index': stream['index'],
+                'language': stream['tags']['language'],
+            })
+    return subtitles
+
+def extract_subtitles(path, language=None):
+    extra = []
+    if language:
+        tracks = get_text_subtitles(path)
+        track = [t for t in tracks if t['language'] == language]
+        if track:
+            extra = ['-map', '0:%s' % track[0]['index']]
+        else:
+            raise Exception("unknown language: %s" % language)
+    cmd = ['ffmpeg', '-loglevel', 'error', '-i', path] + extra + ['-f', 'srt', '-']
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
+    stdout, stderr = p.communicate()
+    return ox.srt.loads(stdout.decode())

pandora/archive/models.py

@@ -376,6 +376,50 @@ class File(models.Model):
             return save_chunk(stream, stream.media, chunk, offset, name, done_cb)
         return False, 0
 
+    def extract_text_data(self):
+        if self.data:
+            for sub in extract.get_text_subtitles(self.data.path):
+                srt = extract.extract_subtitles(self.data.path, sub['language'])
+                # fixme add subtitles, possibly with language!
+            chapters = extract.get_chapters(self.data.path)
+            if chapters:
+                # fixme add chapters as notes
+                pass
+
+    def get_codec(self, type):
+        track = self.info.get(type)
+        if track:
+            return track[0].get('codec')
+
+    MP4_VCODECS = ['h264']
+    MP4_ACODECS = ['aac', None]
+    WEBM_VCODECS = ['vp8', 'vp9']
+    WEBM_ACODECS = ['vorbis', 'opus', None]
+
+    def can_remux(self):
+        config = settings.CONFIG['video']
+        height = self.info['video'][0]['height'] if self.info.get('video') else None
+        max_resolution = max(config['resolutions'])
+        if height <= max_resolution and self.extension in ('mov', 'mkv', 'mp4', 'm4v'):
+            vcodec = self.get_codec('video')
+            acodec = self.get_codec('audio')
+            if vcodec in self.MP4_VCODECS and acodec in self.MP4_ACODECS:
+                return True
+        return False
+
+    def can_stream(self):
+        config = settings.CONFIG['video']
+        height = self.info['video'][0]['height'] if self.info.get('video') else None
+        max_resolution = max(config['resolutions'])
+        if height <= max_resolution and config['formats'][0] == self.extension:
+            vcodec = self.get_codec('video')
+            acodec = self.get_codec('audio')
+            if self.extension in ['mp4', 'm4v'] and vcodec in self.MP4_VCODECS and acodec in self.MP4_ACODECS:
+                return extract.has_faststart(self.data.path)
+            elif self.extension == 'webm' and vcodec in self.WEBM_VCODECS and acodec in self.WEBM_ACODECS:
+                return True
+        return False
+
     def stream_resolution(self):
         config = settings.CONFIG['video']
         height = self.info['video'][0]['height'] if self.info.get('video') else None
@@ -523,7 +567,7 @@ class File(models.Model):
                     n += 1
                 profile = '%sp.%s' % (resolution, config['formats'][0])
                 target = os.path.join(tmp, language + '_' + profile)
-                ok, error = extract.stream(media, target, profile, info, audio_track=i+1, flags=self.flags)
+                ok, error = extract.stream(media, target, profile, info, audio_track=i+1, flags={})
                 if ok:
                     tinfo = ox.avinfo(target)
                     del tinfo['path']
@@ -747,18 +791,31 @@ class Stream(models.Model):
                         derivative.encode()
 
     def encode(self):
+        reuse = settings.CONFIG['video'].get('reuseUpload', False)
         media = self.source.media.path if self.source else self.file.data.path
-
         if not self.media:
             self.media.name = self.path(self.name())
         target = self.media.path
         info = ox.avinfo(media)
+
+        done = False
+        if reuse and not self.source:
+            if self.file.can_stream():
+                ok, error = True, None
+                ox.makedirs(os.path.dirname(target))
+                shutil.move(self.file.data.path, target)
+                self.file.data.name = ''
+                self.file.save()
+            elif self.file.can_remux():
+                ok, error = extract.remux_stream(media, target)
+                done = True
+        if not done:
             ok, error = extract.stream(media, target, self.name(), info, flags=self.flags)
+
         # file could have been moved while encoding
         # get current version from db and update
-        _self = Stream.objects.get(id=self.id)
-        _self.update_status(ok, error)
-        return _self
+        self.refresh_from_db()
+        self.update_status(ok, error)
 
     def get_index(self):
         index = 1

pandora/archive/tasks.py

@@ -128,7 +128,7 @@ def process_stream(fileId):
         stream = streams[0]
         stream.make_timeline()
         stream.extract_derivatives()
-        file = models.File.objects.get(id=fileId)
+        file.refresh_from_db()
         file.encoding = False
         file.save()
     file.item.update_selected()
@@ -158,13 +158,12 @@ def extract_stream(fileId):
         if created:
             file.extract_frames()
             stream.media.name = stream.path(stream.name())
-            stream = stream.encode()
+            stream.encode()
             if stream.available:
                 stream.make_timeline()
                 stream.extract_derivatives()
                 file.extract_tracks()
-                # get current version from db
-                file = models.File.objects.get(id=fileId)
+                file.refresh_from_db()
                 if not file.item.rendered \
                         and not file.item.files.exclude(id=fileId).filter(Q(queued=True) | Q(encoding=True)).count():
                     file.item.update_timeline()
@@ -209,7 +208,8 @@ def download_media(item_id, url):
 @task(queue='default')
 def move_media(data, user):
     from changelog.models import add_changelog
-    from item.models import get_item, Item
+    from item.models import get_item, Item, ItemSort
+    from item.utils import is_imdb_id
     from annotation.models import Annotation
 
     user = models.User.objects.get(username=user)
@@ -218,7 +218,7 @@ def move_media(data, user):
         i = Item.objects.get(public_id=data['item'])
     else:
         data['public_id'] = data.pop('item').strip()
-        if len(data['public_id']) != 7:
+        if not is_imdb_id(data['public_id']):
             del data['public_id']
             if 'director' in data and isinstance(data['director'], string_types):
                 if data['director'] == '':
@@ -228,6 +228,11 @@ def move_media(data, user):
             i = get_item(data, user=user)
         else:
             i = get_item({'imdbId': data['public_id']}, user=user)
+    try:
+        i.sort
+    except ItemSort.DoesNotExist:
+        i.update_sort()
+
     changed = [i.public_id]
     old_item = None
     for f in models.File.objects.filter(oshash__in=data['ids']):

pandora/archive/views.py

@@ -368,7 +368,7 @@ def direct_upload(request):
     return render_to_json_response(response)
 
 
-@login_required_json
+#@login_required_json
 def getTaskStatus(request, data):
     '''
     Gets the status for a given task

pandora/clip/models.py

@@ -121,7 +121,9 @@ class MetaClip(object):
                         annotations = annotations.filter(q)
                 entity_cache = {}
                 j['annotations'] = [
-                    a.json(keys=['value', 'id', 'layer'], entity_cache=entity_cache) for a in annotations
+                    a.json(keys=['value', 'id', 'layer'], entity_cache=entity_cache)
+                    for a in annotations
+                    if a.value
                 ]
             if 'layers' in keys:
                 j['layers'] = self.get_layers()

pandora/clip/views.py

@@ -78,7 +78,7 @@ def findClips(request, data):
     takes {
         query: object, // find clips, query object, see `find`
         itemsQuery: object, // limit to matching items, query object, see `find`
-        keys: [string], // list of properties to return
+        keys: [string], // list of properties to return, include 'annotations' to get all annotations for a clip
         positions: [int], // list of positions
         range: [int, int], // range of results to return
         sort: [object] // list of sort objects, see `find`
@@ -102,8 +102,6 @@ def findClips(request, data):
         subtitles = utils.get_by_key(layers, 'isSubtitles', True)
         layer_ids = [k['id'] for k in layers]
         keys = list(filter(lambda k: k not in layer_ids + ['annotations'], data['keys']))
-        if list(filter(lambda k: k not in models.Clip.clip_keys, keys)):
-            qs = qs.select_related('item__sort')
 
         clips = {}
         response['data']['items'] = clip_jsons = []

pandora/config.0xdb.jsonc

@@ -38,6 +38,7 @@
         "canAddItems": {"staff": true, "admin": true},
         "canAddDocuments": {"staff": true, "admin": true},
         "canDownloadVideo": {"guest": -1, "member": -1, "friend": -1, "staff": -1, "admin": -1},
+        "canDownloadSource": {"guest": -1, "member": -1, "friend": -1, "staff": -1, "admin": -1},
         "canEditAnnotations": {"staff": true, "admin": true},
         "canEditEntities": {"staff": true, "admin": true},
         "canEditDocuments": {"staff": true, "admin": true},
@@ -97,7 +98,7 @@
     text of clips (in grid view, below the icon). Excluding a layer from this
     list means it will not be included in find annotations.
     */
-    "clipLayers": ["subtitles"],
+    "clipLayers": ["subtitles", "keywords"],
     "documentKeys": [
         {
             "id": "*",
@@ -709,6 +710,14 @@
             "advanced": true,
             "find": true
         },
+        {
+            "id": "tags",
+            "title": "Tags",
+            "type": "layer",
+            "autocomplete": true,
+            "filter": true,
+            "find": true
+        },
         {
             "id": "subtitles",
             "title": "Subtitles",
@@ -997,6 +1006,15 @@
             tooltip that appears on mouseover.
     */
     "layers": [
+        {
+            "id": "tags",
+            "title": "Tags",
+            "canAddAnnotations": {"member": true, "staff": true, "admin": true},
+            "item": "Tag",
+            "autocomplete": true,
+            "overlap": true,
+            "type": "string"
+        },
         {
             "id": "privatenotes",
             "title": "Private Notes",

pandora/config.indiancinema.jsonc

@@ -39,6 +39,7 @@
         "canAddItems": {"researcher": true, "staff": true, "admin": true},
         "canAddDocuments": {"researcher": true, "staff": true, "admin": true},
         "canDownloadVideo": {"guest": -1, "member": -1, "researcher": 3, "staff": 3, "admin": 3},
+        "canDownloadSource": {"guest": -1, "member": -1, "researcher": -1, "staff": -1, "admin": -1},
         "canEditAnnotations": {"staff": true, "admin": true},
         "canEditDocuments": {"researcher": true, "staff": true, "admin": true},
         "canEditEntities": {"staff": true, "admin": true},
@@ -75,7 +76,7 @@
         "canSeeExtraItemViews": {"researcher": true, "staff": true, "admin": true},
         "canSeeMedia": {"researcher": true, "staff": true, "admin": true},
         "canSeeDocument": {"guest": 1, "member": 1, "researcher": 2, "staff": 3, "admin": 3},
-        "canSeeItem": {"guest": 3, "member": 3, "researcher": 3, "staff": 3, "admin": 3},
+        "canSeeItem": {"guest": 2, "member": 2, "researcher": 2, "staff": 3, "admin": 3},
         "canSeeSize": {"researcher": true, "staff": true, "admin": true},
         "canSeeSoftwareVersion": {"researcher": true, "staff": true, "admin": true},
         "canSendMail": {"staff": true, "admin": true}
@@ -1695,7 +1696,7 @@
             "annotationsCalendarSize": 128,
             "annotationsHighlight": "none",
             "annotationsMapSize": 128,
-            "annotationsRange": "all",
+            "annotationsRange": "selection",
             "annotationsSize": 256,
             "annotationsSort": "position",
             "calendarFind": "",
@@ -1852,7 +1853,7 @@
             "videoSize": "small",
             "videoSubtitles": true,
             "videoSubtitlesOffset": 0,
-            "videoTimeline": "slitscan",
+            "videoTimeline": "keyframes",
             "videoView": "player",
             "videoVolume": 1
         },

pandora/config.padma.jsonc

@@ -38,6 +38,7 @@
         "canAddItems": {"member": true, "staff": true, "admin": true},
         "canAddDocuments": {"member": true, "staff": true, "admin": true},
         "canDownloadVideo": {"guest": 0, "member": 0, "staff": 4, "admin": 4},
+        "canDownloadSource": {"guest": -1, "member": -1, "staff": 4, "admin": 4},
         "canEditAnnotations": {"staff": true, "admin": true},
         "canEditEntities": {"staff": true, "admin": true},
         "canEditDocuments": {"staff": true, "admin": true},
@@ -1160,7 +1161,7 @@
             "annotationsHighlight": "none",
             "annotationsHighlight": false,
             "annotationsMapSize": 128,
-            "annotationsRange": "position",
+            "annotationsRange": "selection",
             "annotationsSize": 256,
             "annotationsSort": "position",
             "calendarFind": "",
@@ -1310,7 +1311,7 @@
             "videoSize": "large",
             "videoSubtitles": false,
             "videoSubtitlesOffset": 0,
-            "videoTimeline": "antialias",
+            "videoTimeline": "keyframes",
             "videoView": "player",
             "videoVolume": 1
         },

pandora/config.pandora.jsonc

@@ -45,6 +45,7 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
         "canAddItems": {"member": true, "staff": true, "admin": true},
         "canAddDocuments": {"member": true, "staff": true, "admin": true},
         "canDownloadVideo": {"guest": 1, "member": 1, "staff": 4, "admin": 4},
+        "canDownloadSource": {"member": 1, "staff": 4, "admin": 4},
         "canEditAnnotations": {"staff": true, "admin": true},
         "canEditDocuments": {"staff": true, "admin": true},
         "canEditEntities": {"staff": true, "admin": true},
@@ -550,7 +551,7 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
         {
             "id": "country",
             "title": "Country",
-            "type": "string",
+            "type": ["string"],
             "autocomplete": true,
             "columnWidth": 180,
             "filter": true,
@@ -990,11 +991,6 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
         {"name": "Private", "color": [255, 128, 128]}
     ],
     /*
-    "sendReferrer", if set to false, will cause all outgoing links to originate
-    from one single URL
-    */
-    "sendReferrer": false,
-    /*
     "site" contains various settings for this instance. In "email", "contact"
     if the address in the contact form (to), "system" is the address used by
     the system (from).
@@ -1278,6 +1274,6 @@ examples (config.SITENAME.jsonc) that are part of this pan.do/ra distribution.
         "formats": ["webm", "mp4"],
         "previewRatio": 1.3333333333,
         "resolutions": [240, 480],
-        "torrent": true
+        "torrent": false
     }
 }

pandora/document/fulltext.py

@@ -0,0 +1,97 @@
+import subprocess
+
+from django.conf import settings
+
+
+def extract_text(pdf):
+    cmd = ['pdftotext', pdf, '-']
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    stdout = stdout.decode()
+    return stdout.strip()
+
+def ocr_image(path):
+    cmd = ['tesseract', path, '-', 'txt']
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    stdout = stdout.decode()
+    return stdout.strip()
+
+class FulltextMixin:
+    _ES_INDEX = "document-index"
+
+    @classmethod
+    def elasticsearch(cls):
+        from elasticsearch import Elasticsearch
+        es = Elasticsearch(settings.ELASTICSEARCH_HOST)
+        return es
+
+    def extract_fulltext(self):
+        if self.file:
+            if self.extension == 'pdf':
+                return extract_text(self.file.path)
+            elif self.extension in ('png', 'jpg'):
+                return ocr_image(self.file.path)
+        elif self.extension == 'html':
+            return self.data.get('text', '')
+        return ''
+
+    def has_fulltext_key(self):
+        return bool([k for k in settings.CONFIG['documentKeys'] if k.get('fulltext')])
+
+    def delete_fulltext(self):
+        if self.has_fulltext_key():
+            from elasticsearch.exceptions import NotFoundError
+            try:
+                res = self.elasticsearch().delete(index=self._ES_INDEX, doc_type='document', id=self.id)
+            except NotFoundError:
+                pass
+
+    def update_fulltext(self):
+        if self.has_fulltext_key():
+            text = self.extract_fulltext()
+            if text:
+                doc = {
+                    'text': text.lower()
+                }
+                res = self.elasticsearch().index(index=self._ES_INDEX, doc_type='document', id=self.id, body=doc)
+
+    @classmethod
+    def find_fulltext(cls, query):
+        ids = cls.find_fulltext_ids(query)
+        return cls.objects.filter(id__in=ids)
+
+    @classmethod
+    def find_fulltext_ids(cls, query):
+        if not query:
+            return []
+        elif query[0] == '"' and query[-1] == '"':
+            query = {
+                "match_phrase": {
+                    "text": query.lower()[1:-1]
+                },
+            }
+        else:
+            query = {
+                "match": {
+                    "text": {
+                        "query": query.lower(),
+                        "operator": "and"
+                    }
+                }
+            }
+        ids = []
+        res = None
+        from_ = 0
+        es = cls.elasticsearch()
+        while not res or len(ids) < res['hits']['total']['value']:
+            res = es.search(index=cls._ES_INDEX, body={
+                "from": from_,
+                "_source": False,
+                "query": query
+            })
+            if not res['hits']['hits']:
+                break
+            ids += [int(r['_id']) for r in res['hits']['hits']]
+            from_ += len(res['hits']['hits'])
+        return ids

pandora/document/managers.py

@@ -36,6 +36,8 @@ def get_key_type(k):
     }.get(key_type, key_type)
     return key_type
 
+
+
 def parseCondition(condition, user, item=None, owner=None):
     '''
     '''
@@ -68,6 +70,9 @@ def buildCondition(k, op, v, user, exclude=False, owner=None):
         k = 'collection'
 
     key_type = get_key_type(k)
+
+    key_config = (utils.get_by_id(settings.CONFIG['documentKeys'], k) or {'type': 'string'})
+
     facet_keys = models.Document.facet_keys
     if k == 'id':
         if op == '&' and isinstance(v, list):
@@ -128,6 +133,12 @@ def buildCondition(k, op, v, user, exclude=False, owner=None):
             else:
                 q = Q(id=0)
         return q
+    elif key_config.get('fulltext'):
+        qs = models.Document.find_fulltext_ids(v)
+        q = Q(id__in=qs)
+        if exclude:
+            q = ~Q(id__in=qs)
+        return q
     elif key_type == 'boolean':
         q = Q(**{'find__key': k, 'find__value': v})
         if exclude:

pandora/document/models.py

@@ -30,6 +30,8 @@ from user.models import Group
 
 from . import managers
 from . import utils
+from . import tasks
+from .fulltext import FulltextMixin
 
 User = get_user_model()
 
@@ -40,7 +42,7 @@ def get_path(f, x):
     return f.path(x)
 
 @python_2_unicode_compatible
-class Document(models.Model):
+class Document(models.Model, FulltextMixin):
 
     created = models.DateTimeField(auto_now_add=True)
     modified = models.DateTimeField(auto_now=True)
@@ -153,6 +155,8 @@ class Document(models.Model):
                 i = key['id']
                 if i == 'rightslevel':
                     save(i, self.rightslevel)
+                if key.get('fulltext'):
+                    continue
                 elif i not in ('*', 'dimensions') and i not in self.facet_keys:
                     value = data.get(i)
                     if isinstance(value, list):
@@ -409,6 +413,8 @@ class Document(models.Model):
                 and document_key['value'].get('type') == 'map' \
                 and self.get_value(document_key['value']['key']):
             value = re.compile(document_key['value']['map']).findall(self.get_value(document_key['value']['key']))
+            if value and document_key['value'].get('format'):
+                value = [document_key['value']['format'].format(value[0])]
             return value[0] if value else default
         elif key == 'user':
             return self.user.username
@@ -502,6 +508,7 @@ class Document(models.Model):
                     self.oshash = ox.oshash(self.file.path)
                     self.save()
                     self.delete_cache()
+                    tasks.extract_fulltext.delay(self.id)
                 return True, self.file.size
 
             return save_chunk(self, self.file, chunk, offset, name, done_cb)
@@ -518,7 +525,13 @@ class Document(models.Model):
         else:
             path = src
         if self.extension == 'pdf':
+            crop = []
             if page:
+                if ',' in page:
+                    crop = list(map(int, page.split(',')))
+                    page = crop[0]
+                    crop = crop[1:]
+                else:
                     page = int(page)
             if page and page > 1 and page <= self.pages:
                 src = os.path.join(folder, '1024p%d.jpg' % page)
@@ -529,6 +542,18 @@ class Document(models.Model):
                 self.extract_page(page)
             if size:
                 path = os.path.join(folder, '%dp%d.jpg' % (size, page))
+            if len(crop) == 4:
+                path = os.path.join(folder, '%dp%d,%s.jpg' % (1024, page, ','.join(map(str, crop))))
+                if not os.path.exists(path):
+                    img = Image.open(src).crop(crop)
+                    img.save(path)
+                else:
+                    img = Image.open(path)
+                src = path
+                if size < max(img.size):
+                    path = os.path.join(folder, '%dp%d,%s.jpg' % (size, page, ','.join(map(str, crop))))
+                    if not os.path.exists(path):
+                        resize_image(src, path, size=size)
         elif self.extension in ('jpg', 'png', 'gif'):
             if os.path.exists(src):
                 if size and page:
@@ -649,6 +674,7 @@ def delete_document(sender, **kwargs):
     if t.file:
         t.delete_cache()
         t.file.delete(save=False)
+    t.delete_fulltext()
 pre_delete.connect(delete_document, sender=Document)
 
 class ItemProperties(models.Model):

pandora/document/tasks.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+from celery.task import task
+
+@task(queue="encoding")
+def extract_fulltext(id):
+    from . import models
+    d = models.Document.objects.get(id=id)
+    d.update_fulltext()

pandora/document/utils.py

@@ -15,7 +15,7 @@ def pdfinfo(pdf):
     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
     stdout, stderr = p.communicate()
     data = {}
-    for line in stdout.decode('utf-8').strip().split('\n'):
+    for line in stdout.decode('utf-8', 'replace').strip().split('\n'):
         parts = line.split(':')
         key = parts[0].lower().strip()
         if key:

pandora/document/views.py

@@ -210,7 +210,7 @@ def parse_query(data, user):
     for key in ('keys', 'group', 'file', 'range', 'position', 'positions', 'sort'):
         if key in data:
             query[key] = data[key]
-    print(query.get('sort'), data.get('sort'))
+    #print(query.get('sort'), data.get('sort'))
     query['qs'] = models.Document.objects.find(data, user)
     query['item'] = get_item(data.get('query', {}))
     return query
@@ -439,7 +439,7 @@ def upload(request):
 
 def autocompleteDocuments(request, data):
     '''
-    Returns autocomplete strings for a given documeny key and search string
+    Returns autocomplete strings for a given document key and search string
     takes {
         key: string, // document key
         value: string, // search string

pandora/edit/models.py

@@ -248,7 +248,7 @@ class Edit(models.Model):
             clips_query = self.clip_query()
             if clips_query['conditions']:
                 clips = clip.models.Clip.objects.find({'query': clips_query}, user)
-                items = [i['id'] for i in self.get_items(user).values('id')]
+                items = self.get_items(user).values('id')
                 clips = clips.filter(item__in=items)
             else:
                 clips = clip.models.Clip.objects.filter(id=None)

pandora/item/management/commands/sync_itemsort.py

@@ -107,6 +107,8 @@ class Command(BaseCommand):
                     print(sql)
                 cursor.execute(sql)
             transaction.commit()
+            for i in models.Item.objects.filter(sort=None):
+                i.save()
             if rebuild:
                 print("Updating sort values...")
                 ids = [i['id'] for i in models.Item.objects.all().values('id')]
@@ -115,3 +117,5 @@ class Command(BaseCommand):
                     if options['debug']:
                         print(i)
                     i.update_sort()
+        for i in models.Item.objects.filter(sort=None):
+            i.save()

pandora/item/management/commands/update_external.py

@@ -17,9 +17,9 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument('--all', action='store_true', dest='all',
-            default=False, help='update all items, otherwise oldes N'),
+            default=False, help='update all items, otherwise oldes N')
         parser.add_argument('-n', '--items', action='store', dest='items', type=int,
-            default=30, help='number of items ot update'),
+            default=30, help='number of items ot update')
 
     def handle(self, **options):
         offset = 0

pandora/item/managers.py

@@ -165,6 +165,9 @@ def parseCondition(condition, user, owner=None):
                     else:
                         q = Q(id__in=l.items.all())
                     if exclude:
+                        if isinstance(q, list):
+                            q = [~x for x in q]
+                        else:
                             q = ~q
             else:
                 q = Q(id=0)

pandora/item/models.py

@@ -14,14 +14,15 @@ from glob import glob
 
 from six import PY2, string_types
 from six.moves.urllib.parse import quote
-from django.db import models, transaction, connection
-from django.db.models import Q, Sum, Max
+
 from django.conf import settings
 from django.contrib.auth import get_user_model
-
+from django.core.files.temp import NamedTemporaryFile
+from django.db import models, transaction, connection
+from django.db.models import Q, Sum, Max
 from django.db.models.signals import pre_delete
-from django.utils import datetime_safe
 from django.utils.encoding import python_2_unicode_compatible
+from django.utils import datetime_safe
 
 import ox
 from oxdjango.fields import JSONField, to_json
@@ -214,6 +215,8 @@ class Item(models.Model):
                 and item_key['value'].get('type') == 'map' \
                 and self.get(item_key['value']['key']):
             value = re.compile(item_key['value']['map']).findall(self.get(item_key['value']['key']))
+            if value and item_key['value'].get('format'):
+                value = [item_key['value']['format'].format(value[0])]
             return value[0] if value else default
         return default
 
@@ -387,8 +390,9 @@ class Item(models.Model):
             if self.oxdbId != oxdbId:
                 q = Item.objects.filter(oxdbId=oxdbId).exclude(id=self.id)
                 if q.count() != 0:
-                    if len(self.public_id) == 7:
+                    if utils.is_imdb_id(self.public_id):
                         self.oxdbId = None
+                        self.update_sort()
                         q[0].merge_with(self, save=False)
                     else:
                         n = 1
@@ -401,14 +405,14 @@ class Item(models.Model):
                             q = Item.objects.filter(oxdbId=oxdbId).exclude(id=self.id)
                 self.oxdbId = oxdbId
                 update_poster = True
-                if len(self.public_id) != 7:
+                if not utils.is_imdb_id(self.public_id):
                     update_ids = True
 
         # id changed, what about existing item with new id?
-        if settings.USE_IMDB and len(self.public_id) != 7 and self.oxdbId != self.public_id:
+        if settings.USE_IMDB and not utils.is_imdb_id(self.public_id) and self.oxdbId != self.public_id:
             self.public_id = self.oxdbId
             # FIXME: move files to new id here
-        if settings.USE_IMDB and len(self.public_id) == 7:
+        if settings.USE_IMDB and utils.is_imdb_id(self.public_id):
             for key in ('title', 'year', 'director', 'season', 'episode',
                         'seriesTitle', 'episodeTitle'):
                 if key in self.data:
@@ -418,7 +422,7 @@ class Item(models.Model):
         if settings.USE_IMDB:
             defaults = list(filter(lambda k: 'default' in k, settings.CONFIG['itemKeys']))
             for k in defaults:
-                if len(self.public_id) == 7:
+                if utils.is_imdb_id(self.public_id):
                     if k['id'] in self.data and self.data[k['id']] == k['default']:
                         del self.data[k['id']]
                 else:
@@ -637,6 +641,9 @@ class Item(models.Model):
             if self.poster_height:
                 i['posterRatio'] = self.poster_width / self.poster_height
 
+        if keys and 'source' in keys:
+            i['source'] = self.streams().exclude(file__data='').exists()
+
         streams = self.streams()
         i['durations'] = [s.duration for s in streams]
         i['duration'] = sum(i['durations'])
@@ -938,6 +945,8 @@ class Item(models.Model):
         s.oxdbId = self.oxdbId
         if not settings.USE_IMDB and s.public_id.isupper() and s.public_id.isalpha():
             s.public_id = ox.sort_string(str(ox.fromAZ(s.public_id)))
+        else:
+            s.public_id = ox.sort_string(s.public_id)
         s.modified = self.modified or datetime.now()
         s.created = self.created or datetime.now()
         s.rightslevel = self.level
@@ -1041,6 +1050,8 @@ class Item(models.Model):
                     set_value(s, name, value)
                 elif sort_type == 'year':
                     value = self.get(source)
+                    if isinstance(value, str):
+                        value = value[:4]
                     set_value(s, name, value)
                 elif sort_type == 'date':
                     value = value_ = self.get(source)
@@ -1179,6 +1190,37 @@ class Item(models.Model):
                     return None
                 return path
 
+    def extract_clip(self, in_, out, resolution, format, track=None, force=False):
+        streams = self.streams(track)
+        stream = streams[0].get(resolution, format)
+        if streams.count() > 1 and stream.info['duration'] < out:
+            video = NamedTemporaryFile(suffix='.%s' % format)
+            r = self.merge_streams(video.name, resolution, format)
+            if not r:
+                return False
+            path = video.name
+            duration = sum(item.cache['durations'])
+        else:
+            path = stream.media.path
+            duration = stream.info['duration']
+
+        cache_name = '%s_%sp_%s.%s' % (self.public_id, resolution, '%s,%s' % (in_, out), format)
+        cache_path = os.path.join(settings.MEDIA_ROOT, self.path('cache/%s' % cache_name))
+        if os.path.exists(cache_path) and not force:
+            return cache_path
+        if duration >= out:
+            subtitles = utils.get_by_key(settings.CONFIG['layers'], 'isSubtitles', True)
+            if subtitles:
+                srt = self.srt(subtitles['id'], encoder=ox.srt)
+                if len(srt) < 4:
+                    srt = None
+            else:
+                srt = None
+            ox.makedirs(os.path.dirname(cache_path))
+            extract.chop(path, in_, out, subtitles=srt, dest=cache_path, encode=True)
+            return cache_path
+        return False
+
     @property
     def timeline_prefix(self):
         videos = self.streams()

pandora/item/tasks.py

@@ -22,6 +22,7 @@ def cronjob(**kwargs):
     if limit_rate('item.tasks.cronjob', 8 * 60 * 60):
         update_random_sort()
         update_random_clip_sort()
+        clear_cache.delay()
 
 def update_random_sort():
     from . import models
@@ -125,6 +126,33 @@ def load_subtitles(public_id):
         item.update_sort()
         item.update_facets()
 
+
+@task(queue="encoding")
+def extract_clip(public_id, in_, out, resolution, format, track=None):
+    from . import models
+    try:
+        item = models.Item.objects.get(public_id=public_id)
+    except models.Item.DoesNotExist:
+        return False
+    if item.extract_clip(in_, out, resolution, format, track):
+        return True
+    return False
+
+
+@task(queue="encoding")
+def clear_cache(days=60):
+    import subprocess
+    path = os.path.join(settings.MEDIA_ROOT, 'media')
+    cmd = ['find', path, '-iregex', '.*/frames/.*', '-atime', '+%s' % days, '-type', 'f', '-exec', 'rm', '{}', ';']
+    subprocess.check_output(cmd)
+    path = os.path.join(settings.MEDIA_ROOT, 'items')
+    cmd = ['find', path, '-iregex', '.*/cache/.*', '-atime', '+%s' % days, '-type', 'f', '-exec', 'rm', '{}', ';']
+    subprocess.check_output(cmd)
+    path = settings.MEDIA_ROOT
+    cmd = ['find', path, '-type', 'd', '-size', '0', '-prune', '-exec', 'rmdir', '{}', ';']
+    subprocess.check_output(cmd)
+
+
 @task(ignore_results=True, queue='default')
 def update_sitemap(base_url):
     from . import models
@@ -133,13 +161,47 @@ def update_sitemap(base_url):
     def absolute_url(url):
         return base_url + url
 
+    state = {}
+    state['part'] = 1
+    state['count'] = 0
+
+    def new_urlset():
         urlset = ET.Element('urlset')
         urlset.attrib['xmlns'] = "http://www.sitemaps.org/schemas/sitemap/0.9"
         urlset.attrib['xmlns:xsi'] = "http://www.w3.org/2001/XMLSchema-instance"
         urlset.attrib['xsi:schemaLocation'] = "http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"
         urlset.attrib['xmlns:video'] = "http://www.google.com/schemas/sitemap-video/1.1"
+        return urlset
 
-    url = ET.SubElement(urlset, "url")
+    def save_urlset():
+        s = ET.SubElement(sitemap_index, "sitemap")
+        loc = ET.SubElement(s, "loc")
+        loc.text = absolute_url("sitemap%06d.xml" % state['part'])
+        lastmod = ET.SubElement(s, "lastmod")
+        lastmod.text = datetime.now().strftime("%Y-%m-%d")
+        data = b'<?xml version="1.0" encoding="UTF-8"?>\n' + ET.tostring(state['urlset'])
+        path = os.path.abspath(os.path.join(settings.MEDIA_ROOT, 'sitemap%06d.xml.gz' % state['part']))
+        with open(path[:-3], 'wb') as f:
+            f.write(data)
+        with gzip.open(path, 'wb') as f:
+            f.write(data)
+        state['part'] += 1
+        state['count'] = 0
+        state['urlset'] = new_urlset()
+
+    def tick():
+        state['count'] += 1
+        if state['count'] > 40000:
+            save_urlset()
+
+    sitemap_index = ET.Element('sitemapindex')
+    sitemap_index.attrib['xmlns'] = "http://www.sitemaps.org/schemas/sitemap/0.9"
+    sitemap_index.attrib['xmlns:xsi'] = "http://www.w3.org/2001/XMLSchema-instance"
+    sitemap_index.attrib['xsi:schemaLocation'] = "http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"
+
+    state['urlset'] = new_urlset()
+
+    url = ET.SubElement(state['urlset'], "url")
     loc = ET.SubElement(url, "loc")
     loc.text = absolute_url('') 
     # always, hourly, daily, weekly, monthly, yearly, never
@@ -151,9 +213,10 @@ def update_sitemap(base_url):
     # priority of page on site values 0.1 - 1.0
     priority = ET.SubElement(url, "priority")
     priority.text = '1.0'
+    tick()
 
     for page in [s['id'] for s in settings.CONFIG['sitePages']]:
-        url = ET.SubElement(urlset, "url")
+        url = ET.SubElement(state['urlset'], "url")
         loc = ET.SubElement(url, "loc")
         loc.text = absolute_url(page)
         # always, hourly, daily, weekly, monthly, yearly, never
@@ -162,11 +225,12 @@ def update_sitemap(base_url):
         # priority of page on site values 0.1 - 1.0
         priority = ET.SubElement(url, "priority")
         priority.text = '1.0'
+        tick()
 
     allowed_level = settings.CONFIG['capabilities']['canSeeItem']['guest']
     can_play = settings.CONFIG['capabilities']['canPlayVideo']['guest']
     for i in models.Item.objects.filter(level__lte=allowed_level):
-        url = ET.SubElement(urlset, "url")
+        url = ET.SubElement(state['urlset'], "url")
         # URL of the page. This URL must begin with the protocol (such as http)
         loc = ET.SubElement(url, "loc")
         loc.text = absolute_url("%s/info" % i.public_id)
@@ -202,11 +266,12 @@ def update_sitemap(base_url):
                 el.text = "%s" % int(duration)
             el = ET.SubElement(video, "video:live")
             el.text = "no"
+        tick()
 
     # Featured Lists
     from itemlist.models import List
     for l in List.objects.filter(Q(status='featured') | Q(status='public')):
-        url = ET.SubElement(urlset, "url")
+        url = ET.SubElement(state['urlset'], "url")
         # URL of the page. This URL must begin with the protocol (such as http)
         loc = ET.SubElement(url, "loc")
         loc.text = absolute_url("list==%s" % quote(l.get_id()))
@@ -220,10 +285,12 @@ def update_sitemap(base_url):
         # priority of page on site values 0.1 - 1.0
         priority = ET.SubElement(url, "priority")
         priority.text = '1.0' if l.status == 'featured' else '0.75'
+        tick()
+
     # Featured Edits
     from edit.models import Edit
     for l in Edit.objects.filter(Q(status='featured') | Q(status='public')):
-        url = ET.SubElement(urlset, "url")
+        url = ET.SubElement(state['urlset'], "url")
         # URL of the page. This URL must begin with the protocol (such as http)
         loc = ET.SubElement(url, "loc")
         loc.text = absolute_url(l.get_absolute_url()[1:])
@@ -237,10 +304,12 @@ def update_sitemap(base_url):
         # priority of page on site values 0.1 - 1.0
         priority = ET.SubElement(url, "priority")
         priority.text = '1.0' if l.status == 'featured' else '0.75'
+        tick()
+
     # Featured Collections
     from documentcollection.models import Collection
     for l in Collection.objects.filter(Q(status='featured') | Q(status='public')):
-        url = ET.SubElement(urlset, "url")
+        url = ET.SubElement(state['urlset'], "url")
         # URL of the page. This URL must begin with the protocol (such as http)
         loc = ET.SubElement(url, "loc")
         loc.text = absolute_url("documents/collection==%s" % quote(l.get_id()))
@@ -254,10 +323,11 @@ def update_sitemap(base_url):
         # priority of page on site values 0.1 - 1.0
         priority = ET.SubElement(url, "priority")
         priority.text = '1.0' if l.status == 'featured' else '0.75'
+        tick()
 
     from document.models import Document
     for d in Document.objects.filter(rightslevel=0).filter(Q(extension='html') | Q(extension='pdf')):
-        url = ET.SubElement(urlset, "url")
+        url = ET.SubElement(state['urlset'], "url")
         # URL of the page. This URL must begin with the protocol (such as http)
         loc = ET.SubElement(url, "loc")
         loc.text = absolute_url(d.get_id())
@@ -273,8 +343,10 @@ def update_sitemap(base_url):
         priority.text = '0.75'
         if d.collections.filter(Q(status='featured') | Q(status='public')).count():
             priority.text = '1.0'
-
-    data = b'<?xml version="1.0" encoding="UTF-8"?>\n' + ET.tostring(urlset)
+        tick()
+    if state['count']:
+        save_urlset()
+    data = b'<?xml version="1.0" encoding="UTF-8"?>\n' + ET.tostring(sitemap_index)
     with open(sitemap[:-3], 'wb') as f:
         f.write(data)
     with gzip.open(sitemap, 'wb') as f:

pandora/item/urls.py

@@ -16,6 +16,7 @@ urlpatterns = [
     url(r'^(?P<id>[A-Z0-9].*)/download$', views.download),
     url(r'^(?P<id>[A-Z0-9].*)/download/$', views.download),
     url(r'^(?P<id>[A-Z0-9].*)/download/source/(?P<part>\d+)?$', views.download_source),
+    url(r'^(?P<id>[A-Z0-9].*)/download/(?P<resolution>\d+)p(?P<part>\d+)\.(?P<format>webm|ogv|mp4)$', views.download),
     url(r'^(?P<id>[A-Z0-9].*)/download/(?P<resolution>\d+)p\.(?P<format>webm|ogv|mp4)$', views.download),
 
     #video

pandora/item/utils.py

@@ -103,3 +103,7 @@ def normalize_dict(encoding, data):
     elif isinstance(data, list):
         return [normalize_dict(encoding, value) for value in data]
     return data
+
+
+def is_imdb_id(id):
+    return bool(len(id) >= 7 and str(id).isdigit())

pandora/item/views.py

@@ -638,6 +638,32 @@ def edit(request, data):
     return render_to_json_response(response)
 actions.register(edit, cache=False)
 
+
+def extractClip(request, data):
+    '''
+    Extract and cache clip
+
+    takes {
+        item: string
+        resolution: int
+        format: string
+        in: float
+        out: float
+    }
+    returns {
+        taskId: string, // taskId
+    }
+    '''
+    item = get_object_or_404_json(models.Item, public_id=data['item'])
+    if not item.access(request.user):
+        return HttpResponseForbidden()
+
+    response = json_response()
+    t = tasks.extract_clip.delay(data['item'], data['in'], data['out'], data['resolution'], data['format'])
+    response['data']['taskId'] = t.task_id
+    return render_to_json_response(response)
+actions.register(extractClip, cache=False)
+
 @login_required_json
 def remove(request, data):
     '''
@@ -966,6 +992,8 @@ def download_source(request, id, part=None):
         raise Http404
 
     parts = ['%s - %s ' % (item.get('title'), settings.SITENAME), item.public_id]
+    if len(streams) > 1:
+        parts.append('.Part %d' % (part + 1))
     parts.append('.')
     parts.append(f.extension)
     filename = ''.join(parts)
@@ -976,7 +1004,7 @@ def download_source(request, id, part=None):
     response['Content-Disposition'] = "attachment; filename*=UTF-8''%s" % quote(filename.encode('utf-8'))
     return response
 
-def download(request, id, resolution=None, format='webm'):
+def download(request, id, resolution=None, format='webm', part=None):
     item = get_object_or_404(models.Item, public_id=id)
     if not resolution or int(resolution) not in settings.CONFIG['video']['resolutions']:
         resolution = max(settings.CONFIG['video']['resolutions'])
@@ -984,14 +1012,22 @@ def download(request, id, resolution=None, format='webm'):
         resolution = int(resolution)
     if not item.access(request.user) or not item.rendered:
         return HttpResponseForbidden()
+    if part is not None:
+        part = int(part) - 1
+        streams = item.streams()
+        if part > len(streams):
+            raise Http404
     ext = '.%s' % format
     parts = ['%s - %s ' % (item.get('title'), settings.SITENAME), item.public_id]
     if resolution != max(settings.CONFIG['video']['resolutions']):
         parts.append('.%dp' % resolution)
+    if part is not None:
+        parts.append('.Part %d' % (part + 1))
     parts.append(ext)
     filename = ''.join(parts)
     video = NamedTemporaryFile(suffix=ext)
     content_type = mimetypes.guess_type(video.name)[0]
+    if part is None:
         r = item.merge_streams(video.name, resolution, format)
         if not r:
             return HttpResponseForbidden()
@@ -1000,6 +1036,11 @@ def download(request, id, resolution=None, format='webm'):
             response['Content-Length'] = os.path.getsize(video.name)
         else:
             response = HttpFileResponse(r, content_type=content_type)
+    else:
+        stream = streams[part].get(resolution, format)
+        path = stream.media.path
+        content_type = mimetypes.guess_type(path)[0]
+        response = HttpFileResponse(path, content_type=content_type)
     response['Content-Disposition'] = "attachment; filename*=UTF-8''%s" % quote(filename.encode('utf-8'))
     return response
 
@@ -1056,6 +1097,23 @@ def video(request, id, resolution, format, index=None, track=None):
         ext = '.%s' % format
         duration = stream.info['duration']
 
+        filename = u"Clip of %s - %s-%s - %s %s%s" % (
+            item.get('title'),
+            ox.format_duration(t[0] * 1000).replace(':', '.')[:-4],
+            ox.format_duration(t[1] * 1000).replace(':', '.')[:-4],
+            settings.SITENAME.replace('/', '-'),
+            item.public_id,
+            ext
+        )
+        content_type = mimetypes.guess_type(path)[0]
+
+        cache_name = '%s_%sp_%s.%s' % (item.public_id, resolution, '%s,%s' % (t[0], t[1]), format)
+        cache_path = os.path.join(settings.MEDIA_ROOT, item.path('cache/%s' % cache_name))
+        if os.path.exists(cache_path):
+            response = HttpFileResponse(cache_path, content_type=content_type)
+            response['Content-Disposition'] = "attachment; filename*=UTF-8''%s" % quote(filename.encode('utf-8'))
+            return response
+
         # multipart request beyond first part, merge parts and chop that
         if not index and streams.count() > 1 and stream.info['duration'] < t[1]:
             video = NamedTemporaryFile(suffix=ext)
@@ -1065,7 +1123,6 @@ def video(request, id, resolution, format, index=None, track=None):
             path = video.name
             duration = sum(item.cache['durations'])
 
-        content_type = mimetypes.guess_type(path)[0]
         if len(t) == 2 and t[1] > t[0] and duration >= t[1]:
             # FIXME: could be multilingual here
             subtitles = utils.get_by_key(settings.CONFIG['layers'], 'isSubtitles', True)
@@ -1076,20 +1133,12 @@ def video(request, id, resolution, format, index=None, track=None):
             else:
                 srt = None
             response = HttpResponse(extract.chop(path, t[0], t[1], subtitles=srt), content_type=content_type)
-            filename = u"Clip of %s - %s-%s - %s %s%s" % (
-                item.get('title'),
-                ox.format_duration(t[0] * 1000).replace(':', '.')[:-4],
-                ox.format_duration(t[1] * 1000).replace(':', '.')[:-4],
-                settings.SITENAME,
-                item.public_id,
-                ext
-            )
             response['Content-Disposition'] = "attachment; filename*=UTF-8''%s" % quote(filename.encode('utf-8'))
             return response
         else:
             filename = "%s - %s %s%s" % (
                 item.get('title'),
-                settings.SITENAME,
+                settings.SITENAME.replace('/', '-'),
                 item.public_id,
                 ext
             )
@@ -1326,6 +1375,15 @@ def sitemap_xml(request):
     response['Content-Type'] = 'application/xml'
     return response
 
+def sitemap_part_xml(request, part):
+    part = int(part)
+    sitemap = os.path.abspath(os.path.join(settings.MEDIA_ROOT, 'sitemap%06d.xml' % part))
+    if not os.path.exists(sitemap):
+        raise Http404
+    response = HttpFileResponse(sitemap)
+    response['Content-Type'] = 'application/xml'
+    return response
+
 def item_json(request, id):
     level = settings.CONFIG['capabilities']['canSeeItem']['guest']
     if not request.user.is_anonymous():

pandora/itemlist/models.py

@@ -271,6 +271,7 @@ class List(models.Model):
                 self.save()
         for i in self.poster_frames:
             from item.models import Item
+            if 'item' in i:
                 qs = Item.objects.filter(public_id=i['item'])
                 if qs.count() > 0:
                     if i.get('position'):

pandora/manage.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import os
 import signal
 import sys

pandora/person/migrations/0002_auto_20190723_1446.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.22 on 2019-07-23 14:46
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('person', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='person',
+            name='imdbId',
+            field=models.CharField(blank=True, max_length=16),
+        ),
+    ]

pandora/person/models.py

@@ -38,7 +38,7 @@ class Person(models.Model):
     #FIXME: how to deal with aliases
     aliases = fields.TupleField(default=[])
 
-    imdbId = models.CharField(max_length=7, blank=True)
+    imdbId = models.CharField(max_length=16, blank=True)
     wikipediaId = models.CharField(max_length=1000, blank=True)
 
     objects = managers.PersonManager()

pandora/place/managers.py

@@ -141,4 +141,5 @@ class PlaceManager(Manager):
                                      user)
         if conditions:
             qs = qs.filter(conditions)
+        qs = qs.distinct()
         return qs

pandora/place/views.py

@@ -239,8 +239,8 @@ def findPlaces(request, data):
     qs = order_query(query['qs'], query['sort'])
     qs = qs.distinct()
     if 'keys' in data:
-        qs = qs.select_related('user__profile')
         qs = qs[query['range'][0]:query['range'][1]]
+        qs = qs.select_related()
         response['data']['items'] = [p.json(data['keys'], request.user) for p in qs]
     elif 'position' in query:
         ids = [i.get_id() for i in qs]

pandora/settings.py

@@ -178,6 +178,7 @@ CACHES = {
 AUTH_PROFILE_MODULE = 'user.UserProfile'
 AUTH_CHECK_USERNAME = True
 FFMPEG = 'ffmpeg'
+FFPROBE = 'ffprobe'
 FFMPEG_SUPPORTS_VP9 = True
 FFMPEG_DEBUG = False
 
@@ -204,6 +205,9 @@ CELERY_BROKER_URL = 'amqp://pandora:box@localhost:5672//pandora'
 
 SEND_CELERY_ERROR_EMAILS = False
 
+# Elasticsearch
+ELASTICSEARCH_HOST = None
+
 #with apache x-sendfile or lighttpd set this to True
 XSENDFILE = False
 

pandora/taskqueue/models.py

@@ -183,13 +183,17 @@ class Task(models.Model):
     def json(self):
         if self.status != 'canceled':
             self.update()
-        return {
+        data = {
             'started': self.started,
             'ended': self.ended,
             'status': self.status,
-            'title': self.item.get('title'),
-            'item': self.item.public_id,
-            'user': self.user and self.user.username or '',
             'id': self.public_id,
+            'user': self.user and self.user.username or '',
         }
+        try:
+            data['title'] = self.item.get('title')
+            data['item'] = self.item.public_id
+        except:
+            pass
+        return data
 

pandora/text/templates/pdf/viewer.html

@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+<!DOCTYPE html>
 <!--
 Copyright 2012 Mozilla Foundation
 
@@ -41,9 +41,6 @@ See https://github.com/adobe-type-tools/cmap-resources
 <script src="/static/pdf.js/l10n.js"></script>
 <script src="/static/pdf.js/pdf.js"></script>
 
-
-
-    <script src="/static/pdf.js/debugger.js"></script>
     <script type="text/javascript">
         var DEFAULT_URL = '{{url}}',
             embeds = {{embeds|safe}},

pandora/title/migrations/0002_auto_20190723_1446.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.22 on 2019-07-23 14:46
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('title', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='title',
+            name='imdbId',
+            field=models.CharField(blank=True, max_length=16),
+        ),
+    ]

pandora/title/models.py

@@ -32,7 +32,7 @@ class Title(models.Model):
     sortsorttitle = models.CharField(max_length=1000)
     edited = models.BooleanField(default=False)
 
-    imdbId = models.CharField(max_length=7, blank=True)
+    imdbId = models.CharField(max_length=16, blank=True)
 
     objects = managers.TitleManager()
 

pandora/urls.py

@@ -64,6 +64,7 @@ urlpatterns = [
     url(r'^atom.xml$', item.views.atom_xml),
     url(r'^robots.txt$', app.views.robots_txt),
     url(r'^sitemap.xml$', item.views.sitemap_xml),
+    url(r'^sitemap(?P<part>\d+).xml$', item.views.sitemap_part_xml),
     url(r'', include(item.urls)),
 ]
 #sould this not be enabled by default? nginx should handle those

pandora/user/views.py

@@ -6,7 +6,7 @@ random.seed()
 import re
 import json
 
-from django.contrib.auth import authenticate, login, logout
+from django.contrib.auth import authenticate, login, logout, update_session_auth_hash
 from django.template import loader
 from django.conf import settings
 from django.core.mail import send_mail, BadHeaderError, EmailMessage
@@ -719,7 +719,9 @@ def editPreferences(request, data):
         profile.save()
     if 'password' in data:
         change = True
-        request.user.set_password(data['password'])
+        user = request.user
+        user.set_password(data['password'])
+        update_session_auth_hash(request, user)
     if 'script' in data:
         profile = request.user.profile
         profile.preferences['script'] = data['script']

requirements.txt

@@ -1,4 +1,4 @@
-Django==1.11.22
+Django==1.11.28
 simplejson
 chardet
 celery>4
@@ -11,3 +11,4 @@ tornado<5
 geoip2==2.9.0
 youtube-dl>=2019.4.30
 python-memcached
+elasticsearch

static/js/addFilesDialog.js

@@ -8,6 +8,11 @@ pandora.ui.addFilesDialog = function(options) {
     }).bindEvent({
         click: function() {
             $button.options({disabled: true});
+            that.disableCloseButton()
+            var $screen = Ox.LoadingScreen({
+                size: 16
+            });
+            that.options({content: $screen.start()});
             (options.action == 'upload' ? uploadVideos : importVideos)(function() {
                 that.close();
                 pandora.ui.tasksDialog({
@@ -117,14 +122,6 @@ pandora.ui.addFilesDialog = function(options) {
             )
         });
     } else {
-        selectItems.push({
-            id: 'one',
-            title: Ox._(
-                'Create one {0} with multiple parts',
-                [pandora.site.itemName.singular.toLowerCase()]
-            )
-        });
-    }
         if (options.items.length > 1) {
             selectItems.push({
                 id: 'multiple',
@@ -134,6 +131,14 @@ pandora.ui.addFilesDialog = function(options) {
                 )
             });
         }
+        selectItems.push({
+            id: 'one',
+            title: Ox._(
+                'Create one {0} with multiple parts',
+                [pandora.site.itemName.singular.toLowerCase()]
+            )
+        });
+    }
     var $select = Ox.Select({
         items: selectItems,
         width: 256
@@ -144,6 +149,49 @@ pandora.ui.addFilesDialog = function(options) {
     $($select.find('.OxButton')[0]).css({margin: '-1px'});
     $button.parent().parent().append($select);
 
+    function getNewOrEmptyItem(data, callback) {
+        pandora.api.find({
+            query: {
+                conditions: [
+                    {key: 'title', value: data.title, operator: '=='}
+                ]
+            },
+            keys: ['id']
+        }, function(result) {
+            if (!result.data.items.length) {
+                pandora.api.add(data, callback)
+            } else {
+                var isNew = true
+                Ox.serialForEach(result.data.items, function(item, index, items, next) {
+                    isNew && pandora.api.findMedia({
+                        query: {
+                            conditions: [
+                                {key: 'id', value: item.id, operator: '=='}
+                            ]
+                        },
+                        keys: ['id']
+                    }, function(result) {
+                        if (!result.data.items.length) {
+                            isNew = false
+                            callback({
+                                data: {
+                                    title: data.title,
+                                    id: item.id
+                                }
+                            })
+                        }
+                        next()
+                    })
+                }, function() {
+                    if (isNew) {
+                        pandora.api.add(data, callback)
+                    }
+                })
+
+            }
+        })
+    }
+
     function importVideos(callback) {
         var id, title;
         ($select.value() == 'add' ? pandora.api.get : Ox.noop)({
@@ -161,7 +209,7 @@ pandora.ui.addFilesDialog = function(options) {
                 } else {
                     title = items[$select.value() == 'one' ? 0 : index].title;
                 }
-                (isNewItem ? pandora.api.add : Ox.noop)({
+                (isNewItem ? getNewOrEmptyItem : Ox.noop)({
                     title: title
                 }, function(result) {
                     if (isNewItem) {
@@ -201,7 +249,7 @@ pandora.ui.addFilesDialog = function(options) {
                 } else {
                     title = items[$select.value() == 'one' ? 0 : index].title;
                 }
-                (isNewItem ? pandora.api.add : Ox.noop)({
+                (isNewItem ? getNewOrEmptyItem : Ox.noop)({
                     title: title
                 }, function(result) {
                     if (isNewItem) {

static/js/addItemDialog.js

@@ -69,6 +69,15 @@ pandora.ui.addItemDialog = function(options) {
         title: Ox._('Add {0}', [pandora.site.itemName.singular]),
         width: 544
     });
+    if (options.files) {
+        that.options({content: $screen.start()});
+        $button.options({disabled: true});
+        Ox.serialMap(options.files, function(file, index, files, callback) {
+            getFileInfo(file, function(info) {
+                callback(Ox.extend(info, {file: file}));
+            });
+        }, onInfo);
+    }
 
     function createButton() {
         $button = Ox[selected == 'upload' ? 'FileButton' : 'Button']({
@@ -221,6 +230,7 @@ pandora.ui.addItemDialog = function(options) {
         // FIXME: what about pending/aborted uploads
         pandora.api.findMedia({
             keys: ['id', 'item', 'url'],
+            range: [0, items.length],
             query: {
                 conditions: selected == 'upload' ? items.map(function(item) {
                     return {key: 'oshash', operator: '==', value: item.oshash};

static/js/document.js

@@ -66,7 +66,7 @@ pandora.ui.document = function() {
                         ? pandora.user.ui.documents[item.id].position
                         : 1,
                     url: '/documents/' + item.id + '/'
-                        + item.title.replace('?', '_') + '.' + item.extension,
+                        + pandora.safeDocumentName(item.title) + '.' + item.extension,
                     width: that.width(),
                     zoom: 'fit'
                 })
@@ -80,7 +80,7 @@ pandora.ui.document = function() {
                     imageHeight: item.dimensions[1],
                     imagePreviewURL: pandora.getMediaURL('/documents/' + item.id + '/256p.jpg?' + item.modified),
                     imageURL: pandora.getMediaURL('/documents/' + item.id + '/'
-                        + item.title + '.' + item.extension + '?' + item.modified),
+                        + pandora.safeDocumentName(item.title) + '.' + item.extension + '?' + item.modified),
                     imageWidth: item.dimensions[0],
                     width: that.width()
                 }).css({

static/js/documentDialog.js

@@ -196,13 +196,12 @@ pandora.ui.documentDialog = function(options) {
                         ? pandora.user.ui.documents[item.id].position
                         : 1,
                     url: '/documents/' + item.id + '/'
-                        + item.title.replace('?', '_') + '.' + item.extension,
+                        + pandora.safeDocumentName(item.title) + '.' + item.extension,
                     width: dialogWidth,
                     zoom: 'fit'
                 })
                 : item.extension == 'html'
-                ? pandora.ui.textPanel(item).css({
-                })
+                ? pandora.$ui.textPanel = pandora.ui.textPanel(item)
                 : Ox.ImageViewer({
                     area: pandora.user.ui.documents[item.id]
                         ? pandora.user.ui.documents[item.id].position
@@ -211,7 +210,7 @@ pandora.ui.documentDialog = function(options) {
                     imageHeight: item.dimensions[1],
                     imagePreviewURL: pandora.getMediaURL('/documents/' + item.id + '/256p.jpg?' + item.modified),
                     imageURL: pandora.getMediaURL('/documents/' + item.id + '/'
-                        + item.title + '.' + item.extension + '?' + item.modified),
+                        + pandora.safeDocumentName(item.title) + '.' + item.extension + '?' + item.modified),
                     imageWidth: item.dimensions[0],
                     width: dialogWidth
                 })
@@ -243,7 +242,7 @@ pandora.ui.documentDialog = function(options) {
     }
 
     function setTitle() {
-        that.options({title: item.title + '.' + item.extension});
+        that.options({title: item.title + (item.extension == 'html' ? '' : '.' + item.extension)});
     }
 
     that.getItems = function() {

static/js/documentFilter.js

@@ -4,7 +4,7 @@ pandora.ui.documentFilter = function(id) {
     var i = Ox.getIndexById(pandora.user.ui.documentFilters, id),
         filter = Ox.getObjectById(pandora.site.documentFilters, id),
         panelWidth = Ox.$document.width() - (pandora.user.ui.showSidebar * pandora.user.ui.sidebarSize) - 1,
-        title = Ox._(Ox.getObjectById(pandora.site.documentFilters, id).title),
+        title = Ox._(filter.title),
         //width = pandora.getFilterWidth(i, panelWidth),
         that = Ox.TableList({
             _selected: !pandora.user.ui.showFilters

static/js/documentFilterForm.js

@@ -27,7 +27,7 @@ pandora.ui.documentFilterForm = function(options) {
                     if (key.format && key.format.type == 'ColorPercent') {
                         key.format.type = 'percent';
                     }
-                    Ox.print(key);
+                    key.autocomplete = autocompleteFunction(key)
                     return key;
                 }).concat([{
                     id: 'collection',
@@ -37,6 +37,7 @@ pandora.ui.documentFilterForm = function(options) {
                         return item.id;
                     })
                 }]),
+                listName: Ox._('Collection'),
                 list: mode == 'find' ? {
                     sort: pandora.user.ui.collectionSort,
                     view: pandora.user.ui.collectionView
@@ -69,6 +70,24 @@ pandora.ui.documentFilterForm = function(options) {
         that.getList = that.$filter.getList;
         that.value = that.$filter.value;
     });
+    function autocompleteFunction(key) {
+        return key.autocomplete ? function(value, callback) {
+            pandora.api.autocomplete({
+                key: key.id,
+                query: {
+                    conditions: [],
+                    operator: '&'
+                },
+                range: [0, 100],
+                sort: key.autocompleteSort,
+                value: value
+            }, function(result) {
+                callback(result.data.items.map(function(item) {
+                    return Ox.decodeHTMLEntities(item);
+                }));
+            });
+        } : null;
+    }
     that.updateResults = function() {
         if (mode == 'collection') {
             Ox.Request.clearCache(collection.id);
@@ -81,11 +100,9 @@ pandora.ui.documentFilterForm = function(options) {
                     }
                 })
                 .reloadList();
-            /*
-            pandora.$ui.filters && pandora.$ui.filters.forEach(function($filter) {
+            pandora.$ui.documentFilters && pandora.$ui.documentFilters.forEach(function($filter) {
                 $filter.reloadList();
             });
-            */
         } else {
             pandora.UI.set({findDocuments: Ox.clone(that.$filter.options('value'), true)});
             pandora.$ui.findElement.updateElement();

static/js/documentInfoView.js

@@ -28,6 +28,13 @@ pandora.ui.documentInfoView = function(data, isMixed) {
         }).map(function(key){
             return key.id;
         }),
+        displayedKeys = [ // FIXME: can tis be a flag in the config?
+            'title', 'notes', 'name', 'description', 'id',
+            'user', 'rightslevel', 'timesaccessed',
+            'extension', 'dimensions', 'size', 'matches',
+            'created', 'modified', 'accessed',
+            'random', 'entity'
+        ],
         statisticsWidth = 128,
 
         $bar = Ox.Bar({size: 16})
@@ -126,7 +133,7 @@ pandora.ui.documentInfoView = function(data, isMixed) {
                     height: iconHeight + 'px'
                 })
                 .bindEvent({
-                    // singleclick: toggleIconSize
+                    singleclick: toggleIconSize
                 })
                 .appendTo($info),
 
@@ -234,6 +241,10 @@ pandora.ui.documentInfoView = function(data, isMixed) {
 
     Ox.getObjectById(pandora.site.documentKeys, 'keywords') && renderGroup(['keywords'])
 
+    // Render any remaing keys defined in config
+
+    renderRemainingKeys();
+
 
     // Description -------------------------------------------------------------
 
@@ -321,6 +332,7 @@ pandora.ui.documentInfoView = function(data, isMixed) {
 
     }
 
+
     // Extension, Dimensions, Size ---------------------------------------------
 
     ['extension', 'dimensions', 'size'].forEach(function(key) {
@@ -533,6 +545,7 @@ pandora.ui.documentInfoView = function(data, isMixed) {
 
     function renderGroup(keys) {
         var $element;
+        keys.forEach(function(key) { displayedKeys.push(key) });
         if (canEdit || keys.filter(function(key) {
             return data[key];
         }).length) {
@@ -565,6 +578,17 @@ pandora.ui.documentInfoView = function(data, isMixed) {
         return $element;
     }
 
+    function renderRemainingKeys() {
+        var keys = pandora.site.documentKeys.filter(function(item) {
+            return item.id != '*' && !Ox.contains(displayedKeys, item.id);
+        }).map(function(item) {
+            return item.id;
+        });
+        if (keys.length) {
+            renderGroup(keys)
+        }
+    }
+
     function renderRightsLevel() {
         var $rightsLevelElement = getRightsLevelElement(data.rightslevel),
             $rightsLevelSelect;
@@ -612,6 +636,36 @@ pandora.ui.documentInfoView = function(data, isMixed) {
         //renderCapabilities(data.rightslevel);
     }
 
+    function toggleIconSize() {
+        iconSize = iconSize == 256 ? 512 : 256;
+        iconWidth = iconRatio > 1 ? iconSize : Math.round(iconSize * iconRatio);
+        iconHeight = iconRatio < 1 ? iconSize : Math.round(iconSize / iconRatio);
+        iconLeft = iconSize == 256 ? Math.floor((iconSize - iconWidth) / 2) : 0,
+        $icon.animate({
+            left: margin + iconLeft + 'px',
+            width: iconWidth + 'px',
+            height: iconHeight + 'px',
+        }, 250);
+        $reflection.animate({
+            top: margin + iconHeight + 'px',
+            width: iconSize + 'px',
+            height: iconSize / 2 + 'px'
+        }, 250);
+        $reflectionIcon.animate({
+            left: iconLeft + 'px',
+            width: iconWidth + 'px',
+            height: iconHeight + 'px',
+        }, 250);
+        $reflectionGradient.animate({
+            width: iconSize + 'px',
+            height: iconSize / 2 + 'px'
+        }, 250);
+        $text.animate({
+            left: margin + (iconSize == 256 ? 256 : iconWidth) + margin + 'px'
+        }, 250);
+        pandora.UI.set({infoIconSize: iconSize});
+    }
+
     that.reload = function() {
         /*
         var src = '/documents/' + data.id + '/512p.jpg?' + data.modified;

static/js/documentsPanel.js

@@ -1,104 +1,5 @@
 'use strict';
 
-pandora.documentColumns = [
-    {
-        id: 'title',
-        operator: '+',
-        title: Ox._('Title'),
-        find: true,
-        visible: true,
-        width: 256
-    },
-    {
-        id: 'id',
-        operator: '+',
-        title: Ox._('ID'),
-        visible: true,
-        width: 64
-    },
-    {
-        format: function(value) {
-            return value.toUpperCase();
-        },
-        id: 'extension',
-        operator: '+',
-        title: Ox._('Extension'),
-        find: true,
-        visible: true,
-        width: 64
-    },
-    {
-        align: 'right',
-        format: function(value, data) {
-            return Ox.isArray(value)
-                ? Ox.formatDimensions(value, 'px')
-                : Ox.formatCount(value, (data && data.extension == 'html') ? 'word' : 'page');
-        },
-        id: 'dimensions',
-        operator: '-',
-        title: Ox._('Dimensions'),
-        visible: true,
-        width: 128
-    },
-    {
-        align: 'right',
-        format: function(value) {
-            return Ox.formatValue(value, 'B');
-        },
-        id: 'size',
-        operator: '-',
-        title: Ox._('Size'),
-        visible: true,
-        width: 64
-    },
-    {
-        id: 'description',
-        operator: '+',
-        title: Ox._('Description'),
-        find: true,
-        visible: true,
-        width: 256
-    },
-    {
-        align: 'right',
-        id: 'matches',
-        operator: '-',
-        title: Ox._('Matches'),
-        visible: true,
-        width: 64
-    },
-    {
-        id: 'user',
-        operator: '+',
-        title: Ox._('User'),
-        find: true,
-        visible: true,
-        width: 128
-    },
-    {
-        align: 'right',
-        format: function(value) {
-            return Ox.formatDate(value, '%F %T');
-        },
-        id: 'created',
-        operator: '-',
-        title: Ox._('Created'),
-        visible: true,
-        width: 144
-    },
-    {
-        align: 'right',
-        format: function(value) {
-            return Ox.formatDate(value, '%F %T');
-        },
-        id: 'modified',
-        operator: '-',
-        title: Ox._('Modified'),
-        visible: true,
-        width: 144
-    }
-];
-
 pandora.ui.documentSortSelect = function() {
     var ui = pandora.user.ui,
         $orderButton = Ox.Button({
@@ -116,7 +17,9 @@ pandora.ui.documentSortSelect = function() {
             }
         }),
         $sortSelect = Ox.Select({
-            items: pandora.documentColumns.map(function(column) {
+            items: pandora.site.documentKeys.filter(function(key) {
+                return key.sort;
+            }).map(function(column) {
                 return {
                     id: column.id,
                     title: Ox._('Sort by {0}', [column.title])
@@ -130,7 +33,7 @@ pandora.ui.documentSortSelect = function() {
                 var key = data.value;
                 pandora.UI.set({documentsSort: [{
                     key: key,
-                    operator: Ox.getObjectById(pandora.documentColumns, key).operator
+                    operator: Ox.getObjectById(pandora.site.documentKeys, key).operator
                 }]});
             }
         }),
@@ -875,7 +778,32 @@ pandora.ui.documentsPanel = function(options) {
                 unique: 'id'
             };
         return (ui.documentsView == 'list' ? Ox.TableList(Ox.extend(options, {
-            columns: pandora.documentColumns,
+            columns: pandora.site.documentSortKeys.filter(function(key) {
+                return (!key.capability
+                    || pandora.hasCapability(key.capability)) && key.columnWidth;
+            }).map(function(key) {
+                var position = ui.collectionColumns.indexOf(key.id);
+                return {
+                    addable: key.id != 'random',
+                    align: ['string', 'text'].indexOf(
+                        Ox.isArray(key.type) ? key.type[0]: key.type
+                    ) > -1 ? 'left' : key.type == 'list' ? 'center' : 'right',
+                    defaultWidth: key.columnWidth,
+                    format: (function() {
+                        return function(value, data) {
+                            return pandora.formatDocumentKey(key, data);
+                        }
+                    })(),
+                    id: key.id,
+                    operator: pandora.getDocumentSortOperator(key.id),
+                    position: position,
+                    removable: !key.columnRequired,
+                    title: Ox._(key.title),
+                    type: key.type,
+                    visible: position > -1,
+                    width: ui.collectionColumnWidth[key.id] || key.columnWidth
+                };
+            }),
             columnsVisible: true,
             scrollbarVisible: true,
         })) : Ox.IconList(Ox.extend(options, {
@@ -883,7 +811,7 @@ pandora.ui.documentsPanel = function(options) {
                 var sortKey = sort[0].key,
                     infoKey = sortKey == 'title' ? 'extension' : sortKey,
                     info = (
-                        Ox.getObjectById(pandora.documentColumns, infoKey).format || Ox.identity
+                        Ox.getObjectById(pandora.site.documentKeys, infoKey).format || Ox.identity
                     )(data[infoKey]),
                     size = size || 128;
                 return {

static/js/downloadVideoDialog.js

@@ -9,6 +9,10 @@ pandora.ui.downloadVideoDialog = function(options) {
             'mp4': 'MP4',
         },
 
+        parts = options.out ? null : Ox.max(options.video.map(function(video) {
+            return video.index
+        })),
+
         $content = Ox.Element()
             .css({margin: '16px'}),
 
@@ -27,6 +31,9 @@ pandora.ui.downloadVideoDialog = function(options) {
             .css({marginBottom: '16px'})
             .appendTo($content),
 
+        $format,
+        $resolution,
+
         $form = window.$form = Ox.Form({
             items: [
                 Ox.Select({
@@ -36,7 +43,10 @@ pandora.ui.downloadVideoDialog = function(options) {
                                 id: format,
                                 title: formats[format]
                             };
-                        }),
+                        }).concat(!options.out && options.source ? [{
+                            id: 'source',
+                            title: Ox._('Source')
+                        }] : []),
                         label: Ox._('Format'),
                         labelWidth: 120,
                         value: pandora.site.video.downloadFormat,
@@ -44,9 +54,14 @@ pandora.ui.downloadVideoDialog = function(options) {
                     })
                     .bindEvent({
                         change: function(data) {
+                            if (data.value == 'source') {
+                                $resolution.hide()
+                            } else {
+                                $resolution.show()
+                            }
                         }
                     }),
-                Ox.Select({
+                $resolution = Ox.Select({
                         id: 'resolution',
                         items: pandora.site.video.resolutions.map(function(resolution) {
                             return {
@@ -63,9 +78,29 @@ pandora.ui.downloadVideoDialog = function(options) {
                         change: function(data) {
                         }
                     })
-            ]
+            ].concat(parts ? [
+                Ox.Select({
+                        id: 'part',
+                        items: Ox.range(parts + 1).map(function(resolution, idx) {
+                            return {
+                                id: idx + 1,
+                                title: 'Part ' + (idx+1)
+                            };
+                        }),
+                        label: Ox._('Part'),
+                        labelWidth: 120,
+                        value: 1,
+                        width: 240
+                    })
+                    .bindEvent({
+                        change: function(data) {
+                        }
+                    })
+            ] : [])
         }).appendTo($content),
 
+        failed = false,
+
         that = Ox.Dialog({
             buttons: [
                 Ox.Button({
@@ -73,21 +108,64 @@ pandora.ui.downloadVideoDialog = function(options) {
                     title: Ox._('Download')
                 }).bindEvent({
                     click: function() {
+                        if (failed) {
                             that.close();
+                            return
+                        }
                         var values = $form.values(),
                             url
                         if (options.out) {
+                            var $screen = Ox.LoadingScreen({
+                                size: 16
+                            })
+                            that.options({content: $screen.start()});
+                            pandora.api.extractClip({
+                                item: options.item,
+                                resolution: values.resolution,
+                                format: values.format,
+                                'in': options['in'],
+                                out: options.out
+                            }, function(result) {
+                                if (result.data.taskId) {
+                                    pandora.wait(result.data.taskId, function(result) {
+                                        console.log('wait -> ', result)
+                                        if (result.data.result) {
                                             url = '/' + options.item
                                                 + '/' + values.resolution
                                                 + 'p.' + values.format
                                                 + '?t=' + options['in'] + ',' + options.out;
+                                            that.close();
+                                            document.location.href = url
+                                        } else {
+                                        }
+                                    }, 1000)
+                                } else {
+                                    that.options({content: 'Failed to extract clip.'});
+                                    that.options('buttons')[0].options({
+                                        title: Ox._('Close')
+                                    });
+                                    failed = true;
+                                }
+                            })
+
+                        } else {
+                            if (values.format == 'source') {
+                                url = '/' + options.item
+                                    + '/download/source/'
+                                    + (values.part ? values.part : '')
                             } else {
                                 url = '/' + options.item
                                     + '/download/' + values.resolution
-                                + 'p.' + values.format
+                                    + 'p'
+                                    + (values.part ? values.part : '')
+                                    + '.' + values.format
                             }
+                        }
+                        if (url) {
+                            that.close();
                             document.location.href = url
                         }
+                    }
                 })
             ],
             closeButton: true,

static/js/editor.js

@@ -28,7 +28,7 @@ pandora.ui.editor = function(data) {
             enableSetPosterFrame: !pandora.site.media.importFrames && data.editable,
             enableSubtitles: ui.videoSubtitles,
             find: ui.itemFind,
-            findLayer: ui._findState.key,
+            findLayer: pandora.getFindLayer(),
             getFrameURL: function(position) {
                 return pandora.getMediaURL('/' + ui.item + '/' + ui.videoResolution + 'p' + position + '.jpg?' + data.modified);
             },
@@ -189,7 +189,9 @@ pandora.ui.editor = function(data) {
                 pandora.ui.downloadVideoDialog({
                     item: ui.item,
                     rightsLevel: rightsLevel,
-                    title: data.title
+                    source: data.source && pandora.hasCapability('canDownloadSource'),
+                    title: data.title,
+                    video: data.video
                 }).open();
             },
             downloadselection: function(selection) {
@@ -198,7 +200,8 @@ pandora.ui.editor = function(data) {
                     'in': selection['in'],
                     out: selection.out,
                     rightsLevel: rightsLevel,
-                    title: data.title
+                    title: data.title,
+                    video: data.video
                 }).open();
             },
             editannotation: function(data) {

static/js/filterDialog.js

@@ -12,7 +12,9 @@ pandora.ui.filterDialog = function() {
                         click: function() {
                             var list = pandora.$ui.filterForm.getList();
                             if (list.save) {
-                                pandora.api.addList({
+                                pandora.api[
+                                    pandora.user.ui.section == 'documents' ? 'addCollection' : 'addList'
+                                ]({
                                     name: list.name,
                                     query: list.query,
                                     status: 'private',
@@ -20,12 +22,21 @@ pandora.ui.filterDialog = function() {
                                 }, function(result) {
                                     var $list = pandora.$ui.folderList.personal,
                                         id = result.data.id;
+                                    if (pandora.user.ui.section) {
+                                        pandora.UI.set({
+                                            findDocuments: {
+                                                conditions: [{key: 'collection', value: id, operator: '=='}],
+                                                operator: '&'
+                                            }
+                                        });
+                                    } else {
                                         pandora.UI.set({
                                             find: {
                                                 conditions: [{key: 'list', value: id, operator: '=='}],
                                                 operator: '&'
                                             }
                                         });
+                                    }
                                     Ox.Request.clearCache(); // fixme: remove
                                     $list.bindEventOnce({
                                             load: function(data) {

static/js/findDocumentsElement.js

@@ -33,7 +33,7 @@ pandora.ui.findDocumentsElement = function() {
                             }
                         }),
                 ] : [], [
-                    $findSelect = Ox.Select({
+                    pandora.$ui.findDocumentsSelect = $findSelect = Ox.Select({
                             id: 'select',
                             items: [].concat(
                                 pandora.site.documentKeys.filter(function(key) {
@@ -70,7 +70,7 @@ pandora.ui.findDocumentsElement = function() {
                                 }
                             }
                         }),
-                    $findInput = Ox.Input({
+                    pandora.$ui.findDocumentsInput = $findInput = Ox.Input({
                         autocomplete: autocompleteFunction(),
                         autocompleteSelect: true,
                         autocompleteSelectHighlight: true,

static/js/folderList.js

@@ -287,7 +287,9 @@ pandora.ui.folderList = function(id, section) {
         // works when switching back from browser, but won't work on load, since
         // getListData relies on $folderList, so selectList is called in init handler
         selected: pandora.getListData().folder == id
-            ? [ui[ui.section == 'items' ? '_list' : ui.section.slice(0, -1)]]
+            ? [ui[ui.section == 'items' ? '_list'
+                  : ui.section == 'documents' ? '_collection'
+                  : ui.section.slice(0, -1)]]
             : [],
         sort: [{key: 'position', operator: '+'}],
         sortable: id != 'featured' || canEditFeatured,
@@ -389,7 +391,7 @@ pandora.ui.folderList = function(id, section) {
         },
         key_control_d: function() {
             if (that.options('selected').length) {
-                pandora.addFolderItem(ui.section, ui._list);
+                pandora.addFolderItem(ui.section, ui.section == 'documents' ? ui._collection : ui._list);
             }
         },
         key_control_e: function() {

static/js/folders.js

@@ -52,13 +52,13 @@ pandora.ui.folders = function(section) {
                                     [Ox._(folderItem)]),
                                     keyboard: 'shift control n',
                                     disabled:  ui.section == 'documents'
-                                        ? ui.collectionSelection == 0
+                                        ? ui.collectionSelection.length == 0
                                         : ui.listSelection.length == 0
                                 },
                                 { id: 'newsmartlist', title: Ox._('New Smart {0}', [Ox._(folderItem)]), keyboard: 'alt control n' },
                                 { id: 'newsmartlistfromresults', title: Ox._('New Smart {0} from Results', [Ox._(folderItem)]), keyboard: 'shift alt control n' },
                                 {},
-                                { id: 'duplicatelist', title: Ox._('Duplicate Selected {0}', [Ox._(folderItem)]), keyboard: 'control d', disabled: !ui._list },
+                                { id: 'duplicatelist', title: Ox._('Duplicate Selected {0}', [Ox._(folderItem)]), keyboard: 'control d', disabled: ui.section == 'documents' ? !ui._collection : !ui._list },
                                 { id: 'editlist', title: Ox._('Edit Selected {0}...', [Ox._(folderItem)]), keyboard: 'control e', disabled: !editable },
                                 { id: 'deletelist', title: Ox._('Delete Selected {0}...', [Ox._(folderItem)]), keyboard: 'delete', disabled: !editable }
                             ],
@@ -75,13 +75,18 @@ pandora.ui.folders = function(section) {
                                 ], data.id)) {
                                     pandora.addList(data.id.indexOf('smart') > -1, data.id.indexOf('from') > -1);
                                 } else if (data.id == 'duplicatelist') {
-                                    pandora.addList(ui._list);
+                                    pandora.addList(ui.section == 'documents' ? ui._collection : ui._list);
                                 } else if (data.id == 'editlist') {
                                     pandora.ui.listDialog().open();
                                 } else if (data.id == 'deletelist') {
                                     pandora.ui.deleteListDialog().open();
                                 }
                             },
+                            pandora_collectionselection: function(data) {
+                                pandora.$ui.personalListsMenu[
+                                    data.value.length ? 'enableItem' : 'disableItem'
+                                ]('newlistfromselection');
+                            },
                             pandora_find: function() {
                                 // fixme: duplicated
                                 var action = ui._list
@@ -96,6 +101,20 @@ pandora.ui.folders = function(section) {
                                     ui.listSelection.length ? 'enableItem' : 'disableItem'
                                 ]('newlistfromselection');
                             },
+                            pandora_finddocuments: function() {
+                                // fixme: duplicated
+                                var action = ui._collection
+                                    && pandora.getListData(ui._collection).user == pandora.user.username
+                                    ? 'enableItem' : 'disableItem'
+                                pandora.$ui.personalListsMenu[
+                                    ui._collection ? 'enableItem' : 'disableItem'
+                                ]('duplicatelist');
+                                pandora.$ui.personalListsMenu[action]('editlist');
+                                pandora.$ui.personalListsMenu[action]('deletelist');
+                                pandora.$ui.personalListsMenu[
+                                    ui.collectionSelection.length ? 'enableItem' : 'disableItem'
+                                ]('newlistfromselection');
+                            },
                             pandora_listselection: function(data) {
                                 pandora.$ui.personalListsMenu[
                                     data.value.length ? 'enableItem' : 'disableItem'

static/js/idDialog.js

@@ -97,7 +97,7 @@ pandora.ui.idDialog = function(data) {
                         labelWidth: 128,
                         value: Ox.decodeHTMLEntities(key == 'director' && data[key]
                             ? data[key].join(', ')
-                            : ('' + data[key])),
+                            : ('' + (data[key] || ''))),
                         width: formWidth
                     })
                     .css({display: 'inline-block', margin: '3px'})
@@ -201,7 +201,7 @@ pandora.ui.idDialog = function(data) {
         }
         return Ox.filter([
             item.id,
-            item.title + (item.originalTitle ? ' (' + item.originalTitle + ')' : ''),
+            item.title + ((item.originalTitle && item.title != item.originalTitle) ? ' (' + item.originalTitle + ')' : ''),
             item.director ? item.director.join(', ') : '',
             item.year
         ]).join(' — ');

static/js/importAnnotationsDialog.js

@@ -68,7 +68,15 @@ pandora.ui.importAnnotationsDialog = function(options) {
         .bindEvent({
             change: function(data) {
                 $status.empty();
-                data.value.length && $formatSelect.options({value: Ox.last(data.value[0].name.split('.'))});
+                if (data.value.length) {
+                    var format = Ox.last(data.value[0].name.split('.'));
+                    $formatSelect.options({value: format});
+                    var subtitlesLayer = pandora.getSubtitlesLayer()
+                    if (subtitlesLayer && format == 'srt' && Ox.getObjectById(layers, subtitlesLayer))  {
+                        $layerSelect.options({value: subtitlesLayer})
+                    }
+                    updateLanguageSelect();
+                }
                 that[
                     data.value.length ? 'enableButton' : 'disableButton'
                 ]('import');

static/js/importScreen.js

@@ -0,0 +1,54 @@
+'use strict';
+
+pandora.ui.importScreen = function() {
+
+    var that = Ox.Element()
+        .attr({id: 'importScreen'})
+        .css({
+            position: 'absolute',
+            left: 0,
+            top: 0,
+            right: 0,
+            bottom: 0,
+            backgroundColor: 'rgba(0, 0, 0, 0.5)',
+            zIndex: 1000
+        })
+        .on({
+            click: function() {
+                that.remove();
+            },
+            dragleave: function() {
+                that.remove();
+            }
+        });
+
+    Ox.Element()
+        .css({
+            position: 'absolute',
+            left: 0,
+            top: 0,
+            right: 0,
+            bottom: 0,
+            width: pandora.hasCapability('canAddItems') ? 192 : 256,
+            height: 16,
+            padding: '8px 0',
+            borderRadius: 8,
+            margin: 'auto',
+            background: 'rgba(255, 255, 255, 0.9)',
+            fontSize: 13,
+            color: 'rgb(0, 0, 0)',
+            textAlign: 'center'
+        })
+        .text(
+            Ox._(pandora.hasCapability('canAddItems') ? (
+                'Import {0}'
+            ) : (
+                'You are not allowed to import {0}'
+            ),
+            [pandora.user.ui.section == 'documents' ? 'Documents' : pandora.site.itemName.plural])
+        )
+        .appendTo(that);
+
+    return that;
+
+};

static/js/infoView.0xdb.js

@@ -919,12 +919,12 @@ pandora.ui.infoView = function(data, isMixed) {
         });
     }
 
-    function renderList() {
+    function renderFrames() {
         pandora.api.get({
             id: data.id,
-            keys: [ui.icons == 'posters' ? 'posters' : 'frames']
+            keys: ['frames']
         }, 0, function(result) {
-            var images = result.data[ui.icons == 'posters' ? 'posters' : 'frames'].map(function(image) {
+            var images = result.data.frames.map(function(image) {
                     return Ox.extend(image, {index: image.index.toString()});
                 }),
                 selectedImage = images.filter(function(image) {
@@ -932,8 +932,8 @@ pandora.ui.infoView = function(data, isMixed) {
                 })[0],
                 modified = data.modified;
             $list = Ox.IconList({
-                    defaultRatio: ui.icons == 'posters' || !data.stream ? pandora.site.posters.ratio : data.stream.aspectratio,
-                    fixedRatio: ui.icons == 'posters' || !data.stream ? false : data.stream.aspectratio,
+                    defaultRatio: !data.stream ? pandora.site.posters.ratio : data.stream.aspectratio,
+                    fixedRatio: !data.stream ? false : data.stream.aspectratio,
                     item: function(data, sort, size) {
                         var ratio = data.width / data.height;
                         size = size || 128;
@@ -941,17 +941,124 @@ pandora.ui.infoView = function(data, isMixed) {
                             height: ratio <= 1 ? size : size / ratio,
                             id: data.id,
                             info: data.width + ' × ' + data.height + ' px',
-                            title: ui.icons == 'posters' ? data.source : Ox.formatDuration(data.position),
+                            title: Ox.formatDuration(data.position),
+                            url: data.url,
+                            width: ratio >= 1 ? size : size * ratio
+                        }
+                    },
+                    items: images,
+                    keys: ['index', 'position', 'width', 'height', 'url'],
+                    max: 1,
+                    min: 1,
+                    orientation: 'both',
+                    // fixme: should never be undefined
+                    selected: selectedImage ? [selectedImage['index']] : [],
+                    size: 128,
+                    sort: [{key: 'index', operator: '+'}],
+                    unique: 'index'
+                })
+                .addClass('OxMedia')
+                .css({
+                    display: 'block',
+                    position: 'absolute',
+                    left: 0,
+                    top: 0,
+                    width: listWidth + 'px',
+                    height: getHeight() + 'px'
+                })
+                .bindEvent({
+                    select: function(event) {
+                        var index = event.ids[0];
+                        selectedImage = images.filter(function(image) {
+                            return image.index == index;
+                        })[0];
+                        var imageRatio = selectedImage.width / selectedImage.height,
+                            src = selectedImage.url
+                        if ($browserImages.length == 0) {
+                            $browserImages = pandora.$ui.browser.find('img[src*="/' + data.id + '/"]');
+                        }
+                        pandora.api.setPosterFrame({
+                            id: data.id,
+                            // fixme: api slightly inconsistent, this shouldn't be "position"
+                            position: selectedImage.index
+                        }, function() {
+                            var src;
+                            Ox.Request.clearCache();
+                            if (ui.icons == 'frames') {
+                                src = pandora.getMediaURL('/' + data.id + '/icon512.jpg?' + Ox.uid());
+                                $icon.attr({src: src});
+                                $reflectionIcon.attr({src: src});
+                                if (pandora.$ui.videoPreview) {
+                                    pandora.$ui.videoPreview.options({
+                                        position: $list.value(selectedImage.index, 'position')
+                                    });
+                                }
+                            } else if (ui.icons == 'posters' && ui.showSitePosters) {
+                                src = pandora.getMediaURL('/' + data.id + '/siteposter512.jpg?' + Ox.uid());
+                                $icon.attr({src: src});
+                                $reflectionIcon.attr({src: src});
+                            }
+                            $browserImages.each(function() {
+                                $(this).attr({src: pandora.getMediaURL('/' + data.id + '/' + (
+                                    ui.icons == 'posters'
+                                        ? ui.showSitePosters ? 'siteposter' : 'poster'
+                                        : 'icon'
+                                ) + '128.jpg?' + Ox.uid())});
+                            });
+                            if (ui.listSort[0].key == 'modified') {
+                                pandora.$ui.browser.reloadList();
+                            }
+                        });
+                    }
+                })
+                .appendTo($info);
+            $list.size();
+        });
+    }
+
+    function renderList() {
+        if (ui.icons == 'posters' && !ui.showSitePosters) {
+            renderPosters()
+        } else {
+            renderFrames()
+        }
+    }
+
+    function renderPosters() {
+        pandora.api.get({
+            id: data.id,
+            keys: ['posters']
+        }, 0, function(result) {
+            var images = result.data.posters.map(function(image) {
+                    return Ox.extend(image, {index: image.index.toString()});
+                }),
+                selectedImage = images.filter(function(image) {
+                    return image.selected;
+                })[0],
+                modified = data.modified;
+            if (images.length == 1) {
+                renderFrames()
+                return
+            }
+            $list = Ox.IconList({
+                    defaultRatio: pandora.site.posters.ratio,
+                    fixedRatio: false,
+                    item: function(data, sort, size) {
+                        var ratio = data.width / data.height;
+                        size = size || 128;
+                        return {
+                            height: ratio <= 1 ? size : size / ratio,
+                            id: data.id,
+                            info: data.width + ' × ' + data.height + ' px',
+                            title: data.source,
                             url: data.url.replace('http://', '//') + (
-                                ui.icons == 'posters' && data.source == pandora.site.site.url ? '?' + modified : ''
+                                data.source == pandora.site.site.url ? '?' + modified : ''
                             ),
                             width: ratio >= 1 ? size : size * ratio
                         }
                     },
                     items: images,
-                    keys: ui.icons == 'posters'
-                        ? ['index', 'source', 'width', 'height', 'url']
-                        : ['index', 'position', 'width', 'height', 'url'],
+                    keys: ['index', 'source', 'width', 'height', 'url'],
                     max: 1,
                     min: 1,
                     orientation: 'both',
@@ -981,12 +1088,12 @@ pandora.ui.infoView = function(data, isMixed) {
                         if ($browserImages.length == 0) {
                             $browserImages = pandora.$ui.browser.find('img[src*="/' + data.id + '/"]');
                         }
-                        if (ui.icons == 'posters' && !ui.showSitePosters) {
+                        if (!ui.showSitePosters) {
                             $browserImages.each(function() {
                                 var $this = $(this),
                                     size = Math.max($this.width(), $this.height());
                                 $this.attr({src: src});
-                                ui.icons == 'posters' && $this.css(imageRatio < 1 ? {
+                                $this.css(imageRatio < 1 ? {
                                     width: Math.round(size * imageRatio) + 'px',
                                     height: size + 'px'
                                 } : {
@@ -1000,31 +1107,17 @@ pandora.ui.infoView = function(data, isMixed) {
                             iconSize = iconSize == 256 ? 512 : 256;
                             toggleIconSize();
                         }
-                        pandora.api[ui.icons == 'posters' ? 'setPoster' : 'setPosterFrame'](Ox.extend({
-                            id: data.id
-                        }, ui.icons == 'posters' ? {
+                        pandora.api.setPoster({
+                            id: data.id,
                             source: selectedImage.source
-                        } : {
-                             // fixme: api slightly inconsistent, this shouldn't be "position"
-                            position: selectedImage.index
-                        }), function() {
+                        }, function() {
                             var src;
                             Ox.Request.clearCache();
-                            if (ui.icons == 'frames') {
-                                src = pandora.getMediaURL('/' + data.id + '/icon512.jpg?' + Ox.uid());
-                                $icon.attr({src: src});
-                                $reflectionIcon.attr({src: src});
-                                if (pandora.$ui.videoPreview) {
-                                    pandora.$ui.videoPreview.options({
-                                        position: $list.value(selectedImage.index, 'position')
-                                    });
-                                }
-                            }
                             if (!ui.showSitePosters) {
                                 $browserImages.each(function() {
-                                    $(this).attr({src: pandora.getMediaURL('/' + data.id + '/' + (
-                                        ui.icons == 'posters' ? 'poster' : 'icon'
-                                    ) + '128.jpg?' + Ox.uid())});
+                                    $(this).attr({
+                                        src: pandora.getMediaURL('/' + data.id + '/poster128.jpg?' + Ox.uid())
+                                    });
                                 });
                             }
                             if (ui.listSort[0].key == 'modified') {
@@ -1038,6 +1131,7 @@ pandora.ui.infoView = function(data, isMixed) {
         });
     }
 
+
     function renderRightsLevel() {
         var $rightsLevelElement = getRightsLevelElement(data.rightslevel),
             $rightsLevelSelect;

static/js/infoView.indiancinema.js

@@ -621,19 +621,6 @@ pandora.ui.infoView = function(data, isMixed) {
     $('<div>').css({height: '16px'}).appendTo($statistics);
 
 
-    function cleanupDate(value) {
-        if (/\d{2}-\d{2}-\d{4}/.test(value)) {
-            value = Ox.reverse(value.split('-')).join('-')
-        }
-        if (/\d{4}i\/\d{2}\/\d{d}/.test(value)) {
-            value = value.split('/').join('-')
-        }
-        if (/\d{2}\/\d{2}\/\d{4}/.test(value)) {
-            value = Ox.reverse(value.split('/')).join('-')
-        }
-        return value
-    }
-
     function editMetadata(key, value) {
         if (value != data[key]) {
             var itemKey = Ox.getObjectById(pandora.site.itemKeys, key);
@@ -656,12 +643,12 @@ pandora.ui.infoView = function(data, isMixed) {
                     ? Ox.decodeHTMLEntities(value).split('; ').map(Ox.encodeHTMLEntities)
                     : [];
             } else if (key == 'imdbId') {
-                edit[key] = value ? value.match(/\d{7}/)[0] : value;
+                edit[key] = value ? value.match(/\d+/)[0] : value;
             } else {
                 edit[key] = value;
             }
             if (itemKey && itemKey.type && itemKey.type[0] == 'date') {
-                edit[key] = edit[key].map(cleanupDate);
+                edit[key] = edit[key].map(pandora.cleanupDate);
             }
             pandora.api.edit(edit, function(result) {
                 if (!isMultiple) {
@@ -763,14 +750,14 @@ pandora.ui.infoView = function(data, isMixed) {
             specialListKeys.indexOf(key) > -1 && itemKey && itemKey.type[0] == 'date'
         ) {
             ret = value.split('; ').map(function(date) {
-                date = cleanupDate(date)
+                date = pandora.cleanupDate(date)
                 return date ? formatLink(Ox.formatDate(date,
                     ['', '%Y', '%B %Y', '%B %e, %Y'][date.split('-').length],
                     true
                 ), key, date) : '';
             }).join('; ');
         } else if (['releasedate'].indexOf(key) > -1) {
-            value = cleanupDate(value);
+            value = pandora.cleanupDate(value);
             ret = value ? Ox.formatDate(value,
                 ['', '%Y', '%B %Y', '%B %e, %Y'][value.split('-').length],
                 true

static/js/infoView.js

@@ -38,6 +38,15 @@ pandora.ui.infoView = function(data, isMixed) {
             })
         ),
         posterKeys = nameKeys.concat(['title', 'year']),
+        displayedKeys = [ // FIXME: can tis be a flag in the config?
+            'title', 'notes', 'name', 'summary', 'id',
+            'hue', 'saturation', 'lightness', 'cutsperminute', 'volume',
+            'user', 'rightslevel', 'bitrate', 'timesaccessed',
+            'numberoffiles', 'numberofannotations', 'numberofcuts', 'words', 'wordsperminute',
+            'duration', 'aspectratio', 'pixels', 'size', 'resolution',
+            'created', 'modified', 'accessed',
+            'random'
+        ],
         statisticsWidth = 128,
 
         $bar = Ox.Bar({size: 16})
@@ -236,13 +245,17 @@ pandora.ui.infoView = function(data, isMixed) {
         )
         .appendTo($text);
 
-    // Director, Year and Country ----------------------------------------------
+    // Director, Year and Country, Language --------------------------------
 
-    renderGroup(['director', 'year', 'country']);
+    renderGroup(['director', 'year', 'country', 'language']);
 
     // Featuring ----------------------------------------------
 
-    renderGroup(['featuring']);
+    Ox.getObjectById(pandora.site.itemKeys, 'featuring') && renderGroup(['featuring']);
+
+    // Render any remaing keys defined in config
+
+    renderRemainingKeys();
 
     // Summary -----------------------------------------------------------------
 
@@ -278,6 +291,7 @@ pandora.ui.infoView = function(data, isMixed) {
             .appendTo($text);
     }
 
+
     // Duration, Aspect Ratio --------------------------------------------------
     if (!isMultiple) {
         ['duration', 'aspectratio'].forEach(function(key) {
@@ -354,7 +368,7 @@ pandora.ui.infoView = function(data, isMixed) {
             .append(
                 Ox.EditableContent({
                         height: 128,
-                        placeholder: formatLight(Ox._(isMixed ? 'Mixed notes' : 'No notes')),
+                        placeholder: formatLight(Ox._(isMixed.notes ? 'Mixed notes' : 'No notes')),
                         tooltip: pandora.getEditTooltip(),
                         type: 'textarea',
                         value: data.notes || '',
@@ -371,19 +385,6 @@ pandora.ui.infoView = function(data, isMixed) {
 
     $('<div>').css({height: '16px'}).appendTo($statistics);
 
-    function cleanupDate(value) {
-        if (/\d{2}-\d{2}-\d{4}/.test(value)) {
-            value = Ox.reverse(value.split('-')).join('-')
-        }
-        if (/\d{4}i\/\d{2}\/\d{d}/.test(value)) {
-            value = value.split('/').join('-')
-        }
-        if (/\d{2}\/\d{2}\/\d{4}/.test(value)) {
-            value = Ox.reverse(value.split('/')).join('-')
-        }
-        return value
-    }
-
     function editMetadata(key, value) {
         if (value != data[key]) {
             var itemKey = Ox.getObjectById(pandora.site.itemKeys, key);
@@ -400,7 +401,7 @@ pandora.ui.infoView = function(data, isMixed) {
                 edit[key] = value ? value : null;
             }
             if (itemKey && itemKey.type && itemKey.type[0] == 'date') {
-                edit[key] = edit[key].map(cleanupDate);
+                edit[key] = edit[key].map(pandora.cleanupDate);
             }
             pandora.api.edit(edit, function(result) {
                 if (!isMultiple) {
@@ -474,7 +475,7 @@ pandora.ui.infoView = function(data, isMixed) {
             listKeys.indexOf(key) > -1 && Ox.getObjectById(pandora.site.itemKeys, key).type[0] == 'date'
         ) {
             ret = value.split('; ').map(function(date) {
-                date = cleanupDate(date)
+                date = pandora.cleanupDate(date)
                 return date ? formatLink(Ox.formatDate(date,
                     ['', '%Y', '%B %Y', '%B %e, %Y'][date.split('-').length],
                     true
@@ -588,6 +589,7 @@ pandora.ui.infoView = function(data, isMixed) {
 
     function renderGroup(keys) {
         var $element;
+        keys.forEach(function(key) { displayedKeys.push(key) });
         if (canEdit || keys.filter(function(key) {
             return data[key];
         }).length) {
@@ -619,6 +621,17 @@ pandora.ui.infoView = function(data, isMixed) {
         }
     }
 
+    function renderRemainingKeys() {
+        var keys = pandora.site.itemKeys.filter(function(item) {
+            return item.id != '*' && item.type != 'layer' && !Ox.contains(displayedKeys, item.id);
+        }).map(function(item) {
+            return item.id;
+        });
+        if (keys.length) {
+            renderGroup(keys)
+        }
+    }
+
     function renderRightsLevel() {
         var $rightsLevelElement = getRightsLevelElement(data.rightslevel),
             $rightsLevelSelect;

static/js/infoViewUtils.js

@@ -0,0 +1,16 @@
+'use strict';
+
+
+pandora.cleanupDate = function(value) {
+    if (/\d{2}-\d{2}-\d{4}/.test(value)) {
+        value = Ox.reverse(value.split('-')).join('-')
+    }
+    if (/\d{4}i\/\d{2}\/\d{d}/.test(value)) {
+        value = value.split('/').join('-')
+    }
+    if (/\d{2}\/\d{2}\/\d{4}/.test(value)) {
+        value = Ox.reverse(value.split('/')).join('-')
+    }
+    return value
+};
+

static/js/item.js

@@ -41,11 +41,28 @@ pandora.ui.item = function() {
                     pandora.user.ui.itemView.slice(0, 1)
                 ) > -1 ? 'an': 'a') + ' '
                 +'{1} view.', [result.data.title, Ox._(pandora.user.ui.itemView)]);
-            pandora.$ui.contentPanel.replaceElement(1,
-                Ox.Element()
+
+            var note = Ox.Element()
                 .css({marginTop: '32px', fontSize: '12px', textAlign: 'center'})
-                    .html(html)
-            );
+            pandora.$ui.contentPanel.replaceElement(1, note);
+            if (pandora.user.username == result.data.user || pandora.hasCapability('canSeeAllTasks')) {
+                pandora.api.getTasks({
+                    user: pandora.hasCapability('canSeeAllTasks') ? '' : pandora.user.username
+                }, function(result_) {
+                    var tasks = result_.data.items.filter(function(task) { return task.item == item})
+                    if (tasks.length > 0) {
+                        html = Ox._(
+                            '<i>{0}</i> is currently processed. '
+                            + '{1} view will be available in a moment.',
+                            [result.data.title, Ox._(pandora.user.ui.itemView)]
+                        )
+                    }
+                    note.html(html)
+                })
+            } else {
+                note.html(html)
+            }
+
             pandora.site.itemViews.filter(function(view) {
                 return view.id == 'documents';
             }).length && pandora.api.get({

static/js/mainMenu.js

@@ -52,170 +52,7 @@ pandora.ui.mainMenu = function() {
                     ] },
                     getListMenu(),
                     getItemMenu(),
-                    { id: 'viewMenu', title: Ox._('View'), items: [
-                        { id: 'section', title: Ox._('Section'), items: [
-                            { group: 'viewsection', min: 1, max: 1, items: Object.keys(pandora.site.sectionFolders).map(function(section) {
-                                return {
-                                    id: section,
-                                    title: section == 'items' ? Ox._(pandora.site.itemName.plural) : Ox._(Ox.toTitleCase(section)),
-                                    checked: ui.section == section
-                                };
-                            }) }
-                        ] },
-                        {},
-                        { id: 'movies', title: Ox._('View {0}', [Ox._(pandora.site.itemName.plural)]), items: [
-                            { group: 'listview', min: 1, max: 1, items: pandora.site.listViews.map(function(view) {
-                                return Ox.extend({
-                                    checked: ui.listView == view.id
-                                }, view, {
-                                    keyboard: listViewKey <= 10
-                                        ? 'shift ' + (listViewKey++%10)
-                                        : void 0,
-                                    title: Ox._(view.title)
-                                });
-                            }) },
-                        ]},
-                        { id: 'icons', title: Ox._('Icons'), items: [].concat([
-                                { group: 'viewicons', min: 1, max: 1, items: ['posters', 'frames'].map(function(icons) {
-                                    return {id: icons, title: Ox._(Ox.toTitleCase(icons)), checked: ui.icons == icons};
-                                }) },
-                                {},
-                            ], pandora.site.media.importPosters ? [
-                                { id: 'showsiteposters', title: Ox._('Always Show {0} Poster', [pandora.site.site.name]), checked: ui.showSitePosters },
-                                {}
-                            ] : [], [
-                                { id: 'showreflections', title: Ox._('Show Reflections'), checked: true, disabled: true }
-                            ]
-                        ) },
-                        { id: 'timelines', title: Ox._('Timelines'), items: [
-                            { group: 'viewtimelines', min: 1, max: 1, items: pandora.site.timelines.map(function(mode) {
-                                return {id: mode.id, title: Ox._(mode.title), checked: ui.videoTimeline == mode.id};
-                            }) }
-                        ]},
-                        { id: 'columns', title: Ox._('Columns'), items: [
-                            { id: 'loadcolumns', title: Ox._('Load Layout...'), disabled: true },
-                            { id: 'savecolumns', title: Ox._('Save Layout...'), disabled: true },
-                            {},
-                            { id: 'resetcolumns', title: Ox._('Reset Layout'), disabled: true }
-                        ] },
-                        { id: 'filters', title: Ox._('Filters'), disabled: ui.section != 'items', items: [
-                            { group: 'filters', min: 5, max: 5, items: pandora.site.filters.map(function(filter) {
-                                return Ox.extend({
-                                    checked: Ox.getIndexById(ui.filters, filter.id) > -1
-                                }, filter, {
-                                    title: Ox._(filter.title)
-                                });
-                            }) },
-                            {},
-                            { id: 'resetfilters', title: Ox._('Reset Filters') }
-                        ] },
-                        {},
-                        { id: 'item', title: [
-                                Ox._('Open {0}', [Ox._(pandora.site.itemName.singular)]),
-                                Ox._('Open {0}', [Ox._(pandora.site.itemName.plural)])
-                            ], items: [
-                            { group: 'itemview', min: 1, max: 1, items: pandora.site.itemViews.filter(function(view) {
-                                return view.id != 'data' && view.id != 'media' ||
-                                    pandora.hasCapability('canSeeExtraItemViews');
-                            }).map(function(view) {
-                                return Ox.extend({
-                                    checked: ui.itemView == view.id
-                                }, view, {
-                                    keyboard: itemViewKey <= 10
-                                        ? 'shift ' + (itemViewKey++%10)
-                                        : void 0,
-                                    title: Ox._(view.title)
-                                });
-                            }) },
-                        ] },
-                        { id: 'clips', title: Ox._('Open Clips'), items: [
-                            { group: 'videoview', min: 1, max: 1, items: ['player', 'editor', 'timeline'].map(function(view) {
-                                return {id: view, title: Ox._(Ox.toTitleCase(view)), checked: ui.videoView == view};
-                            }) }
-                        ] },
-                        { id: 'documents', title: Ox._('Open Documents'), items: [
-                            { group: 'documentview', min: 1, max: 1, items: ['info', 'view'].map(function(id) {
-                                return {
-                                    id: id,
-                                    checked: ui.documentView == id,
-                                    keyboard: documentViewKey <= 10
-                                        ? 'shift ' + (documentViewKey++%10)
-                                        : void 0,
-                                    title: Ox._(Ox.toTitleCase(id))
-                                }
-                            }) }
-                        ] },
-                        {},
-                        {
-                            id: 'showsidebar',
-                            title: Ox._((ui.showSidebar ? 'Hide' : 'Show') + ' Sidebar'),
-                            keyboard: 'shift s'
-                        },
-                        {
-                            id: 'showinfo',
-                            title: Ox._((ui.showInfo ? 'Hide' : 'Show') + ' Info'),
-                            disabled: !ui.showSidebar, keyboard: 'shift i'
-                        },
-                        {
-                            id: 'showfilters',
-                            title: Ox._((ui.showFilters ? 'Hide' : 'Show') + ' Filters'),
-                            disabled: ui.section != 'items' || !!ui.item, keyboard: 'shift f'
-                        },
-                        {
-                            id: 'showbrowser',
-                            title: Ox._((ui.showBrowser ? 'Hide': 'Show') + ' {0} Browser', [Ox._(pandora.site.itemName.singular)]),
-                            disabled: !ui.item, keyboard: 'shift b'
-                        },
-                        {
-                            id: 'showdocument',
-                            title: Ox._((ui.showDocument ? 'Hide' : 'Show') + ' Document'),
-                            disabled: !hasDocument(), keyboard: 'shift d'
-                        },
-                        {
-                            id: 'showtimeline',
-                            title: Ox._((ui.showTimeline ? 'Hide' : 'Show') + ' Timeline'),
-                            disabled: !hasTimeline(), keyboard: 'shift t'
-                        },
-                        {
-                            id: 'showannotations',
-                            title: Ox._((ui.showAnnotations ? 'Hide' : 'Show') + ' Annotations'),
-                            disabled: !hasAnnotations(), keyboard: 'shift a'
-                        },
-                        {
-                            id: 'showclips',
-                            title: Ox._((ui.showClips ? 'Hide' : 'Show') + ' Clips'),
-                            disabled: !hasClips(), keyboard: 'shift c'
-                        },
-                        {},
-                        {
-                            id: 'togglefullscreen',
-                            title: Ox._((fullscreenState ? 'Exit' : 'Enter') + ' Fullscreen'),
-                            disabled: fullscreenState === void 0,
-                            keyboard: /^Mac/.test(window.navigator.platform)
-                                ? 'shift alt f'
-                                : 'F11'
-
-                        },
-                        {
-                            id: 'entervideofullscreen',
-                            title: Ox._('Enter Video Fullscreen'),
-                            disabled: !ui.item || ui.itemView != 'player'
-                        },
-                        {},
-                        { id: 'theme', title: Ox._('Theme'), items: [
-                            { group: 'settheme', min: 1, max: 1, items: pandora.site.themes.map(function(theme) {
-                                return {id: theme, title: Ox.Theme.getThemeData(theme).themeName, checked: ui.theme == theme}
-                            }) }
-                        ] },
-                        { id: 'locale',
-                          title: Ox._('Language'), items: [
-                            { group: 'setlocale', min: 1, max: 1, items: pandora.site.languages.map(function(locale) {
-                                return {id: locale, title: Ox.LOCALE_NAMES[locale], checked: ui.locale == locale}
-                            }) }
-                        ] },
-                        {},
-                        { id: 'embed', title: Ox._('Embed...') }
-                    ]},
+                    getViewMenu(),
                     getSortMenu(),
                     getFindMenu(),
                     { id: 'dataMenu', title: Ox._('Data'), items: [
@@ -229,8 +66,10 @@ pandora.ui.mainMenu = function() {
                         { id: 'names', title: Ox._('Manage Names...'), disabled: !pandora.hasCapability('canManageTitlesAndNames') },
                         { id: 'translations', title: Ox._('Manage Translations...'), disabled: !pandora.hasCapability('canManageTranslations') },
                         {},
-                        { id: 'places', title: Ox._('Manage Places...'), disabled: !pandora.hasCapability('canManagePlacesAndEvents') },
-                        { id: 'events', title: Ox._('Manage Events...'), disabled: !pandora.hasCapability('canManagePlacesAndEvents') },
+                        pandora.hasView('map')
+                        ? [{ id: 'places', title: Ox._('Manage Places...'), disabled: !pandora.hasCapability('canManagePlacesAndEvents') }] : [],
+                        pandora.hasView('calendar')
+                        ? [{ id: 'events', title: Ox._('Manage Events...'), disabled: !pandora.hasCapability('canManagePlacesAndEvents') }] : [],
                         {},
                         { id: 'users', title: Ox._('Manage Users...'), disabled: !pandora.hasCapability('canManageUsers') },
                         { id: 'statistics', title: Ox._('Statistics...'), disabled: !pandora.hasCapability('canManageUsers') },
@@ -312,6 +151,9 @@ pandora.ui.mainMenu = function() {
                     pandora.UI.set({listSort: [{key: value, operator: pandora.getSortOperator(value)}]});
                 } else if (data.id == 'itemview') {
                     pandora.UI.set({itemView: value});
+                } else if (data.id == 'collectionview') {
+                    var set = {collectionView: value};
+                    pandora.UI.set(set);
                 } else if (data.id == 'listview') {
                     var set = {listView: value};
                     if (
@@ -560,6 +402,47 @@ pandora.ui.mainMenu = function() {
                             });
                         }
                     } else if (ui.section == 'documents') {
+                        var files, ids = [];
+                        if (ui.document) {
+                            files = [pandora.$ui.document.info()];
+                            ids = [files[0].id];
+                        } else {
+                            files = pandora.$ui.list.options('selected').map(function(id) {
+                                ids.push(id)
+                                return pandora.$ui.list.value(id);
+                            });
+                        }
+                        if (ui._collection) {
+                            //fixme use history
+                            //pandora.doHistory('delete', files, ui._collection, function() {
+                            pandora.api.removeCollectionItems({
+                                collection: ui._collection,
+                                items: ids
+                            }, function() {
+                                pandora.UI.set({collectionSelection: []});
+                                pandora.reloadList();
+                            });
+                        } else {
+                            pandora.ui.deleteDocumentDialog(
+                                files,
+                                function() {
+                                    Ox.Request.clearCache();
+                                    if (ui.document) {
+                                        pandora.UI.set({document: ''});
+                                    } else {
+                                        pandora.$ui.list.reloadList()
+                                    }
+                                }
+                            ).open();
+                        }
+                    } else if (ui.section == 'edits') {
+                        var clips = pandora.$ui.editPanel.getSelectedClips();
+                        pandora.doHistory('delete', clips, ui.edit, function(result) {
+                            pandora.$ui.editPanel.updatePanel(function() {});
+                        });
+                    }
+                } else if (data.id == 'deletefromarchive') {
+                    if (ui.section == 'documents') {
                         var files;
                         if (ui.document) {
                             files = [pandora.$ui.document.info()];
@@ -579,11 +462,6 @@ pandora.ui.mainMenu = function() {
                                 }
                             }
                         ).open();
-                    } else if (ui.section == 'edits') {
-                        var clips = pandora.$ui.editPanel.getSelectedClips();
-                        pandora.doHistory('delete', clips, ui.edit, function(result) {
-                            pandora.$ui.editPanel.updatePanel(function() {});
-                        });
                     }
                 } else if (data.id == 'undo') {
                     fromMenu = true;
@@ -595,10 +473,17 @@ pandora.ui.mainMenu = function() {
                     fromMenu = true;
                     pandora.history.clear();
                 } else if (data.id == 'resetfilters') {
+                    if (ui.section == 'documents') {
+                        pandora.UI.set({
+                            documentFilters: pandora.site.user.ui.documentFilters
+                        });
+                        pandora.$ui.documentContentPanel.replaceElement(0, pandora.$ui.documentBrowser = pandora.ui.documentBrowser());
+                    } else {
                         pandora.UI.set({
                             filters: pandora.site.user.ui.filters
                         });
                         pandora.$ui.contentPanel.replaceElement(0, pandora.$ui.browser = pandora.ui.browser());
+                    }
                 } else if (data.id == 'showsidebar') {
                     pandora.UI.set({showSidebar: !ui.showSidebar});
                 } else if (data.id == 'showinfo') {
@@ -838,6 +723,9 @@ pandora.ui.mainMenu = function() {
                     );
                 }
             },
+            pandora_collectionview: function(data) {
+                that.checkItem('viewMenu_documents_' + data.value);
+            },
             pandora_listview: function(data) {
                 that.checkItem('viewMenu_movies_' + data.value);
                 if (
@@ -855,6 +743,7 @@ pandora.ui.mainMenu = function() {
             },
             pandora_section: function(data) {
                 lists = {};
+                that.replaceMenu('viewMenu', getViewMenu());
                 that.checkItem('viewMenu_section_' + data.value);
                 that.replaceMenu('listMenu', getListMenu());
                 that.replaceMenu('itemMenu', getItemMenu());
@@ -1192,7 +1081,11 @@ pandora.ui.mainMenu = function() {
             { id: 'paste', title: clipboardItems == 0 ? Ox._('Paste') : Ox._('Paste {0}', [clipboardItemName]), disabled: !canPaste, keyboard: 'control v' },
             { id: 'clearclipboard', title: Ox._('Clear Clipboard'), disabled: !clipboardItems},
             {},
-            { id: 'delete', title: Ox._('{0} {1} {2}', [deleteVerb, selectionItemName, listName]), disabled: !canDelete, keyboard: 'delete' },
+            [
+                { id: 'delete', title: Ox._('{0} {1} {2}', [deleteVerb, selectionItemName, listName]), disabled: !canDelete, keyboard: 'delete' }
+            ].concat(ui._collection ? [
+                { id: 'deletefromarchive', title: Ox._('{0} {1} {2}', [Ox._('Delete'), selectionItemName, Ox._('from Archive')]), disabled: !canDelete }
+            ] : []),
             {},
             { id: 'undo', title: undoText ? Ox._('Undo {0}', [undoText]) : Ox._('Undo'), disabled: !undoText, keyboard: 'control z' },
             { id: 'redo', title: redoText ? Ox._('Redo {0}', [redoText]) : Ox._('Redo'), disabled: !redoText, keyboard: 'shift control z' },
@@ -1555,6 +1448,123 @@ pandora.ui.mainMenu = function() {
         ] };
     }
 
+    function getSectionViews() {
+        if (ui.section == 'documents') {
+            return [
+                { id: 'documents', title: Ox._('View Documents'), items: [
+                    { group: 'collectionview', min: 1, max: 1, items: pandora.site.listViews.filter(function(view) {
+                        return Ox.contains(['list', 'grid'], view.id)
+                    }).map(function(view) {
+                        return Ox.extend({
+                            checked: ui.collectionView == view.id
+                        }, view, {
+                            keyboard: listViewKey <= 10
+                                ? 'shift ' + (listViewKey++%10)
+                                : void 0,
+                            title: Ox._(view.title)
+                        });
+                    }) },
+                ]},
+                { id: 'filters', title: Ox._('Filters'), items: [
+                    { group: 'filters', min: 5, max: 5, items: pandora.site.documentFilters.map(function(filter) {
+                        return Ox.extend({
+                            checked: Ox.getIndexById(ui.documentFilters, filter.id) > -1
+                        }, filter, {
+                            title: Ox._(filter.title)
+                        });
+                    }) },
+                    {},
+                    { id: 'resetfilters', title: Ox._('Reset Filters') }
+                ] },
+            ]
+        } else {
+            return [
+                { id: 'movies', title: Ox._('View {0}', [Ox._(pandora.site.itemName.plural)]), items: [
+                    { group: 'listview', min: 1, max: 1, items: pandora.site.listViews.map(function(view) {
+                        return Ox.extend({
+                            checked: ui.listView == view.id
+                        }, view, {
+                            keyboard: listViewKey <= 10
+                                ? 'shift ' + (listViewKey++%10)
+                                : void 0,
+                            title: Ox._(view.title)
+                        });
+                    }) },
+                ]},
+                { id: 'icons', title: Ox._('Icons'), items: [].concat([
+                        { group: 'viewicons', min: 1, max: 1, items: ['posters', 'frames'].map(function(icons) {
+                            return {id: icons, title: Ox._(Ox.toTitleCase(icons)), checked: ui.icons == icons};
+                        }) },
+                        {},
+                    ], pandora.site.media.importPosters ? [
+                        { id: 'showsiteposters', title: Ox._('Always Show {0} Poster', [pandora.site.site.name]), checked: ui.showSitePosters },
+                        {}
+                    ] : [], [
+                        { id: 'showreflections', title: Ox._('Show Reflections'), checked: true, disabled: true }
+                    ]
+                ) },
+                { id: 'timelines', title: Ox._('Timelines'), items: [
+                    { group: 'viewtimelines', min: 1, max: 1, items: pandora.site.timelines.map(function(mode) {
+                        return {id: mode.id, title: Ox._(mode.title), checked: ui.videoTimeline == mode.id};
+                    }) }
+                ]},
+                { id: 'columns', title: Ox._('Columns'), items: [
+                    { id: 'loadcolumns', title: Ox._('Load Layout...'), disabled: true },
+                    { id: 'savecolumns', title: Ox._('Save Layout...'), disabled: true },
+                    {},
+                    { id: 'resetcolumns', title: Ox._('Reset Layout'), disabled: true }
+                ] },
+                { id: 'filters', title: Ox._('Filters'), disabled: ui.section != 'items', items: [
+                    { group: 'filters', min: 5, max: 5, items: pandora.site.filters.map(function(filter) {
+                        return Ox.extend({
+                            checked: Ox.getIndexById(ui.filters, filter.id) > -1
+                        }, filter, {
+                            title: Ox._(filter.title)
+                        });
+                    }) },
+                    {},
+                    { id: 'resetfilters', title: Ox._('Reset Filters') }
+                ] },
+                {},
+                { id: 'item', title: [
+                        Ox._('Open {0}', [Ox._(pandora.site.itemName.singular)]),
+                        Ox._('Open {0}', [Ox._(pandora.site.itemName.plural)])
+                    ], items: [
+                    { group: 'itemview', min: 1, max: 1, items: pandora.site.itemViews.filter(function(view) {
+                        return view.id != 'data' && view.id != 'media' ||
+                            pandora.hasCapability('canSeeExtraItemViews');
+                    }).map(function(view) {
+                        return Ox.extend({
+                            checked: ui.itemView == view.id
+                        }, view, {
+                            keyboard: itemViewKey <= 10
+                                ? 'shift ' + (itemViewKey++%10)
+                                : void 0,
+                            title: Ox._(view.title)
+                        });
+                    }) },
+                ] },
+                { id: 'clips', title: Ox._('Open Clips'), items: [
+                    { group: 'videoview', min: 1, max: 1, items: ['player', 'editor', 'timeline'].map(function(view) {
+                        return {id: view, title: Ox._(Ox.toTitleCase(view)), checked: ui.videoView == view};
+                    }) }
+                ] },
+                { id: 'documents', title: Ox._('Open Documents'), items: [
+                    { group: 'documentview', min: 1, max: 1, items: ['info', 'view'].map(function(id) {
+                        return {
+                            id: id,
+                            checked: ui.documentView == id,
+                            keyboard: documentViewKey <= 10
+                                ? 'shift ' + (documentViewKey++%10)
+                                : void 0,
+                            title: Ox._(Ox.toTitleCase(id))
+                        }
+                    }) }
+                ] }
+            ]
+        }
+    }
+
     function getSortMenu() {
 
         if (ui.section == 'documents') {
@@ -1634,6 +1644,94 @@ pandora.ui.mainMenu = function() {
         ] };
     }
 
+    function getViewMenu() {
+        return { id: 'viewMenu', title: Ox._('View'), items: [
+            { id: 'section', title: Ox._('Section'), items: [
+                { group: 'viewsection', min: 1, max: 1, items: pandora.site.sections.map(function(section) {
+                    section = Ox.extend({}, section)
+                    section.checked = section.id == ui.section;
+                    return section;
+                }) }
+            ] },
+            {},
+            getSectionViews(),
+            {},
+            {
+                id: 'showsidebar',
+                title: Ox._((ui.showSidebar ? 'Hide' : 'Show') + ' Sidebar'),
+                keyboard: 'shift s'
+            },
+            {
+                id: 'showinfo',
+                title: Ox._((ui.showInfo ? 'Hide' : 'Show') + ' Info'),
+                disabled: !ui.showSidebar, keyboard: 'shift i'
+            },
+            {
+                id: 'showfilters',
+                title: Ox._((ui.showFilters ? 'Hide' : 'Show') + ' Filters'),
+                disabled: (
+                    !Ox.contains(['items', 'documents'], ui.section) ||
+                    (ui.section == 'items' && !!ui.item) ||
+                    (ui.section == 'documents' && !!ui.document)
+                ), keyboard: 'shift f'
+            },
+            {
+                id: 'showbrowser',
+                title: Ox._((ui.showBrowser ? 'Hide': 'Show') + ' {0} Browser', [Ox._(pandora.site.itemName.singular)]),
+                disabled: !ui.item, keyboard: 'shift b'
+            },
+            {
+                id: 'showdocument',
+                title: Ox._((ui.showDocument ? 'Hide' : 'Show') + ' Document'),
+                disabled: !hasDocument(), keyboard: 'shift d'
+            },
+            {
+                id: 'showtimeline',
+                title: Ox._((ui.showTimeline ? 'Hide' : 'Show') + ' Timeline'),
+                disabled: !hasTimeline(), keyboard: 'shift t'
+            },
+            {
+                id: 'showannotations',
+                title: Ox._((ui.showAnnotations ? 'Hide' : 'Show') + ' Annotations'),
+                disabled: !hasAnnotations(), keyboard: 'shift a'
+            },
+            {
+                id: 'showclips',
+                title: Ox._((ui.showClips ? 'Hide' : 'Show') + ' Clips'),
+                disabled: !hasClips(), keyboard: 'shift c'
+            },
+            {},
+            {
+                id: 'togglefullscreen',
+                title: Ox._((fullscreenState ? 'Exit' : 'Enter') + ' Fullscreen'),
+                disabled: fullscreenState === void 0,
+                keyboard: /^Mac/.test(window.navigator.platform)
+                    ? 'shift alt f'
+                    : 'F11'
+
+            },
+            {
+                id: 'entervideofullscreen',
+                title: Ox._('Enter Video Fullscreen'),
+                disabled: !ui.item || ui.itemView != 'player'
+            },
+            {},
+            { id: 'theme', title: Ox._('Theme'), items: [
+                { group: 'settheme', min: 1, max: 1, items: pandora.site.themes.map(function(theme) {
+                    return {id: theme, title: Ox.Theme.getThemeData(theme).themeName, checked: ui.theme == theme}
+                }) }
+            ] },
+            { id: 'locale',
+              title: Ox._('Language'), items: [
+                { group: 'setlocale', min: 1, max: 1, items: pandora.site.languages.map(function(locale) {
+                    return {id: locale, title: Ox.LOCALE_NAMES[locale], checked: ui.locale == locale}
+                }) }
+            ] },
+            {},
+            { id: 'embed', title: Ox._('Embed...') }
+        ]}
+    }
+
     function hasAnnotations() {
         return ui.section == 'items' && ui.item && pandora.isVideoView();
     }

static/js/mainPanel.js

@@ -23,9 +23,40 @@ pandora.ui.mainPanel = function() {
         .bindEvent({
             pandora_finddocuments: function() {
                 var previousUI = pandora.UI.getPrevious();
-                if (!previousUI.document && ui._list == previousUI._list) {
+                Ox.Log('FIND', 'finddocuments handled in mainPanel', previousUI.document, previousUI._collection)
+                if (!previousUI.document && ui._collection == previousUI._collection) {
+                    pandora.$ui.list.reloadList();
+
+                    // FIXME: why is this being handled _here_?
+                    ui._documentFilterState.forEach(function(data, i) {
+                        if (!Ox.isEqual(data.selected, previousUI._documentFilterState[i].selected)) {
+                            pandora.$ui.documentFilters[i].options(
+                                ui.showFilters ? {
+                                    selected: data.selected
+                                } : {
+                                    _selected: data.selected,
+                                    selected: []
+                                }
+                            );
+                        }
+                        if (!Ox.isEqual(data.find, previousUI._documentFilterState[i].find)) {
+                            if (!ui.showFilters) {
+                                pandora.$ui.documentFilters[i].options({
+                                    _selected: data.selected
+                                });
+                            }
+                            // we can call reloadList here, since the items function
+                            // handles the hidden filters case without making requests
+                            pandora.$ui.documentFilters[i].reloadList();
+                        }
+                    });
+                } else {
+                    if (pandora.stayInItemView) {
+                        pandora.stayInItemView = false;
+                    } else {
                         that.replaceElement(1, pandora.$ui.documentPanel = pandora.ui.documentPanel());
                     }
+                }
             },
             pandora_document: function(data) {
                 if (!data.value || !data.previousValue) {
@@ -37,7 +68,7 @@ pandora.ui.mainPanel = function() {
             },
             pandora_find: function() {
                 var previousUI = pandora.UI.getPrevious();
-                Ox.Log('FIND', 'handled in mainPanel', previousUI.item, previousUI._list)
+                Ox.Log('FIND', 'find handled in mainPanel', previousUI.item, previousUI._list)
                 if (!previousUI.item && ui._list == previousUI._list) {
                     if (['map', 'calendar'].indexOf(ui.listView) > -1) {
                         pandora.$ui.contentPanel.replaceElement(1,

static/js/mediaView.js

@@ -406,7 +406,7 @@ pandora.ui.mediaView = function(options) {
                 var conditions, matches;
                 if (key == 'id' && data.value.substr(0, 2) != '0x') {
                     if (pandora.site.site.id == '0xdb') {
-                        matches = data.value.match(/\d{7}/);
+                        matches = data.value.match(/\d+/);
                     } else {
                         matches = data.value.match(/[A-Z]+/);
                     }

static/js/metadataDialog.js

@@ -181,6 +181,9 @@ pandora.ui.metadataDialog = function(data) {
     }
 
     function getKey(key) {
+        if (Ox.getObjectById(pandora.site.itemKeys, key) && mapKeys[key]) {
+            return key
+        }
         return mapKeys[key] || key;
     }
 
@@ -222,7 +225,7 @@ pandora.ui.metadataDialog = function(data) {
             if (result.data) {
                 imdb = Ox.clone(result.data, true);
                 if (!Ox.contains(keys, 'originalTitle')) {
-                    if (imdb.originalTitle) {
+                    if (imdb.originalTitle && imdb.originalTitle != imdb.title) {
                         imdb.alternativeTitles = [[imdb.title, []]].concat(imdb.alternativeTitles || []);
                         imdb.title = imdb.originalTitle;
                     }

static/js/pandora.js

@@ -281,6 +281,37 @@ appPanel
             resize: pandora.resizeWindow,
             unload: pandora.unloadWindow
         })
+        Ox.$document.on({
+            dragenter: function(event) {
+                if (Ox.contains(event.originalEvent.dataTransfer.types, 'Files')) {
+                    event.originalEvent.preventDefault();
+                    event.originalEvent.stopPropagation();
+                    if (!$('#importScreen').length) {
+                        pandora.ui.importScreen().appendTo(Ox.$body);
+                    }
+                } else {
+                    console.log(event.originalEvent.dataTransfer);
+                }
+            },
+            dragover: function(event) {
+                event.originalEvent.preventDefault();
+                event.originalEvent.stopPropagation();
+            },
+            dragstart: function(event) {
+                event.originalEvent.preventDefault();
+                event.originalEvent.stopPropagation();
+            },
+            drop: function(event) {
+                $('#importScreen').remove();
+                if (pandora.hasCapability('canAddItems')) {
+                    if (event.originalEvent.dataTransfer.files.length) {
+                        event.originalEvent.preventDefault();
+                        event.originalEvent.stopPropagation();
+                        pandora.uploadDroppedFiles(event.originalEvent.dataTransfer.files)
+                    }
+                }
+            }
+        });
         Ox.extend(pandora, {
             $ui: {},
             site: data.site,
@@ -364,6 +395,11 @@ appPanel
             }) ? 'manual' : data.site.layers.some(function(layer) {
                 return layer.hasPlaces;
             }) ? 'auto' : 'none',
+            sections: [
+                {id: 'items', title: Ox._(pandora.site.itemName.plural)},
+                {id: 'edits', title: Ox._('Edits')},
+                {id: 'documents', title: Ox._('Documents')}
+            ],
             sectionFolders: {
                 items: [
                     {id: 'personal', title: 'Personal Lists'},

static/js/preferencesDialog.js

@@ -43,11 +43,19 @@ pandora.ui.preferencesDialog = function() {
                                         id: 'password',
                                         label: Ox._('New Password'),
                                         labelWidth: 120,
-                                        type: 'password',
+                                        type: 'text',
                                         validate: pandora.validateNewPassword,
                                         width: 320
                                     })
+                                    .attr({
+                                        autocomplete: 'new-password'
+                                    })
                                     .bindEvent({
+                                        focus: function(data) {
+                                            this.options({
+                                                type: 'password'
+                                            })
+                                        },
                                         validate: function(data) {
                                             data.valid && pandora.api.editPreferences({password: data.value});
                                         }

static/js/sectionButtons.js

@@ -2,11 +2,7 @@
 
 pandora.ui.sectionButtons = function(section) {
     var that = Ox.ButtonGroup({
-            buttons: [
-                {id: 'items', title: Ox._(pandora.site.itemName.plural)},
-                {id: 'edits', title: Ox._('Edits')},
-                {id: 'documents', title: Ox._('Documents')}
-            ],
+            buttons: pandora.site.sections,
             id: 'sectionButtons',
             selectable: true,
             value: section || pandora.user.ui.section

static/js/sectionSelect.js

@@ -4,11 +4,7 @@ pandora.ui.sectionSelect = function(section) {
     // fixme: duplicated
     var that = Ox.Select({
             id: 'sectionSelect',
-            items: [
-                {id: 'items', title: Ox._(pandora.site.itemName.plural)},
-                {id: 'edits', title: Ox._('Edits')},
-                {id: 'documents', title: Ox._('Documents')}
-            ],
+            items: pandora.site.sections,
             value: section || pandora.user.ui.section
         }).css({
             float: 'left',

static/js/textPanel.js

@@ -1,16 +1,30 @@
 'use strict';
 
 pandora.ui.textPanel = function(text, $toolbar) {
+    if (Ox.isUndefined(text.text)) {
+        var that = Ox.Element().append(Ox.LoadingScreen().start())
+        pandora.api.getDocument({
+            id: text.id,
+            keys: ['text']
+        }, function(result) {
+            text.text = result.data.text
+            if (text.text) {
+                pandora.$ui.textPanel.replaceWith(pandora.$ui.textPanel = pandora.ui.textPanel(text, $toolbar))
+            }
+        })
+        return that;
+    }
+
     var textElement,
         textEmbed,
         embedURLs = getEmbedURLs(text.text),
         that = Ox.SplitPanel({
             elements: [
                 {
-                    element: textElement = pandora.$ui.textElement = pandora.ui.textHTML(text)
+                    element: textElement = pandora.ui.textHTML(text)
                 },
                 {
-                    element: textEmbed = pandora.ui.textEmbed(textElement),
+                    element: textEmbed = pandora.ui.textEmbed(),
                     collapsed: !embedURLs.length,
                     size: pandora.user.ui.embedSize,
                     resizable: true,
@@ -124,7 +138,6 @@ pandora.ui.textPanel = function(text, $toolbar) {
             0),
             position = 100 * scrollTop / Math.max(1, textElement[0].scrollHeight);
         textElement.scrollTo(position);
-        window.text = textElement;
     }
 
     that.selectEmbed = function(index, scroll) {
@@ -431,7 +444,7 @@ pandora.ui.textHTML = function(text) {
 
 };
 
-pandora.ui.textEmbed = function(textElement) {
+pandora.ui.textEmbed = function(textEmbed) {
 
     var that = Ox.Element()
             .bindEvent({

static/js/uploadDocumentDialog.js

@@ -5,6 +5,8 @@ pandora.ui.uploadDocumentDialog = function(options, callback) {
         extensions = files.map(function(file) {
             return file.name.split('.').pop().toLowerCase()
         }),
+        existingFiles = [],
+        uploadFiles = [],
 
         supportedExtensions = ['gif', 'jpg', 'jpeg', 'pdf', 'png'],
 
@@ -56,9 +58,9 @@ pandora.ui.uploadDocumentDialog = function(options, callback) {
                 height: 112,
                 keys: {escape: 'close'},
                 width: 288,
-                title: files.length == 1
+                title: uploadFiles.length == 1
                     ? Ox._('Upload Document')
-                    : Ox._('Upload {0} Documents', [files.length])
+                    : Ox._('Upload {0} Documents', [uploadFiles.length])
             })
             .bindEvent({
                 open: function() {
@@ -73,12 +75,17 @@ pandora.ui.uploadDocumentDialog = function(options, callback) {
             Ox._('Supported file types are GIF, JPG, PNG and PDF.')
         );
     } else {
-        var valid = true;
+        var oshashes = [];
         Ox.parallelForEach(files, function(file, index, array, callback) {
             var extension = file.name.split('.').pop().toLowerCase(),
                 filename = file.name.split('.').slice(0, -1).join('.') + '.'
                     + (extension == 'jpeg' ? 'jpg' : extension);
-            valid && Ox.oshash(file, function(oshash) {
+            Ox.oshash(file, function(oshash) {
+                // skip duplicate files
+                if (Ox.contains(oshashes, oshash)) {
+                    callback();
+                } else {
+                    oshashes.push(oshash)
                     pandora.api.findDocuments({
                         keys: ['id', 'user', 'title', 'extension'],
                         query: {
@@ -95,7 +102,25 @@ pandora.ui.uploadDocumentDialog = function(options, callback) {
                         if (result.data.items.length) {
                             var id = result.data.items[0].title + '.'
                                 + result.data.items[0].extension;
-                        valid && errorDialog(
+                            existingFiles.push({
+                                id: id,
+                                filename: filename
+
+                            })
+                        } else {
+                            uploadFiles.push(file)
+                        }
+                        callback();
+                    })
+                }
+            });
+        } ,function() {
+            if (uploadFiles.length) {
+                $uploadDialog.open();
+            } else if (existingFiles.length) {
+                var filename = existingFiles[0].filename
+                var id = existingFiles[0].id
+                errorDialog(
                     filename == id
                     ? Ox._(
                         'The file "{0}" already exists.',
@@ -106,13 +131,7 @@ pandora.ui.uploadDocumentDialog = function(options, callback) {
                         [filename, id]
                     )
                 ).open();
-                        valid = false;
             }
-                    callback();
-                })
-            });
-        } ,function() {
-            valid && $uploadDialog.open();
         });
         return {open: Ox.noop};
     }
@@ -138,7 +157,7 @@ pandora.ui.uploadDocumentDialog = function(options, callback) {
     function uploadFile(part) {
         var data = {
             },
-            file = files[part],
+            file = uploadFiles[part],
             extension = file.name.split('.').pop().toLowerCase(),
             filename = file.name.split('.').slice(0, -1).join('.') + '.'
                 + (extension == 'jpeg' ? 'jpg' : extension);
@@ -158,7 +177,7 @@ pandora.ui.uploadDocumentDialog = function(options, callback) {
                     if (data.progress == 1) {
                         part++;
                         ids.push(data.response.id);
-                        if (part == files.length) {
+                        if (part == uploadFiles.length) {
                             $progress.options({progress: data.progress});
                             callback && callback({ids: ids});
                             $uploadDialog.close();
@@ -174,7 +193,7 @@ pandora.ui.uploadDocumentDialog = function(options, callback) {
                 },
                 progress: function(data) {
                     var progress = data.progress || 0;
-                    progress = part / files.length + 1 / files.length * progress;
+                    progress = part / uploadFiles.length + 1 / uploadFiles.length * progress;
                     $progress.options({progress: progress});
                 }
             });

static/js/uploadVideoDialog.js

@@ -1,599 +0,0 @@
-'use strict';
-
-pandora.ui.uploadVideoDialog = function(data) {
-
-    var cancelled = false,
-        file,
-        hasFirefogg = !(Ox.isUndefined(window.Firefogg)) && (
-            $.browser.version < '35' || Firefogg().version >= 334
-        ),
-        infoContent = Ox._('Please select the video file that you want to upload.'),
-        itemView = pandora.hasCapability('canSeeExtraItemViews') ? 'media' : 'info',
-        selectFile,
-        $actionButton,
-        $closeButton,
-        $content = Ox.Element().css({margin: '16px'}),
-        $info = $('<div>')
-            .css({padding: '4px'})
-            .html(infoContent),
-        $progress,
-        $status = $('<div>').css({padding: '4px', paddingTop: '8px'}),
-        that = Ox.Dialog({
-            buttons: [
-                $closeButton = Ox.Button({
-                    id: 'close',
-                    title: Ox._('Close')
-                }).css({
-                    float: 'left'
-                }).bindEvent({
-                    click: function() {
-                        if ($closeButton.options('title') == Ox._('Cancel')) {
-                            cancelled = true;
-                            $info.html(infoContent);
-                            $status.html('');
-                            pandora.firefogg && pandora.firefogg.cancel();
-                            pandora.$ui.upload && pandora.$ui.upload.abort();
-                            $closeButton.options('title', Ox._('Close'));
-                            if ($actionButton.options('title') == Ox._('Upload')) {
-                                $closeButton.options('title', Ox._('Close'));
-                                $actionButton.replaceWith($actionButton = hasFirefogg
-                                    ? getFirefoggButton()
-                                    : getSelectVideoButton()
-                                );
-                            }
-                            $actionButton.show();
-                        } else {
-                            that.triggerEvent('close');
-                        }
-                    }
-                }),
-                $actionButton = hasFirefogg ? getFirefoggButton() : getSelectVideoButton()
-            ],
-            content: $content,
-            height: 128,
-            removeOnClose: true,
-            width: 368,
-            title: Ox._('Upload Video'),
-        })
-        .bindEvent({
-            close: function(data) {
-                if (pandora.firefogg) {
-                    pandora.firefogg.cancel();
-                    delete pandora.firefogg;
-                }
-                that.close();
-            }
-        });
-
-    if (!pandora.site.itemRequiresVideo && !pandora.user.ui.item) {
-        $info.html(Ox._(
-            'You can only upload a video to an existing {0}.'
-            + ' Please check if an entry for the {0}'
-            + ' you want to upload exists, and create one otherwise.',
-            [pandora.site.itemName.singular.toLowerCase()]
-        ));
-        $actionButton.hide();
-    }
-    $content.append($info);
-    $content.append($status);
-
-    function aspectratio(ratio) {
-        var denominator, numerator;
-        ratio = ratio.split(':');
-        numerator = ratio[0];
-        if (ratio.length == 2) {
-            denominator = ratio[1];
-        }
-        if (Math.abs(numerator / denominator - 4/3) < 0.03) {
-            numerator = 4;
-            denominator = 3;
-        } else if (Math.abs(numerator / denominator - 16/9) < 0.02) {
-            numerator = 16;
-            denominator = 9;
-        }
-        return {
-            denominator: denominator,
-            'float': numerator / denominator,
-            numerator: numerator,
-            ratio: numerator + ':' + denominator
-        };
-    }
-
-    function resetProgress(status) {
-        $progress = Ox.Progressbar({
-            progress: 0,
-            showPercent: true,
-            showTime: true,
-            width: 304
-        });
-        $status.html(status || '').append($progress);
-    }
-
-    function directUpload(file, info) {
-        resetProgress();
-        pandora.api.addMedia({
-            filename: info.name,
-            id: info.oshash,
-            item: pandora.site.itemRequiresVideo
-                ? undefined
-                : pandora.user.ui.item
-        }, function(result) {
-            uploadStream(result.data.item, info, file);
-        });
-    }
-
-    function encode() {
-        var filename = pandora.firefogg.sourceFilename,
-            info = JSON.parse(pandora.firefogg.sourceInfo),
-            item,
-            oshash = info.oshash;
-        $info.html('<b>' + filename + '</b><br>' + Ox._('encoding...'));
-        resetProgress();
-        pandora.api.addMedia({
-            filename: filename,
-            id: oshash,
-            info: info,
-            item: pandora.site.itemRequiresVideo
-                ? undefined
-                : pandora.user.ui.item
-        }, function(result) {
-            item = result.data.item;
-            pandora.firefogg.encode(
-                getEncodingOptions(info),
-                function(result, file) {
-                    result = JSON.parse(result);
-                    if (result.progress != 1) {
-                        $status.html(
-                            cancelled
-                            ? Ox._('Encoding cancelled.')
-                            : Ox._('Encoding failed.')
-                        );
-                        delete pandora.firefogg;
-                        return;
-                    }
-                    setTimeout(function() {
-                        $info.html(
-                            '<b>' + filename + '</b><br>'
-                            + Ox._('uploading...')
-                        );
-                        uploadStream(item, info, file);
-                    });
-                },
-                Ox.throttle(function(progress) {
-                    progress = JSON.parse(progress).progress || 0;
-                    $progress.options({progress: progress});
-                }, 1000)
-            );
-        });
-    }
-
-    function getInfo(file, callback) {
-        Ox.oshash(file, function(oshash) {
-            var $video = $('<video>'),
-                url = URL.createObjectURL(file),
-                info = {
-                    audio: [],
-                    direct: false,
-                    oshash: oshash,
-                    name: file.name,
-                    size: file.size,
-                    video: []
-                };
-            $video.one('error', function(event) {
-                callback(info);
-            });
-            $video.one('loadedmetadata', function(event) {
-                info.duration = $video[0].duration;
-                if ($video[0].videoHeight > 0) {
-                    info.video.push({
-                        height: $video[0].videoHeight,
-                        width: $video[0].videoWidth
-                    });
-                }
-                if (info.duration) {
-                    info.bitrate = info.size * 8 / info.duration / 1000;
-                }
-                var format = pandora.site.video.formats[0],
-                    resolution = getResolution(info);
-                info.direct = Ox.endsWith(info.name, format)
-                    && info.video.length > 0
-                    && info.video[0].height <= resolution;
-                callback(info);
-            });
-            $video[0].src = url;
-        });
-    }
-
-    function getResolution(info) {
-        var height = info.video && info.video.length
-                ? info.video[0].height
-                : Ox.max(pandora.site.video.resolutions),
-            resolution = Ox.sort(pandora.site.video.resolutions)
-                .filter(function(resolution) {
-                    return height <= resolution;
-                })[0] || Ox.max(pandora.site.video.resolutions);
-        return resolution;
-    }
-
-    function getFirefoggButton() {
-        return Ox.Button({
-            id: 'action',
-            title: Ox._('Select Video')
-        }).bindEvent({
-            click: function() {
-                if ($actionButton.options('title') == Ox._('Select Video')) {
-                    if (selectVideo()) {
-                        $closeButton.options('title', Ox._('Cancel'));
-                        $actionButton.options('title', Ox._('Upload'));
-                    }
-                } else if ($actionButton.options('title') == Ox._('Cancel')) {
-                    cancelled = true;
-                    pandora.firefogg && pandora.firefogg.cancel();
-                    pandora.$ui.upload && pandora.$ui.upload.abort();
-                    $actionButton.options('title', Ox._('Select Video'));
-                    $closeButton.show();
-                } else {
-                    $closeButton.options('title', Ox._('Cancel'));
-                    $actionButton.hide().options('title', Ox._('Select Video'));
-                    encode();
-                }
-            }
-        })
-    }
-
-    function getSelectVideoButton() {
-        return Ox.FileButton({
-            id: 'action',
-            title: Ox._('Select Video'),
-            maxFiles: 1,
-            width: 96
-        }).css({
-            float: 'left'
-        }).bindEvent({
-            click: function(data) {
-                if (data.files.length) {
-                    cancelled = false;
-                    $actionButton.replaceWith($actionButton = Ox.Button({
-                        id: 'action',
-                        title: 'Upload',
-                        disabled: true
-                    }).css({
-                        float: 'left'
-                    }));
-                    getInfo(data.files[0], function(info) {
-                        $actionButton.options({
-                            disabled: false
-                        }).bindEvent({
-                            click: function() {
-                                $actionButton.replaceWith($actionButton = getSelectVideoButton().hide());
-                                info.direct
-                                    ? directUpload(data.files[0], info)
-                                    : upload(data.files[0]);
-                            }
-                        });
-                        $info.html(formatVideoInfo(info));
-                        $status.html(
-                            info.direct
-                            ? Ox._(
-                                'Your video will be used directly.'
-                            )
-                            : Ox._(
-                                'Your video will be transcoded on the server.'
-                            )
-                        );
-                    });
-                    $closeButton.options('title', Ox._('Cancel'));
-                }
-            }
-        });
-    }
-
-    function uploadStream(item, info, file) {
-        var oshash = info.oshash,
-            format = pandora.site.video.formats[0],
-            resolution = getResolution(info);
-        pandora.$ui.upload = pandora.chunkupload({
-            file: file,
-            url: '/api/upload/?profile=' + resolution + 'p.'
-                + format + '&id=' + oshash,
-            data: {}
-        }).bindEvent({
-            done: function(data) {
-                if (data.progress == 1) {
-                    Ox.Request.clearCache();
-                    if (
-                        pandora.user.ui.item == item
-                        && pandora.user.ui.itemView == itemView
-                    ) {
-                        pandora.$ui.item.reload();
-                    } else {
-                        pandora.UI.set({
-                            item: item,
-                            itemView: itemView
-                        });
-                    }
-                    delete pandora.firefogg;
-                    that.close();
-                } else {
-                    $status.html(Ox._('Upload failed.'));
-                    pandora.api.logError({
-                        text: data.responseText,
-                        url: '/' + item,
-                        line: 1
-                    });
-                }
-            },
-            progress: function(data) {
-                $progress.options({progress: data.progress || 0});
-            },
-        });
-    }
-
-    function upload(file) {
-        resetProgress();
-        $info.html(Ox._('Uploading {0}', [file.name]));
-        Ox.oshash(file, function(oshash) {
-            pandora.api.findMedia({
-                query: {
-                    conditions: [{key: 'oshash', value: oshash}]
-                },
-                keys: ['id', 'item', 'available']
-            }, function(result) {
-                if (
-                    result.data.items.length === 0
-                    || !result.data.items[0].available
-                ) {
-                    pandora.api.addMedia({
-                        filename: file.name,
-                        id: oshash,
-                        item: pandora.site.itemRequiresVideo
-                            ? undefined
-                            : pandora.user.ui.item
-                    }, function(result) {
-                        var item = result.data.item;
-                        pandora.$ui.upload = pandora.chunkupload({
-                            file: file,
-                            url: '/api/upload/direct/',
-                            data: {
-                                id: oshash
-                            }
-                        }).bindEvent({
-                            done: function(data) {
-                                if (data.progress == 1) {
-                                    Ox.Request.clearCache();
-                                    if (
-                                        pandora.user.ui.item == item
-                                        && pandora.user.ui.itemView == itemView
-                                    ) {
-                                        pandora.$ui.item.reload();
-                                    } else {
-                                        pandora.UI.set({
-                                            item: item,
-                                            itemView: itemView
-                                        });
-                                    }
-                                    that.close();
-                                } else {
-                                    $status.html(
-                                        cancelled
-                                        ? Ox._('Upload cancelled.')
-                                        : Ox._('Upload failed.')
-                                    );
-                                    !cancelled && pandora.api.logError({
-                                        text: data.responseText,
-                                        url: '/' + item,
-                                        line: 1
-                                    });
-                                }
-                            },
-                            progress: function(data) {
-                                $progress.options({
-                                    progress: data.progress || 0
-                                });
-                            }
-                        });
-                    });
-                } else {
-                    pandora.UI.set({
-                        item: result.data.items[0].item,
-                        itemView: itemView
-                    });
-                    that.close();
-                }
-            });
-        });
-    }
-
-    function getEncodingOptions(info) {
-        var bpp = 0.17,
-            dar,
-            format = pandora.site.video.formats[0],
-            fps,
-            options = {},
-            resolution = getResolution(info);
-        if (format == 'webm') {
-            options.videoCodec = 'vp8';
-            options.audioCodec = 'vorbis';
-        } else if (format == 'ogv') {
-            options.videoCodec = 'theora';
-            options.audioCodec = 'vorbis';
-        }
-        if (resolution == 720) {
-            options.height = 720;
-            options.samplerate = 48000;
-            options.audioQuality = 5;
-        } else if (resolution == 480) {
-            options.height = 480;
-            options.samplerate = 44100;
-            options.audioQuality = 3;
-            options.channels = 2;
-        } else if (resolution == 432) {
-            options.height = 432;
-            options.samplerate = 44100;
-            options.audioQuality = 3;
-            options.channels = 2;
-        } else if (resolution == 360) {
-            options.height = 320;
-            options.samplerate = 44100;
-            options.audioQuality = 1;
-            options.channels = 1;
-        } else if (resolution == 288) {
-            options.height = 288;
-            options.samplerate = 44100;
-            options.audioQuality = 0;
-            options.channels = 1;
-        } else if (resolution == 240) {
-            options.height = 240;
-            options.samplerate = 44100;
-            options.audioQuality = 0;
-            options.channels = 1;
-        } else if (resolution == 144) {
-            options.height = 144;
-            options.samplerate = 22050;
-            options.audioQuality = -1;
-            options.audioBitrate = 22;
-            options.channels = 1;
-        } else if (resolution == 96) {
-            options.height = 96;
-            options.samplerate = 22050;
-            options.audioQuality = -1;
-            options.audioBitrate = 22;
-            options.channels = 1;
-        }
-        if (info.video && info.video.length) {
-            info.video.forEach(function(video) {
-                if (!video.display_aspect_ratio) {
-                    video.display_aspect_ratio = video.width + ':' + video.height;
-                    video.pixel_aspect_ratio = '1:1';
-                }
-            });
-            dar = aspectratio(info.video[0].display_aspect_ratio);
-            fps = aspectratio(info.video[0].framerate).float;
-            options.width = parseInt(dar.float * options.height, 10);
-            options.width += options.width % 2;
-            // interlaced hdv material is detected with double framerates
-            if (fps == 50) {
-                fps = options.framerate = 25;
-            } else if (fps == 60) {
-                fps = options.framerate = 30;
-            }
-            if (Math.abs(options.width/options.height - dar.float) < 0.02) {
-                options.aspect = options.width + ':' + options.height;
-            } else {
-                options.aspect = dar.ratio;
-            }
-            options.videoBitrate = Math.round(
-                options.height * options.width * fps * bpp / 1000
-            );
-            options.denoise = true; 
-            options.deinterlace = true; 
-        } else {
-            options.noVideo = true;
-        }
-        if (info.audio) {
-            if (options.cannels && info.audio[0].channels < options.channels) {
-                delete options.channels;
-            }
-        } else {
-            options.noAudio = true;
-            delete options.samplerate;
-            delete options.audioQuality;
-            delete options.channels;
-        }
-        options.noUpscaling = true;
-        if (
-            (!info.video.length || (
-                info.video[0].codec == options.videoCodec
-                && info.video[0].height <= options.height
-            ))
-            && (!info.audio.length || info.audio[0].codec == options.audioCodec)
-        ) {
-            options = {passthrough: true};
-        }
-        return JSON.stringify(options);
-    }
-
-    function formatInfo(info) {
-        var html = '<b>' + info.path + '</b><br>';
-        if (info.video && info.video.length > 0) {
-            var video = info.video[0];
-            html += video.width + '×' + video.height  + ' (' + video.codec + ')';
-        }
-        if (
-            info.video && info.video.length > 0
-            && info.audio && info.audio.length > 0
-        ) {
-            html += ' / ';
-        }
-        if (info.audio && info.audio.length > 0) {
-            var audio = info.audio[0];
-            html += {1: 'mono', 2: 'stereo', 6: '5.1'}[audio.channels]
-                + ' ' + audio.samplerate / 1000 + ' kHz (' + audio.codec + ')';
-        }
-        html += '<br>' + Ox.formatValue(info.size, 'B')
-            + ' / ' + Ox.formatDuration(info.duration);
-        return html; 
-    }
-
-    function formatVideoInfo(info) {
-        var html = '<b>' + info.name + '</b><br>';
-        if (info.video && info.video.length > 0) {
-            html += info.video[0].width + '×' + info.video[0].height;
-        }
-        html += '<br>' + Ox.formatValue(info.size, 'B');
-        if(info.duration) {
-            html += ' / ' + Ox.formatDuration(info.duration);
-        }
-        return html; 
-    }
-
-    function selectVideo() {
-        cancelled = false;
-        pandora.firefogg = new Firefogg();
-        pandora.firefogg.setFormat(pandora.site.video.formats[0]);
-        if (pandora.firefogg.selectVideo()) {
-            var info = JSON.parse(pandora.firefogg.sourceInfo),
-                options = JSON.parse(getEncodingOptions(info)),
-                oshash = info.oshash,
-                filename = pandora.firefogg.sourceFilename,
-                item;
-            pandora.api.findMedia({
-                query: {
-                    conditions: [{key: 'oshash', value: oshash}]
-                },
-                keys: ['id', 'available']
-            }, function(result) {
-                if (
-                    result.data.items.length === 0
-                    || !result.data.items[0].available
-                ) {
-                    $info.html(formatInfo(info));
-                    $status.html(
-                        options.passthrough
-                        ? Ox._('Your video will be uploaded directly.')
-                        : Ox._('Your video will be transcoded before upload.')
-                    );
-                } else {
-                    pandora.api.find({
-                        query: {
-                            conditions: [{key: 'oshash', value: oshash}]
-                        },
-                        keys: ['id']
-                    }, function(result) {
-                        pandora.UI.set({
-                            item: result.data.items[0].id,
-                            itemView: itemView
-                        });
-                        delete pandora.firefogg;
-                        that.close();
-                    });
-                }
-            });
-            return true;
-        }
-        return false;
-    }
-
-    return that;
-
-};

static/js/utils.js

@@ -39,7 +39,6 @@ pandora.addFolderItem = function(section) {
                 if (isItems) {
                     data.items = ui.listSelection;
                 } else if (section == 'documents') {
-                    //fixme
                     data.items = ui.collectionSelection;
                 } else {
                     data.clips = pandora.getClipData(
@@ -49,7 +48,7 @@ pandora.addFolderItem = function(section) {
                     );
                 }
             } else {
-                data.query = ui.find;
+                data.query = section == 'documents' ? ui.findDocuments : ui.find;
             }
         }
         if (ui.section == 'items' && section == 'edits') {
@@ -82,18 +81,28 @@ pandora.addFolderItem = function(section) {
             data.description = result.data.items[0].description;
             if (data.type == 'static') {
                 var query;
-                if (isItems) {
+                if (Ox.contains(['items', 'documents'], section)) {
                     query = {
                         conditions: [{
-                            key: 'list',
+                            key: {
+                                items: 'list',
+                                documents: 'collection'
+                            }[section],
                             operator: '==',
                             value: list
                         }],
                         operator: '&'
                     };
-                    pandora.api.find({query: query}, function(result) {
+
+                    pandora.api[{
+                        items: 'find',
+                        documents: 'findDocuments'
+                    }[section]]({query: query}, function(result) {
                         if (result.data.items) {
-                            pandora.api.find({
+                            pandora.api[{
+                                items: 'find',
+                                documents: 'findDocuments'
+                            }[section]]({
                                 query: query,
                                 keys: ['id'],
                                 sort: [{key: 'id', operator: ''}],
@@ -108,9 +117,6 @@ pandora.addFolderItem = function(section) {
                             addList();
                         }
                     });
-                } else if(section == 'documents') {
-                    //fixme
-                    addList();
                 } else {
                     pandora.api.getEdit({
                         id: list,
@@ -145,7 +151,7 @@ pandora.addFolderItem = function(section) {
     }
     function getPosterFrames(newList) {
         var query,
-            sortKey = Ox.getObjectById(pandora.site.itemKeys, 'votes')
+            sortKey = section == 'items' && Ox.getObjectById(pandora.site.itemKeys, 'votes')
                 ? 'votes' : 'timesaccessed';
         if (!isDuplicate) {
             ({
@@ -404,6 +410,34 @@ pandora.createLinks = function($element) {
     });
 };
 
+pandora.uploadDroppedFiles = function(files) {
+    var documentExtensions = ['pdf', /* 'epub', 'txt', */ 'png', 'gif', 'jpg', 'jpeg'];
+    files = Ox.map(files, function(file) { return file });
+
+    if (files.every(function(file) {
+        var extension = file.name.split('.').pop().toLowerCase()
+        return Ox.contains(documentExtensions, extension)
+    })) {
+        pandora.ui.uploadDocumentDialog({
+            files: files
+        }, function(files) {
+            if (files) {
+                Ox.Request.clearCache('findDocuments');
+                if (pandora.user.ui.document || pandora.user.ui.section != 'documents') {
+                    pandora.UI.set({section: 'documents', document: ''});
+                } else {
+                    pandora.$ui.list && pandora.$ui.list.reloadList();
+                }
+            }
+        }).open();
+    } else {
+        pandora.ui.addItemDialog({
+            files: files,
+            selected: 'upload'
+        }).open()
+    }
+};
+
 (function() {
 
     pandora.doHistory = function(action, items, targets, index, callback) {
@@ -1415,6 +1449,14 @@ pandora.getFoldersWidth = function(section) {
     return width;
 };
 
+pandora.getFindLayer = function() {
+    var key = pandora.user.ui._findState.key
+    if (!Ox.getObjectById(pandora.site.layers, key)) {
+        key = '*'
+    }
+    return key
+};
+
 pandora.getHash = function(state, callback) {
     // FIXME: remove this
     var embedKeys = [
@@ -2335,6 +2377,7 @@ pandora.VIDEO_OPTIONS_KEYS = [
     'rendered',
     'rightslevel',
     'size',
+    'source',
     'streams',
     'title',
     'videoRatio'
@@ -2456,6 +2499,9 @@ pandora.hasPlacesLayer = function() {
     });
 };
 
+pandora.hasView = function(id) {
+    return !!(Ox.getObjectById(pandora.site.itemViews, id) || Ox.getObjectById(pandora.site.listViews, id))
+};
 
 pandora.isClipView = function(view, item) {
     if (pandora.user.ui.section == 'items') {
@@ -2601,6 +2647,13 @@ pandora.openURL = function(url) {
     }
 };
 
+pandora.safeDocumentName = function(name) {
+    ['?', '#', '%'].forEach(function(c) {
+        name = name.replace(c, '_');
+    })
+    return name;
+};
+
 pandora.saveURL = function(url, name) {
     var link = document.createElement('a');
     if (typeof link.download === 'string') {
@@ -2660,7 +2713,11 @@ pandora.reloadList = function() {
     Ox.Log('', 'reloadList')
     var listData = pandora.getListData();
     Ox.Request.clearCache(); // fixme: remove
-    if (pandora.$ui.filters) {
+    if (pandora.user.ui.section == 'documents' && pandora.$ui.documentFilters) {
+        pandora.$ui.documentFilters.forEach(function($filter) {
+            $filter.reloadList();
+        });
+    } else if (pandora.$ui.filters) {
         pandora.$ui.filters.forEach(function($filter) {
             $filter.reloadList();
         });
@@ -2976,6 +3033,7 @@ pandora.resizeWindow = function() {
             }
         }
     } else if (pandora.user.ui.section == 'documents') {
+        pandora.resizeFilters(pandora.$ui.documentPanel.width());
         if (pandora.user.ui.document) {
             pandora.$ui.document && pandora.$ui.document.update();
         } else {
@@ -3193,9 +3251,8 @@ pandora.updateStatus = function(item) {
         return ui.item == item && [
             'info', 'player', 'editor', 'timeline'
         ].indexOf(ui.itemView) > -1 && !(
-            // fixme: still wrong
-            pandora.$ui.uploadVideoDialog
-            && pandora.$ui.uploadVideoDialog.is('::visible')
+            pandora.$ui.addItemDialog
+            && pandora.$ui.addItemDialog.is('::visible')
         );
     }
 };

static/pdf.js/index.html

@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+<!DOCTYPE html>
 <!--
 Copyright 2012 Mozilla Foundation
 
@@ -43,8 +43,6 @@ See https://github.com/adobe-type-tools/cmap-resources
 
 
 
-    <script src="/static/pdf.js/debugger.js"></script>
-    
     <script src="/static/pdf.js/embeds.js"></script>
     <script src="/static/pdf.js/viewer.js"></script>
     <link rel="stylesheet" type="text/css" href="/static/pdf.js/css/videopdf.css"/>

update.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 from __future__ import print_function
 import json
 import os
@@ -127,6 +127,9 @@ def get_branch(path=None):
 
 
 if __name__ == "__main__":
+    if os.stat(__file__).st_uid != os.getuid() or os.getuid() == 0:
+        print('you must run update.py as the pandora user')
+        sys.exit(1)
     base = os.path.normpath(os.path.abspath(os.path.dirname(__file__)))
     os.chdir(base)
     activate_venv(base)
@@ -332,6 +335,7 @@ if __name__ == "__main__":
             '-- Model missing for table: djcelery_intervalschedule\n',
             '-- Model missing for table: djcelery_workerstate\n',
             '-- Model missing for table: djcelery_taskstate\n',
+            '-- Model missing for table: south_migrationhistory\n',
             '-- Model missing for table: cache\n',
         ]:
             if row in diff:

vm/LXC_README.md

@@ -4,7 +4,7 @@
 
 # Installing pan.do/ra inside LXC
 
-1) Install lxc on the host (Ubuntu 16.04 or later):
+1) Install lxc on the host (Ubuntu 18.04 or later):
 
     sudo apt-get install lxc
 
@@ -15,7 +15,7 @@
 
 2) Create a new container, use different names if installing multiple instances:
 
-    sudo lxc-create -n pandora -t ubuntu -- -r xenial
+    sudo lxc-create -n pandora -t ubuntu-cloud -- -r bionic
 
     or
 
@@ -28,12 +28,12 @@
 4) Attach to container and install pan.do/ra
 
     sudo lxc-attach -n pandora --clear-env
-    apt-get update -qq && apt-get upgrade -y
-    apt-get -y install curl ca-certificates
     sed -i s/ubuntu/pandora/g /etc/passwd /etc/shadow /etc/group
     mv /home/ubuntu /home/pandora
     echo "pandora:pandora" | chpasswd
     echo PasswordAuthentication no >> /etc/ssh/sshd_config
+    apt-get update -qq && apt-get upgrade -y
+    apt-get -y install curl ca-certificates
     locale-gen en_US.UTF-8
     update-locale LANG=en_US.UTF-8
     export LANG=en_US.UTF-8

vm/build.sh

@@ -5,11 +5,11 @@ BASE=`pwd`
 VERSION=`cd ..;git rev-list HEAD --count`
 TARGET=${BASE}/pandora-r${VERSION}.vdi
 
-img=xenial-server-cloudimg-amd64-disk1.img
+img=bionic-server-cloudimg-amd64.img
 
 if [ ! -e $img ]; then
     echo downloading $img
-    curl -s -O https://cloud-images.ubuntu.com/xenial/current/$img
+    curl -s -O https://cloud-images.ubuntu.com/bionic/current/$img
 fi
 echo preparing ${TARGET}.img
 cp -a $img ${TARGET}.img
@@ -19,7 +19,7 @@ qemu-img resize ${TARGET}.img +998G
 
 echo boot image and install pandora
 kvm -m 1024 \
-    -smp 4 \
+    -smp 2 \
     -cdrom seed.img \
     -device e1000,netdev=user.0 \
     -netdev user,id=user.0,hostfwd=tcp::5555-:22,hostfwd=tcp::2620-:80 \

vm/install_elasticsearch.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+curl -sL https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
+echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" > /etc/apt/sources.list.d/elasticsearch.list
+apt-get update -qq
+apt-get -y install elasticsearch
+systemctl enable elasticsearch.service
+systemctl start elasticsearch.service
+#curl -X GET "http://localhost:9200/?pretty"

vm/pandora_install.sh

@@ -91,6 +91,7 @@ apt-get install -y \
     python3-lxml \
     python3-html5lib \
     python3-ox \
+    python3-elasticsearch \
     oxframe \
     ffmpeg \
     mkvtoolnix \
@@ -98,6 +99,8 @@ apt-get install -y \
     imagemagick \
     poppler-utils \
     ipython3 \
+    tesseract-ocr \
+    tesseract-ocr-eng \
     postfix \
     postgresql-client $EXTRA
 
@@ -126,6 +129,7 @@ fi
 git clone https://git.0x2620.org/pandora.git /srv/pandora
 cd /srv/pandora
 git checkout $BRANCH
+chown -R $PANDORA:$PANDORA /srv/pandora
 ./ctl init
 
 # create config.jsonc from templates in git