pandora/app/config.py

@@ -71,7 +71,7 @@ def load_config(init=False):
         if getattr(settings, 'SITEURL', False):
             config['site']['url'] = settings.SITEURL
         settings.URL = config['site']['url']
-        settings.EMAIL_SUBJECT_PREFIX = '[%s]' % settings.SITENAME
+        settings.EMAIL_SUBJECT_PREFIX = '[%s]'%settings.SITENAME
         settings.DEFAULT_FROM_EMAIL = config['site']['email']['system']
         settings.SERVER_EMAIL = config['site']['email']['system']
         config['site']['videoprefix'] = settings.VIDEO_PREFIX
@@ -79,9 +79,9 @@ def load_config(init=False):
         config['site']['googleapikey'] = getattr(settings, 'GOOGLE_API_KEY')
         config['site']['version'] = get_version()
         config['site']['dontValidateUser'] = not settings.AUTH_CHECK_USERNAME
-        if 'folderdepth' not in config['site']:
+        if not 'folderdepth' in config['site']:
             config['site']['folderdepth'] = settings.USE_IMDB and 4 or 3
-        if 'sendReferrer' in config and 'sendReferrer' not in config['site']:
+        if 'sendReferrer' in config and not 'sendReferrer' in config['site']:
             config['site']['sendReferrer'] = config.pop('sendReferrer')
 
         # enable default filters if needed
@@ -91,13 +91,6 @@ def load_config(init=False):
                 key['filter'] = True
                 sys.stderr.write('enabled filter for "%s" since its used as default filter.\n' % (key['id']))
 
-        # enable default document filters if needed
-        default_filters = [f['id'] for f in config['user']['ui']['documentFilters']]
-        for key in config['documentKeys']:
-            if key['id'] in default_filters and not key.get('filter'):
-                key['filter'] = True
-                sys.stderr.write('enabled filter for documeny key "%s" since its used as default filter.\n' % (key['id']))
-
         config['keys'] = {}
         for key in config['itemKeys']:
             config['keys'][key['id']] = key

pandora/document/fulltext.py

@@ -1,85 +0,0 @@
-import subprocess
-
-from django.conf import settings
-
-
-def extract_text(pdf):
-    cmd = ['pdftotext', pdf, '-']
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    stdout, stderr = p.communicate()
-    stdout = stdout.decode()
-    return stdout.strip()
-
-def ocr_image(path):
-    cmd = ['tesseract', path, '-', 'txt']
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    stdout, stderr = p.communicate()
-    stdout = stdout.decode()
-    return stdout.strip()
-
-class FulltextMixin:
-    _ES_INDEX = "document-index"
-
-    @classmethod
-    def elasticsearch(cls):
-        from elasticsearch import Elasticsearch
-        es = Elasticsearch(settings.ELASTICSEARCH_HOST)
-        return es
-
-    def extract_fulltext(self):
-        if self.extension == 'pdf':
-            return extract_text(self.file.path)
-        elif self.extension in ('png', 'jpg'):
-            return ocr_image(self.file.path)
-        elif self.extension == 'html':
-            return self.data.get('text', '')
-        return ''
-
-    def delete_fulltext(self):
-        res = self.elasticsearch().delete(index=self._ES_INDEX, doc_type='document', id=self.id)
-
-    def update_fulltext(self):
-        text = self.extract_fulltext()
-        if text:
-            doc = {
-                'text': text.lower()
-            }
-            res = self.elasticsearch().index(index=self._ES_INDEX, doc_type='document', id=self.id, body=doc)
-
-    @classmethod
-    def find_fulltext(cls, query):
-        ids = cls.find_fulltext_ids(query)
-        return cls.objects.filter(id__in=ids)
-
-    @classmethod
-    def find_fulltext_ids(cls, query):
-        if query[0] == '"' and query[-1] == '"':
-            query = {
-                "match_phrase": {
-                    "text": query.lower()[1:-1]
-                },
-            }
-        else:
-            query = {
-                "match": {
-                    "text": {
-                        "query": query.lower(),
-                        "operator": "and"
-                    }
-                }
-            }
-        ids = []
-        res = None
-        from_ = 0
-        es = cls.elasticsearch()
-        while not res or len(ids) < res['hits']['total']['value']:
-            res = es.search(index=cls._ES_INDEX, body={
-                "from": from_,
-                "_source": False,
-                "query": query
-            })
-            if not res['hits']['hits']:
-                break
-            ids += [int(r['_id']) for r in res['hits']['hits']]
-            from_ += len(res['hits']['hits'])
-        return ids

pandora/document/managers.py

@@ -128,12 +128,6 @@ def buildCondition(k, op, v, user, exclude=False, owner=None):
             else:
                 q = Q(id=0)
         return q
-    elif key_type == 'fulltext':
-        qs = models.Document.find_fulltext_ids(v)
-        q = Q(id__in=qs)
-        if exclude:
-            q = ~Q(id__in=qs)
-        return q
     elif key_type == 'boolean':
         q = Q(**{'find__key': k, 'find__value': v})
         if exclude:

pandora/document/models.py

@@ -30,7 +30,6 @@ from user.models import Group
 
 from . import managers
 from . import utils
-from .fulltext import FulltextMixin
 
 User = get_user_model()
 
@@ -41,7 +40,7 @@ def get_path(f, x):
     return f.path(x)
 
 @python_2_unicode_compatible
-class Document(models.Model, FulltextMixin):
+class Document(models.Model):
 
     created = models.DateTimeField(auto_now_add=True)
     modified = models.DateTimeField(auto_now=True)
@@ -154,7 +153,7 @@ class Document(models.Model, FulltextMixin):
                 i = key['id']
                 if i == 'rightslevel':
                     save(i, self.rightslevel)
-                elif i not in ('*', 'dimensions', 'fulltext') and i not in self.facet_keys:
+                elif i not in ('*', 'dimensions') and i not in self.facet_keys:
                     value = data.get(i)
                     if isinstance(value, list):
                         value = u'\n'.join(value)
@@ -278,7 +277,6 @@ class Document(models.Model, FulltextMixin):
                 self.update_sort()
                 self.update_find()
                 self.update_facets()
-                self.update_fulltext()
             new = False
         else:
             new = True

pandora/settings.py

@@ -204,9 +204,6 @@ CELERY_BROKER_URL = 'amqp://pandora:box@localhost:5672//pandora'
 
 SEND_CELERY_ERROR_EMAILS = False
 
-# Elasticsearch
-ELASTICSEARCH_HOST = None
-
 #with apache x-sendfile or lighttpd set this to True
 XSENDFILE = False
 

requirements.txt

@@ -11,4 +11,3 @@ tornado<5
 geoip2==2.9.0
 youtube-dl>=2019.4.30
 python-memcached
-elasticsearch

static/js/documentFilter.js

@@ -4,7 +4,7 @@ pandora.ui.documentFilter = function(id) {
     var i = Ox.getIndexById(pandora.user.ui.documentFilters, id),
         filter = Ox.getObjectById(pandora.site.documentFilters, id),
         panelWidth = Ox.$document.width() - (pandora.user.ui.showSidebar * pandora.user.ui.sidebarSize) - 1,
-        title = Ox._(filter.title),
+        title = Ox._(Ox.getObjectById(pandora.site.documentFilters, id).title),
         //width = pandora.getFilterWidth(i, panelWidth),
         that = Ox.TableList({
             _selected: !pandora.user.ui.showFilters

static/js/documentInfoView.js

@@ -28,13 +28,6 @@ pandora.ui.documentInfoView = function(data, isMixed) {
         }).map(function(key){
             return key.id;
         }),
-        displayedKeys = [ // FIXME: can tis be a flag in the config?
-            'title', 'notes', 'name', 'description', 'id',
-            'user', 'rightslevel', 'timesaccessed',
-            'extension', 'dimensions', 'size', 'matches',
-            'created', 'modified', 'accessed',
-            'random', 'entity'
-        ],
         statisticsWidth = 128,
 
         $bar = Ox.Bar({size: 16})
@@ -241,10 +234,6 @@ pandora.ui.documentInfoView = function(data, isMixed) {
 
     Ox.getObjectById(pandora.site.documentKeys, 'keywords') && renderGroup(['keywords'])
 
-    // Render any remaing keys defined in config
-
-    renderRemainingKeys();
-
 
     // Description -------------------------------------------------------------
 
@@ -332,7 +321,6 @@ pandora.ui.documentInfoView = function(data, isMixed) {
 
     }
 
-
     // Extension, Dimensions, Size ---------------------------------------------
 
     ['extension', 'dimensions', 'size'].forEach(function(key) {
@@ -545,7 +533,6 @@ pandora.ui.documentInfoView = function(data, isMixed) {
 
     function renderGroup(keys) {
         var $element;
-        keys.forEach(function(key) { displayedKeys.push(key) });
         if (canEdit || keys.filter(function(key) {
             return data[key];
         }).length) {
@@ -578,17 +565,6 @@ pandora.ui.documentInfoView = function(data, isMixed) {
         return $element;
     }
 
-    function renderRemainingKeys() {
-        var keys = pandora.site.documentKeys.filter(function(item) {
-            return item.id != '*' && !Ox.contains(displayedKeys, item.id);
-        }).map(function(item) {
-            return item.id;
-        });
-        if (keys.length) {
-            renderGroup(keys)
-        }
-    }
-
     function renderRightsLevel() {
         var $rightsLevelElement = getRightsLevelElement(data.rightslevel),
             $rightsLevelSelect;

vm/pandora_install.sh

@@ -91,7 +91,6 @@ apt-get install -y \
     python3-lxml \
     python3-html5lib \
     python3-ox \
-    python3-elasticsearch \
     oxframe \
     ffmpeg \
     mkvtoolnix \
@@ -99,8 +98,6 @@ apt-get install -y \
     imagemagick \
     poppler-utils \
     ipython3 \
-    tesseract-ocr \
-    tesseract-ocr-eng \
     postfix \
     postgresql-client $EXTRA