pandora/pandora/document/models.py
2017-01-25 00:13:37 +01:00

713 lines
26 KiB
Python

# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from __future__ import division, print_function, absolute_import
import os
import re
from glob import glob
import unicodedata
from six import PY2, string_types
from six.moves.urllib.parse import quote, unquote
from django.db import models, transaction
from django.db.models import Q, Sum, Max
from django.contrib.auth.models import User, Group
from django.db.models.signals import pre_delete
from django.conf import settings
from PIL import Image
import ox
from oxdjango import fields
from oxdjango.sortmodel import get_sort_field
from person.models import get_name_sort
from item.models import Item
from archive.extract import resize_image
from archive.chunk import save_chunk
from . import managers
from . import utils
if not PY2:
unicode = str
def get_path(f, x):
return f.path(x)
class Document(models.Model):
created = models.DateTimeField(auto_now_add=True)
modified = models.DateTimeField(auto_now=True)
user = models.ForeignKey(User, related_name='documents')
groups = models.ManyToManyField(Group, blank=True, related_name='documents')
extension = models.CharField(max_length=255)
size = models.IntegerField(default=0)
matches = models.IntegerField(default=0)
ratio = models.FloatField(default=640/1024)
pages = models.IntegerField(default=-1)
width = models.IntegerField(default=-1)
height = models.IntegerField(default=-1)
oshash = models.CharField(max_length=16, unique=True, null=True)
file = models.FileField(default=None, blank=True, null=True, upload_to=get_path)
objects = managers.DocumentManager()
uploading = models.BooleanField(default=False)
items = models.ManyToManyField(Item, through='ItemProperties', related_name='documents')
rightslevel = models.IntegerField(db_index=True, default=0)
data = fields.DictField(default={})
def update_access(self, user):
if not user.is_authenticated():
user = None
access, created = Access.objects.get_or_create(document=self, user=user)
if not created:
access.save()
def update_facet(self, key):
current_values = self.get_value(key, [])
if key == 'name':
current_values = []
for k in settings.CONFIG['documentKeys']:
if k.get('sortType') == 'person':
current_values += self.get(k['id'], [])
if not isinstance(current_values, list):
if not current_values:
current_values = []
else:
current_values = [unicode(current_values)]
filter_map = utils.get_by_id(settings.CONFIG['documentKeys'], key).get('filterMap')
if filter_map:
filter_map = re.compile(filter_map)
_current_values = []
for value in current_values:
value = filter_map.findall(value)
if value:
_current_values.append(value[0])
current_values = _current_values
current_values = list(set(current_values))
current_values = [ox.decode_html(ox.strip_tags(v)) for v in current_values]
current_values = [unicodedata.normalize('NFKD', v) for v in current_values]
self.update_facet_values(key, current_values)
def update_facet_values(self, key, current_values):
current_sortvalues = set([value.lower() for value in current_values])
saved_values = [i.value.lower() for i in Facet.objects.filter(document=self, key=key)]
removed_values = filter(lambda i: i not in current_sortvalues, saved_values)
if removed_values:
q = Q()
for v in removed_values:
q |= Q(value__iexact=v)
Facet.objects.filter(document=self, key=key).filter(q).delete()
for value in current_values:
if value.lower() not in saved_values:
sortvalue = value
if key in self.person_keys + ['name']:
sortvalue = get_name_sort(value)
sortvalue = utils.sort_string(sortvalue).lower()[:900]
f, created = Facet.objects.get_or_create(document=self, key=key, value=value, sortvalue=sortvalue)
if created:
Facet.objects.filter(document=self, key=key, value__iexact=value).exclude(value=value).delete()
Facet.objects.filter(key=key, value__iexact=value).exclude(value=value).update(value=value)
saved_values.append(value.lower())
def update_facets(self):
for key in set(self.facet_keys + ['title']):
self.update_facet(key)
def update_find(self):
def save(key, value):
if value not in ('', None):
f, created = Find.objects.get_or_create(document=self, key=key)
if isinstance(value, bool):
value = value and 'true' or 'false'
if isinstance(value, string_types):
value = ox.decode_html(ox.strip_tags(value.strip()))
value = unicodedata.normalize('NFKD', value).lower()
f.value = value
f.save()
else:
Find.objects.filter(document=self, key=key).delete()
with transaction.atomic():
data = self.json()
for key in settings.CONFIG['documentKeys']:
i = key['id']
if i == 'rightslevel':
save(i, self.rightslevel)
elif i not in ('*', 'dimensions') and i not in self.facet_keys:
value = data.get(i)
if isinstance(value, list):
value = u'\n'.join(value)
save(i, value)
base_keys = ('id', 'size', 'dimensions', 'extension', 'matches')
def update_sort(self):
try:
s = self.sort
except Sort.DoesNotExist:
s = Sort(document=self)
s.id = self.id
s.extension = self.extension
s.size = self.size
s.matches = self.matches
if self.extension == 'pdf':
prefix = 2
value = self.pages
else:
if self.extension == 'html':
prefix = 1
value = self.dimensions
else:
prefix = 0
value = self.width * self.height
if value < 0:
value = 0
s.dimensions = ox.sort_string('%d' % prefix) + ox.sort_string('%d' % value)
def sortNames(values):
sort_value = u''
if values:
sort_value = u'; '.join([get_name_sort(name) for name in values])
if not sort_value:
sort_value = u''
return sort_value
def set_value(s, name, value):
if isinstance(value, string_types):
value = ox.decode_html(value.lower())
if not value:
value = None
setattr(s, name, value)
def get_value(source, key):
if 'value' in key and 'layer' in key['value']:
value = [a.value for a in self.annotations.filter(layer=key['value']['layer']).exclude(value='')]
else:
value = self.get_value(source)
return value
def get_words(source, key):
value = get_value(source, key)
if isinstance(value, list):
value = '\n'.join(value)
value = len(value.split(' ')) if value else 0
return value
for key in filter(lambda k: k.get('sort', False), settings.CONFIG['documentKeys']):
name = key['id']
if name not in self.base_keys:
source = name
sort_type = key.get('sortType', key['type'])
if 'value' in key:
if 'key' in key['value']:
source = key['value']['key']
sort_type = key['value'].get('type', sort_type)
if isinstance(sort_type, list):
sort_type = sort_type[0]
if sort_type == 'title':
value = self.get_value(source, u'Untitled')
value = utils.sort_title(value)[:955]
set_value(s, name, value)
elif sort_type == 'person':
value = sortNames(self.get_value(source, []))
value = utils.sort_string(value)[:955]
set_value(s, name, value)
elif sort_type == 'string':
value = self.get_value(source, u'')
if isinstance(value, list):
value = u','.join(value)
value = utils.sort_string(value)[:955]
set_value(s, name, value)
elif sort_type == 'words':
value = get_words(source, key) if s.duration else None
set_value(s, name, value)
elif sort_type == 'wordsperminute':
value = get_words(source, key)
value = value / (s.duration / 60) if value and s.duration else None
set_value(s, name, value)
elif sort_type in ('length', 'integer', 'time', 'float'):
# can be length of strings or length of arrays, i.e. keywords
if 'layer' in key.get('value', []):
value = self.annotations.filter(layer=key['value']['layer']).count()
else:
value = self.get_value(source)
if isinstance(value, list):
value = len(value)
set_value(s, name, value)
elif sort_type == 'year':
value = self.get_value(source)
set_value(s, name, value)
elif sort_type == 'date':
value = self.get_value(source)
if isinstance(value, string_types):
value = datetime_safe.datetime.strptime(value, '%Y-%m-%d')
set_value(s, name, value)
s.save()
def save(self, *args, **kwargs):
if not self.uploading:
if self.file:
self.size = self.file.size
self.get_info()
if self.extension == 'html':
self.size = len(self.data.get('text', ''))
is_ready = not self.uploading and (self.file or self.extension == 'html')
if self.id:
if is_ready:
self.update_sort()
self.update_find()
self.update_facets()
new = False
else:
new = True
super(Document, self).save(*args, **kwargs)
if new:
if is_ready:
self.update_sort()
self.update_find()
self.update_facets()
self.update_matches()
def __unicode__(self):
return self.get_id()
def add(self, item):
p, created = ItemProperties.objects.get_or_create(item=item, document=self)
if created:
p.index = ItemProperties.objects.filter(item=item).aggregate(Max('index'))['index__max'] + 1
p.save()
p.document.update_matches()
item.update_sort()
def remove(self, item):
ItemProperties.objects.filter(item=item, document=self).delete()
@classmethod
def get(cls, id):
return cls.objects.get(pk=ox.fromAZ(id))
def get_absolute_url(self):
return ('/documents/%s' % quote(self.get_id())).replace('%3A', ':')
def get_id(self):
return ox.toAZ(self.id)
def access(self, user):
if user.is_anonymous():
level = 'guest'
else:
level = user.profile.get_level()
editable = self.editable(user)
if editable:
return True
allowed_level = settings.CONFIG['capabilities']['canSeeDocument'][level]
if self.rightslevel <= allowed_level:
return True
return False
def editable(self, user, item=None):
if not user or user.is_anonymous():
return False
if self.user == user or \
user.is_staff or \
user.profile.capability('canEditDocuments') is True or \
(item and item.editable(user)):
return True
return False
def edit(self, data, user, item=None):
if item:
p, created = ItemProperties.objects.get_or_create(item=item, document=self)
if 'description' in data:
p.description = ox.sanitize_html(data['description'])
p.save()
else:
for key in data:
k = list(filter(lambda i: i['id'] == key, settings.CONFIG['documentKeys']))
ktype = k and k[0].get('type') or ''
if key == 'text' and self.extension == 'html':
self.data['text'] = ox.sanitize_html(data['text'], global_attributes=[
'data-name',
'data-type',
'data-value',
'lang'
])
elif key == 'rightslevel':
setattr(self, key, int(data[key]))
elif ktype == 'text':
self.data[key] = ox.sanitize_html(data[key])
elif ktype == '[text]':
self.data[key] = [ox.sanitize_html(t) for t in data[key]]
elif ktype == '[string]':
self.data[key] = [ox.escape_html(t) for t in data[key]]
elif isinstance(data[key], string_types):
self.data[key] = ox.escape_html(data[key])
elif isinstance(data[key], list):
def cleanup(i):
if isinstance(i, string_types):
i = ox.escape_html(i)
return i
self.data[key] = [cleanup(i) for i in data[key]]
elif isinstance(data[key], int) or isinstance(data[key], float):
self.data[key] = data[key]
else:
self.data[key] = ox.escape_html(data[key])
@property
def dimensions(self):
if self.extension == 'pdf':
return self.pages
elif self.extension == 'html':
return len(self.data.get('text', '').split(' '))
else:
return self.resolution
@property
def resolution(self):
return [self.width, self.height]
def get_value(self, key, default=None):
if key in (
'extension',
'id',
'matches',
'ratio',
'size',
'rightslevel',
):
return getattr(self, key)
elif key == 'user':
return self.user.username
else:
return self.data.get(key, default)
def json(self, keys=None, user=None, item=None):
if not keys:
keys = [
'description',
'dimensions',
'editable',
'entities',
'extension',
'id',
'oshash',
'title',
'ratio',
'matches',
'size',
'user',
]
if self.extension in ('html', 'txt'):
keys.append('text')
for key in settings.CONFIG['documentKeys']:
if key['id'] in ('*', ):
continue
if key['id'] not in keys:
keys.append(key['id'])
response = {}
_map = {
}
for key in keys:
if key == 'id':
response[key] = self.get_id()
elif key == 'editable':
response[key] = self.editable(user)
elif key == 'user':
response[key] = self.user.username
elif key == 'accessed':
response[key] = self.accessed.aggregate(Max('access'))['access__max']
elif key == 'timesaccessed':
response[key] = self.accessed.aggregate(Sum('accessed'))['accessed__sum']
elif key == 'entities':
dps = self.documentproperties.select_related('entity').order_by('index')
response[key] = entity_jsons = []
for dp in dps:
entity_json = dp.entity.json(['id', 'type', 'name'])
entity_json['data'] = dp.data
entity_jsons.append(entity_json)
elif key == 'items':
response[key] = [i['public_id'] for i in self.items.all().values('public_id')]
elif key in self.data:
response[key] = self.data[key]
elif hasattr(self, _map.get(key, key)):
response[key] = getattr(self, _map.get(key, key)) or ''
if self.extension == 'html':
response['text'] = self.data.get('text', '')
if item:
if isinstance(item, string_types):
item = Item.objects.get(public_id=item)
d = self.descriptions.filter(item=item)
if d.exists():
if 'description' in keys and d[0].description:
response['description'] = d[0].description
response['index'] = d[0].index
if keys:
for key in list(response):
if key not in keys:
del response[key]
return response
def path(self, name=''):
h = ox.toAZ(self.id)
h = (7-len(h))*'0' + h
return os.path.join('documents', h[:2], h[2:4], h[4:6], h[6:], name)
def save_chunk(self, chunk, offset=None, done=False):
if self.uploading:
name = 'data.%s' % self.extension
name = self.path(name)
def done_cb():
if done:
self.uploading = False
self.get_info()
self.get_ratio()
self.oshash = ox.oshash(self.file.path)
self.save()
self.delete_cache()
return True, self.file.size
return save_chunk(self, self.file, chunk, offset, name, done_cb)
return False, 0
def thumbnail(self, size=None, page=None):
if not self.file:
return os.path.join(settings.STATIC_ROOT, 'png/cover.png')
return os.path.join(settings.STATIC_ROOT, 'jpg/list256.jpg')
src = self.file.path
folder = os.path.dirname(src)
if size:
size = int(size)
path = os.path.join(folder, '%d.jpg' % size)
else:
path = src
if self.extension == 'pdf':
if page:
page = int(page)
if page and page > 1 and page <= self.pages:
src = os.path.join(folder, '1024p%d.jpg' % page)
else:
src = os.path.join(folder, '1024p1.jpg')
page = 1
if not os.path.exists(src):
self.extract_page(page)
if size:
path = os.path.join(folder, '%dp%d.jpg' % (size, page))
elif self.extension in ('jpg', 'png', 'gif'):
if os.path.exists(src):
if size and page:
crop = list(map(int, page.split(',')))
if len(crop) == 4:
path = os.path.join(folder, '%s.jpg' % ','.join(map(str, crop)))
if not os.path.exists(path):
img = Image.open(src).crop(crop)
img.save(path)
else:
img = Image.open(path)
src = path
if size < max(img.size):
path = os.path.join(folder, '%sp%s.jpg' % (size, ','.join(map(str, crop))))
if not os.path.exists(path):
resize_image(src, path, size=size)
if os.path.exists(src) and not os.path.exists(path):
image_size = max(self.width, self.height)
if image_size == -1:
image_size = max(*Image.open(src).size)
if size > image_size:
path = src
else:
resize_image(src, path, size=size)
return path
def extract_page(self, page):
pdf = self.file.path
image = os.path.join(os.path.dirname(pdf), '1024p%d.jpg' % page)
utils.extract_pdfpage(pdf, image, page)
def get_info(self):
if self.extension == 'pdf':
self.thumbnail(1024)
if self.pages == -1:
self.width = -1
self.height = -1
self.pages = utils.pdfpages(self.file.path)
elif self.width == -1:
self.pages = -1
self.width, self.height = Image.open(self.file.path).size
def get_ratio(self):
if self.extension == 'pdf':
image = self.thumbnail(1024)
try:
size = Image.open(image).size
except:
size = [1, 1]
else:
if self.width > 0:
size = self.resolution
else:
size = [640, 1024]
self.ratio = size[0] / size[1]
return self.ratio
def update_matches(self):
import annotation.models
import item.models
import text.models
urls = [self.get_absolute_url()]
url = unquote(urls[0])
if url != urls[0]:
urls.append(url)
matches = self.items.count() + self.entities.count()
for url in urls:
matches += annotation.models.Annotation.objects.filter(value__contains=url).count()
matches += item.models.Item.objects.filter(data__contains=url).count()
matches += text.models.Text.objects.filter(text__contains=url).count()
if matches != self.matches:
Document.objects.filter(id=self.id).update(matches=matches)
self.matches = matches
def delete_cache(self):
if self.file:
folder = os.path.dirname(self.file.path)
for f in glob('%s/*' % folder):
if f != self.file.path:
os.unlink(f)
def delete_document(sender, **kwargs):
t = kwargs['instance']
if t.file:
t.delete_cache()
t.file.delete(save=False)
pre_delete.connect(delete_document, sender=Document)
class ItemProperties(models.Model):
class Meta:
unique_together = ("item", "document")
created = models.DateTimeField(auto_now_add=True)
modified = models.DateTimeField(auto_now=True)
item = models.ForeignKey(Item)
document = models.ForeignKey(Document, related_name='descriptions')
description = models.TextField(default="")
index = models.IntegerField(default=0)
description_sort = models.CharField(max_length=512, null=True)
def save(self, *args, **kwargs):
if self.description:
self.description_sort = ox.sort_string(self.description)[:512].lower()
else:
self.description_sort = self.document.sort.description
super(ItemProperties, self).save(*args, **kwargs)
class Access(models.Model):
class Meta:
unique_together = ("document", "user")
access = models.DateTimeField(auto_now=True)
document = models.ForeignKey(Document, related_name='accessed')
user = models.ForeignKey(User, null=True, related_name='accessed_documents')
accessed = models.IntegerField(default=0)
def save(self, *args, **kwargs):
if not self.accessed:
self.accessed = 0
self.accessed += 1
super(Access, self).save(*args, **kwargs)
timesaccessed = Access.objects.filter(document=self.document).aggregate(Sum('accessed'))['accessed__sum']
Sort.objects.filter(document=self.document).update(timesaccessed=timesaccessed, accessed=self.access)
def __unicode__(self):
if self.user:
return u"%s/%s/%s" % (self.user, self.document, self.access)
return u"%s/%s" % (self.item, self.access)
class Facet(models.Model):
'''
used for keys that can have multiple values like people, languages etc.
does not perform to well if total number of items goes above 10k
this happens for keywords in 0xdb right now
'''
class Meta:
unique_together = ("document", "key", "value")
document = models.ForeignKey('Document', related_name='facets')
key = models.CharField(max_length=200, db_index=True)
value = models.CharField(max_length=1000, db_index=True)
sortvalue = models.CharField(max_length=1000, db_index=True)
def __unicode__(self):
return u"%s=%s" % (self.key, self.value)
def save(self, *args, **kwargs):
if not self.sortvalue:
self.sortvalue = utils.sort_string(self.value).lower()[:900]
self.sotvalue = self.sortvalue.lower()
super(Facet, self).save(*args, **kwargs)
Document.facet_keys = []
for key in settings.CONFIG['documentKeys']:
if 'autocomplete' in key and 'autocompleteSortKey' not in key or \
key.get('filter'):
Document.facet_keys.append(key['id'])
Document.person_keys = []
for key in settings.CONFIG['itemKeys']:
if key.get('sortType') == 'person':
Document.person_keys.append(key['id'])
class Find(models.Model):
class Meta:
unique_together = ('document', 'key')
document = models.ForeignKey('Document', related_name='find', db_index=True)
key = models.CharField(max_length=200, db_index=True)
value = models.TextField(blank=True, db_index=settings.DB_GIN_TRGM)
def __unicode__(self):
return u'%s=%s' % (self.key, self.value)
'''
Sort
table constructed based on info in settings.CONFIG['documentKeys']
'''
attrs = {
'__module__': 'document.models',
'document': models.OneToOneField('Document', related_name='sort', primary_key=True),
'created': models.DateTimeField(null=True, blank=True, db_index=True),
}
for key in filter(lambda k: k.get('sort', False) or k['type'] in ('integer', 'time', 'float', 'date', 'enum'), settings.CONFIG['documentKeys']):
name = key['id']
sort_type = key.get('sortType', key['type'])
if isinstance(sort_type, list):
sort_type = sort_type[0]
field = get_sort_field(sort_type)
if name not in attrs:
attrs[name] = field[0](**field[1])
Sort = type('Sort', (models.Model,), attrs)
Sort.fields = [f.name for f in Sort._meta.fields]