openmedialibrary/oml/media/pdf.py

# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4


import sys
import tempfile
import subprocess
import os
import shutil
from glob import glob
from datetime import datetime

from PyPDF2 import PdfFileReader
import ox

import settings
from utils import get_language, to_isbn13, find_isbns

import logging
logger = logging.getLogger(__name__)

def cover(pdf):
    if sys.platform == 'darwin':
        return ql_cover(pdf)
    else:
        return page(pdf, 1)

def ql_cover(pdf):
    tmp = tempfile.mkdtemp()
    cmd = [
        'qlmanage',
        '-t',
        '-s',
        '1024',
        '-o',
        tmp,
        pdf
    ]
    p = subprocess.Popen(cmd, close_fds=True)
    p.wait()
    image = glob('%s/*' % tmp)
    if image:
        image = image[0]
        with open(image, 'rb') as fd:
            data = fd.read()
    else:
        logger.debug('qlmanage did not create cover for %s', pdf)
        data = None
    shutil.rmtree(tmp)
    return data

def page(pdf, page):
    tmp = tempfile.mkdtemp()
    cmd = [
        'pdftocairo',
        pdf,
        '-jpeg',
        '-f',  str(page), '-l', str(page),
        '-scale-to', '1024', '-cropbox',
        os.path.join(tmp, 'page')
    ]
    p = subprocess.Popen(cmd, close_fds=True)
    p.wait()
    image = glob('%s/*' % tmp)
    if image:
        image = image[0]
        with open(image, 'rb') as fd:
            data = fd.read()
    else:
        logger.debug('pdftocairo %s %s', pdf, ' '.join(cmd))
        data = None
    shutil.rmtree(tmp)
    return data

'''
def page(pdf, page):
    image = tempfile.mkstemp('.jpg')[1]
    cmd = [
        'gs', '-q',
        '-dBATCH', '-dSAFER', '-dNOPAUSE', '-dNOPROMPT',
        '-dMaxBitmap=500000000',
        '-dAlignToPixels=0', '-dGridFitTT=2',
        '-sDEVICE=jpeg', '-dTextAlphaBits=4', '-dGraphicsAlphaBits=4',
        '-r72',
        '-dUseCropBox',
        '-dFirstPage=%d' % page,
        '-dLastPage=%d' % page,
        '-sOutputFile=%s' % image,
        pdf
    ]
    p = subprocess.Popen(cmd, close_fds=True)
    p.wait()
    with open(image, 'rb') as fd:
        data = fd.read()
    os.unlink(image)
    return data
'''

def parse_tableofcontents(reader):
    titles = []
    try:
        if '/Root' in reader.trailer and '/Outlines' in reader.trailer['/Root']:
            outlines = reader.trailer['/Root']['/Outlines']
            if '/First' in outlines:
                title = outlines['/First']
                while title:
                    if '/Title' in title:
                        titles.append(title['/Title'])
                    if '/Next' in title:
                        title = title['/Next']
                    else:
                        title = None
    except:
        logger.debug('failed to parse pdf outline', exc_info=True)
    try:
        titles = [title.decode('utf-8','ignore').strip() if isinstance(title, bytes) else title for title in titles]
        toc = '\n'.join(titles).strip()
    except:
        logger.debug('failed to decode outline', exc_info=True)
        titles = []
    return toc

def info(pdf):
    data = {}
    with open(pdf, 'rb') as fd:
        try:
            pdfreader = PdfFileReader(fd)
            data['pages'] = pdfreader.numPages
            if pdfreader.getIsEncrypted():
                pdfreader.decrypt('')
            toc = parse_tableofcontents(pdfreader)
            if toc:
                data['tableofcontents'] = toc
            info = pdfreader.getDocumentInfo()
            if info:
                for key in info:
                    if info[key]:
                        try:
                            value = info[key]
                            if len(value) == 1:
                                value = value[0]
                            if isinstance(value, bytes):
                                value = value.decode('utf-16')
                            data[key[1:].lower()] = value
                        except:
                            pass

            xmp = pdfreader.getXmpMetadata()
            if xmp:
                for key in dir(xmp):
                    if key.startswith('dc_'):
                        try:
                            value = getattr(xmp, key)
                        except:
                            continue
                        if isinstance(value, dict) and 'x-default' in value:
                            value = value['x-default']
                        elif isinstance(value, list):
                            value = [v.strip() if isinstance(v, str) else v for v in value if v]
                            value = [v.strftime('%Y-%m-%d') if isinstance(v, datetime) else v for v in value]
                            if len(value) == 1:
                                value = value[0]
                        _key = key[3:]
                        if value and _key not in data:
                            data[_key] = value
        except:
            logger.debug('FAILED TO PARSE %s', pdf, exc_info=1)

    '''
    cmd = ['pdfinfo', pdf]
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
    stdout, stderr = p.communicate()
    for line in stdout.strip().split('\n'):
        parts = line.split(':')
        key = parts[0].lower().strip()
        if key:
            data[key] = ':'.join(parts[1:]).strip()
    for key in data.keys():
        if not data[key]:
            del data[key]
    '''
    if 'identifier' in data:
        value = to_isbn13(data['identifier'])
        if value:
            data['isbn'] = value
            del data['identifier']
    for key, value in data.items():
        if isinstance(value, dict):
            value = ' '.join(list(value.values()))
            data[key] = value.strip()
    for key in list(data):
        if data[key] in ('Unknown',):
            del data[key]
        if key == 'language':
            data[key] = get_language(data[key])
    text = extract_text(pdf)
    data['textsize'] = len(text)
    if settings.server['extract_text']:
        if not 'isbn' in data:
            isbn = extract_isbn(text)
            if isbn:
                data['isbn'] = isbn
    if 'date' in data and len(data['date']) == 8 and data['date'].isdigit():
        d = data['date']
        data['date'] = '%s-%s-%s' % (d[:4], d[4:6], d[6:])
    if 'author' in data and isinstance(data['author'], str):
        data['author'] = [ox.normalize_name(data['author'])]
    return data

'''
    #possbile alternative with gs
    tmp = tempfile.mkstemp('.txt')[1]
    cmd = ['gs', '-dBATCH', '-dNOPAUSE', '-sDEVICE=txtwrite', '-dFirstPage=3', '-dLastPage=5', '-sOutputFile=%s'%tmp, pdf]

'''
def extract_text(pdf):
    if sys.platform == 'darwin':
        cmd = ['/usr/bin/mdimport', '-d2', pdf]
    else:
        cmd = ['pdftotext', pdf, '-']
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
    stdout, stderr = p.communicate()
    stdout = stdout.decode()
    stderr = stderr.decode()
    if sys.platform == 'darwin':
        if 'kMDItemTextContent' in stderr:
            stdout = stderr.split('kMDItemTextContent = "')[-1].split('\n')[0][:-2]
        else:
            stdout = ''
    return stdout.strip()


def pypdf_extract_text(path):
    '''
        slow and bad results
    '''
    pdf = PdfFileReader(path)
    content = []
    for i in range(0, pdf.getNumPages()):
        try:
            extracted_text = pdf.getPage(i).extractText()
            content.append(extracted_text)
        except:
            pass
    content = "\n".join(content).replace("\xa0", " ").strip()
    return content

def extract_isbn(text):
    isbns = find_isbns(text)
    if isbns:
        return isbns[0]