store metadata per user. remove primaryid. only store isbn13

oml/media/__init__.py

@@ -15,7 +15,7 @@ from . import epub
 from . import txt
 from . import opf
 
-from meta.utils import decode_html_data
+from meta.utils import decode_html_data, to_isbn13
 
 def get_id(f=None, data=None):
     if data:
@@ -23,7 +23,6 @@ def get_id(f=None, data=None):
     else:
         return base64.b32encode(codecs.decode(ox.sha1sum(f, cached=True), 'hex')).decode()
 
-
 def metadata(f, from_=None):
     ext = f.split('.')[-1]
     data = {}
@@ -64,10 +63,17 @@ def metadata(f, from_=None):
                 data[key] = data[key].replace('\x00', '')
             elif isinstance(data[key], list):
                 data[key] = [e.replace('\x00', '') if isinstance(e, str) else e for e in data[key]]
-    if 'isbn' in data:
-        data['primaryid'] = ['isbn', data['isbn'][0]]
-    elif 'asin' in data:
-        data['primaryid'] = ['asin', data['asin'][0]]
+    if 'isbn' in data and isinstance(data['isbn'], list):
+        isbns = set()
+        for i in data['isbn']:
+            i = to_isbn13(i)
+            if i:
+                isbns.add(i)
+        if isbns:
+            data['isbn'] = list(isbns)[0]
+        else:
+            del data['isbn']
+
     if 'author' in data:
         if isinstance(data['author'], str):
             if data['author'].strip():

oml/media/epub.py

@@ -5,14 +5,13 @@
 import os
 import xml.etree.ElementTree as ET
 import zipfile
-from io import BytesIO
 import re
 from urllib.parse import unquote
+import lxml.html
 
-import stdnum.isbn
 from ox import strip_tags, decode_html
 
-from utils import normalize_isbn, find_isbns, get_language
+from utils import find_isbns, get_language, to_isbn13
 
 import logging
 logger = logging.getLogger(__name__)
@@ -104,13 +103,24 @@ def info(epub):
                     }.get(key, key)
                     value = e.text.strip()
                     if key == 'identifier':
-                        value = normalize_isbn(value)
-                        if stdnum.isbn.is_valid(value):
-                            data['isbn'] = [value]
+                        value = to_isbn13(value)
+                        if value:
+                            data['isbn'] = value
                     elif key == 'author':
                         data[key] = value.split(', ')
                     else:
                         data[key] = value
+        guide = info.findall('{http://www.idpf.org/2007/opf}guide')
+        if guide:
+            for ref in guide[0].findall('{http://www.idpf.org/2007/opf}reference'):
+                if ref.attrib.get('type') == 'toc':
+                    filename = unquote(ref.attrib['href'])
+                    filename = os.path.normpath(os.path.join(os.path.dirname(opf[0]), filename))
+                    toc = z.read(filename)
+                    if toc:
+                        doc = lxml.html.document_fromstring(toc)
+                        data['tableofcontents'] = '\n'.join([a.text_content() for a in doc.xpath('//a')])
+
     if 'description' in data:
         data['description'] = strip_tags(decode_html(data['description']))
     text = extract_text(epub)
@@ -118,7 +128,7 @@ def info(epub):
     if not 'isbn' in data:
         isbn = extract_isbn(text)
         if isbn:
-            data['isbn'] = [isbn]
+            data['isbn'] = isbn
     if 'date' in data and 'T' in data['date']:
         data['date'] = data['date'].split('T')[0]
     if 'language' in data and isinstance(data['language'], str):
@@ -139,4 +149,3 @@ def extract_isbn(data):
     isbns = find_isbns(data)
     if isbns:
         return isbns[0]
-

oml/media/opf.py

@@ -4,11 +4,9 @@
 
 import xml.etree.ElementTree as ET
 
-import stdnum.isbn
 
-from utils import normalize_isbn, get_language
+from utils import get_language, to_isbn13
 from ox import strip_tags
-import ox.iso
 
 import logging
 logger = logging.getLogger(__name__)
@@ -31,12 +29,9 @@ def info(opf):
             }.get(key, key)
             value = e.text
             if key == 'identifier':
-                isbn = normalize_isbn(value)
-                if stdnum.isbn.is_valid(isbn):
-                    if not 'isbn' in data:
-                        data['isbn'] = [isbn]
-                    else:
-                        data['isbn'].append(isbn)
+                isbn = to_isbn13(value)
+                if isbn:
+                    data['isbn'] = isbn
                 if e.attrib.get(ns + 'scheme') == 'AMAZON':
                     if not 'asin' in data:
                         data['asin'] = [value]

oml/media/pdf.py

@@ -11,11 +11,10 @@ from glob import glob
 from datetime import datetime
 
 from PyPDF2 import PdfFileReader
-import stdnum.isbn
 import ox
 
 import settings
-from utils import normalize_isbn, find_isbns, get_language
+from utils import get_language, to_isbn13, find_isbns
 
 import logging
 logger = logging.getLogger(__name__)
@@ -151,9 +150,9 @@ def info(pdf):
             del data[key]
     '''
     if 'identifier' in data:
-        value = normalize_isbn(data['identifier'])
-        if stdnum.isbn.is_valid(value):
-            data['isbn'] = [value]
+        value = to_isbn13(data['identifier'])
+        if value:
+            data['isbn'] = value
             del data['identifier']
     for key, value in data.items():
         if isinstance(value, dict):
@@ -170,9 +169,7 @@ def info(pdf):
         if not 'isbn' in data:
             isbn = extract_isbn(text)
             if isbn:
-                data['isbn'] = [isbn]
-    if 'isbn' in data and isinstance(data['isbn'], str):
-        data['isbn'] = [data['isbn']]
+                data['isbn'] = isbn
     if 'date' in data and len(data['date']) == 8 and data['date'].isdigit():
         d = data['date']
         data['date'] = '%s-%s-%s' % (d[:4], d[4:6], d[6:])

oml/media/txt.py

@@ -23,7 +23,7 @@ def info(path):
     text = extract_text(path)
     isbn = extract_isbn(text)
     if isbn:
-        data['isbn'] = [isbn]
+        data['isbn'] = isbn
     data['textsize'] = len(text)
     return data