pandora_client_eightfiftyeight/import_raw.py

#!/usr/bin/python3
#
# apt install convert ufraw-batch
#
from __future__ import print_function

import os
import subprocess
from glob import glob

import ox
import pandora_client
from pandora_client.utils import hash_prefix

def convert_raw(raw, jpg):
    ox.makedirs(os.path.dirname(jpg))
    cmd = ['convert', raw, jpg]
    print(' '.join(cmd))
    subprocess.call(cmd)

def get_raw(client):
    files = []
    for volume in client.active_volumes():
        query = {
            'conditions': [
                {'key': 'list', 'value': volume, 'operator': '=='},
                {
                    'conditions': [
                        {'key': 'filename', 'value': '.cr2', 'operator': ''},
                        {'key': 'filename', 'value': '.pef', 'operator': ''},
                        {'key': 'filename', 'value': '.nef', 'operator': ''},
                    ],
                    'operator': '|'
                },
            ],
            'operator': '&'
        }
        n = client.api.findMedia({'query': query})['data']['items']
        if n:
            o = 0
            chunk = 5000
            while o < n:
                files += client.api.findMedia({
                    'query': query,
                    'keys': ['item', 'id', 'extension'],
                    'range': [o, o+chunk]
                })['data']['items']
                o += chunk
    files = [f for f in files if f['extension'].lower() in ('cr2', 'nef', 'pef')]
    return files

def raw_exists(client, oshash):
    r = client.api.findDocuments({
        'keys': ['id', 'description'],
        'query': {
            'conditions': [
                {'key': 'description', 'value': '[%s]' % oshash, 'operator': '='}
            ],
            'operator': '&'
        }
    })
    if r['data']['items'] and \
            'Converted from' in r['data']['items'][0]['description']:
        return True
    return False

def import_raw(client):
    files = get_raw(client)
    print('got', len(files), 'raw files')
    for f in files:
        oshash = f['id']
        print(oshash)
        cache = os.path.join(client.media_cache(), os.path.join(*hash_prefix(oshash)))
        if isinstance(cache, bytes):
            cache = cache.decode()
        jpg = glob('%s/*.jpg' % cache)
        if jpg:
            continue
        elif raw_exists(client, oshash):
            continue
        else:
            jpg = None
            for path in client.path(oshash):
                if os.path.exists(path):
                    jpg = os.path.join(cache, '%s.jpg' % os.path.basename(path).split('.')[0])
                    convert_raw(path, jpg)
                    break
            if jpg and os.path.exists(jpg):
                r = client._add_document(jpg, f['item'])
                doc = client.find_document(ox.oshash(jpg))
                if doc:
                    client.api.editDocument({
                        'id': doc,
                        'description': 'Converted from %s [%s]' % (os.path.basename(path), oshash)
                    })
                print('added', oshash, 'to', f['item'])


if __name__ == '__main__':
    client = pandora_client.Client(os.path.expanduser('~/.ox/client.json'), False)
    import_raw(client)