oxjs/tools/build/build.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#vim: et:ts=4:sw=4:sts=4

import json
import os
import ox
import re
import shutil
import subprocess
import sys
import tarfile
import time

def build_oxjs(downloads=False, geo=False):

    base_path = os.path.dirname(__file__)
    if base_path:
        os.chdir(base_path)

    root_path = '../../'
    source_path = root_path + 'source/'
    build_path = root_path + 'build/'
    dev_path = root_path + 'dev/'

    locales = {}
    version = '0.1.%s' % subprocess.Popen(
        ['bzr', 'revno'],
        stdout=subprocess.PIPE
    ).communicate()[0].strip()
    year = time.strftime('%Y', time.gmtime())
    comment = ' OxJS %s (c) %s 0x2620, dual-licensed GPL/MIT, see https://oxjs.org for details ' % (version, year)

    # Empty build and dev
    for path in [build_path, dev_path]:
        if os.path.exists(path):
            for item in os.listdir(path):
                full_path = '%s%s' % (path, item)
                if os.path.isdir(full_path):
                    if not (geo == False and item == 'Ox.Geo'):
                        shutil.rmtree(full_path)
                else:
                    os.remove(full_path)

    # Ox.UI Theme Data
    theme_data = {}
    themes = [dirname for dirname in os.listdir(source_path + 'Ox.UI/themes/') if not dirname[0] in '._']
    for theme in themes:
        theme_data[theme] = read_jsonc(source_path + 'Ox.UI/themes/%s/json/theme.jsonc' % theme)
        theme_data[theme]['themeClass'] = 'OxTheme' + theme[0].upper() + theme[1:]

    # Ox.UI CSS
    css = read_file(source_path + 'Ox.UI/css/Ox.UI.css')
    css = css.replace('$import', '\n'.join([
        '@import url("../themes/%s/css/theme.css?%s");' % (theme, version) for theme in themes
    ]))
    write_file('%sOx.UI/css/Ox.UI.css' % build_path, css)
    write_file('%sOx.UI/css/Ox.UI.css' % dev_path, css)

    # Ox.UI Theme CSS
    css = read_file(source_path + 'Ox.UI/css/theme.css')
    for theme in themes:
        theme_css = parse_css(css, theme_data[theme])
        theme_css = theme_css.replace('.png)', '.png?%s)' % version)
        write_file('%sOx.UI/themes/%s/css/theme.css' % (build_path, theme), theme_css)
        write_file('%sOx.UI/themes/%s/css/theme.css' % (dev_path, theme), theme_css)

    # Ox.UI SVGs
    ui_images = {}
    path = source_path + 'Ox.UI/svg/'
    for filename in [filename for filename in os.listdir(path) if not filename[0] in '._']:
        svg = read_file(path + filename)
        svg = re.sub('\n\s*', '', svg)
        svg = re.sub('<!--.+?-->', '', svg)
        # temporary fix for Chrome SVG bug, remove later!
        svg = re.sub('width="256" height="256"', 'width="10" height="10" viewBox="0 0 255 255"', svg)
        # end fix
        ui_images[filename[:-4]] = svg
        if filename.startswith('symbolLoading'):
            for theme in themes:
                theme_svg = re.sub('#808080', format_hex(theme_data[theme]['symbolDefaultColor']), svg)
                write_file('%sOx.UI/themes/%s/svg/%s' % (build_path, theme, filename), theme_svg)
                write_file('%sOx.UI/themes/%s/svg/%s' % (dev_path, theme, filename), theme_svg)

    # copy & link
    ui_files = {'build': [], 'dev': []}
    for path, dirnames, filenames in os.walk(source_path):
        for filename in filenames:
            if not '_' in path and not filename[0] in '._' \
                    and not filename.endswith('~') \
                    and not filename.endswith('.css') \
                    and not '/Ox.UI/svg' in path \
                    and (geo or not '/Ox.Geo/' in path):
                # write copies in build path
                source = os.path.join(path, filename)
                is_jquery = re.search('^jquery-[\d\.]+\.js$', filename)
                is_jquery_min = re.search('^jquery-[\d\.]+\.min\.js$', filename)
                is_jquery_plugin = re.search('^jquery\..*?\.js$', filename)
                is_jsonc = re.search('\.jsonc$', filename)
                if is_jquery or is_jquery_min:
                    target = os.path.join(path.replace(source_path, build_path), 'jquery.js')
                else:
                    target = os.path.join(path.replace(source_path, build_path), filename)
                    if is_jquery_plugin:
                        ui_files['build'].append(target.replace(build_path, ''))
                        ui_files['dev'].append(target.replace(build_path, ''))
                if not '/Ox/js/' in source and not '/Ox.UI/js/' in source and not is_jquery:
                    if re.match('^Ox\..+\.js$', filename) or is_jsonc:
                        js = read_file(source)
                        write_file(target, ox.js.minify(js, '' if is_jsonc else comment))
                    else:
                        copy_file(source, target)
                # write links in dev path
                parts = os.path.join(path.replace(source_path, ''), filename).split('/')
                for i, part in enumerate(parts):
                    if i < len(parts) - 1:
                        parts[i] = '..'
                link_source = '/'.join(parts).replace(filename, os.path.join(path, filename))[3:]
                link_target = target.replace(build_path, dev_path)
                if not is_jquery_min:
                    write_link(link_source, link_target)
                # locales
                match = re.search('/Ox\.?(\w*?)/json/locale.(\w+).json', source)
                if match:
                    module = match.group(1)
                    locale = match.group(2)
                    if not module in locales:
                        locales[module] = []
                    locales[module].append(locale)
    # remove dangling links from dev tree that might
    # be left over from renamed or removed files
    for path, dirnames, filenames in os.walk(dev_path):
        for f in filenames:
            f = os.path.join(path, f)
            if os.path.islink(f) and not os.path.exists(f):
                os.unlink(f)
    # Ox.js
    filenames = [
        [
            'Core.js' # has to run first so that Ox is defined
        ],
        [
            'Function.js', # getSortValue (Array.js) depends on Ox.cache
            'Polyfill.js' # FIXME: not clear if needed here
        ],
        [
            'Array.js', # Ox.slice (Collection.js) depends on Ox.toArray, salt (HTML.js) depends on Ox.range
            'String.js', # salt (HTML.js) depends on Ox.char
            'Type.js' # Ox.typeOf needed in Collection.js FF3.6 for Ox.slice fallback
        ],
        [
            'Collection.js', # Ox.PATH (Constants.js) depends on Ox.slice
            'Math.js' # Ox.MAX_LATITUDE (Constants.js) depends on Ox.sinh
        ]
    ]
    js = ''
    js_dir = 'Ox/js/'
    ox_files = []
    for group in filenames:
        ox_files.append([])
        for filename in group:
            ox_files[-1].append(js_dir + filename)
    ox_files.append([])
    filenames = sum(filenames, []) # flatten
    for filename in sorted(os.listdir(source_path + js_dir)):
        if not filename in filenames \
                and not filename.startswith('.') \
                and not filename.startswith('_') \
                and not filename.endswith('~'):
            filenames.append(filename)
    for filename in filenames:
        js += read_file(source_path + js_dir + filename) + '\n'
        if not js_dir + filename in sum(ox_files, []):
            ox_files[-1].append(js_dir + filename)
    js = re.sub(
        'Ox.LOCALES = \{\}',
        'Ox.LOCALES = ' + json.dumps(locales, indent=4, sort_keys=True),
        js
    )
    js = re.sub(
        "Ox.VERSION = '([\d\.]+)'",
        "Ox.VERSION = '%s'" % version,
        js
    )
    write_file(build_path + 'Ox.js', ox.js.minify(js, comment))
    write_file(dev_path + '/Ox/json/' + 'Ox.json', json.dumps({
        'files': ox_files,
        'locales': locales,
        'version': version
    }, indent=4, sort_keys=True))

    # Ox.UI
    js = ''
    root = source_path + 'Ox.UI/'
    for path, dirnames, filenames in os.walk(root):
        for filename in sorted(filenames):
            # jquery gets included by Ox.UI loader
            # locale json files are loaded lazily
            # Ox.UI.css imports all other css files
            # svgs are loaded as URLs or dataURLs
            # Ox.UI PNGs are loaded on demand
            if path != root and not '_' in path and not filename[0] in '._' \
                    and not filename.endswith('~') \
                    and not 'jquery' in filename \
                    and not 'locale' in filename \
                    and not filename.endswith('theme.css') \
                    and not filename.endswith('.svg') \
                    and not 'Ox.UI/png' in path:
                ui_files['dev'].append(os.path.join(path.replace(source_path, ''), filename))
                if not '/js/' in path:
                    ui_files['build'].append(os.path.join(path.replace(source_path, ''), filename))
                if filename.endswith('.js'):
                    js += read_file(os.path.join(path, filename)) + '\n'
    filename = build_path + 'Ox.UI/js/Ox.UI.js'
    write_file(filename, ox.js.minify(js, comment))
    ui_files['build'].append(filename.replace(build_path, ''))
    write_file(build_path + 'Ox.UI/json/Ox.UI.json', json.dumps({
        'files': sorted(ui_files['build']),
        'images': ui_images
    }, sort_keys=True))
    write_file(dev_path + 'Ox.UI/json/Ox.UI.json', json.dumps({
        'files': sorted(ui_files['dev']),
        'images': ui_images
    }, indent=4, sort_keys=True))
    ui_files['dev'].append('Ox.UI/Ox.UI.js')

    # index
    data = {
        # sum(list, []) is flatten
        'documentation': sorted(sum(ox_files, [])) + sorted(filter(
            lambda x: re.search('\.js$', x),
            ui_files['dev']
        ) + map(
            lambda x: 'Ox.%s/Ox.%s.js' % (x, x),
            ['Geo', 'Image', 'Unicode']
        )),
        'examples': sorted(sum(map(
            lambda x: filter(
                lambda x: not re.search('/[._]', x),
                map(
                    lambda y: x + '/' + y,
                    os.listdir(root_path + 'examples/' + x)
                )
            ),
            filter(
                lambda x: not re.search('^[._]', x),
                os.listdir(root_path + 'examples/')
            )
        ), [])),
        'readme': map(
            lambda x: {
                'date': time.strftime(
                    '%Y-%m-%dT%H:%M:%SZ',
                    time.gmtime(os.path.getmtime(root_path + 'readme/' + x))
                ),
                'id': x.split('.')[0],
                'title': get_title(root_path + 'readme/' + x)
            },
            filter(
                lambda x: not re.search('^[._]', x) and re.search('\.html$', x),
                os.listdir(root_path + 'readme/')
            )
        )
    }
    write_file(root_path + 'index.json', json.dumps(data, indent=4, sort_keys=True))

    # downloads
    if downloads:
        data = {
            'date': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
            'size': {'minified': os.path.getsize(build_path + 'Ox.js')},
            'version': version
        }
        download_path = root_path + 'downloads/'
        # source
        source_file = download_path + 'OxJS.%s.source.tar.gz' % version
        data['size']['source'] = write_tarfile(source_file, root_path, 'OxJS', filter_source)
        write_link(source_file.replace(download_path, ''), source_file.replace(version, 'latest'))
        # build
        build_file = download_path + 'OxJS.%s.build.tar.gz' % version
        data['size']['build'] = write_tarfile(build_file, root_path, 'OxJS', filter_build)
        write_link(build_file.replace(download_path, ''), build_file.replace(version, 'latest'))
        # json
        write_file(download_path + 'downloads.json', json.dumps(data, indent=4, sort_keys=True))


def copy_file(source, target):
    print 'copying', source, 'to', target
    write_file(target, read_file(source))

def filter_build(tarinfo):
    name = tarinfo.name
    if name == 'OxJS' or re.search('^OxJS/build', name):
        return tarinfo
    return None

def filter_source(tarinfo):
    name = tarinfo.name
    if re.search('^[._]', name) or re.search('/[._]', name) or re.search('~$', name):
        return None
    if re.search('^OxJS/downloads', name):
        return None
    if name == 'OxJS/tools/geo/png/icons.png':
        return None
    if re.search('^OxJS/tools/geo/png/icons/', name) and (
        not re.search('4096', name) or not os.path.exists(
            name.replace('OxJS/', '../../').replace('icons/4096', 'flags')
        )
    ):
        return None
    return tarinfo

def format_hex(rgb):
    return '#%s' % ''.join([hex(c)[-2:].replace('x', '0').upper() for c in rgb])

def get_title(file):
    match = re.search('<h1>(.+)</h1>', read_file(file))
    return match.groups()[0] if match else 'Untitled'

def parse_css(css, values):
    def sub(match):
        key = match.group(1)
        index = match.group(2)
        value = values[key] if index == None else values[key][int(index[1:-1])]
        if isinstance(value, str):
            string = value
        else:
            if isinstance(value[0], int):
                value = [value]
            string = ', '.join(
                ['rgb%s(%s)' % (
                    'a' if len(vals) == 4 else '',
                    ', '.join([str(val) for val in vals])
                ) for vals in value]
            )
        return string
    return re.sub('\$(\w+)(\[\d+\])?', sub, css)

def read_file(file):
    print 'reading', file
    f = open(file)
    data = f.read()
    f.close()
    return data

def read_jsonc(file):
    return ox.jsonc.loads(read_file(file))

def write_file(file, data):
    print 'writing', file
    write_path(file)
    f = open(file, 'w')
    f.write(data)
    f.close()
    return len(data)

def write_link(source, target):
    print 'linking', source, 'to', target
    write_path(target)
    # remove files, symlinks *and broken symlinks*
    if os.path.exists(target) or os.path.lexists(target):
        os.unlink(target)
    os.symlink(source, target)

def write_path(file):
    path = os.path.split(file)[0]
    if path and not os.path.exists(path):
        os.makedirs(path)

def write_tarfile(file, path, arcname, filter):
    print 'writing', file
    build_tar = tarfile.open(file, 'w:gz')
    build_tar.add(path, arcname=arcname, filter=filter)
    build_tar.close()
    return os.path.getsize(file)

if __name__ == '__main__':
    build_oxjs(downloads='-downloads' in sys.argv, geo=not '-nogeo' in sys.argv)