oxjs/source/Ox/js/JavaScript.js

'use strict';

/*@
Ox.doc <f> Generates documentation for annotated JavaScript
    (source) -> <[o]> Array of doc objects
        arguments <[o]|u> Arguments (array of doc objects)
            Present if the <code>type</code> of the item is
            <code>"function"</code>.
        description <s|u> Multi-line description with optional markup
            See Ox.parseHTML for details
        events <[o]|u> Events (array of doc objects)
            Present if the item fires any events
        file <s> File name
        line <n> Line number
        name <s> Name of the item
        properties <[o]|u> Properties (array of doc objects)
            Present if the <code>type</code> of the item is
            <code>"event"</code>, <code>"function"</code>
            or <code>"object"</code>.
        section <s|u> Section in the file
        source <[o]> Source code (array of tokens)
            length <n> Length of the token
            offset <n> Offset of the token
            type <s> Type of the token
                See Ox.tokenize for list of types
        summary <s> One-line summary
        usage <[o]> Usage (array of doc objects)
            Present if the <code>type</code> of the item is
            <code>"function"</code>.
        type <s> Type of the item
    (file, callback) -> <u> undefined
    (files, callback) -> <u> undefined
    source <s> JavaScript source code
    file <s> JavaScript file
    files <[s]> Array of javascript files
    callback <f> Callback function
        doc <[o]> Array of doc objects
    # > Ox.doc("//@ My.FOO <n> Magic constant\nMy.FOO = 23;")
    # [{"name": "Ox.foo", "summary": "just some string", "type": "string"}]
@*/

Ox.doc = (function() {
    // fixme: dont require the trailing '@'
    var re = {
            item: /^(.+?) <(.+?)> (.+)$/,
            multiline: /^\/\*\@.*?\n([\w\W]+)\n.*?\@\*\/$/,
            script: /\n(\s*<script>s*\n[\w\W]+\n\s*<\/script>s*)/g,
            singleline: /^\/\/@\s*(.*?)\s*$/,
            test: /\n(\s*> .+\n.+?)/g,
            usage: /\(.*?\)/
        },
        types = {
            a: 'array', b: 'boolean', d: 'date',
            e: 'element', f: 'function', n: 'number',
            o: 'object', r: 'regexp', s: 'string',
            u: 'undefined', '*': 'value', '!': 'event'
        };
    function decodeLinebreaks(match, submatch) {
        return (submatch || match).replace(/\u21A9/g, '\n');
    }
    function encodeLinebreaks(match, submatch) {
        return '\n' + (submatch || match).replace(/\n/g, '\u21A9');
    }
    function getIndent(str) {
        var indent = -1;
        while (str[++indent] == ' ') {}
        return indent;
    }
    function parseItem(str) {
        var matches = re.item.exec(str);
        // to tell a variable with default value, like
        //     name <string|'<a href="...">foo</a>'> summary
        // from a line of description with tags, like
        //     some <a href="...">description</a> text
        // we need to check if there is either no forward slash
        // or if the second last char is a single or double quote
        return matches && (
            matches[2].indexOf('/') == -1 ||
            '\'"'.indexOf(matches[2].substr(-2, 1)) > -1
        ) ? Ox.extend({
            name: parseName(matches[1].trim()),
            summary: matches[3].trim()
        }, parseType(matches[2])) : null;
    }
    function parseName(str) {
        var matches = re.usage.exec(str);
        return matches ? matches[0] : str;
    }
    function parseNode(node) {
        var item = parseItem(node.line), subitem;
        node.nodes && node.nodes.forEach(function(node) {
            var key, line = node.line, subitem;
            if (!/^#/.test(node.line)) {
                if (/^<script>/.test(line)) {
                    item.examples = [parseScript(line)];
                } else if (/^>/.test(line)) {
                    item.examples = item.examples || [];
                    item.examples.push(parseTest(line));
                } else if ((subitem = parseItem(line))) {
                    if (/^\(/.test(subitem.name)) {
                        item.usage = item.usage || [];
                        item.usage.push(parseNode(node));
                    } else if (subitem.types[0] == 'event') {
                        item.events = item.events || [];
                        item.events.push(parseNode(node));                            
                    } else {
                        key = item.types[0] == 'function'
                            ? 'arguments' : 'properties'
                        item[key] = item[key] || [];
                        item[key].push(parseNode(node));                            
                    }
                } else {
                    item.description = item.description
                        ? item.description + ' ' + line : line
                }
            }
        });
        return item;
    }
    function parseScript(str) {
        // remove script tags and extra indentation
        var lines = decodeLinebreaks(str).split('\n'),
            indent = getIndent(lines[1]);
        return {
            statement: Ox.map(lines, function(line, i) {
                return i && i < lines.length - 1 ?
                    line.substr(indent) : null;
            }).join('\n')
        };
    }
    function parseSource(source, file) {
        var blocks = [],
            items = [],
            section = '',
            tokens = [];
        Ox.tokenize(source).forEach(function(token) {
            var match;
            token.source = source.substr(token.offset, token.length);
            if (token.type == 'comment' && (
                match = re.multiline.exec(token.source)
                || re.singleline.exec(token.source)
            )) {
                blocks.push(match[1]);
                tokens.push([]);
            } else if (tokens.length) {
                tokens[tokens.length - 1].push(token);
            }
        });
        blocks.forEach(function(block, i) {
            var item, lastItem,
                lines = block
                    .replace(re.script, encodeLinebreaks)
                    .replace(re.test, encodeLinebreaks)
                    .split('\n'),
                tree = parseTree(lines);
            if (re.item.test(tree.line)) {
                // parse the tree's root node
                item = parseNode(tree);
                item.file = file || '';
                if (section) {
                    item.section = section;
                }
                if (/^[A-Z]/.test(item.name)) {
                    // main item
                    // include leading whitespace
                    item.source = parseTokens(tokens[i]);
                    item.line = source.substr(0, item.source[0].offset)
                        .split('\n').length;
                    items.push(item);
                } else {
                    // property of a function item
                    lastItem = items[items.length - 1];
                    lastItem.properties = lastItem.properties || [];
                    lastItem.properties.push(item);
                    // include leading linebreaks and whitespace
                    lastItem.source = Ox.merge(
                        lastItem.source, parseTokens(tokens[i], true)
                    );
                }
            } else {
                section = tree.line.split(' ')[0]
            }
        });
        return items;        
    }
    function parseTest(str) {
        // fixme: we cannot properly handle tests where a string contains '\n '
        var lines = decodeLinebreaks(str).split('\n ');
        return {
            statement: lines[0].substr(2),
            result: lines[1].trim()
        };
    }
    function parseTokens(tokens, includeLeadingLinebreaks) {
        var isLeading = true,
            isTrailing = false,
            tokens_ = [],
            types = ['linebreak', 'whitespace'];
        tokens.forEach(function(token) {
            if (isLeading && types.indexOf(token.type) > -1) {
                if (token.type == 'linebreak') {
                    if (includeLeadingLinebreaks) {
                        tokens_.push(token);
                    } else {
                        tokens_ = [];
                    }
                } else {
                    tokens_.push(token);
                }
            } else {
                tokens_.push(token);
                isLeading = false;
                if (types.indexOf(token.type) == -1) {
                    isTrailing = true;
                }
            }
        });
        if (isTrailing) {
            while (types.indexOf(tokens_[tokens_.length - 1].type) > -1) {
                tokens_.pop();
            }
        }
        return tokens_;
    }
    function parseTree(lines) {
        // parses indented lines into a tree structure, like
        // {line: "...", nodes: [{line: "...", nodes: [...]}]}
        var branches = [],
            indent,
            node = {
                // chop the root line
                line: lines.shift().trim()
            };
        if (lines.length) {
            indent = getIndent(lines[0]);
            lines.forEach(function(line) {
                if (getIndent(line) == indent) {
                    // line is a child,
                    // make it the root line of a new branch
                    branches.push([line]);
                } else {
                    // line is a descendant of the last child,
                    // add it to the last branch
                    branches[branches.length - 1].push(line);
                }
            });
            node.nodes = branches.map(function(lines) {
                return parseTree(lines);
            });
        }
        return node;
    }
    function parseType(str) {
        // returns {types: [""]}
        // or {types: [""], default: ""}
        // or {types: [""], super: ""}
        var isArray,
            ret = {types: []},
            split,
            type;
        // only split by ':' if there is no default string value
        if ('\'"'.indexOf(str.substr(-2, 1)) == -1) {
            split = str.split(':');
            str = split[0];
            if (split.length == 2) {
                ret.super = split[1];
            }
        }
        str.split('|').forEach(function(str) {
            var unwrapped = unwrap(str);
            if (unwrapped in types) {
                ret.types.push(wrap(types[unwrapped]))
            } else if (
                (type = Ox.filter(Ox.values(types), function(type) {
                    return Ox.startsWith(type, unwrapped);
                })).length
            ) {
                ret.types.push(wrap(type[0]));
            } else {
                ret['default'] = str;
            }
        });
        function unwrap(str) {
            return (isArray = /^\[.+\]$/.test(str)) ? Ox.sub(str, 1, -1) : str;
        }
        function wrap(str) {
            return isArray ? '[' + str + 's' + ']' : str;
        }
        return ret;
    }
    return function(/* source | file, callback | files, callback*/) {
        var source = arguments.length == 1 ? arguments[0] : void 0,
            files = arguments.length == 2 ? Ox.toArray(arguments[0]) : void 0,
            callback = arguments[1],
            counter = 0, items = [];
        files && files.forEach(function(file) {
            Ox.get(file, function(source) {
                items = Ox.merge(items, parseSource(source, file));
                ++counter == files.length && callback(items);
            });                
        });
        return source ? parseSource(source) : void 0;
    }
}());

/*@
Ox.minify <f> Minifies JavaScript
    (source) -> <s> Minified JavaScript
    (file, callback) -> <u> undefined
    source <s> JavaScript source
    file <s> JavaScript file
    callback <f> Callback function
    > Ox.minify('for (a in b)\n{\t\tc = void 0;\n}')
    'for(a in b)\n{c=void 0;}'
    > Ox.minify('return a; return 0; return "";')
    'return a;return 0;return"";'
    > Ox.minify('return\na;\nreturn\n0;\nreturn\n"";')
    'return\na;\nreturn\n0;\nreturn\n"";\n'
@*/
Ox.minify = function() {
    // see https://github.com/douglascrockford/JSMin/blob/master/README
    // and http://inimino.org/~inimino/blog/javascript_semicolons
    if (arguments.length == 1) {
        return minify(arguments[0]);
    } else {
        Ox.get(arguments[0], function(source) {
            arguments[1](minify(source));
        });
    }
    function minify(source) {
        var tokens = Ox.tokenize(source),
            length = tokens.length;
        function isIdentifierOrNumber(token) {
            return [
                'constant', 'identifier', 'keyword',
                'number', 'method', 'object', 'property'
            ].indexOf(token.type) > -1;
        }
        return Ox.map(tokens, function(token, i) {
            var ret = source.substr(token.offset, token.length);
            if (token.type == 'comment') {
                if (ret[1] == '/') {
                    // replace line comment with newline
                    ret = i == length - 1 || tokens[i + 1].type == 'linebreak'
                        ? null : '\n';
                } else {
                    // replace block comment with space
                    ret = i == length - 1 || tokens[i + 1].type == 'whitespace'
                        ? null : ' ';
                }
            }
            if (token.type == 'linebreak' || token.type == 'whitespace') {
                // remove consecutive linebreaks or whitespace
                ret = ret[0];
            }
            if (
                // strip linebreaks, except between two tokens that
                // are both either identifier or number or string
                // or identifier in brackets or identifier with
                // unary operator or array literal or object literal
                //
                // strip linebreaks, except after "break", "continue",
                // "return", "++" or "--", where the linebreak becomes
                // a semicolon
                token.type == 'linebreak' && (
                    i == 0 || i == length - 1
                    || (
                        !isIdentifierOrNumber(tokens[i - 1])
                        && !tokens[i - 1].type == 'string'
                        && [')', ']', '}', '++', '--'].indexOf(
                            source.substr(tokens[i - 1].offset, tokens[i - 1].length)
                        ) == -1
                    )
                    || (
                        !isIdentifierOrNumber(tokens[i + 1])
                        && ['(', '[', '{', '+', '-', '++', '--'].indexOf(
                            source.substr(tokens[i + 1].offset, tokens[i + 1].length)
                        ) == -1
                    )
                )
            ) {
                ret = null;
            } else if (
                // strip whitespace, except between two tokens
                // that are both either identifier or number
                token.type == 'whitespace' && (
                    i == 0 || i == length - 1
                    || !isIdentifierOrNumber(tokens[i - 1]) 
                    || !isIdentifierOrNumber(tokens[i + 1]) 
                )
            ) {
                ret = null;
            }
            return ret;
        }).join('');
    }
};

/*@
Ox.test <f> Takes JavaScript, runs inline tests, returns results
@*/
Ox.test = function(file, callback) {
    Ox.doc(file, function(items) {
        var tests = [];
        items.forEach(function(item) {
            item.examples && item.examples.forEach(function(example) {
                Ox.Log('TEST', example.statement)
                var actual = eval(example.statement);
                if (example.result) {
                    tests.push({
                        actual: JSON.stringify(actual),
                        expected: example.result,
                        name: item.name,
                        section: item.section,
                        statement: example.statement,
                        passed: Ox.isEqual(eval(
                            'Ox.test.result = ' + example.result
                        ), actual)
                    });
                }
            });
        });
        callback(tests);
    });
};

/*@
Ox.tokenize <f> Tokenizes JavaScript
    (source) -> <[o]> Array of tokens
        length <n> Length of the token
        offset <n> Offset of the token
        type <s> Type of the token
            Type can be <code>"comment"</code>, <code>"constant"</code>,
            <code>"identifier"</code>, <code>"keyword"</code>,
            <code>"linebreak"</code>, <code>"method"</code>,
            <code>"number"</code>, <code>"object"</code>, 
            <code>"operator"</code>, <code>"property"</code>,
            <code>"regexp"</code>, <code>"string"</code>
            or <code>"whitespace"</code>
    source <s> JavaScript source code
@*/
// FIXME: constant/method/object/property is of interest
// for syntax highlighting, but may not belong here
Ox.tokenize = (function() {

    // see https://github.com/mozilla/narcissus/blob/master/lib/jslex.js
    // and https://developer.mozilla.org/en/JavaScript/Reference

    var identifier = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ$_',
        linebreak = '\n\r',
        number = '0123456789',
        operator = [
            // arithmetic
            '+', '-', '*', '/', '%', '++', '--',
            // assignment
            '=', '+=', '-=', '*=', '/=', '%=',
            '&=', '|=', '^=', '<<=', '>>=', '>>>=',
            // bitwise
            '&', '|', '^', '~', '<<', '>>', '>>>',
            // comparison
            '==', '!=', '===', '!==', '>', '>=', '<', '<=',
            // conditional
            '?', ':',
            // grouping
            '(', ')', '[', ']', '{', '}',
            // logical
            '&&', '||', '!',
            // other
            '.', ',', ';'
        ],
        regexp = 'abcdefghijklmnopqrstuvwxyz',
        string = '\'"',
        whitespace = ' \t',
        word = {
            constant: [
                // Math
                'E', 'LN2', 'LN10', 'LOG2E', 'LOG10E', 'PI', 'SQRT1_2', 'SQRT2',
                // Number
                'MAX_VALUE', 'MIN_VALUE', 'NEGATIVE_INFINITY', 'POSITIVE_INFINITY'
            ],
            keyword: [
                'break',
                'case', 'catch', 'class', 'const', 'continue',
                'debugger', 'default', 'delete', 'do',
                'else', 'enum', 'export', 'extends',
                'false', 'finally', 'for', 'function',
                'if', 'implements', 'import', 'in', 'instanceof', 'interface',
                'let', 'module',
                'new', 'null',
                'package', 'private', 'protected', 'public',
                'return',
                'super', 'switch', 'static',
                'this', 'throw', 'true', 'try', 'typeof',
                'var', 'void',
                'yield',
                'while', 'with',
            ],
            method: [
                // Array
                'concat',
                'every',
                'filter', 'forEach',
                'join',
                'lastIndexOf',
                'indexOf', 'isArray',
                'map',
                'pop', 'push',
                'reduce', 'reduceRight', 'reverse',
                'shift', 'slice', 'some', 'sort', 'splice',
                'unshift',
                // Date
                'getDate', 'getDay', 'getFullYear', 'getHours',
                'getMilliseconds', 'getMinutes', 'getMonth', 'getSeconds',
                'getTime', 'getTimezoneOffset',
                'getUTCDate', 'getUTCDay', 'getUTCFullYear', 'getUTCHours',
                'getUTCMilliseconds', 'getUTCMinutes', 'getUTCMonth', 'getUTCSeconds',
                'now',
                'parse',
                'setDate', 'setFullYear', 'setHours', 'setMilliseconds',
                'setMinutes', 'setMonth', 'setSeconds', 'setTime',
                'setUTCDate', 'setUTCFullYear', 'setUTCHours', 'setUTCMilliseconds',
                'setUTCMinutes', 'setUTCMonth', 'setUTCSeconds',
                'toDateString', 'toJSON', 'toLocaleDateString', 'toLocaleString',
                'toLocaleTimeString', 'toTimeString', 'toUTCString',
                'UTC',
                // Function
                'apply', 'bind', 'call', 'isGenerator',
                // JSON
                'parse', 'stringify',
                // Math
                'abs', 'acos', 'asin', 'atan', 'atan2',
                'ceil', 'cos',
                'exp',
                'floor',
                'log',
                'max', 'min',
                'pow',
                'random', 'round',
                'sin', 'sqrt',
                'tan',
                // Number
                'toExponential', 'toFixed', 'toLocaleString', 'toPrecision',
                // Object
                'create',
                'defineProperty', 'defineProperties',
                'freeze',
                'getOwnPropertyDescriptor', 'getOwnPropertyNames', 'getPrototypeOf',
                'hasOwnProperty',
                'isExtensible', 'isFrozen', 'isPrototypeOf', 'isSealed',
                'keys',
                'preventExtensions', 'propertyIsEnumerable',
                'seal',
                'toLocaleString', 'toString',
                'valueOf',
                // RegExp
                'exec', 'test',
                // String
                'charAt', 'charCodeAt', 'concat',
                'fromCharCode',
                'indexOf',
                'lastIndexOf', 'localeCompare',
                'match',
                'replace',
                'search', 'slice', 'split', 'substr', 'substring',
                'toLocaleLowerCase', 'toLocaleUpperCase', 'toLowerCase', 'toUpperCase', 'trim',
                // Window
                'addEventListener', 'alert', 'atob',
                'blur', 'btoa',
                'clearInterval', 'clearTimeout', 'close', 'confirm',
                'dispatchEvent',
                'escape',
                'find', 'focus',
                'getComputedStyle', 'getSelection',
                'moveBy', 'moveTo',
                'open',
                'postMessage', 'print', 'prompt',
                'removeEventListener', 'resizeBy', 'resizeTo',
                'scroll', 'scrollBy', 'scrollTo',
                'setCursor', 'setInterval', 'setTimeout', 'stop',
                'unescape'
            ],
            object: [
                'Array',
                'Boolean',
                'Date', 'decodeURI', 'decodeURIComponent',
                'encodeURI', 'encodeURIComponent', 'Error', 'eval', 'EvalError',
                'Function',
                'Infinity', 'isFinite', 'isNaN',
                'JSON',
                'Math',
                'NaN', 'Number',
                'Object',
                'parseFloat', 'parseInt',
                'RangeError', 'ReferenceError', 'RegExp',
                'String', 'SyntaxError',
                'TypeError',
                'undefined', 'URIError',
                'window'
            ],
            property: [
                // Function
                'constructor', 'length', 'prototype',
                // RegExp
                'global', 'ignoreCase', 'lastIndex', 'multiline', 'source',
                // Window
                'applicationCache',
                'closed', 'console', 'content', 'crypto',
                'defaultStatus', 'document',
                'frameElement', 'frames',
                'history',
                'innerHeight', 'innerWidth',
                'length', 'location', 'locationbar', 'localStorage',
                'menubar',
                'name', 'navigator',
                'opener', 'outerHeight', 'outerWidth',
                'pageXOffset', 'pageYOffset', 'parent', 'personalbar',
                'screen', 'screenX', 'screenY', 'scrollbars', 'scrollX', 'scrollY',
                'self', 'sessionStorage', 'status', 'statusbar',
                'toolbar', 'top'
            ]
        };

    return function(source) {

        source = source.replace(/\r\n/g, '\n').replace(/\r/g, '\n');

        var cursor = 0,
            tokenize = {
                comment: function() {
                    while (char = source[++cursor]) {
                        if (next == '/' && char == '\n') {
                            break;
                        } else if (next == '*' && char + source[cursor + 1] == '*/') {
                            cursor += 2;
                            break;
                        }
                    }
                },
                identifier: function() {
                    var str;
                    while ((identifier + number).indexOf(source[++cursor]) > -1) {}
                    str = source.substring(start, cursor);
                    Ox.forEach(word, function(value, key) {
                        if (value.indexOf(str) > -1) {
                            type = key;
                            return false;
                        }
                    });
                },
                linebreak: function() {
                    while (linebreak.indexOf(source[++cursor]) > -1) {}
                },
                number: function() {
                    while ((number + '.').indexOf(source[++cursor]) > -1) {}
                },
                operator: function() {
                    while (operator.indexOf(char += source[++cursor]) > -1) {}
                },
                regexp: function() {
                    while ((char = source[++cursor]) != '/') {
                        char == '\\' && ++cursor;
                        if (cursor == source.length) {
                            break;
                        }
                    }
                    while (regexp.indexOf(source[++cursor]) > -1) {}
                },
                string: function() {
                    var delimiter = char;
                    while ((char = source[++cursor]) != delimiter) {
                        char == '\\' && ++cursor;
                        if (cursor == source.length) {
                            break;
                        }
                    }
                    ++cursor;
                },
                whitespace: function() {
                    while (whitespace.indexOf(source[++cursor]) > -1) {}
                }
            },
            tokens = [],
            type;

        while (cursor < source.length) {
            var char = source[cursor],
                next = source[cursor + 1],
                start = cursor;
            if (char == '/' && (next == '/' || next == '*')) {
                type = 'comment';
            } else if (identifier.indexOf(char) > -1) {
                type = 'identifier';
            } else if (linebreak.indexOf(char) > -1) {
                type = 'linebreak';
            } else if (number.indexOf(char) > -1) {
                type = 'number';
            } else if (string.indexOf(char) > -1) {
                type = 'string';
            } else if (whitespace.indexOf(char) > -1) {
                type = 'whitespace';
            } else if (char == '/') {
                type = isRegExp() ? 'regexp' : 'operator';
            } else if (operator.indexOf(char) > -1) {
                type = 'operator';
            }
            tokenize[type]();
            tokens.push({
                length: cursor - start,
                offset: start,
                type: type,
            });
        }

        function isRegExp() {
            // checks if a forward slash is the beginning of a regexp,
            // as opposed to the beginning of an operator
            // see http://www.mozilla.org/js/language/js20-2000-07/rationale/syntax.html#regular-expressions
            var index = tokens.length,
                isRegExp = false,
                offset = 0,
                prevToken,
                prevString;
            // scan back to the previous significant token,
            // or the beginning of the source
            while (
                tokens[--index] !== void 0 && [
                    'comment', 'linebreak', 'whitespace'
                ].indexOf(tokens[index].type) > -1
            ) {
                offset += tokens[index].length;
            }
            if (typeof tokens[index] == 'undefined') {
                // source begins with forward slash
                isRegExp = true;
            } else {
                prevToken = tokens[index];
                prevString = source.substr(
                    cursor - prevToken.length - offset, prevToken.length
                );
                isRegExp = (
                    prevToken.type == 'keyword'
                    && ['false', 'null', 'true'].indexOf(prevString) == -1
                ) || (
                    prevToken.type == 'operator'
                    && ['++', '--', ')', ']', '}'].indexOf(prevString) == -1
                );
            }
            return isRegExp;
        }

        return tokens;

    };

}());