724 lines
27 KiB
JavaScript
724 lines
27 KiB
JavaScript
'use strict';
|
|
|
|
/*@
|
|
Ox.doc <f> Generates documentation for annotated JavaScript
|
|
(source) -> <[o]> Array of doc objects
|
|
arguments <[o]|u> Arguments (array of doc objects)
|
|
Present if the <code>type</code> of the item is
|
|
<code>"function"</code>.
|
|
description <s|u> Multi-line description with optional markup
|
|
See Ox.parseHTML for details
|
|
events <[o]|u> Events (array of doc objects)
|
|
Present if the item fires any events
|
|
file <s> File name
|
|
line <n> Line number
|
|
name <s> Name of the item
|
|
properties <[o]|u> Properties (array of doc objects)
|
|
Present if the <code>type</code> of the item is
|
|
<code>"event"</code>, <code>"function"</code>
|
|
or <code>"object"</code>.
|
|
section <s|u> Section in the file
|
|
source <[o]> Source code (array of tokens)
|
|
column <n> Column
|
|
line <n> Line
|
|
type <s> Type (see Ox.tokenize for a list of types)
|
|
value <s> Value
|
|
summary <s> One-line summary
|
|
usage <[o]> Usage (array of doc objects)
|
|
Present if the <code>type</code> of the item is
|
|
<code>"function"</code>.
|
|
type <s> Type of the item
|
|
(file, callback) -> <u> undefined
|
|
(files, callback) -> <u> undefined
|
|
source <s> JavaScript source code
|
|
file <s> JavaScript file
|
|
files <[s]> Array of javascript files
|
|
callback <f> Callback function
|
|
doc <[o]> Array of doc objects
|
|
# > Ox.doc("//@ My.FOO <n> Magic constant\nMy.FOO = 23;")
|
|
# [{"name": "Ox.foo", "summary": "just some string", "type": "string"}]
|
|
@*/
|
|
Ox.doc = (function() {
|
|
// fixme: dont require the trailing '@'
|
|
var re = {
|
|
item: /^(.+?) <(.+?)> (.+?)$/,
|
|
multiline: /^\/\*\@.*?\n([\w\W]+)\n.*?\@\*\/$/,
|
|
script: /\n(\s*<script>s*\n[\w\W]+\n\s*<\/script>s*)/g,
|
|
singleline: /^\/\/@\s*(.*?)\s*$/,
|
|
test: /\n(\s*> .+\n.+?)/g,
|
|
usage: /\(.*?\)/
|
|
},
|
|
types = {
|
|
a: 'array', b: 'boolean', d: 'date',
|
|
e: 'element', f: 'function', n: 'number',
|
|
o: 'object', r: 'regexp', s: 'string',
|
|
u: 'undefined', '*': 'value', '!': 'event'
|
|
};
|
|
function decodeLinebreaks(match, submatch) {
|
|
return (submatch || match).replace(/\u21A9/g, '\n');
|
|
}
|
|
function encodeLinebreaks(match, submatch) {
|
|
return '\n' + (submatch || match).replace(/\n/g, '\u21A9');
|
|
}
|
|
function getIndent(string) {
|
|
var indent = -1;
|
|
while (string[++indent] == ' ') {}
|
|
return indent;
|
|
}
|
|
function parseItem(string) {
|
|
var matches = re.item.exec(string);
|
|
// to tell a variable with default value, like
|
|
// name <string|'<a href="...">foo</a>'> summary
|
|
// from a line of description with tags, like
|
|
// some <a href="...">description</a> text
|
|
// we need to check if there is either no forward slash
|
|
// or if the second last char is a single or double quote
|
|
return matches && (
|
|
matches[2].indexOf('/') == -1 ||
|
|
'\'"'.indexOf(matches[2].slice(-2, -1)) > -1
|
|
) ? Ox.extend({
|
|
name: parseName(matches[1].trim()),
|
|
summary: matches[3].trim()
|
|
}, parseType(matches[2])) : null;
|
|
}
|
|
function parseName(string) {
|
|
var matches = re.usage.exec(string);
|
|
return matches ? matches[0] : string;
|
|
}
|
|
function parseNode(node) {
|
|
var item = parseItem(node.line), subitem;
|
|
node.nodes && node.nodes.forEach(function(node) {
|
|
var key, line = node.line, subitem;
|
|
if (!/^#/.test(node.line)) {
|
|
if (/^<script>/.test(line)) {
|
|
item.examples = [parseScript(line)];
|
|
} else if (/^>/.test(line)) {
|
|
item.examples = item.examples || [];
|
|
item.examples.push(parseTest(line));
|
|
} else if ((subitem = parseItem(line))) {
|
|
if (/^\(/.test(subitem.name)) {
|
|
item.usage = item.usage || [];
|
|
item.usage.push(parseNode(node));
|
|
} else if (subitem.types[0] == 'event') {
|
|
item.events = item.events || [];
|
|
item.events.push(parseNode(node));
|
|
} else {
|
|
key = item.types[0] == 'function'
|
|
? 'arguments' : 'properties'
|
|
item[key] = item[key] || [];
|
|
item[key].push(parseNode(node));
|
|
}
|
|
} else {
|
|
item.description = item.description
|
|
? item.description + ' ' + line : line
|
|
}
|
|
}
|
|
});
|
|
return item;
|
|
}
|
|
function parseScript(string) {
|
|
// remove script tags and extra indentation
|
|
var lines = decodeLinebreaks(string).split('\n'),
|
|
indent = getIndent(lines[1]);
|
|
return {
|
|
statement: lines.slice(1, -1).map(function(line, i) {
|
|
return line.slice(indent);
|
|
}).join('\n')
|
|
};
|
|
}
|
|
function parseSource(source, file) {
|
|
var blocks = [],
|
|
items = [],
|
|
section = '',
|
|
tokens = [];
|
|
Ox.tokenize(source).forEach(function(token) {
|
|
var match;
|
|
if (token.type == 'comment' && (
|
|
match = re.multiline.exec(token.value)
|
|
|| re.singleline.exec(token.value)
|
|
)) {
|
|
blocks.push(match[1]);
|
|
tokens.push([]);
|
|
} else if (tokens.length) {
|
|
tokens[tokens.length - 1].push(token);
|
|
}
|
|
});
|
|
blocks.forEach(function(block, i) {
|
|
var item, lastItem,
|
|
lines = block
|
|
.replace(re.script, encodeLinebreaks)
|
|
.replace(re.test, encodeLinebreaks)
|
|
.split('\n'),
|
|
tree = parseTree(lines);
|
|
if (re.item.test(tree.line)) {
|
|
// parse the tree's root node
|
|
item = parseNode(tree);
|
|
item.file = file || '';
|
|
if (section) {
|
|
item.section = section;
|
|
}
|
|
if (/^[A-Z]/.test(item.name)) {
|
|
// main item
|
|
// include leading whitespace
|
|
item.source = parseTokens(tokens[i]);
|
|
item.line = source.slice(0, item.source[0].offset)
|
|
.split('\n').length;
|
|
items.push(item);
|
|
} else {
|
|
// property of a function item
|
|
lastItem = items[items.length - 1];
|
|
lastItem.properties = lastItem.properties || [];
|
|
lastItem.properties.push(item);
|
|
// include leading linebreaks and whitespace
|
|
lastItem.source = lastItem.source.concat(
|
|
parseTokens(tokens[i], true)
|
|
);
|
|
}
|
|
} else {
|
|
section = tree.line.split(' ')[0]
|
|
}
|
|
});
|
|
return items;
|
|
}
|
|
function parseTest(string) {
|
|
// fixme: we cannot properly handle tests where a string contains '\n '
|
|
var lines = decodeLinebreaks(string).split('\n ');
|
|
return {
|
|
statement: lines[0].slice(2),
|
|
result: lines[1].trim()
|
|
};
|
|
}
|
|
function parseTokens(tokens, includeLeadingLinebreaks) {
|
|
var isLeading = true,
|
|
isTrailing = false,
|
|
tokens_ = [],
|
|
types = ['linebreak', 'whitespace'];
|
|
tokens.forEach(function(token) {
|
|
if (isLeading && types.indexOf(token.type) > -1) {
|
|
if (token.type == 'linebreak') {
|
|
if (includeLeadingLinebreaks) {
|
|
tokens_.push(token);
|
|
} else {
|
|
tokens_ = [];
|
|
}
|
|
} else {
|
|
tokens_.push(token);
|
|
}
|
|
} else {
|
|
tokens_.push(token);
|
|
isLeading = false;
|
|
if (types.indexOf(token.type) == -1) {
|
|
isTrailing = true;
|
|
}
|
|
}
|
|
});
|
|
if (isTrailing) {
|
|
while (types.indexOf(tokens_[tokens_.length - 1].type) > -1) {
|
|
tokens_.pop();
|
|
}
|
|
}
|
|
return tokens_;
|
|
}
|
|
function parseTree(lines) {
|
|
// parses indented lines into a tree structure, like
|
|
// {line: "...", nodes: [{line: "...", nodes: [...]}]}
|
|
var branches = [],
|
|
indent,
|
|
node = {
|
|
// chop the root line
|
|
line: lines.shift().trim()
|
|
};
|
|
if (lines.length) {
|
|
indent = getIndent(lines[0]);
|
|
lines.forEach(function(line) {
|
|
if (getIndent(line) == indent) {
|
|
// line is a child,
|
|
// make it the root line of a new branch
|
|
branches.push([line]);
|
|
} else {
|
|
// line is a descendant of the last child,
|
|
// add it to the last branch
|
|
branches[branches.length - 1].push(line);
|
|
}
|
|
});
|
|
node.nodes = branches.map(function(lines) {
|
|
return parseTree(lines);
|
|
});
|
|
}
|
|
return node;
|
|
}
|
|
function parseType(string) {
|
|
// returns {types: [""]}
|
|
// or {types: [""], default: ""}
|
|
// or {types: [""], super: ""}
|
|
var array,
|
|
isArray,
|
|
ret = {types: []},
|
|
type;
|
|
// only split by ':' if there is no default string value
|
|
if ('\'"'.indexOf(string.slice(-2, -1)) == -1) {
|
|
array = string.split(':');
|
|
string = array[0];
|
|
if (array.length == 2) {
|
|
ret['super'] = array[1];
|
|
}
|
|
}
|
|
string.split('|').forEach(function(string) {
|
|
var unwrapped = unwrap(string);
|
|
if (unwrapped in types) {
|
|
ret.types.push(wrap(types[unwrapped]))
|
|
} else if (
|
|
(type = Ox.filter(Ox.values(types), function(type) {
|
|
return Ox.startsWith(type, unwrapped);
|
|
})).length
|
|
) {
|
|
ret.types.push(wrap(type[0]));
|
|
} else {
|
|
ret['default'] = string;
|
|
}
|
|
});
|
|
function unwrap(string) {
|
|
return (isArray = /^\[.+\]$/.test(string))
|
|
? string.slice(1, -1) : string;
|
|
}
|
|
function wrap(string) {
|
|
return isArray ? '[' + string + 's' + ']' : string;
|
|
}
|
|
return ret;
|
|
}
|
|
return function(/* source | file, callback | files, callback*/) {
|
|
var source = arguments.length == 1 ? arguments[0] : void 0,
|
|
files = arguments.length == 2 ? Ox.makeArray(arguments[0]) : void 0,
|
|
callback = arguments[1],
|
|
counter = 0, items = [];
|
|
files && files.forEach(function(file) {
|
|
Ox.get(file, function(source) {
|
|
items = items.concat(parseSource(source, file));
|
|
++counter == files.length && callback(items);
|
|
});
|
|
});
|
|
return source ? parseSource(source) : void 0;
|
|
}
|
|
}());
|
|
|
|
/*@
|
|
Ox.identify <f> Returns the type of a JavaScript identifier
|
|
(str) -> <s> Type
|
|
Type can be <code>constant</code>, <code>identifier</code>,
|
|
<code>keyword</code>, <code>method</code>, <code>object</code> or
|
|
<code>property</code>
|
|
@*/
|
|
Ox.identify = (function() {
|
|
// see https://developer.mozilla.org/en/JavaScript/Reference
|
|
var identifiers = {
|
|
constant: [
|
|
// Math
|
|
'E', 'LN2', 'LN10', 'LOG2E', 'LOG10E', 'PI', 'SQRT1_2', 'SQRT2',
|
|
// Number
|
|
'MAX_VALUE', 'MIN_VALUE', 'NEGATIVE_INFINITY', 'POSITIVE_INFINITY'
|
|
],
|
|
method: [
|
|
// Array
|
|
'concat',
|
|
'every',
|
|
'filter', 'forEach',
|
|
'join',
|
|
'lastIndexOf',
|
|
'indexOf', 'isArray',
|
|
'map',
|
|
'pop', 'push',
|
|
'reduce', 'reduceRight', 'reverse',
|
|
'shift', 'slice', 'some', 'sort', 'splice',
|
|
'unshift',
|
|
// Date
|
|
'getDate', 'getDay', 'getFullYear', 'getHours',
|
|
'getMilliseconds', 'getMinutes', 'getMonth', 'getSeconds',
|
|
'getTime', 'getTimezoneOffset',
|
|
'getUTCDate', 'getUTCDay', 'getUTCFullYear', 'getUTCHours',
|
|
'getUTCMilliseconds', 'getUTCMinutes', 'getUTCMonth', 'getUTCSeconds',
|
|
'now',
|
|
'parse',
|
|
'setDate', 'setFullYear', 'setHours', 'setMilliseconds',
|
|
'setMinutes', 'setMonth', 'setSeconds', 'setTime',
|
|
'setUTCDate', 'setUTCFullYear', 'setUTCHours', 'setUTCMilliseconds',
|
|
'setUTCMinutes', 'setUTCMonth', 'setUTCSeconds',
|
|
'toDateString', 'toJSON', 'toLocaleDateString', 'toLocaleString',
|
|
'toLocaleTimeString', 'toTimeString', 'toUTCString',
|
|
'UTC',
|
|
// Function
|
|
'apply', 'bind', 'call', 'isGenerator',
|
|
// JSON
|
|
'parse', 'stringify',
|
|
// Math
|
|
'abs', 'acos', 'asin', 'atan', 'atan2',
|
|
'ceil', 'cos',
|
|
'exp',
|
|
'floor',
|
|
'log',
|
|
'max', 'min',
|
|
'pow',
|
|
'random', 'round',
|
|
'sin', 'sqrt',
|
|
'tan',
|
|
// Number
|
|
'toExponential', 'toFixed', 'toLocaleString', 'toPrecision',
|
|
// Object
|
|
'create',
|
|
'defineProperty', 'defineProperties',
|
|
'freeze',
|
|
'getOwnPropertyDescriptor', 'getOwnPropertyNames', 'getPrototypeOf',
|
|
'hasOwnProperty',
|
|
'isExtensible', 'isFrozen', 'isPrototypeOf', 'isSealed',
|
|
'keys',
|
|
'preventExtensions', 'propertyIsEnumerable',
|
|
'seal',
|
|
'toLocaleString', 'toString',
|
|
'valueOf',
|
|
// RegExp
|
|
'exec', 'test',
|
|
// String
|
|
'charAt', 'charCodeAt', 'concat',
|
|
'fromCharCode',
|
|
'indexOf',
|
|
'lastIndexOf', 'localeCompare',
|
|
'match',
|
|
'replace',
|
|
'search', 'slice', 'split', 'substr', 'substring',
|
|
'toLocaleLowerCase', 'toLocaleUpperCase',
|
|
'toLowerCase', 'toUpperCase', 'trim',
|
|
// Window
|
|
'addEventListener', 'alert', 'atob',
|
|
'blur', 'btoa',
|
|
'clearInterval', 'clearTimeout', 'close', 'confirm',
|
|
'dispatchEvent',
|
|
'escape',
|
|
'find', 'focus',
|
|
'getComputedStyle', 'getSelection',
|
|
'moveBy', 'moveTo',
|
|
'open',
|
|
'postMessage', 'print', 'prompt',
|
|
'removeEventListener', 'resizeBy', 'resizeTo',
|
|
'scroll', 'scrollBy', 'scrollTo',
|
|
'setCursor', 'setInterval', 'setTimeout', 'stop',
|
|
'unescape'
|
|
],
|
|
object: [
|
|
'Array',
|
|
'Boolean',
|
|
'Date', 'decodeURI', 'decodeURIComponent',
|
|
'encodeURI', 'encodeURIComponent', 'Error', 'eval', 'EvalError',
|
|
'Function',
|
|
'Infinity', 'isFinite', 'isNaN',
|
|
'JSON',
|
|
'Math',
|
|
'NaN', 'Number',
|
|
'Object',
|
|
'parseFloat', 'parseInt',
|
|
'RangeError', 'ReferenceError', 'RegExp',
|
|
'String', 'SyntaxError',
|
|
'TypeError',
|
|
'undefined', 'URIError',
|
|
'window'
|
|
],
|
|
property: [
|
|
// Function
|
|
'constructor', 'length', 'prototype',
|
|
// RegExp
|
|
'global', 'ignoreCase', 'lastIndex', 'multiline', 'source',
|
|
// Window
|
|
'applicationCache',
|
|
'closed', 'console', 'content', 'crypto',
|
|
'defaultStatus', 'document',
|
|
'frameElement', 'frames',
|
|
'history',
|
|
'innerHeight', 'innerWidth',
|
|
'length', 'location', 'locationbar', 'localStorage',
|
|
'menubar',
|
|
'name', 'navigator',
|
|
'opener', 'outerHeight', 'outerWidth',
|
|
'pageXOffset', 'pageYOffset', 'parent', 'personalbar',
|
|
'screen', 'screenX', 'screenY', 'scrollbars', 'scrollX', 'scrollY',
|
|
'self', 'sessionStorage', 'status', 'statusbar',
|
|
'toolbar', 'top'
|
|
]
|
|
};
|
|
return function(identifier) {
|
|
var ret;
|
|
if (Ox.KEYWORDS.indexOf(identifier) > -1) {
|
|
ret = 'keyword'
|
|
} else {
|
|
ret = 'identifier'
|
|
Ox.forEach(identifiers, function(words, type) {
|
|
if (words.indexOf(identifier) > -1) {
|
|
ret = type;
|
|
Ox.Break();
|
|
}
|
|
});
|
|
}
|
|
return ret;
|
|
};
|
|
}());
|
|
|
|
/*@
|
|
Ox.minify <f> Minifies JavaScript
|
|
(source) -> <s> Minified JavaScript
|
|
(file, callback) -> <u> undefined
|
|
source <s> JavaScript source
|
|
file <s> JavaScript file
|
|
callback <f> Callback function
|
|
> Ox.minify('for (a in b)\n{\t\tc = void 0;\n}')
|
|
'for(a in b)\n{c=void 0;}'
|
|
> Ox.minify('return a; return 0; return "";')
|
|
'return a;return 0;return"";'
|
|
> Ox.minify('return\na;\nreturn\n0;\nreturn\n"";')
|
|
'return\na;return\n0;return\n"";'
|
|
@*/
|
|
Ox.minify = function() {
|
|
// see https://github.com/douglascrockford/JSMin/blob/master/README
|
|
// and http://inimino.org/~inimino/blog/javascript_semicolons
|
|
if (arguments.length == 1) {
|
|
return minify(arguments[0]);
|
|
} else {
|
|
Ox.get(arguments[0], function(source) {
|
|
arguments[1](minify(source));
|
|
});
|
|
}
|
|
function minify(source) {
|
|
var tokens = Ox.tokenize(source),
|
|
length = tokens.length,
|
|
ret = '';
|
|
tokens.forEach(function(token, i) {
|
|
var next, nextToken, prevToken;
|
|
if (['linebreak', 'whitespace'].indexOf(token.type) > -1) {
|
|
prevToken = i == 0 ? null : tokens[i - 1];
|
|
next = i + 1;
|
|
while (
|
|
next < length && ['comment', 'linebreak', 'whitespace']
|
|
.indexOf(tokens[next].type) > -1
|
|
) {
|
|
next++;
|
|
}
|
|
nextToken = next == length ? null : tokens[next];
|
|
}
|
|
if (token.type == 'linebreak') {
|
|
// replace a linebreak between two tokens that are identifiers
|
|
// or numbers or strings or unary operators or grouping
|
|
// operators with a single newline, otherwise remove it
|
|
if (
|
|
prevToken && nextToken && (
|
|
['identifier', 'number', 'string'].indexOf(prevToken.type) > -1
|
|
|| ['++', '--', ')', ']', '}'].indexOf(prevToken.value) > -1
|
|
) && (
|
|
['identifier', 'number', 'string'].indexOf(nextToken.type) > -1
|
|
|| ['+', '-', '++', '--', '~', '!', '(', '[', '{'].indexOf(nextToken.value) > -1
|
|
)
|
|
) {
|
|
ret += '\n';
|
|
}
|
|
} else if (token.type == 'whitespace') {
|
|
// replace whitespace between two tokens that are identifiers or
|
|
// numbers, or between a token that ends with "+" or "-" and one
|
|
// that begins with "+" or "-", with a single space, otherwise
|
|
// remove it
|
|
if (
|
|
prevToken && nextToken && ((
|
|
['identifier', 'number'].indexOf(prevToken.type) > -1
|
|
&& ['identifier', 'number'].indexOf(nextToken.type) > -1
|
|
) || (
|
|
['+', '-', '++', '--'].indexOf(prevToken.value) > -1
|
|
&& ['+', '-', '++', '--'].indexOf(nextToken.value) > -1
|
|
))
|
|
) {
|
|
ret += ' ';
|
|
}
|
|
} else if (token.type != 'comment') {
|
|
// remove comments and leave all other tokens untouched
|
|
ret += token.value;
|
|
}
|
|
});
|
|
return ret;
|
|
}
|
|
};
|
|
|
|
/*@
|
|
Ox.test <f> Takes JavaScript, runs inline tests, returns results
|
|
@*/
|
|
Ox.test = function(file, callback) {
|
|
Ox.doc(file, function(items) {
|
|
var tests = [];
|
|
items.forEach(function(item) {
|
|
item.examples && item.examples.some(function(example) {
|
|
return example.result;
|
|
}) && item.examples.forEach(function(example) {
|
|
Ox.Log('TEST', example.statement);
|
|
var actual = eval(example.statement);
|
|
if (example.result) {
|
|
tests.push({
|
|
actual: JSON.stringify(actual),
|
|
expected: example.result,
|
|
name: item.name,
|
|
section: item.section,
|
|
statement: example.statement,
|
|
passed: Ox.isEqual(eval(
|
|
'Ox.test.result = ' + example.result
|
|
), actual)
|
|
});
|
|
}
|
|
});
|
|
});
|
|
callback(tests);
|
|
});
|
|
};
|
|
|
|
/*@
|
|
Ox.tokenize <f> Tokenizes JavaScript
|
|
(source) -> <[o]> Array of tokens
|
|
column <n> Column of the token
|
|
line <n> Line of the token
|
|
type <s> Type of the token
|
|
Type can be <code>"comment"</code>, <code>"identifier"</code>,
|
|
<code>"linebreak"</code>, <code>"number"</code>,
|
|
<code>"operator"</code>, <code>"regexp"</code>,
|
|
<code>"string"</code> or <code>"whitespace"</code>
|
|
value <s> Value of the token
|
|
source <s> JavaScript source code
|
|
> Ox.tokenize('// comment\nvar foo = bar / baz;').length
|
|
14
|
|
> Ox.tokenize('return /foo/g;')[2].value.length
|
|
6
|
|
@*/
|
|
// FIXME: numbers (hex, exp, etc.)
|
|
Ox.tokenize = (function() {
|
|
|
|
// see https://github.com/mozilla/narcissus/blob/master/lib/lexer.js
|
|
|
|
var comment = ['//', '/*'],
|
|
identifier = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ$_',
|
|
linebreak = '\n\r',
|
|
number = '0123456789',
|
|
operator = [
|
|
// arithmetic
|
|
'+', '-', '*', '/', '%', '++', '--',
|
|
// assignment
|
|
'=', '+=', '-=', '*=', '/=', '%=',
|
|
'&=', '|=', '^=', '<<=', '>>=', '>>>=',
|
|
// bitwise
|
|
'&', '|', '^', '~', '<<', '>>', '>>>',
|
|
// comparison
|
|
'==', '!=', '===', '!==', '>', '>=', '<', '<=',
|
|
// conditional
|
|
'?', ':',
|
|
// grouping
|
|
'(', ')', '[', ']', '{', '}',
|
|
// logical
|
|
'&&', '||', '!',
|
|
// other
|
|
'.', ',', ';'
|
|
],
|
|
regexp = 'abcdefghijklmnopqrstuvwxyz',
|
|
string = '\'"',
|
|
whitespace = ' \t';
|
|
|
|
function isRegExp(tokens) {
|
|
// Returns true if the current token is the beginning of a RegExp, as
|
|
// opposed to the beginning of an operator
|
|
var i = tokens.length - 1, isRegExp, token
|
|
// Scan back to the previous significant token, or to the beginning of
|
|
// the source
|
|
while (i >= 0 && [
|
|
'comment', 'linebreak', 'whitespace'
|
|
].indexOf(tokens[i].type) > -1) {
|
|
i--;
|
|
}
|
|
if (i == -1) {
|
|
// Source begins with a forward slash
|
|
isRegExp = true;
|
|
} else {
|
|
token = tokens[i];
|
|
isRegExp = (
|
|
token.type == 'identifier'
|
|
&& Ox.identify(token.value) == 'keyword'
|
|
&& ['false', 'null', 'true'].indexOf(token.value) == -1
|
|
) || (
|
|
token.type == 'operator'
|
|
&& ['++', '--', ')', ']', '}'].indexOf(token.value) == -1
|
|
)
|
|
}
|
|
return isRegExp;
|
|
}
|
|
|
|
return function(source) {
|
|
var char,
|
|
column = 0,
|
|
cursor = 0,
|
|
delimiter,
|
|
length = source.length,
|
|
line = 0,
|
|
lines,
|
|
next,
|
|
tokens = [],
|
|
start,
|
|
type,
|
|
value;
|
|
source = source.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
|
|
while (cursor < length) {
|
|
start = cursor;
|
|
char = source[cursor];
|
|
if (comment.indexOf(delimiter = char + source[cursor + 1]) > -1) {
|
|
type = 'comment';
|
|
++cursor;
|
|
while (char = source[++cursor]) {
|
|
if (delimiter == '//' && char == '\n') {
|
|
break;
|
|
} else if (delimiter == '/*' && char + source[cursor + 1] == '*/') {
|
|
cursor += 2;
|
|
break;
|
|
}
|
|
}
|
|
} else if (identifier.indexOf(char) > -1) {
|
|
type = 'identifier';
|
|
while ((identifier + number).indexOf(source[++cursor]) > -1) {}
|
|
} else if (linebreak.indexOf(char) > -1) {
|
|
type = 'linebreak';
|
|
while (linebreak.indexOf(source[++cursor]) > -1) {}
|
|
} else if (number.indexOf(char) > -1) {
|
|
type = 'number';
|
|
while ((number + '.').indexOf(source[++cursor]) > -1) {}
|
|
} else if (char == '/' && isRegExp(tokens)) {
|
|
type = 'regexp';
|
|
while ((char = source[++cursor]) != '/' && cursor < length) {
|
|
char == '\\' && ++cursor;
|
|
}
|
|
while (regexp.indexOf(source[++cursor]) > -1) {}
|
|
} else if (operator.indexOf(char) > -1) {
|
|
type = 'operator';
|
|
while (operator.indexOf(char += source[++cursor]) > -1 && cursor < length) {}
|
|
} else if (string.indexOf(delimiter = char) > -1) {
|
|
type = 'string';
|
|
while ((char = source[++cursor]) != delimiter && cursor < length) {
|
|
char == '\\' && ++cursor;
|
|
}
|
|
++cursor;
|
|
} else if (whitespace.indexOf(char) > -1) {
|
|
type = 'whitespace';
|
|
while (whitespace.indexOf(source[++cursor]) > -1) {}
|
|
} else {
|
|
break;
|
|
}
|
|
value = source.slice(start, cursor);
|
|
tokens.push({column: column, line: line, type: type, value: value});
|
|
if (type == 'comment') {
|
|
lines = value.split('\n');
|
|
column = lines[lines.length - 1].length;
|
|
line += lines.length - 1;
|
|
} else if (type == 'linebreak') {
|
|
column = 0;
|
|
line += value.length;
|
|
} else {
|
|
column += value.length;
|
|
}
|
|
}
|
|
return tokens;
|
|
};
|
|
|
|
}());
|