753 lines
28 KiB
JavaScript
753 lines
28 KiB
JavaScript
'use strict';
|
|
|
|
/*@
|
|
Ox.doc <f> Generates documentation for annotated JavaScript
|
|
(source) -> <[o]> Array of doc objects
|
|
arguments <[o]|u> Arguments (array of doc objects)
|
|
Present if the <code>type</code> of the item is
|
|
<code>"function"</code>.
|
|
description <s|u> Multi-line description with optional markup
|
|
See Ox.parseHTML for details
|
|
events <[o]|u> Events (array of doc objects)
|
|
Present if the item fires any events
|
|
file <s> File name
|
|
line <n> Line number
|
|
name <s> Name of the item
|
|
properties <[o]|u> Properties (array of doc objects)
|
|
Present if the <code>type</code> of the item is
|
|
<code>"event"</code>, <code>"function"</code>
|
|
or <code>"object"</code>.
|
|
section <s|u> Section in the file
|
|
source <[o]> Source code (array of tokens)
|
|
length <n> Length of the token
|
|
offset <n> Offset of the token
|
|
type <s> Type of the token
|
|
See Ox.tokenize for list of types
|
|
summary <s> One-line summary
|
|
usage <[o]> Usage (array of doc objects)
|
|
Present if the <code>type</code> of the item is
|
|
<code>"function"</code>.
|
|
type <s> Type of the item
|
|
(file, callback) -> <u> undefined
|
|
(files, callback) -> <u> undefined
|
|
source <s> JavaScript source code
|
|
file <s> JavaScript file
|
|
files <[s]> Array of javascript files
|
|
callback <f> Callback function
|
|
doc <[o]> Array of doc objects
|
|
# > Ox.doc("//@ My.FOO <n> Magic constant\nMy.FOO = 23;")
|
|
# [{"name": "Ox.foo", "summary": "just some string", "type": "string"}]
|
|
@*/
|
|
|
|
Ox.doc = (function() {
|
|
// fixme: dont require the trailing '@'
|
|
var re = {
|
|
item: /^(.+?) <(.+?)> (.+?)$/,
|
|
multiline: /^\/\*\@.*?\n([\w\W]+)\n.*?\@\*\/$/,
|
|
script: /\n(\s*<script>s*\n[\w\W]+\n\s*<\/script>s*)/g,
|
|
singleline: /^\/\/@\s*(.*?)\s*$/,
|
|
test: /\n(\s*> .+\n.+?)/g,
|
|
usage: /\(.*?\)/
|
|
},
|
|
types = {
|
|
a: 'array', b: 'boolean', d: 'date',
|
|
e: 'element', f: 'function', n: 'number',
|
|
o: 'object', r: 'regexp', s: 'string',
|
|
u: 'undefined', '*': 'value', '!': 'event'
|
|
};
|
|
function decodeLinebreaks(match, submatch) {
|
|
return (submatch || match).replace(/\u21A9/g, '\n');
|
|
}
|
|
function encodeLinebreaks(match, submatch) {
|
|
return '\n' + (submatch || match).replace(/\n/g, '\u21A9');
|
|
}
|
|
function getIndent(str) {
|
|
var indent = -1;
|
|
while (str[++indent] == ' ') {}
|
|
return indent;
|
|
}
|
|
function parseItem(str) {
|
|
var matches = re.item.exec(str);
|
|
// to tell a variable with default value, like
|
|
// name <string|'<a href="...">foo</a>'> summary
|
|
// from a line of description with tags, like
|
|
// some <a href="...">description</a> text
|
|
// we need to check if there is either no forward slash
|
|
// or if the second last char is a single or double quote
|
|
return matches && (
|
|
matches[2].indexOf('/') == -1 ||
|
|
'\'"'.indexOf(matches[2].slice(-2, -1)) > -1
|
|
) ? Ox.extend({
|
|
name: parseName(matches[1].trim()),
|
|
summary: matches[3].trim()
|
|
}, parseType(matches[2])) : null;
|
|
}
|
|
function parseName(str) {
|
|
var matches = re.usage.exec(str);
|
|
return matches ? matches[0] : str;
|
|
}
|
|
function parseNode(node) {
|
|
var item = parseItem(node.line), subitem;
|
|
node.nodes && node.nodes.forEach(function(node) {
|
|
var key, line = node.line, subitem;
|
|
if (!/^#/.test(node.line)) {
|
|
if (/^<script>/.test(line)) {
|
|
item.examples = [parseScript(line)];
|
|
} else if (/^>/.test(line)) {
|
|
item.examples = item.examples || [];
|
|
item.examples.push(parseTest(line));
|
|
} else if ((subitem = parseItem(line))) {
|
|
if (/^\(/.test(subitem.name)) {
|
|
item.usage = item.usage || [];
|
|
item.usage.push(parseNode(node));
|
|
} else if (subitem.types[0] == 'event') {
|
|
item.events = item.events || [];
|
|
item.events.push(parseNode(node));
|
|
} else {
|
|
key = item.types[0] == 'function'
|
|
? 'arguments' : 'properties'
|
|
item[key] = item[key] || [];
|
|
item[key].push(parseNode(node));
|
|
}
|
|
} else {
|
|
item.description = item.description
|
|
? item.description + ' ' + line : line
|
|
}
|
|
}
|
|
});
|
|
return item;
|
|
}
|
|
function parseScript(str) {
|
|
// remove script tags and extra indentation
|
|
var lines = decodeLinebreaks(str).split('\n'),
|
|
indent = getIndent(lines[1]);
|
|
return {
|
|
statement: lines.slice(1, -1).map(function(line, i) {
|
|
return line.slice(indent);
|
|
}).join('\n')
|
|
};
|
|
}
|
|
function parseSource(source, file) {
|
|
var blocks = [],
|
|
items = [],
|
|
section = '',
|
|
tokens = [];
|
|
Ox.tokenize(source).forEach(function(token) {
|
|
var match;
|
|
token.source = source.substr(token.offset, token.length);
|
|
if (token.type == 'comment' && (
|
|
match = re.multiline.exec(token.source)
|
|
|| re.singleline.exec(token.source)
|
|
)) {
|
|
blocks.push(match[1]);
|
|
tokens.push([]);
|
|
} else if (tokens.length) {
|
|
tokens[tokens.length - 1].push(token);
|
|
}
|
|
});
|
|
blocks.forEach(function(block, i) {
|
|
var item, lastItem,
|
|
lines = block
|
|
.replace(re.script, encodeLinebreaks)
|
|
.replace(re.test, encodeLinebreaks)
|
|
.split('\n'),
|
|
tree = parseTree(lines);
|
|
if (re.item.test(tree.line)) {
|
|
// parse the tree's root node
|
|
item = parseNode(tree);
|
|
item.file = file || '';
|
|
if (section) {
|
|
item.section = section;
|
|
}
|
|
if (/^[A-Z]/.test(item.name)) {
|
|
// main item
|
|
// include leading whitespace
|
|
item.source = parseTokens(tokens[i]);
|
|
item.line = source.slice(0, item.source[0].offset)
|
|
.split('\n').length;
|
|
items.push(item);
|
|
} else {
|
|
// property of a function item
|
|
lastItem = items[items.length - 1];
|
|
lastItem.properties = lastItem.properties || [];
|
|
lastItem.properties.push(item);
|
|
// include leading linebreaks and whitespace
|
|
lastItem.source = lastItem.source.concat(
|
|
parseTokens(tokens[i], true)
|
|
);
|
|
}
|
|
} else {
|
|
section = tree.line.split(' ')[0]
|
|
}
|
|
});
|
|
return items;
|
|
}
|
|
function parseTest(str) {
|
|
// fixme: we cannot properly handle tests where a string contains '\n '
|
|
var lines = decodeLinebreaks(str).split('\n ');
|
|
return {
|
|
statement: lines[0].slice(2),
|
|
result: lines[1].trim()
|
|
};
|
|
}
|
|
function parseTokens(tokens, includeLeadingLinebreaks) {
|
|
var isLeading = true,
|
|
isTrailing = false,
|
|
tokens_ = [],
|
|
types = ['linebreak', 'whitespace'];
|
|
tokens.forEach(function(token) {
|
|
if (isLeading && types.indexOf(token.type) > -1) {
|
|
if (token.type == 'linebreak') {
|
|
if (includeLeadingLinebreaks) {
|
|
tokens_.push(token);
|
|
} else {
|
|
tokens_ = [];
|
|
}
|
|
} else {
|
|
tokens_.push(token);
|
|
}
|
|
} else {
|
|
tokens_.push(token);
|
|
isLeading = false;
|
|
if (types.indexOf(token.type) == -1) {
|
|
isTrailing = true;
|
|
}
|
|
}
|
|
});
|
|
if (isTrailing) {
|
|
while (types.indexOf(tokens_[tokens_.length - 1].type) > -1) {
|
|
tokens_.pop();
|
|
}
|
|
}
|
|
return tokens_;
|
|
}
|
|
function parseTree(lines) {
|
|
// parses indented lines into a tree structure, like
|
|
// {line: "...", nodes: [{line: "...", nodes: [...]}]}
|
|
var branches = [],
|
|
indent,
|
|
node = {
|
|
// chop the root line
|
|
line: lines.shift().trim()
|
|
};
|
|
if (lines.length) {
|
|
indent = getIndent(lines[0]);
|
|
lines.forEach(function(line) {
|
|
if (getIndent(line) == indent) {
|
|
// line is a child,
|
|
// make it the root line of a new branch
|
|
branches.push([line]);
|
|
} else {
|
|
// line is a descendant of the last child,
|
|
// add it to the last branch
|
|
branches[branches.length - 1].push(line);
|
|
}
|
|
});
|
|
node.nodes = branches.map(function(lines) {
|
|
return parseTree(lines);
|
|
});
|
|
}
|
|
return node;
|
|
}
|
|
function parseType(str) {
|
|
// returns {types: [""]}
|
|
// or {types: [""], default: ""}
|
|
// or {types: [""], super: ""}
|
|
var isArray,
|
|
ret = {types: []},
|
|
split,
|
|
type;
|
|
// only split by ':' if there is no default string value
|
|
if ('\'"'.indexOf(str.slice(-2, -1)) == -1) {
|
|
split = str.split(':');
|
|
str = split[0];
|
|
if (split.length == 2) {
|
|
ret.super = split[1];
|
|
}
|
|
}
|
|
str.split('|').forEach(function(str) {
|
|
var unwrapped = unwrap(str);
|
|
if (unwrapped in types) {
|
|
ret.types.push(wrap(types[unwrapped]))
|
|
} else if (
|
|
(type = Ox.filter(Ox.values(types), function(type) {
|
|
return Ox.startsWith(type, unwrapped);
|
|
})).length
|
|
) {
|
|
ret.types.push(wrap(type[0]));
|
|
} else {
|
|
ret['default'] = str;
|
|
}
|
|
});
|
|
function unwrap(str) {
|
|
return (isArray = /^\[.+\]$/.test(str)) ? str.slice(1, -1) : str;
|
|
}
|
|
function wrap(str) {
|
|
return isArray ? '[' + str + 's' + ']' : str;
|
|
}
|
|
return ret;
|
|
}
|
|
return function(/* source | file, callback | files, callback*/) {
|
|
var source = arguments.length == 1 ? arguments[0] : void 0,
|
|
files = arguments.length == 2 ? Ox.makeArray(arguments[0]) : void 0,
|
|
callback = arguments[1],
|
|
counter = 0, items = [];
|
|
files && files.forEach(function(file) {
|
|
Ox.get(file, function(source) {
|
|
items = items.concat(parseSource(source, file));
|
|
++counter == files.length && callback(items);
|
|
});
|
|
});
|
|
return source ? parseSource(source) : void 0;
|
|
}
|
|
}());
|
|
|
|
/*@
|
|
Ox.minify <f> Minifies JavaScript
|
|
(source) -> <s> Minified JavaScript
|
|
(file, callback) -> <u> undefined
|
|
source <s> JavaScript source
|
|
file <s> JavaScript file
|
|
callback <f> Callback function
|
|
> Ox.minify('for (a in b)\n{\t\tc = void 0;\n}')
|
|
'for(a in b)\n{c=void 0;}'
|
|
> Ox.minify('return a; return 0; return "";')
|
|
'return a;return 0;return"";'
|
|
> Ox.minify('return\na;\nreturn\n0;\nreturn\n"";')
|
|
'return\na;\nreturn\n0;\nreturn\n"";\n'
|
|
@*/
|
|
Ox.minify = function() {
|
|
// see https://github.com/douglascrockford/JSMin/blob/master/README
|
|
// and http://inimino.org/~inimino/blog/javascript_semicolons
|
|
if (arguments.length == 1) {
|
|
return minify(arguments[0]);
|
|
} else {
|
|
Ox.get(arguments[0], function(source) {
|
|
arguments[1](minify(source));
|
|
});
|
|
}
|
|
function minify(source) {
|
|
var tokens = Ox.tokenize(source),
|
|
length = tokens.length;
|
|
function isIdentifierOrNumber(token) {
|
|
return [
|
|
'constant', 'identifier', 'keyword',
|
|
'number', 'method', 'object', 'property'
|
|
].indexOf(token.type) > -1;
|
|
}
|
|
return Ox.filter(Ox.map(tokens, function(token, i) {
|
|
var ret = source.substr(token.offset, token.length);
|
|
if (token.type == 'comment') {
|
|
if (ret[1] == '/') {
|
|
// replace line comment with newline
|
|
ret = i == length - 1 || tokens[i + 1].type == 'linebreak'
|
|
? null : '\n';
|
|
} else {
|
|
// replace block comment with space
|
|
ret = i == length - 1 || tokens[i + 1].type == 'whitespace'
|
|
? null : ' ';
|
|
}
|
|
} else if (token.type == 'linebreak' || token.type == 'whitespace') {
|
|
// remove consecutive linebreaks or whitespace
|
|
ret = ret[0];
|
|
}
|
|
if (
|
|
// strip linebreaks, except between two tokens that
|
|
// are both either identifier or number or string
|
|
// or identifier in brackets or identifier with
|
|
// unary operator or array literal or object literal
|
|
//
|
|
// strip linebreaks, except after "break", "continue",
|
|
// "return", "++" or "--", where the linebreak becomes
|
|
// a semicolon
|
|
token.type == 'linebreak' && (
|
|
i == 0 || i == length - 1
|
|
|| (
|
|
!isIdentifierOrNumber(tokens[i - 1])
|
|
&& !tokens[i - 1].type == 'string'
|
|
&& [')', ']', '}', '++', '--'].indexOf(
|
|
source.substr(tokens[i - 1].offset, tokens[i - 1].length)
|
|
) == -1
|
|
)
|
|
|| (
|
|
!isIdentifierOrNumber(tokens[i + 1])
|
|
&& ['(', '[', '{', '+', '-', '++', '--'].indexOf(
|
|
source.substr(tokens[i + 1].offset, tokens[i + 1].length)
|
|
) == -1
|
|
)
|
|
)
|
|
) {
|
|
ret = null;
|
|
} else if (
|
|
// strip whitespace, except between two tokens
|
|
// that are both either identifier or number
|
|
token.type == 'whitespace' && (
|
|
i == 0 || i == length - 1
|
|
|| !isIdentifierOrNumber(tokens[i - 1])
|
|
|| !isIdentifierOrNumber(tokens[i + 1])
|
|
)
|
|
) {
|
|
ret = null;
|
|
}
|
|
return ret;
|
|
})).join('');
|
|
}
|
|
};
|
|
|
|
/*@
|
|
Ox.test <f> Takes JavaScript, runs inline tests, returns results
|
|
@*/
|
|
Ox.test = function(file, callback) {
|
|
Ox.doc(file, function(items) {
|
|
var tests = [];
|
|
items.forEach(function(item) {
|
|
item.examples && item.examples.forEach(function(example) {
|
|
Ox.Log('TEST', example.statement);
|
|
var actual = eval(example.statement);
|
|
if (example.result) {
|
|
tests.push({
|
|
actual: JSON.stringify(actual),
|
|
expected: example.result,
|
|
name: item.name,
|
|
section: item.section,
|
|
statement: example.statement,
|
|
passed: Ox.isEqual(eval(
|
|
'Ox.test.result = ' + example.result
|
|
), actual)
|
|
});
|
|
}
|
|
});
|
|
});
|
|
callback(tests);
|
|
});
|
|
};
|
|
|
|
/*@
|
|
Ox.tokenize <f> Tokenizes JavaScript
|
|
(source) -> <[o]> Array of tokens
|
|
length <n> Length of the token
|
|
offset <n> Offset of the token
|
|
type <s> Type of the token
|
|
Type can be <code>"comment"</code>, <code>"constant"</code>,
|
|
<code>"identifier"</code>, <code>"keyword"</code>,
|
|
<code>"linebreak"</code>, <code>"method"</code>,
|
|
<code>"number"</code>, <code>"object"</code>,
|
|
<code>"operator"</code>, <code>"property"</code>,
|
|
<code>"regexp"</code>, <code>"string"</code>
|
|
or <code>"whitespace"</code>
|
|
source <s> JavaScript source code
|
|
@*/
|
|
// FIXME: constant/method/object/property is of interest
|
|
// for syntax highlighting, but may not belong here
|
|
Ox.tokenize = (function() {
|
|
|
|
// see https://github.com/mozilla/narcissus/blob/master/lib/jslex.js
|
|
// and https://developer.mozilla.org/en/JavaScript/Reference
|
|
|
|
var identifier = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ$_',
|
|
linebreak = '\n\r',
|
|
number = '0123456789',
|
|
operator = [
|
|
// arithmetic
|
|
'+', '-', '*', '/', '%', '++', '--',
|
|
// assignment
|
|
'=', '+=', '-=', '*=', '/=', '%=',
|
|
'&=', '|=', '^=', '<<=', '>>=', '>>>=',
|
|
// bitwise
|
|
'&', '|', '^', '~', '<<', '>>', '>>>',
|
|
// comparison
|
|
'==', '!=', '===', '!==', '>', '>=', '<', '<=',
|
|
// conditional
|
|
'?', ':',
|
|
// grouping
|
|
'(', ')', '[', ']', '{', '}',
|
|
// logical
|
|
'&&', '||', '!',
|
|
// other
|
|
'.', ',', ';'
|
|
],
|
|
regexp = 'abcdefghijklmnopqrstuvwxyz',
|
|
string = '\'"',
|
|
whitespace = ' \t',
|
|
word = {
|
|
constant: [
|
|
// Math
|
|
'E', 'LN2', 'LN10', 'LOG2E', 'LOG10E', 'PI', 'SQRT1_2', 'SQRT2',
|
|
// Number
|
|
'MAX_VALUE', 'MIN_VALUE', 'NEGATIVE_INFINITY', 'POSITIVE_INFINITY'
|
|
],
|
|
keyword: [
|
|
'break',
|
|
'case', 'catch', 'class', 'const', 'continue',
|
|
'debugger', 'default', 'delete', 'do',
|
|
'else', 'enum', 'export', 'extends',
|
|
'false', 'finally', 'for', 'function',
|
|
'if', 'implements', 'import', 'in', 'instanceof', 'interface',
|
|
'let', 'module',
|
|
'new', 'null',
|
|
'package', 'private', 'protected', 'public',
|
|
'return',
|
|
'super', 'switch', 'static',
|
|
'this', 'throw', 'true', 'try', 'typeof',
|
|
'var', 'void',
|
|
'yield',
|
|
'while', 'with',
|
|
],
|
|
method: [
|
|
// Array
|
|
'concat',
|
|
'every',
|
|
'filter', 'forEach',
|
|
'join',
|
|
'lastIndexOf',
|
|
'indexOf', 'isArray',
|
|
'map',
|
|
'pop', 'push',
|
|
'reduce', 'reduceRight', 'reverse',
|
|
'shift', 'slice', 'some', 'sort', 'splice',
|
|
'unshift',
|
|
// Date
|
|
'getDate', 'getDay', 'getFullYear', 'getHours',
|
|
'getMilliseconds', 'getMinutes', 'getMonth', 'getSeconds',
|
|
'getTime', 'getTimezoneOffset',
|
|
'getUTCDate', 'getUTCDay', 'getUTCFullYear', 'getUTCHours',
|
|
'getUTCMilliseconds', 'getUTCMinutes', 'getUTCMonth', 'getUTCSeconds',
|
|
'now',
|
|
'parse',
|
|
'setDate', 'setFullYear', 'setHours', 'setMilliseconds',
|
|
'setMinutes', 'setMonth', 'setSeconds', 'setTime',
|
|
'setUTCDate', 'setUTCFullYear', 'setUTCHours', 'setUTCMilliseconds',
|
|
'setUTCMinutes', 'setUTCMonth', 'setUTCSeconds',
|
|
'toDateString', 'toJSON', 'toLocaleDateString', 'toLocaleString',
|
|
'toLocaleTimeString', 'toTimeString', 'toUTCString',
|
|
'UTC',
|
|
// Function
|
|
'apply', 'bind', 'call', 'isGenerator',
|
|
// JSON
|
|
'parse', 'stringify',
|
|
// Math
|
|
'abs', 'acos', 'asin', 'atan', 'atan2',
|
|
'ceil', 'cos',
|
|
'exp',
|
|
'floor',
|
|
'log',
|
|
'max', 'min',
|
|
'pow',
|
|
'random', 'round',
|
|
'sin', 'sqrt',
|
|
'tan',
|
|
// Number
|
|
'toExponential', 'toFixed', 'toLocaleString', 'toPrecision',
|
|
// Object
|
|
'create',
|
|
'defineProperty', 'defineProperties',
|
|
'freeze',
|
|
'getOwnPropertyDescriptor', 'getOwnPropertyNames', 'getPrototypeOf',
|
|
'hasOwnProperty',
|
|
'isExtensible', 'isFrozen', 'isPrototypeOf', 'isSealed',
|
|
'keys',
|
|
'preventExtensions', 'propertyIsEnumerable',
|
|
'seal',
|
|
'toLocaleString', 'toString',
|
|
'valueOf',
|
|
// RegExp
|
|
'exec', 'test',
|
|
// String
|
|
'charAt', 'charCodeAt', 'concat',
|
|
'fromCharCode',
|
|
'indexOf',
|
|
'lastIndexOf', 'localeCompare',
|
|
'match',
|
|
'replace',
|
|
'search', 'slice', 'split', 'substr', 'substring',
|
|
'toLocaleLowerCase', 'toLocaleUpperCase', 'toLowerCase', 'toUpperCase', 'trim',
|
|
// Window
|
|
'addEventListener', 'alert', 'atob',
|
|
'blur', 'btoa',
|
|
'clearInterval', 'clearTimeout', 'close', 'confirm',
|
|
'dispatchEvent',
|
|
'escape',
|
|
'find', 'focus',
|
|
'getComputedStyle', 'getSelection',
|
|
'moveBy', 'moveTo',
|
|
'open',
|
|
'postMessage', 'print', 'prompt',
|
|
'removeEventListener', 'resizeBy', 'resizeTo',
|
|
'scroll', 'scrollBy', 'scrollTo',
|
|
'setCursor', 'setInterval', 'setTimeout', 'stop',
|
|
'unescape'
|
|
],
|
|
object: [
|
|
'Array',
|
|
'Boolean',
|
|
'Date', 'decodeURI', 'decodeURIComponent',
|
|
'encodeURI', 'encodeURIComponent', 'Error', 'eval', 'EvalError',
|
|
'Function',
|
|
'Infinity', 'isFinite', 'isNaN',
|
|
'JSON',
|
|
'Math',
|
|
'NaN', 'Number',
|
|
'Object',
|
|
'parseFloat', 'parseInt',
|
|
'RangeError', 'ReferenceError', 'RegExp',
|
|
'String', 'SyntaxError',
|
|
'TypeError',
|
|
'undefined', 'URIError',
|
|
'window'
|
|
],
|
|
property: [
|
|
// Function
|
|
'constructor', 'length', 'prototype',
|
|
// RegExp
|
|
'global', 'ignoreCase', 'lastIndex', 'multiline', 'source',
|
|
// Window
|
|
'applicationCache',
|
|
'closed', 'console', 'content', 'crypto',
|
|
'defaultStatus', 'document',
|
|
'frameElement', 'frames',
|
|
'history',
|
|
'innerHeight', 'innerWidth',
|
|
'length', 'location', 'locationbar', 'localStorage',
|
|
'menubar',
|
|
'name', 'navigator',
|
|
'opener', 'outerHeight', 'outerWidth',
|
|
'pageXOffset', 'pageYOffset', 'parent', 'personalbar',
|
|
'screen', 'screenX', 'screenY', 'scrollbars', 'scrollX', 'scrollY',
|
|
'self', 'sessionStorage', 'status', 'statusbar',
|
|
'toolbar', 'top'
|
|
]
|
|
};
|
|
|
|
return function(source) {
|
|
|
|
source = source.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
|
|
|
|
var cursor = 0,
|
|
tokenize = {
|
|
comment: function() {
|
|
while (char = source[++cursor]) {
|
|
if (next == '/' && char == '\n') {
|
|
break;
|
|
} else if (next == '*' && char + source[cursor + 1] == '*/') {
|
|
cursor += 2;
|
|
break;
|
|
}
|
|
}
|
|
},
|
|
identifier: function() {
|
|
var str;
|
|
while ((identifier + number).indexOf(source[++cursor]) > -1) {}
|
|
str = source.slice(start, cursor);
|
|
Ox.forEach(word, function(value, key) {
|
|
if (value.indexOf(str) > -1) {
|
|
type = key;
|
|
Ox.Break();
|
|
}
|
|
});
|
|
},
|
|
linebreak: function() {
|
|
while (linebreak.indexOf(source[++cursor]) > -1) {}
|
|
},
|
|
number: function() {
|
|
while ((number + '.').indexOf(source[++cursor]) > -1) {}
|
|
},
|
|
operator: function() {
|
|
while (operator.indexOf(char += source[++cursor]) > -1) {}
|
|
},
|
|
regexp: function() {
|
|
while ((char = source[++cursor]) != '/') {
|
|
char == '\\' && ++cursor;
|
|
if (cursor == source.length) {
|
|
break;
|
|
}
|
|
}
|
|
while (regexp.indexOf(source[++cursor]) > -1) {}
|
|
},
|
|
string: function() {
|
|
var delimiter = char;
|
|
while ((char = source[++cursor]) != delimiter) {
|
|
char == '\\' && ++cursor;
|
|
if (cursor == source.length) {
|
|
break;
|
|
}
|
|
}
|
|
++cursor;
|
|
},
|
|
whitespace: function() {
|
|
while (whitespace.indexOf(source[++cursor]) > -1) {}
|
|
}
|
|
},
|
|
tokens = [],
|
|
type;
|
|
|
|
while (cursor < source.length) {
|
|
var char = source[cursor],
|
|
next = source[cursor + 1],
|
|
start = cursor;
|
|
if (char == '/' && (next == '/' || next == '*')) {
|
|
type = 'comment';
|
|
} else if (identifier.indexOf(char) > -1) {
|
|
type = 'identifier';
|
|
} else if (linebreak.indexOf(char) > -1) {
|
|
type = 'linebreak';
|
|
} else if (number.indexOf(char) > -1) {
|
|
type = 'number';
|
|
} else if (string.indexOf(char) > -1) {
|
|
type = 'string';
|
|
} else if (whitespace.indexOf(char) > -1) {
|
|
type = 'whitespace';
|
|
} else if (char == '/') {
|
|
type = isRegExp() ? 'regexp' : 'operator';
|
|
} else if (operator.indexOf(char) > -1) {
|
|
type = 'operator';
|
|
}
|
|
tokenize[type]();
|
|
tokens.push({
|
|
length: cursor - start,
|
|
offset: start,
|
|
type: type,
|
|
});
|
|
}
|
|
|
|
function isRegExp() {
|
|
// checks if a forward slash is the beginning of a regexp,
|
|
// as opposed to the beginning of an operator
|
|
// see http://www.mozilla.org/js/language/js20-2000-07/rationale/syntax.html#regular-expressions
|
|
var index = tokens.length,
|
|
isRegExp = false,
|
|
offset = 0,
|
|
prevToken,
|
|
prevString;
|
|
// scan back to the previous significant token,
|
|
// or the beginning of the source
|
|
while (
|
|
tokens[--index] !== void 0 && [
|
|
'comment', 'linebreak', 'whitespace'
|
|
].indexOf(tokens[index].type) > -1
|
|
) {
|
|
offset += tokens[index].length;
|
|
}
|
|
if (typeof tokens[index] == 'undefined') {
|
|
// source begins with forward slash
|
|
isRegExp = true;
|
|
} else {
|
|
prevToken = tokens[index];
|
|
prevString = source.substr(
|
|
cursor - prevToken.length - offset, prevToken.length
|
|
);
|
|
isRegExp = (
|
|
prevToken.type == 'keyword'
|
|
&& ['false', 'null', 'true'].indexOf(prevString) == -1
|
|
) || (
|
|
prevToken.type == 'operator'
|
|
&& ['++', '--', ')', ']', '}'].indexOf(prevString) == -1
|
|
);
|
|
}
|
|
return isRegExp;
|
|
}
|
|
|
|
return tokens;
|
|
|
|
};
|
|
|
|
}());
|