0x2620/oxjs
2
0
Fork 1
Code Issues 172 Pull requests Releases Wiki Activity

Ox.tokenize: handle hex numbers and numbers with exponent

Browse source
This commit is contained in:
rolux 2012-06-04 15:36:25 +02:00
parent c4ad59bdb6
commit 56c6ef98c7
1 changed files with 10 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
source/Ox/js/JavaScript.js
View file

@ -892,8 +892,9 @@ Ox.tokenize <f> Tokenizes JavaScript
14 14
> Ox.tokenize('return /foo/g;')[2].value.length > Ox.tokenize('return /foo/g;')[2].value.length
6 6
> Ox.tokenize('[0xFF, 1e1, 1e+1, 1e-1, 1e+1+1]').length
17
@*/ @*/
// FIXME: numbers (hex, exp, etc.)
Ox.tokenize = (function() { Ox.tokenize = (function() {
// see https://github.com/mozilla/narcissus/blob/master/lib/lexer.js // see https://github.com/mozilla/narcissus/blob/master/lib/lexer.js
@ -989,7 +990,14 @@ Ox.tokenize = (function() {
while (linebreak.indexOf(source[++cursor]) > -1) {} while (linebreak.indexOf(source[++cursor]) > -1) {}
} else if (number.indexOf(char) > -1) { } else if (number.indexOf(char) > -1) {
type = 'number'; type = 'number';
while ((number + '.').indexOf(source[++cursor]) > -1) {} while ((number + '.abcdefxABCDEFX+-').indexOf(source[++cursor]) > -1) {
if (
source[cursor - 1] != 'e' && source[cursor - 1] != 'E'
&& (source[cursor] == '+' || source[cursor] == '-')
) {
break;
}
}
} else if (char == '/' && isRegExp(tokens)) { } else if (char == '/' && isRegExp(tokens)) {
type = 'regexp'; type = 'regexp';
while ((char = source[++cursor]) != '/' && cursor < length) { while ((char = source[++cursor]) != '/' && cursor < length) {