From 56c6ef98c701f54309a0a027fc4a7ad0dfc43f22 Mon Sep 17 00:00:00 2001
From: rolux <rolux@rolux.org>
Date: Mon, 4 Jun 2012 15:36:25 +0200
Subject: [PATCH] Ox.tokenize: handle hex numbers and numbers with exponent

---
 source/Ox/js/JavaScript.js | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/source/Ox/js/JavaScript.js b/source/Ox/js/JavaScript.js
index 5b52f637..c14bc46a 100644
--- a/source/Ox/js/JavaScript.js
+++ b/source/Ox/js/JavaScript.js
@@ -892,8 +892,9 @@ Ox.tokenize <f> Tokenizes JavaScript
     14
     > Ox.tokenize('return /foo/g;')[2].value.length
     6
+    > Ox.tokenize('[0xFF, 1e1, 1e+1, 1e-1, 1e+1+1]').length
+    17
 @*/
-// FIXME: numbers (hex, exp, etc.)
 Ox.tokenize = (function() {
 
     // see https://github.com/mozilla/narcissus/blob/master/lib/lexer.js
@@ -989,7 +990,14 @@ Ox.tokenize = (function() {
                 while (linebreak.indexOf(source[++cursor]) > -1) {}
             } else if (number.indexOf(char) > -1) {
                 type = 'number';
-                while ((number + '.').indexOf(source[++cursor]) > -1) {}
+                while ((number + '.abcdefxABCDEFX+-').indexOf(source[++cursor]) > -1) {
+                    if (
+                        source[cursor - 1] != 'e' && source[cursor - 1] != 'E'
+                        && (source[cursor] == '+' || source[cursor] == '-')
+                    ) {
+                        break;
+                    }
+                }
             } else if (char == '/' && isRegExp(tokens)) {
                 type = 'regexp';
                 while ((char = source[++cursor]) != '/' && cursor < length) {