From 74b9a25387b2b98f79bb5784022bf1aff26ed086 Mon Sep 17 00:00:00 2001 From: rolux Date: Thu, 28 Apr 2011 20:34:19 +0200 Subject: [PATCH] Ox.tokenize, Ox.SyntaxHighlighter (+demo) --- demos/syntax/index.html | 10 + demos/syntax/js/syntax.js | 50 ++++ source/Ox.UI/css/Ox.UI.css | 28 ++ source/Ox.UI/js/Core/Ox.SyntaxHighlighter.js | 137 +++++++++ source/Ox.UI/js/Video/Ox.VideoEditorPlayer.js | 1 + source/Ox.UI/themes/classic/css/classic.css | 64 ++++ source/Ox.UI/themes/modern/css/modern.css | 62 ++++ source/Ox.js | 280 ++++++++++++++++++ 8 files changed, 632 insertions(+) create mode 100644 demos/syntax/index.html create mode 100644 demos/syntax/js/syntax.js create mode 100644 source/Ox.UI/js/Core/Ox.SyntaxHighlighter.js diff --git a/demos/syntax/index.html b/demos/syntax/index.html new file mode 100644 index 00000000..dcbd56cd --- /dev/null +++ b/demos/syntax/index.html @@ -0,0 +1,10 @@ + + + + OxJS SyntaxHighlighter Demo + + + + + + \ No newline at end of file diff --git a/demos/syntax/js/syntax.js b/demos/syntax/js/syntax.js new file mode 100644 index 00000000..4a702d89 --- /dev/null +++ b/demos/syntax/js/syntax.js @@ -0,0 +1,50 @@ +Ox.load('UI', { + debug: true, + theme: 'classic' +}, function() { + + Ox.Theme('classic'); + + var $body = $('body'), + $textarea = new Ox.Input({ + height: 400, + type: 'textarea', + width: 400 + }) + .css({ + fontFamily: 'Menlo, Monaco, Courier, Courier New' + }) + .appendTo($body), + $button = new Ox.Button({ + title: 'Run', + width: 40 + }) + .css({ + position: 'absolute', + left: '8px', + top: '416px', + }) + .bindEvent({ + click: function() { + $div.empty(); + new Ox.SyntaxHighlighter({ + showLinebreaks: true, + showTabs: true, + showWhitespace: true, + source: $textarea.value(), + //stripComments: true, + //stripLinebreaks: true, + //stripWhitespace: true, + }).appendTo($div); + } + }) + .appendTo($body), + $div = $('
') + .css({ + position: 'absolute', + left: '416px', + top: '8px' + }) + .appendTo($body); + +}); \ No newline at end of file diff --git a/source/Ox.UI/css/Ox.UI.css b/source/Ox.UI/css/Ox.UI.css index 5ae11ac4..9f0f9b31 100644 --- a/source/Ox.UI/css/Ox.UI.css +++ b/source/Ox.UI/css/Ox.UI.css @@ -1437,6 +1437,34 @@ Scrollbars -webkit-border-radius: 6px; } +/* +================================================================================ +SyntaxHightlighter +================================================================================ +*/ + +.OxSyntaxHighlighter { + position: absolute; + overflow: auto; +} +.OxSyntaxHighlighter > div { + position: absolute; + font-family: Menlo, Monaco, DejaVu Sans Mono, Bitstream Vera Sans Mono, Consolas, Lucida Console; + line-height: 14px; +} +.OxSyntaxHighlighter > .OxLineNumbers { + text-align: right; +} +.OxSyntaxHighlighter > .OxSourceCode { + //display: table-cell; + -moz-user-select: text; + -webkit-user-select: text; +} +.OxSyntaxHighlighter > .OxSourceCode .OxLinebreak { + -moz-user-select: none; + -webkit-user-select: none; +} + /* ================================================================================ Video diff --git a/source/Ox.UI/js/Core/Ox.SyntaxHighlighter.js b/source/Ox.UI/js/Core/Ox.SyntaxHighlighter.js new file mode 100644 index 00000000..707b283c --- /dev/null +++ b/source/Ox.UI/js/Core/Ox.SyntaxHighlighter.js @@ -0,0 +1,137 @@ +// vim: et:ts=4:sw=4:sts=4:ft=js + +/*@ + +@*/ + +Ox.SyntaxHighlighter = function(options, self) { + + self = self || {}; + var that = new Ox.Element('div', self) + .defaults({ + height: 40, + lineLength: 80, //@ number of characters per line + showLinebreaks: false, //@ show linebreak symbols + showTabs: false, //@ show tab symbols + showWhitespace: false, //@ show irregular leading or trailing whitespace + source: '', //@ JavaScript source + stripComments: false, //@ strip all comments + stripLinebreaks: false, //@ strip multiple linebreaks, NOT IMPLEMENTED + stripWhitespace: false, //@ strip all whitespace, NOT IMPLEMENTED + tabLength: 4, //@ number of spaces per tab + width: 80 + }) + .options(options || {}) + .addClass('OxSyntaxHighlighter'); + + var foo = $('
') + //.css({marginTop: '-1000px'}) + .html(Ox.repeat('_', 80)) + .appendTo(that.$element); + //alert(foo.width()); + foo.remove(); + + self.options.source = self.options.source + .replace(/\r\n/g, '\n') + .replace(/\r/g, '\n'); + + self.cursor = 0; + self.source = ''; + self.tokens = Ox.tokenize(self.options.source); + self.tokens.forEach(function(token, i) { + var classNames, tokenString; + if ( + !(self.options.stripComments && token.type == 'comment') + ) { + classNames = 'Ox' + Ox.toTitleCase(token.type); + tokenString = self.options.source.substr(self.cursor, token.length); + if (token.type == 'whitespace') { + if (isAfterLinebreak() && hasIrregularSpaces()) { + classNames += ' OxLeading' + } else if (isBeforeLinebreak()) { + classNames += ' OxTrailing' + } + } + self.source += '' + + encodeToken(tokenString, token.type) + ''; + } + self.cursor += token.length; + function isAfterLinebreak() { + return i == 0 || + self.tokens[i - 1].type == 'linebreak'; + } + function isBeforeLinebreak() { + return i == self.tokens.length - 1 || + self.tokens[i + 1].type == 'linebreak'; + } + function hasIrregularSpaces() { + return tokenString.split('').reduce(function(prev, curr) { + return prev + (curr == ' ' ? 1 : 0); + }, 0) % self.options.tabLength; + } + }); + self.lines = self.source.split('
'); + self.lineNumbersWidth = self.lines.length.toString().length * 7 + 7; + self.sourceCodeWidth = 80 * 7 + ( + self.lines.length > 40 ? Ox.UI.SCROLLBAR_SIZE : 0 + ); + self.height = 40 * 14 + ( + Math.max.apply(null, self.lines.map(function(line) { + return line.length; + })) > 80 ? Ox.UI.SCROLLBAR_SIZE : 0 + ); + + that.css({ + width: self.lineNumbersWidth + self.sourceCodeWidth, + height: self.height + }); + + self.$lineNumbers = new Ox.Element() + .addClass('OxLineNumbers') + .css({ + width: self.lineNumbersWidth + 'px', + height: (self.lines.length * 14) + 'px' + }) + .html( + Ox.range(self.lines.length).map(function(line) { + return (line + 1) + ' '; + }).join('
') + ) + .appendTo(that); + self.$source = new Ox.Element() + .addClass('OxSourceCode') + .css({ + left: self.lineNumbersWidth + 'px', + width: self.sourceCodeWidth + 'px', + height: (self.lines.length * 14) + 'px' + }) + .html(self.source) + .appendTo(that); + + function encodeToken(str, type) { + var linebreak = '
', + tab = Ox.repeat(' ', self.options.tabLength); + if (self.options.showLinebreaks) { + if (type == 'linebreak') { + linebreak = '¶' + linebreak; + } else { + linebreak = '' + linebreak; + } + } + if (self.options.showTabs) { + tab = '\u2192' + tab.substr(6) + ''; + } + str = Ox.encodeHTML(str) + .replace(/ /g, ' ') + .replace(/\t/g, tab) + .replace(/\n/g, linebreak); + return str; + } + + self.onChange = function() { + + }; + + return that; + +}; \ No newline at end of file diff --git a/source/Ox.UI/js/Video/Ox.VideoEditorPlayer.js b/source/Ox.UI/js/Video/Ox.VideoEditorPlayer.js index b87b8576..fc5dd9a8 100644 --- a/source/Ox.UI/js/Video/Ox.VideoEditorPlayer.js +++ b/source/Ox.UI/js/Video/Ox.VideoEditorPlayer.js @@ -1,4 +1,5 @@ // vim: et:ts=4:sw=4:sts=4:ft=js + Ox.VideoEditorPlayer = function(options, self) { var self = self || {}, diff --git a/source/Ox.UI/themes/classic/css/classic.css b/source/Ox.UI/themes/classic/css/classic.css index b956be93..0e21b544 100644 --- a/source/Ox.UI/themes/classic/css/classic.css +++ b/source/Ox.UI/themes/classic/css/classic.css @@ -362,6 +362,70 @@ Scrollbars background: rgb(208, 208, 208); } +/* +================================================================================ +SyntaxHighlighter +================================================================================ +*/ + +.OxThemeClassic .OxSyntaxHighlighter .OxSourceCode { + background-color: rgb(255, 255, 255); +} +.OxThemeClassic .OxSyntaxHighlighter .OxLineNumbers { + background-color: rgb(224, 224, 224); + color: rgb(128, 128, 128); +} +.OxThemeClassic .OxSyntaxHighlighter .OxComment { + color: rgb(128, 128, 128); + font-style: italic; +} +.OxThemeClassic .OxSyntaxHighlighter .OxConstant { + color: rgb(128, 0, 0); + font-weight: bold; +} +.OxThemeClassic .OxSyntaxHighlighter .OxIdentifier { + color: rgb(0, 0, 0); +} +.OxThemeClassic .OxSyntaxHighlighter .OxKeyword { + color: rgb(0, 0, 128); + font-weight: bold; +} +.OxThemeClassic .OxSyntaxHighlighter .OxLinebreak { + color: rgb(192, 192, 192); + font-weight: normal; + font-style: normal; +} +.OxThemeClassic .OxSyntaxHighlighter .OxMethod { + color: rgb(0, 128, 128); +} +.OxThemeClassic .OxSyntaxHighlighter .OxNumber { + color: rgb(128, 0, 0); +} +.OxThemeClassic .OxSyntaxHighlighter .OxObject { + color: rgb(0, 128, 128); + font-weight: bold; +} +.OxThemeClassic .OxSyntaxHighlighter .OxOperator { + color: rgb(0, 0, 128); +} +.OxThemeClassic .OxSyntaxHighlighter .OxProperty { + color: rgb(0, 128, 0); + font-weight: bold; +} +.OxThemeClassic .OxSyntaxHighlighter .OxRegexp { + color: rgb(128, 128, 0); +} +.OxThemeClassic .OxSyntaxHighlighter .OxString { + color: rgb(0, 128, 0); +} +.OxThemeClassic .OxSyntaxHighlighter .OxTab { + color: rgb(192, 192, 192); +} +.OxThemeClassic .OxSyntaxHighlighter .OxWhitespace.OxLeading, +.OxThemeClassic .OxSyntaxHighlighter .OxWhitespace.OxTrailing { + background: rgb(255, 128, 128); +} + /* ================================================================================ Video diff --git a/source/Ox.UI/themes/modern/css/modern.css b/source/Ox.UI/themes/modern/css/modern.css index 733647b3..e4c98e51 100644 --- a/source/Ox.UI/themes/modern/css/modern.css +++ b/source/Ox.UI/themes/modern/css/modern.css @@ -371,6 +371,68 @@ Scrollbars background: rgb(64, 64, 64); } +/* +================================================================================ +SyntaxHighlighter +================================================================================ +*/ + +.OxThemeModern .OxSyntaxHighlighter .OxSourceCode { + background-color: rgb(0, 0, 0); +} +.OxThemeModern .OxSyntaxHighlighter .OxLineNumbers { + background-color: rgb(32, 32, 32); + color: rgb(128, 128, 128); +} +.OxThemeModern .OxSyntaxHighlighter .OxComment { + color: rgb(128, 128, 128); + font-style: italic; +} +.OxThemeModern .OxSyntaxHighlighter .OxConstant { + color: rgb(255, 128, 128); + font-weight: bold; +} +.OxThemeModern .OxSyntaxHighlighter .OxIdentifier { + color: rgb(255, 255, 255); +} +.OxThemeModern .OxSyntaxHighlighter .OxKeyword { + color: rgb(128, 128, 255); + font-weight: bold; +} +.OxThemeModern .OxSyntaxHighlighter .OxLinebreak { + color: rgb(64, 64, 64); + font-weight: normal; + font-style: normal; +} +.OxThemeModern .OxSyntaxHighlighter .OxMethod { + color: rgb(128, 255, 255); +} +.OxThemeModern .OxSyntaxHighlighter .OxNumber { + color: rgb(255, 128, 128); +} +.OxThemeModern .OxSyntaxHighlighter .OxObject { + color: rgb(128, 255, 255); + font-weight: bold; +} +.OxThemeModern .OxSyntaxHighlighter .OxOperator { + color: rgb(128, 128, 255); +} +.OxThemeModern .OxSyntaxHighlighter .OxProperty { + color: rgb(128, 255, 128); + font-weight: bold; +} +.OxThemeModern .OxSyntaxHighlighter .OxRegexp { + color: rgb(255, 255, 128); +} +.OxThemeModern .OxSyntaxHighlighter .OxString { + color: rgb(128, 255, 128); +} +.OxThemeModern .OxSyntaxHighlighter .OxWhitespace { +} +.OxThemeModern .OxSyntaxHighlighter .OxWhitespace.OxTrailing { + background: rgb(255, 255, 255); +} + /* ================================================================================ diff --git a/source/Ox.js b/source/Ox.js index a0383369..53ab8e6e 100644 --- a/source/Ox.js +++ b/source/Ox.js @@ -2701,6 +2701,286 @@ Ox.toDashes = function(str) { }); }; +Ox.tokenize = (function() { + + // see https://github.com/mozilla/narcissus/blob/master/lib/jslex.js + + var identifier = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ$_', + // see https://developer.mozilla.org/en/JavaScript/Reference/Reserved_Words + linebreak = '\n\r', + number = '0123456789', + // see https://developer.mozilla.org/en/JavaScript/Reference + operator = [ + // arithmetic + '+', '-', '*', '/', '%', '++', '--', + // assignment + '=', '+=', '-=', '*=', '/=', '%=', + '&=', '|=', '^=', '<<=', '>>=', '>>>=', + // bitwise + '&', '|', '^', '~', '<<', '>>', '>>>', + // comparison + '==', '!=', '===', '!==', '>', '>=', '<', '<=', + // conditional + '?', ':', + // grouping + '(', ')', '[', ']', '{', '}', + // logical + '&&', '||', '!', + // other + '.', ',', ';' + ], + whitespace = ' \t', + word = { + constant: [ + // Math + 'E', 'LN2', 'LN10', 'LOG2E', 'LOG10E', 'PI', 'SQRT1_2', 'SQRT2', + // Number + 'MAX_VALUE', 'MIN_VALUE', 'NEGATIVE_INFINITY', 'POSITIVE_INFINITY' + ], + keyword: [ + 'break', + 'case', 'catch', 'class', 'const', 'continue', + 'debugger', 'default', 'delete', 'do', + 'else', 'enum', 'export', 'extends', + 'false', 'finally', 'for', 'function', + 'if', 'implements', 'import', 'in', 'instanceof', 'interface', + 'let', 'module', + 'new', 'null', + 'package', 'private', 'protected', 'public', + 'return', + 'super', 'switch', 'static', + 'this', 'throw', 'true', 'try', 'typeof', + 'var', 'void', + 'yield', + 'while', 'with', + ], + method: [ + // Array + 'concat', + 'every', + 'filter', 'forEach', + 'join', + 'lastIndexOf', + 'indexOf', 'isArray', + 'map', + 'pop', 'push', + 'reduce', 'reduceRight', 'reverse', + 'shift', 'slice', 'some', 'sort', 'splice', + 'unshift', + // Date + 'getDate', 'getDay', 'getFullYear', 'getHours', 'getMilliseconds', + 'getMinutes', 'getMonth', 'getSeconds', 'getTime', 'getTimezoneOffset', + 'getUTCDate', 'getUTCDay', 'getUTCFullYear', 'getUTCHours', 'getUTCMilliseconds', + 'getUTCMinutes', 'getUTCMonth', 'getUTCSeconds', + 'now', + 'parse', + 'setDate', 'setFullYear', 'setHours', 'setMilliseconds', 'setMinutes', + 'setMonth', 'setSeconds', 'setTime', + 'setUTCDate', 'setUTCFullYear', 'setUTCHours', 'setUTCMilliseconds', 'setUTCMinutes', + 'setUTCMonth', 'setUTCSeconds', + 'toDateString', 'toJSON', 'toLocaleDateString', 'toLocaleString', 'toLocaleTimeString', + 'toTimeString', 'toUTCString', + 'UTC', + // Function + 'apply', 'bind', 'call', 'isGenerator', + // JSON + 'parse', 'stringify', + // Math + 'abs', 'acos', 'asin', 'atan', 'atan2', + 'ceil', 'cos', + 'exp', + 'floor', + 'log', + 'max', 'min', + 'pow', + 'random', 'round', + 'sin', 'sqrt', + 'tan', + // Number + 'toExponential', 'toFixed', 'toLocaleString', 'toPrecision', + // Object + 'create', + 'defineProperty', 'defineProperties', + 'freeze', + 'getOwnPropertyDescriptor', 'getOwnPropertyNames', 'getPrototypeOf', + 'hasOwnProperty', + 'isExtensible', 'isFrozen', 'isPrototypeOf', 'isSealed', + 'keys', + 'preventExtensions', 'propertyIsEnumerable', + 'seal', + 'toLocaleString', 'toString', + 'valueOf', + // RegExp + 'exec', 'test', + // String + 'charAt', 'charCodeAt', 'concat', + 'fromCharCode', + 'indexOf', + 'lastIndexOf', 'localeCompare', + 'match', + 'replace', + 'search', 'slice', 'split', 'substr', 'substring', + 'toLocaleLowerCase', 'toLocaleUpperCase', 'toLowerCase', 'toUpperCase', 'trim' + ], + object: [ + 'Array', + 'Boolean', + 'Date', 'decodeURI', 'decodeURIComponent', + 'encodeURI', 'encodeURIComponent', 'Error', 'eval', 'EvalError', + 'Function', + 'Infinity', 'isFinite', 'isNaN', + 'JSON', + 'Math', + 'NaN', 'Number', + 'Object', + 'parseFloat', 'parseInt', + 'RangeError', 'ReferenceError', 'RegExp', + 'String', 'SyntaxError', + 'TypeError', + 'undefined', 'URIError' + ], + property: [ + // Function + 'constructor', 'length', 'prototype', + // RegExp + 'global', 'ignoreCase', 'lastIndex', 'multiline', 'source' + ] + }; + + return function(source) { + + //source = source.replace(/\r\n/g, '\n').replace(/\r/g, '\n'); + + var cursor = 0, + tokenize = { + comment: function() { + while (char = source[++cursor]) { + if (next == '/' && char == '\n') { + break; + } else if (next == '*' && char == '*' && source[cursor + 1] == '/') { + cursor += 2; + break; + } + } + }, + identifier: function() { + var str; + while (identifier.indexOf(source[++cursor]) > -1) {} + str = source.substring(start, cursor); + Ox.forEach(word, function(value, key) { + if (value.indexOf(str) > -1) { + type = key; + return false; + } + }); + }, + linebreak: function() { + while (linebreak.indexOf(source[++cursor]) > -1) {} + }, + number: function() { + while ((number + '.').indexOf(source[++cursor]) > -1) {} + }, + operator: function() { + if (operator.indexOf(char + source[++cursor]) > -1) { + if (operator.indexOf(char + next + source[++cursor]) > 1) { + ++cursor; + } + } + }, + regexp: function() { + while ((char = source[++cursor]) != '/') { + char == '\\' && ++cursor; + if (cursor == source.length) { + break; + } + } + while (identifier.indexOf(source[++cursor]) > -1) {} + }, + string: function() { + var delimiter = char; + while ((char = source[++cursor]) != delimiter) { + char == '\\' && ++cursor; + if (cursor == source.length) { + break; + } + } + ++cursor; + }, + whitespace: function() { + while (whitespace.indexOf(source[++cursor]) > -1) {} + } + }, + tokens = [], + type; + + while (cursor < source.length) { + var char = source[cursor], + next = source[cursor + 1], + start = cursor; + if (char == '/' && (next == '/' || next == '*')) { + type = 'comment'; + } else if (identifier.indexOf(char) > -1) { + type = 'identifier'; + } else if (linebreak.indexOf(char) > -1) { + type = 'linebreak'; + } else if (number.indexOf(char) > -1) { + type = 'number'; + } else if (char == "'" || char == '"') { + type = 'string'; + } else if (whitespace.indexOf(char) > -1) { + type = 'whitespace'; + } else if (char == '/') { + type = isRegExp() ? 'regexp' : 'operator'; + } else if (operator.indexOf(char) > -1) { + type = 'operator'; + } + tokenize[type](); + tokens.push({ + length: cursor - start, + type: type, + }); + } + + function isRegExp() { + // checks if a forward slash is the beginning of a regexp, + // as opposed to the beginning of an operator + // see http://www.mozilla.org/js/language/js20-2000-07/rationale/syntax.html#regular-expressions + var index = tokens.length, + isRegExp = false + offset = 0; + // scan back to the previous significant token, + // or the beginning of the source + while ( + typeof tokens[--index] != 'undefined' && + ['comment', 'linebreak', 'whitespace'].indexOf(tokens[index].type) > -1 + ) { + offset += tokens[index].length; + } + if (typeof tokens[index] == 'undefined') { + // source begins with forward slash + isRegExp = true; + } else { + prevToken = tokens[index]; + prevString = source.substr(cursor - prevToken.length - offset, prevToken.length); + Ox.print('forward slash |', prevToken, prevToken.type, '"'+prevString+'"'); + isRegExp = ( + prevToken.type == 'keyword' && + ['false', 'null', 'true'].indexOf(prevString) == -1 + ) || ( + prevToken.type == 'operator' && + ['++', '--', ')', ']', '}'].indexOf(prevString) == -1 + ); + } + return isRegExp; + } + + return tokens; + + }; + +}()); + + Ox.toSlashes = function(str) { /* >>> Ox.toSlashes("fooBarBaz")