Ox.tokenize, Ox.SyntaxHighlighter (+demo)

This commit is contained in:
rolux 2011-04-28 20:34:19 +02:00
commit 74b9a25387
8 changed files with 632 additions and 0 deletions

View file

@ -1437,6 +1437,34 @@ Scrollbars
-webkit-border-radius: 6px;
}
/*
================================================================================
SyntaxHightlighter
================================================================================
*/
.OxSyntaxHighlighter {
position: absolute;
overflow: auto;
}
.OxSyntaxHighlighter > div {
position: absolute;
font-family: Menlo, Monaco, DejaVu Sans Mono, Bitstream Vera Sans Mono, Consolas, Lucida Console;
line-height: 14px;
}
.OxSyntaxHighlighter > .OxLineNumbers {
text-align: right;
}
.OxSyntaxHighlighter > .OxSourceCode {
//display: table-cell;
-moz-user-select: text;
-webkit-user-select: text;
}
.OxSyntaxHighlighter > .OxSourceCode .OxLinebreak {
-moz-user-select: none;
-webkit-user-select: none;
}
/*
================================================================================
Video

View file

@ -0,0 +1,137 @@
// vim: et:ts=4:sw=4:sts=4:ft=js
/*@
@*/
Ox.SyntaxHighlighter = function(options, self) {
self = self || {};
var that = new Ox.Element('div', self)
.defaults({
height: 40,
lineLength: 80, //@ number of characters per line
showLinebreaks: false, //@ show linebreak symbols
showTabs: false, //@ show tab symbols
showWhitespace: false, //@ show irregular leading or trailing whitespace
source: '', //@ JavaScript source
stripComments: false, //@ strip all comments
stripLinebreaks: false, //@ strip multiple linebreaks, NOT IMPLEMENTED
stripWhitespace: false, //@ strip all whitespace, NOT IMPLEMENTED
tabLength: 4, //@ number of spaces per tab
width: 80
})
.options(options || {})
.addClass('OxSyntaxHighlighter');
var foo = $('<div>')
//.css({marginTop: '-1000px'})
.html(Ox.repeat('_', 80))
.appendTo(that.$element);
//alert(foo.width());
foo.remove();
self.options.source = self.options.source
.replace(/\r\n/g, '\n')
.replace(/\r/g, '\n');
self.cursor = 0;
self.source = '';
self.tokens = Ox.tokenize(self.options.source);
self.tokens.forEach(function(token, i) {
var classNames, tokenString;
if (
!(self.options.stripComments && token.type == 'comment')
) {
classNames = 'Ox' + Ox.toTitleCase(token.type);
tokenString = self.options.source.substr(self.cursor, token.length);
if (token.type == 'whitespace') {
if (isAfterLinebreak() && hasIrregularSpaces()) {
classNames += ' OxLeading'
} else if (isBeforeLinebreak()) {
classNames += ' OxTrailing'
}
}
self.source += '<span class="' + classNames + '">' +
encodeToken(tokenString, token.type) + '</span>';
}
self.cursor += token.length;
function isAfterLinebreak() {
return i == 0 ||
self.tokens[i - 1].type == 'linebreak';
}
function isBeforeLinebreak() {
return i == self.tokens.length - 1 ||
self.tokens[i + 1].type == 'linebreak';
}
function hasIrregularSpaces() {
return tokenString.split('').reduce(function(prev, curr) {
return prev + (curr == ' ' ? 1 : 0);
}, 0) % self.options.tabLength;
}
});
self.lines = self.source.split('<br/>');
self.lineNumbersWidth = self.lines.length.toString().length * 7 + 7;
self.sourceCodeWidth = 80 * 7 + (
self.lines.length > 40 ? Ox.UI.SCROLLBAR_SIZE : 0
);
self.height = 40 * 14 + (
Math.max.apply(null, self.lines.map(function(line) {
return line.length;
})) > 80 ? Ox.UI.SCROLLBAR_SIZE : 0
);
that.css({
width: self.lineNumbersWidth + self.sourceCodeWidth,
height: self.height
});
self.$lineNumbers = new Ox.Element()
.addClass('OxLineNumbers')
.css({
width: self.lineNumbersWidth + 'px',
height: (self.lines.length * 14) + 'px'
})
.html(
Ox.range(self.lines.length).map(function(line) {
return (line + 1) + '&nbsp;';
}).join('<br/>')
)
.appendTo(that);
self.$source = new Ox.Element()
.addClass('OxSourceCode')
.css({
left: self.lineNumbersWidth + 'px',
width: self.sourceCodeWidth + 'px',
height: (self.lines.length * 14) + 'px'
})
.html(self.source)
.appendTo(that);
function encodeToken(str, type) {
var linebreak = '<br/>',
tab = Ox.repeat('&nbsp;', self.options.tabLength);
if (self.options.showLinebreaks) {
if (type == 'linebreak') {
linebreak = '¶' + linebreak;
} else {
linebreak = '<span class="OxLinebreak">¶</span>' + linebreak;
}
}
if (self.options.showTabs) {
tab = '<span class="OxTab">\u2192' + tab.substr(6) + '</span>';
}
str = Ox.encodeHTML(str)
.replace(/ /g, '&nbsp;')
.replace(/\t/g, tab)
.replace(/\n/g, linebreak);
return str;
}
self.onChange = function() {
};
return that;
};

View file

@ -1,4 +1,5 @@
// vim: et:ts=4:sw=4:sts=4:ft=js
Ox.VideoEditorPlayer = function(options, self) {
var self = self || {},

View file

@ -362,6 +362,70 @@ Scrollbars
background: rgb(208, 208, 208);
}
/*
================================================================================
SyntaxHighlighter
================================================================================
*/
.OxThemeClassic .OxSyntaxHighlighter .OxSourceCode {
background-color: rgb(255, 255, 255);
}
.OxThemeClassic .OxSyntaxHighlighter .OxLineNumbers {
background-color: rgb(224, 224, 224);
color: rgb(128, 128, 128);
}
.OxThemeClassic .OxSyntaxHighlighter .OxComment {
color: rgb(128, 128, 128);
font-style: italic;
}
.OxThemeClassic .OxSyntaxHighlighter .OxConstant {
color: rgb(128, 0, 0);
font-weight: bold;
}
.OxThemeClassic .OxSyntaxHighlighter .OxIdentifier {
color: rgb(0, 0, 0);
}
.OxThemeClassic .OxSyntaxHighlighter .OxKeyword {
color: rgb(0, 0, 128);
font-weight: bold;
}
.OxThemeClassic .OxSyntaxHighlighter .OxLinebreak {
color: rgb(192, 192, 192);
font-weight: normal;
font-style: normal;
}
.OxThemeClassic .OxSyntaxHighlighter .OxMethod {
color: rgb(0, 128, 128);
}
.OxThemeClassic .OxSyntaxHighlighter .OxNumber {
color: rgb(128, 0, 0);
}
.OxThemeClassic .OxSyntaxHighlighter .OxObject {
color: rgb(0, 128, 128);
font-weight: bold;
}
.OxThemeClassic .OxSyntaxHighlighter .OxOperator {
color: rgb(0, 0, 128);
}
.OxThemeClassic .OxSyntaxHighlighter .OxProperty {
color: rgb(0, 128, 0);
font-weight: bold;
}
.OxThemeClassic .OxSyntaxHighlighter .OxRegexp {
color: rgb(128, 128, 0);
}
.OxThemeClassic .OxSyntaxHighlighter .OxString {
color: rgb(0, 128, 0);
}
.OxThemeClassic .OxSyntaxHighlighter .OxTab {
color: rgb(192, 192, 192);
}
.OxThemeClassic .OxSyntaxHighlighter .OxWhitespace.OxLeading,
.OxThemeClassic .OxSyntaxHighlighter .OxWhitespace.OxTrailing {
background: rgb(255, 128, 128);
}
/*
================================================================================
Video

View file

@ -371,6 +371,68 @@ Scrollbars
background: rgb(64, 64, 64);
}
/*
================================================================================
SyntaxHighlighter
================================================================================
*/
.OxThemeModern .OxSyntaxHighlighter .OxSourceCode {
background-color: rgb(0, 0, 0);
}
.OxThemeModern .OxSyntaxHighlighter .OxLineNumbers {
background-color: rgb(32, 32, 32);
color: rgb(128, 128, 128);
}
.OxThemeModern .OxSyntaxHighlighter .OxComment {
color: rgb(128, 128, 128);
font-style: italic;
}
.OxThemeModern .OxSyntaxHighlighter .OxConstant {
color: rgb(255, 128, 128);
font-weight: bold;
}
.OxThemeModern .OxSyntaxHighlighter .OxIdentifier {
color: rgb(255, 255, 255);
}
.OxThemeModern .OxSyntaxHighlighter .OxKeyword {
color: rgb(128, 128, 255);
font-weight: bold;
}
.OxThemeModern .OxSyntaxHighlighter .OxLinebreak {
color: rgb(64, 64, 64);
font-weight: normal;
font-style: normal;
}
.OxThemeModern .OxSyntaxHighlighter .OxMethod {
color: rgb(128, 255, 255);
}
.OxThemeModern .OxSyntaxHighlighter .OxNumber {
color: rgb(255, 128, 128);
}
.OxThemeModern .OxSyntaxHighlighter .OxObject {
color: rgb(128, 255, 255);
font-weight: bold;
}
.OxThemeModern .OxSyntaxHighlighter .OxOperator {
color: rgb(128, 128, 255);
}
.OxThemeModern .OxSyntaxHighlighter .OxProperty {
color: rgb(128, 255, 128);
font-weight: bold;
}
.OxThemeModern .OxSyntaxHighlighter .OxRegexp {
color: rgb(255, 255, 128);
}
.OxThemeModern .OxSyntaxHighlighter .OxString {
color: rgb(128, 255, 128);
}
.OxThemeModern .OxSyntaxHighlighter .OxWhitespace {
}
.OxThemeModern .OxSyntaxHighlighter .OxWhitespace.OxTrailing {
background: rgb(255, 255, 255);
}
/*
================================================================================

View file

@ -2701,6 +2701,286 @@ Ox.toDashes = function(str) {
});
};
Ox.tokenize = (function() {
// see https://github.com/mozilla/narcissus/blob/master/lib/jslex.js
var identifier = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ$_',
// see https://developer.mozilla.org/en/JavaScript/Reference/Reserved_Words
linebreak = '\n\r',
number = '0123456789',
// see https://developer.mozilla.org/en/JavaScript/Reference
operator = [
// arithmetic
'+', '-', '*', '/', '%', '++', '--',
// assignment
'=', '+=', '-=', '*=', '/=', '%=',
'&=', '|=', '^=', '<<=', '>>=', '>>>=',
// bitwise
'&', '|', '^', '~', '<<', '>>', '>>>',
// comparison
'==', '!=', '===', '!==', '>', '>=', '<', '<=',
// conditional
'?', ':',
// grouping
'(', ')', '[', ']', '{', '}',
// logical
'&&', '||', '!',
// other
'.', ',', ';'
],
whitespace = ' \t',
word = {
constant: [
// Math
'E', 'LN2', 'LN10', 'LOG2E', 'LOG10E', 'PI', 'SQRT1_2', 'SQRT2',
// Number
'MAX_VALUE', 'MIN_VALUE', 'NEGATIVE_INFINITY', 'POSITIVE_INFINITY'
],
keyword: [
'break',
'case', 'catch', 'class', 'const', 'continue',
'debugger', 'default', 'delete', 'do',
'else', 'enum', 'export', 'extends',
'false', 'finally', 'for', 'function',
'if', 'implements', 'import', 'in', 'instanceof', 'interface',
'let', 'module',
'new', 'null',
'package', 'private', 'protected', 'public',
'return',
'super', 'switch', 'static',
'this', 'throw', 'true', 'try', 'typeof',
'var', 'void',
'yield',
'while', 'with',
],
method: [
// Array
'concat',
'every',
'filter', 'forEach',
'join',
'lastIndexOf',
'indexOf', 'isArray',
'map',
'pop', 'push',
'reduce', 'reduceRight', 'reverse',
'shift', 'slice', 'some', 'sort', 'splice',
'unshift',
// Date
'getDate', 'getDay', 'getFullYear', 'getHours', 'getMilliseconds',
'getMinutes', 'getMonth', 'getSeconds', 'getTime', 'getTimezoneOffset',
'getUTCDate', 'getUTCDay', 'getUTCFullYear', 'getUTCHours', 'getUTCMilliseconds',
'getUTCMinutes', 'getUTCMonth', 'getUTCSeconds',
'now',
'parse',
'setDate', 'setFullYear', 'setHours', 'setMilliseconds', 'setMinutes',
'setMonth', 'setSeconds', 'setTime',
'setUTCDate', 'setUTCFullYear', 'setUTCHours', 'setUTCMilliseconds', 'setUTCMinutes',
'setUTCMonth', 'setUTCSeconds',
'toDateString', 'toJSON', 'toLocaleDateString', 'toLocaleString', 'toLocaleTimeString',
'toTimeString', 'toUTCString',
'UTC',
// Function
'apply', 'bind', 'call', 'isGenerator',
// JSON
'parse', 'stringify',
// Math
'abs', 'acos', 'asin', 'atan', 'atan2',
'ceil', 'cos',
'exp',
'floor',
'log',
'max', 'min',
'pow',
'random', 'round',
'sin', 'sqrt',
'tan',
// Number
'toExponential', 'toFixed', 'toLocaleString', 'toPrecision',
// Object
'create',
'defineProperty', 'defineProperties',
'freeze',
'getOwnPropertyDescriptor', 'getOwnPropertyNames', 'getPrototypeOf',
'hasOwnProperty',
'isExtensible', 'isFrozen', 'isPrototypeOf', 'isSealed',
'keys',
'preventExtensions', 'propertyIsEnumerable',
'seal',
'toLocaleString', 'toString',
'valueOf',
// RegExp
'exec', 'test',
// String
'charAt', 'charCodeAt', 'concat',
'fromCharCode',
'indexOf',
'lastIndexOf', 'localeCompare',
'match',
'replace',
'search', 'slice', 'split', 'substr', 'substring',
'toLocaleLowerCase', 'toLocaleUpperCase', 'toLowerCase', 'toUpperCase', 'trim'
],
object: [
'Array',
'Boolean',
'Date', 'decodeURI', 'decodeURIComponent',
'encodeURI', 'encodeURIComponent', 'Error', 'eval', 'EvalError',
'Function',
'Infinity', 'isFinite', 'isNaN',
'JSON',
'Math',
'NaN', 'Number',
'Object',
'parseFloat', 'parseInt',
'RangeError', 'ReferenceError', 'RegExp',
'String', 'SyntaxError',
'TypeError',
'undefined', 'URIError'
],
property: [
// Function
'constructor', 'length', 'prototype',
// RegExp
'global', 'ignoreCase', 'lastIndex', 'multiline', 'source'
]
};
return function(source) {
//source = source.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
var cursor = 0,
tokenize = {
comment: function() {
while (char = source[++cursor]) {
if (next == '/' && char == '\n') {
break;
} else if (next == '*' && char == '*' && source[cursor + 1] == '/') {
cursor += 2;
break;
}
}
},
identifier: function() {
var str;
while (identifier.indexOf(source[++cursor]) > -1) {}
str = source.substring(start, cursor);
Ox.forEach(word, function(value, key) {
if (value.indexOf(str) > -1) {
type = key;
return false;
}
});
},
linebreak: function() {
while (linebreak.indexOf(source[++cursor]) > -1) {}
},
number: function() {
while ((number + '.').indexOf(source[++cursor]) > -1) {}
},
operator: function() {
if (operator.indexOf(char + source[++cursor]) > -1) {
if (operator.indexOf(char + next + source[++cursor]) > 1) {
++cursor;
}
}
},
regexp: function() {
while ((char = source[++cursor]) != '/') {
char == '\\' && ++cursor;
if (cursor == source.length) {
break;
}
}
while (identifier.indexOf(source[++cursor]) > -1) {}
},
string: function() {
var delimiter = char;
while ((char = source[++cursor]) != delimiter) {
char == '\\' && ++cursor;
if (cursor == source.length) {
break;
}
}
++cursor;
},
whitespace: function() {
while (whitespace.indexOf(source[++cursor]) > -1) {}
}
},
tokens = [],
type;
while (cursor < source.length) {
var char = source[cursor],
next = source[cursor + 1],
start = cursor;
if (char == '/' && (next == '/' || next == '*')) {
type = 'comment';
} else if (identifier.indexOf(char) > -1) {
type = 'identifier';
} else if (linebreak.indexOf(char) > -1) {
type = 'linebreak';
} else if (number.indexOf(char) > -1) {
type = 'number';
} else if (char == "'" || char == '"') {
type = 'string';
} else if (whitespace.indexOf(char) > -1) {
type = 'whitespace';
} else if (char == '/') {
type = isRegExp() ? 'regexp' : 'operator';
} else if (operator.indexOf(char) > -1) {
type = 'operator';
}
tokenize[type]();
tokens.push({
length: cursor - start,
type: type,
});
}
function isRegExp() {
// checks if a forward slash is the beginning of a regexp,
// as opposed to the beginning of an operator
// see http://www.mozilla.org/js/language/js20-2000-07/rationale/syntax.html#regular-expressions
var index = tokens.length,
isRegExp = false
offset = 0;
// scan back to the previous significant token,
// or the beginning of the source
while (
typeof tokens[--index] != 'undefined' &&
['comment', 'linebreak', 'whitespace'].indexOf(tokens[index].type) > -1
) {
offset += tokens[index].length;
}
if (typeof tokens[index] == 'undefined') {
// source begins with forward slash
isRegExp = true;
} else {
prevToken = tokens[index];
prevString = source.substr(cursor - prevToken.length - offset, prevToken.length);
Ox.print('forward slash |', prevToken, prevToken.type, '"'+prevString+'"');
isRegExp = (
prevToken.type == 'keyword' &&
['false', 'null', 'true'].indexOf(prevString) == -1
) || (
prevToken.type == 'operator' &&
['++', '--', ')', ']', '}'].indexOf(prevString) == -1
);
}
return isRegExp;
}
return tokens;
};
}());
Ox.toSlashes = function(str) {
/*
>>> Ox.toSlashes("fooBarBaz")