add js module (tokenize, minify)
This commit is contained in:
parent
0050c403ab
commit
409d5da90d
2 changed files with 157 additions and 11 deletions
|
@ -1,17 +1,18 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
# vi:si:et:sw=4:sts=4:ts=4
|
# vi:si:et:sw=4:sts=4:ts=4
|
||||||
# GPL 2008
|
# GPL 2011
|
||||||
__version__ = '2.0.0'
|
__version__ = '2.0.1'
|
||||||
|
|
||||||
from file import *
|
|
||||||
from format import *
|
|
||||||
from html import *
|
|
||||||
from iso import *
|
|
||||||
from text import *
|
|
||||||
from form import *
|
|
||||||
import cache
|
import cache
|
||||||
import net
|
import net
|
||||||
|
|
||||||
from torrent import *
|
from file import *
|
||||||
|
from form import *
|
||||||
import location
|
from format import *
|
||||||
|
from html import *
|
||||||
|
from image import *
|
||||||
|
from js import *
|
||||||
|
from location import *
|
||||||
|
from normalize import *
|
||||||
|
from text import *
|
||||||
|
from torrent import *
|
145
ox/js.py
Normal file
145
ox/js.py
Normal file
|
@ -0,0 +1,145 @@
|
||||||
|
#!/usr/bin/python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# vi:si:et:sw=4:sts=4:ts=4
|
||||||
|
|
||||||
|
def minify(source, comment=''):
|
||||||
|
# see https://github.com/douglascrockford/JSMin/blob/master/README
|
||||||
|
def get_next_non_whitespace_token():
|
||||||
|
pass
|
||||||
|
tokens = tokenize(source)
|
||||||
|
length = len(tokens)
|
||||||
|
minified = '/*' + comment + '*/' if comment else ''
|
||||||
|
for i, token in enumerate(tokens):
|
||||||
|
if token['type'] in ['linebreak', 'whitespace']:
|
||||||
|
prevToken = None if i == 0 else tokens[i - 1]
|
||||||
|
next = i + 1
|
||||||
|
while next < length and tokens[next]['type'] in ['comment', 'linebreak', 'whitespace']:
|
||||||
|
next += 1
|
||||||
|
nextToken = None if next == length else tokens[next]
|
||||||
|
if token['type'] == 'linebreak':
|
||||||
|
# replace a linebreak between two tokens that are identifiers or
|
||||||
|
# numbers or strings or unary operators or grouping operators
|
||||||
|
# with a single newline, otherwise remove it
|
||||||
|
if prevToken and nextToken\
|
||||||
|
and (prevToken['type'] in ['identifier', 'number', 'string']\
|
||||||
|
or prevToken['value'] in ['++', '--', ')', ']', '}'])\
|
||||||
|
and (nextToken['type'] in ['identifier', 'number', 'string']\
|
||||||
|
or nextToken['value'] in ['+', '-', '++', '--', '~', '!', '(', '[', '{']):
|
||||||
|
minified += '\n'
|
||||||
|
elif token['type'] == 'whitespace':
|
||||||
|
# replace whitespace between two tokens that are identifiers or
|
||||||
|
# numbers, or between a token that ends with "+" or "-" and one that
|
||||||
|
# begins with "+" or "-", with a single space, otherwise remove it
|
||||||
|
if prevToken and nextToken\
|
||||||
|
and ((prevToken['type'] in ['identifier', 'number']\
|
||||||
|
and nextToken['type'] in ['identifier', 'number'])
|
||||||
|
or (prevToken['value'] in ['+', '-', '++', '--']
|
||||||
|
and nextToken['value'] in ['+', '-', '++', '--'])):
|
||||||
|
minified += ' '
|
||||||
|
elif token['type'] != 'comment':
|
||||||
|
# remove comments and leave all other tokens untouched
|
||||||
|
minified += token['value']
|
||||||
|
return minified
|
||||||
|
|
||||||
|
def tokenize(source):
|
||||||
|
# see https://github.com/mozilla/narcissus/blob/master/lib/jslex.js
|
||||||
|
IDENTIFIER = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ$_'
|
||||||
|
LINEBREAK = '\n\r'
|
||||||
|
NUMBER = '01234567890'
|
||||||
|
OPERATOR = [
|
||||||
|
# arithmetic
|
||||||
|
'+', '-', '*', '/', '%', '++', '--',
|
||||||
|
# assignment
|
||||||
|
'=', '+=', '-=', '*=', '/=', '%=',
|
||||||
|
'&=', '|=', '^=', '<<=', '>>=', '>>>=',
|
||||||
|
# bitwise
|
||||||
|
'&', '|', '^', '~', '<<', '>>', '>>>',
|
||||||
|
# comparison
|
||||||
|
'==', '!=', '===', '!==', '>', '>=', '<', '<=',
|
||||||
|
# conditional
|
||||||
|
'?', ':',
|
||||||
|
# grouping
|
||||||
|
'(', ')', '[', ']', '{', '}',
|
||||||
|
# logical
|
||||||
|
'&&', '||', '!',
|
||||||
|
# other
|
||||||
|
'.', ',', ';'
|
||||||
|
]
|
||||||
|
REGEXP = 'abcdefghijklmnopqrstuvwxyz'
|
||||||
|
STRING = '\'"'
|
||||||
|
WHITESPACE = ' \t'
|
||||||
|
def is_regexp():
|
||||||
|
# checks if a forward slash is the beginning of a regexp,
|
||||||
|
# as opposed to the beginning of an operator
|
||||||
|
i = len(tokens) - 1
|
||||||
|
# scan back to the previous significant token,
|
||||||
|
# or to the beginnig of the source
|
||||||
|
while i >= 0 and tokens[i]['type'] in ['comment', 'linebreak', 'whitespace']:
|
||||||
|
i -= 1
|
||||||
|
if i == -1:
|
||||||
|
# source begins with forward slash
|
||||||
|
is_regexp = True
|
||||||
|
else:
|
||||||
|
token = tokens[i]
|
||||||
|
is_regexp = (
|
||||||
|
token['type'] == 'keyword' and not token['value'] in ['false', 'null', 'true']
|
||||||
|
) or (
|
||||||
|
token['type'] == 'operator' and not token['value'] in ['++', '--', ')', ']', '}']
|
||||||
|
)
|
||||||
|
return is_regexp
|
||||||
|
cursor = 0
|
||||||
|
length = len(source)
|
||||||
|
tokens = []
|
||||||
|
while cursor < length:
|
||||||
|
char = source[cursor]
|
||||||
|
start = cursor
|
||||||
|
cursor += 1
|
||||||
|
if char == '/' and cursor < length - 1 and source[cursor] in '/*':
|
||||||
|
type = 'comment'
|
||||||
|
cursor += 1
|
||||||
|
while cursor < length:
|
||||||
|
cursor += 1
|
||||||
|
if source[start + 1] == '/' and source[cursor] == '\n':
|
||||||
|
break
|
||||||
|
elif source[start + 1] == '*' and source[cursor:cursor + 2] == '*/':
|
||||||
|
cursor += 2
|
||||||
|
break
|
||||||
|
elif char in IDENTIFIER:
|
||||||
|
type = 'identifier'
|
||||||
|
while cursor < length and source[cursor] in IDENTIFIER + NUMBER:
|
||||||
|
cursor += 1
|
||||||
|
elif char in LINEBREAK:
|
||||||
|
type = 'linebreak'
|
||||||
|
while cursor < length and source[cursor] in LINEBREAK:
|
||||||
|
cursor += 1
|
||||||
|
elif char in NUMBER:
|
||||||
|
type = 'number'
|
||||||
|
while cursor < length and source[cursor] in NUMBER + '.':
|
||||||
|
cursor += 1
|
||||||
|
elif char == '/' and is_regexp():
|
||||||
|
type = 'regexp'
|
||||||
|
while cursor < length and source[cursor] != '/':
|
||||||
|
cursor += (2 if source[cursor] == '\\' else 1)
|
||||||
|
cursor += 1
|
||||||
|
while cursor < length and source[cursor] in REGEXP:
|
||||||
|
cursor += 1
|
||||||
|
elif char in OPERATOR:
|
||||||
|
type = 'operator'
|
||||||
|
string = char + source[cursor]
|
||||||
|
while cursor < length and string in OPERATOR:
|
||||||
|
cursor += 1
|
||||||
|
string += source[cursor]
|
||||||
|
elif char in STRING:
|
||||||
|
type = 'string'
|
||||||
|
while cursor < length and source[cursor] != source[start]:
|
||||||
|
cursor += (2 if source[cursor] == '\\' else 1)
|
||||||
|
cursor += 1
|
||||||
|
elif char in WHITESPACE:
|
||||||
|
type = 'whitespace'
|
||||||
|
while cursor < length and source[cursor] in WHITESPACE:
|
||||||
|
cursor += 1
|
||||||
|
tokens.append({
|
||||||
|
'type': type,
|
||||||
|
'value': source[start:cursor]
|
||||||
|
})
|
||||||
|
return tokens
|
Loading…
Reference in a new issue