add column, line to javascript tokenizer tokens
This commit is contained in:
parent
be9424036f
commit
8563ea8239
1 changed files with 17 additions and 2 deletions
19
ox/js.py
19
ox/js.py
|
@ -109,9 +109,11 @@ def tokenize(source):
|
||||||
token['type'] == 'operator' and not token['value'] in ['++', '--', ')', ']', '}']
|
token['type'] == 'operator' and not token['value'] in ['++', '--', ')', ']', '}']
|
||||||
)
|
)
|
||||||
return is_regexp
|
return is_regexp
|
||||||
|
column = 1
|
||||||
cursor = 0
|
cursor = 0
|
||||||
length = len(source)
|
length = len(source)
|
||||||
tokens = []
|
tokens = []
|
||||||
|
line = 1
|
||||||
while cursor < length:
|
while cursor < length:
|
||||||
char = source[cursor]
|
char = source[cursor]
|
||||||
start = cursor
|
start = cursor
|
||||||
|
@ -161,8 +163,21 @@ def tokenize(source):
|
||||||
type = 'whitespace'
|
type = 'whitespace'
|
||||||
while cursor < length and source[cursor] in WHITESPACE:
|
while cursor < length and source[cursor] in WHITESPACE:
|
||||||
cursor += 1
|
cursor += 1
|
||||||
|
value = source[start:cursor]
|
||||||
tokens.append({
|
tokens.append({
|
||||||
|
'column': column,
|
||||||
|
'line': line,
|
||||||
'type': type,
|
'type': type,
|
||||||
'value': source[start:cursor]
|
'value': value
|
||||||
})
|
})
|
||||||
return tokens
|
if type == 'comment':
|
||||||
|
lines = value.split('\n');
|
||||||
|
column = len(lines[-1])
|
||||||
|
line += len(lines) - 1
|
||||||
|
elif type == 'linebreak':
|
||||||
|
column = 1
|
||||||
|
column = 1
|
||||||
|
line += len(value)
|
||||||
|
else:
|
||||||
|
column += len(value)
|
||||||
|
return tokens
|
||||||
|
|
Loading…
Reference in a new issue