add column, line to javascript tokenizer tokens

This commit is contained in:
j 2013-05-10 13:00:32 +00:00
parent be9424036f
commit 8563ea8239

View file

@ -109,9 +109,11 @@ def tokenize(source):
token['type'] == 'operator' and not token['value'] in ['++', '--', ')', ']', '}'] token['type'] == 'operator' and not token['value'] in ['++', '--', ')', ']', '}']
) )
return is_regexp return is_regexp
column = 1
cursor = 0 cursor = 0
length = len(source) length = len(source)
tokens = [] tokens = []
line = 1
while cursor < length: while cursor < length:
char = source[cursor] char = source[cursor]
start = cursor start = cursor
@ -161,8 +163,21 @@ def tokenize(source):
type = 'whitespace' type = 'whitespace'
while cursor < length and source[cursor] in WHITESPACE: while cursor < length and source[cursor] in WHITESPACE:
cursor += 1 cursor += 1
value = source[start:cursor]
tokens.append({ tokens.append({
'column': column,
'line': line,
'type': type, 'type': type,
'value': source[start:cursor] 'value': value
}) })
return tokens if type == 'comment':
lines = value.split('\n');
column = len(lines[-1])
line += len(lines) - 1
elif type == 'linebreak':
column = 1
column = 1
line += len(value)
else:
column += len(value)
return tokens