fix a bug where the js tokenizer would fail if the last character of the source was an operator

This commit is contained in:
rolux 2011-10-07 00:42:31 +02:00
parent 409d5da90d
commit a7aba40790

View file

@ -125,10 +125,11 @@ def tokenize(source):
cursor += 1
elif char in OPERATOR:
type = 'operator'
string = char + source[cursor]
while cursor < length and string in OPERATOR:
cursor += 1
string += source[cursor]
if cursor < length:
string = char + source[cursor]
while cursor < length and string in OPERATOR:
cursor += 1
string += source[cursor]
elif char in STRING:
type = 'string'
while cursor < length and source[cursor] != source[start]: