fixed

1 files changed, 10 insertions, 4 deletions

diff --git a/src/tokenizer.cc b/src/tokenizer.cc
index 498bca3..6f30659 100644
--- a/src/tokenizer.cc
+++ b/src/tokenizer.cc
@@ -71,12 +71,19 @@ static bool tokenizer_get_buffer(Tokenizer* tokenizer, usize* cursor) {
   return true;
 }
 
+static void tokenizer_advance(usize* offset, usize nbytes) {
+  assert_neq(offset, nullptr);
+  *offset += nbytes;
+}
+
 static Token tokenizer_lex_identifier(Tokenizer* tokenizer, usize start, usize *offset) {
   assert_neq(tokenizer, nullptr);
 
   wchar c;
-  while (tokenizer_advance(tokenizer, offset, &c))
+  while (tokenizer_char(tokenizer, *offset, &c)) {
     if (!utf8_is_alnum(c) && c != '_') break;
+    tokenizer_advance(offset, 1);
+  }
 
   return tokenizer_make_token(tokenizer, Token_Kind_Identifier, start, *offset);
 }
@@ -90,9 +97,8 @@ bool tokenizer_next(Tokenizer* tokenizer, Token* out) {
   usize advance = cursor;
 
   wchar c;
-  (void)tokenizer_advance(tokenizer, &advance,
-                          &c);  // We just checked that we are not at the end
-
+  (void)tokenizer_char(tokenizer, advance, &c);
+  
   if (utf8_is_identifier(c)) {
     *out = tokenizer_lex_identifier(tokenizer, cursor, &advance);
     goto out;