From 8a46601a74462bf70c12253a5232bc21fc7f1427 Mon Sep 17 00:00:00 2001 From: Fabrice Date: Tue, 3 Mar 2026 17:09:45 +0100 Subject: attempt --- src/tokenizer.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tokenizer.cc b/src/tokenizer.cc index 59c0cd5..221ea12 100644 --- a/src/tokenizer.cc +++ b/src/tokenizer.cc @@ -64,7 +64,7 @@ static Buffer* tokenizer_get_buffer(Tokenizer* tokenizer, usize* cursor) { assert_ste(curr_offset, text.length); if (curr_offset == text.length) return false; - unsigned char c = *text[curr_offset]; + unsigned char c = *text[curr_offset - 1]; u8 nobytes = utf8_nobytes(c); if (nobytes > 1) panic("no support for multi-byte chars: %c:%d", c, nobytes); @@ -78,7 +78,7 @@ static Token tokenizer_lex_identifier(Tokenizer* tokenizer, usize start, usize * wchar c; while (tokenizer_advance(tokenizer, offset, &c)) - if (!utf8_is_alnum(c) || c == '_') break; + if (!utf8_is_alnum(c) && c != '_') break; return tokenizer_make_token(tokenizer, Token_Kind_Identifier, start, *offset); } -- cgit v1.2.3