From c1cbfb1feefc309108d6e547122c64f66dfaedb1 Mon Sep 17 00:00:00 2001 From: Fabrice Date: Tue, 3 Mar 2026 17:23:50 +0100 Subject: fixed --- src/tokenizer.cc | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/tokenizer.cc b/src/tokenizer.cc index 498bca3..6f30659 100644 --- a/src/tokenizer.cc +++ b/src/tokenizer.cc @@ -71,12 +71,19 @@ static bool tokenizer_get_buffer(Tokenizer* tokenizer, usize* cursor) { return true; } +static void tokenizer_advance(usize* offset, usize nbytes) { + assert_neq(offset, nullptr); + *offset += nbytes; +} + static Token tokenizer_lex_identifier(Tokenizer* tokenizer, usize start, usize *offset) { assert_neq(tokenizer, nullptr); wchar c; - while (tokenizer_advance(tokenizer, offset, &c)) + while (tokenizer_char(tokenizer, *offset, &c)) { if (!utf8_is_alnum(c) && c != '_') break; + tokenizer_advance(offset, 1); + } return tokenizer_make_token(tokenizer, Token_Kind_Identifier, start, *offset); } @@ -90,9 +97,8 @@ bool tokenizer_next(Tokenizer* tokenizer, Token* out) { usize advance = cursor; wchar c; - (void)tokenizer_advance(tokenizer, &advance, - &c); // We just checked that we are not at the end - + (void)tokenizer_char(tokenizer, advance, &c); + if (utf8_is_identifier(c)) { *out = tokenizer_lex_identifier(tokenizer, cursor, &advance); goto out; -- cgit v1.2.3