From 5db0522e8e8e0931bb0a6a031db194e20b2216cb Mon Sep 17 00:00:00 2001 From: Fabrice Date: Tue, 3 Mar 2026 08:46:27 +0100 Subject: working on lexing --- src/tokenizer.cc | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'src/tokenizer.cc') diff --git a/src/tokenizer.cc b/src/tokenizer.cc index 54634de..274f572 100644 --- a/src/tokenizer.cc +++ b/src/tokenizer.cc @@ -28,14 +28,12 @@ static inline const Buffer* tokenizer_get_buffer(Tokenizer* tokenizer) { static inline char tokenizer_advance(const Tokenizer* tokenizer, usize* offset) { const String text = tokenizer->buffer->content; - const unsigned char* c = text[*offset]; - wchar wc = (wchar)*c; - - u8 nobytes = utf8_nobytes(wc); - if(nobytes > 1) panic("no support for multi-byte chars: %d:%d", wc, nobytes); + unsigned char c = *text[*offset]; + u8 nobytes = utf8_nobytes(c); + if(nobytes > 1) panic("no support for multi-byte chars: %c:%d", c, nobytes); offset += nobytes; - return (char)wc; + return c; } bool tokenizer_next(Tokenizer* tokenizer, Token* out) { @@ -52,7 +50,11 @@ again: goto again; } - return false; + tokenizer_advance(tokenizer, &offset); + + const unsigned char* str = buffer->content[offset]; + *out = Token(Token_Kind_Eof, String(str, 1), Span(buffer->file, 0, 0)); + return true; } #endif -- cgit v1.2.3