diff options
| -rw-r--r-- | src/tokenizer.cc | 16 |
1 files changed, 7 insertions, 9 deletions
diff --git a/src/tokenizer.cc b/src/tokenizer.cc index 029d440..59c0cd5 100644 --- a/src/tokenizer.cc +++ b/src/tokenizer.cc @@ -29,16 +29,15 @@ static inline String tokenizer_make_lexeme(const Tokenizer* tokenizer, return String(buffer->content[start], end - start); } -static inline void tokenizer_make_token(const Tokenizer* tokenizer, - Token* token, Token_Kind kind, +static inline Token tokenizer_make_token(const Tokenizer* tokenizer, + Token_Kind kind, usize start, usize end) { assert_neq(tokenizer, nullptr); - assert_neq(token, nullptr); String lexeme = tokenizer_make_lexeme(tokenizer, start, end); Span span = tokenizer_make_span(tokenizer, start, end); - *token = Token(kind, lexeme, span); + return Token(kind, lexeme, span); } static Buffer* tokenizer_get_buffer(Tokenizer* tokenizer, usize* cursor) { @@ -74,14 +73,14 @@ static Buffer* tokenizer_get_buffer(Tokenizer* tokenizer, usize* cursor) { return true; } -static void tokenizer_lex_identifier(Tokenizer* tokenizer, usize* offset) { +static Token tokenizer_lex_identifier(Tokenizer* tokenizer, usize start, usize *offset) { assert_neq(tokenizer, nullptr); - assert_neq(offset, nullptr); wchar c; while (tokenizer_advance(tokenizer, offset, &c)) if (!utf8_is_alnum(c) || c == '_') break; + return tokenizer_make_token(tokenizer, Token_Kind_Identifier, start, *offset); } bool tokenizer_next(Tokenizer* tokenizer, Token* out) { @@ -98,14 +97,13 @@ bool tokenizer_next(Tokenizer* tokenizer, Token* out) { &c); // We just checked that we are not at the end if (utf8_is_identifier(c)) { - tokenizer_lex_identifier(tokenizer, &advance); - tokenizer_make_token(tokenizer, out, Token_Kind_Identifier, cursor, advance); + *out = tokenizer_lex_identifier(tokenizer, cursor, &advance); goto out; } switch (c) { default: - tokenizer_make_token(tokenizer, out, Token_Kind_Invalid_Char, cursor, + *out = tokenizer_make_token(tokenizer, Token_Kind_Invalid_Char, cursor, advance); } |