1 files changed, 17 insertions, 8 deletions

diff --git a/src/tokenizer.cc b/src/tokenizer.cc
index 274f572..a1a365c 100644
--- a/src/tokenizer.cc
+++ b/src/tokenizer.cc
@@ -7,13 +7,13 @@
 #include "token.cc"
 
 struct Tokenizer {
-  const Buffer* buffer;
+  Buffer* buffer;
   Buffer_Stack* stack;
 
   Tokenizer(Buffer_Stack* stack) : buffer(nullptr), stack(stack) {}
 };
 
-static inline const Buffer* tokenizer_get_buffer(Tokenizer* tokenizer) {
+static inline Buffer* tokenizer_get_buffer(Tokenizer* tokenizer) {
   assert_neq(tokenizer, nullptr);
 
   if(tokenizer->buffer != nullptr) return tokenizer->buffer;
@@ -36,24 +36,33 @@ static inline char tokenizer_advance(const Tokenizer* tokenizer, usize* offset)
   return c;
 }
 
+static inline String tokenizer_make_lexeme(const Tokenizer* tokenizer, usize start, usize end) {
+  assert_neq(tokenizer, nullptr);
+
+  Buffer* buffer = tokenizer->buffer;
+  return String(buffer->content[start], end - start);
+}
+
 bool tokenizer_next(Tokenizer* tokenizer, Token* out) {
   assert_neq(tokenizer, nullptr);
   assert_neq(out, nullptr);
 
 again:
-  const Buffer* buffer = tokenizer_get_buffer(tokenizer);
+  Buffer* buffer = tokenizer_get_buffer(tokenizer);
   if(buffer == nullptr) return false;
 
-  usize offset = buffer->cursor;
-  if(offset == buffer->content.length) {
+  usize start = buffer->cursor;
+  if(start == buffer->content.length) {
     tokenizer->buffer = nullptr;
     goto again;
   }
-
+  
+  usize offset = start;
   tokenizer_advance(tokenizer, &offset);
 
-  const unsigned char* str = buffer->content[offset];  
-  *out = Token(Token_Kind_Eof, String(str, 1), Span(buffer->file, 0, 0));
+  String lexeme = tokenizer_make_lexeme(tokenizer, start, offset);
+  *out = Token(Token_Kind_Eof, lexeme, Span(buffer->file, 0, 0));
+  buffer->cursor = offset;
   return true;
 }