not good

1 files changed, 16 insertions, 19 deletions

diff --git a/src/tokenizer.cc b/src/tokenizer.cc
index 1432961..3451f69 100644
--- a/src/tokenizer.cc
+++ b/src/tokenizer.cc
@@ -41,16 +41,20 @@ static inline void tokenizer_make_token(const Tokenizer* tokenizer,
   *token = Token(kind, lexeme, span);
 }
 
-static Buffer* tokenizer_get_buffer(Tokenizer* tokenizer) {
+static Buffer* tokenizer_get_buffer(Tokenizer* tokenizer, usize* cursor) {
   assert_neq(tokenizer, nullptr);
 
-  if (tokenizer->buffer != nullptr) return tokenizer->buffer;
+  Buffer* curr = tokenizer->buffer;
+  if (likely(curr != nullptr)) {
+    *cursor = curr->cursor;
+    if(*cursor < curr->content.length) return curr;
+  }
 
-  Buffer* buffer = nullptr;
-  if (!buffer_stack_pop(tokenizer->stack, &buffer)) return nullptr;
+  if (!buffer_stack_pop(tokenizer->stack, &curr)) return nullptr;
 
-  tokenizer->buffer = buffer;
-  return buffer;
+  tokenizer->buffer = curr;
+  *cursor = curr->cursor;
+  return curr;
 }
 
 static char tokenizer_advance(const Tokenizer* tokenizer,
@@ -69,24 +73,17 @@ bool tokenizer_next(Tokenizer* tokenizer, Token* out) {
   assert_neq(tokenizer, nullptr);
   assert_neq(out, nullptr);
 
-again:
-  Buffer* buffer = tokenizer_get_buffer(tokenizer);
+  usize cursor, advance;
+  Buffer* buffer = tokenizer_get_buffer(tokenizer, &cursor);
   if (buffer == nullptr) return false;
 
-  usize start = buffer->cursor;
-  if (start == buffer->content.length) {
-    tokenizer->buffer = nullptr;
-    goto again;
-  }
-
-  usize offset = start;
-  tokenizer_advance(tokenizer, &offset);
-
+  tokenizer_advance(tokenizer, &advance);
+  
   Token token = {};
-  tokenizer_make_token(tokenizer, &token, Token_Kind_Eof, start, offset);
+  tokenizer_make_token(tokenizer, &token, Token_Kind_Eof, cursor, advance);
 
   *out = token;
-  buffer->cursor = offset;
+  buffer->cursor = advance;
   return true;
 }