1 files changed, 10 insertions, 8 deletions
diff --git a/src/tokenizer.cc b/src/tokenizer.cc
index f2efda4..26763eb 100644
--- a/src/tokenizer.cc
+++ b/src/tokenizer.cc
@@ -47,7 +47,7 @@ static Buffer* tokenizer_get_buffer(Tokenizer* tokenizer, usize* cursor) {
   Buffer* curr = tokenizer->buffer;
   if (likely(curr != nullptr)) {
     *cursor = curr->cursor;
-    if(*cursor < curr->content.length) return curr;
+    if (*cursor < curr->content.length) return curr;
   }
 
   if (!buffer_stack_pop(tokenizer->stack, &curr)) return nullptr;
@@ -57,8 +57,7 @@ static Buffer* tokenizer_get_buffer(Tokenizer* tokenizer, usize* cursor) {
   return curr;
 }
 
-static char tokenizer_advance(const Tokenizer* tokenizer,
-                                     usize* offset) {
+static wchar tokenizer_advance(const Tokenizer* tokenizer, usize* offset) {
   const String text = tokenizer->buffer->content;
 
   unsigned char c = *text[*offset];
@@ -66,7 +65,7 @@ static char tokenizer_advance(const Tokenizer* tokenizer,
   if (nobytes > 1) panic("no support for multi-byte chars: %c:%d", c, nobytes);
 
   *offset += nobytes;
-  return c;
+  return (wchar)c;
 }
 
 bool tokenizer_next(Tokenizer* tokenizer, Token* out) {
@@ -76,13 +75,16 @@ bool tokenizer_next(Tokenizer* tokenizer, Token* out) {
   usize cursor;
   Buffer* buffer = tokenizer_get_buffer(tokenizer, &cursor);
   if (buffer == nullptr) return false;
+
   usize advance = cursor;
+  wchar c = tokenizer_advance(tokenizer, &advance);
 
-  tokenizer_advance(tokenizer, &advance);
-  Token token = {};
-  tokenizer_make_token(tokenizer, &token, Token_Kind_Eof, cursor, advance);
+  switch (c) {
+    default:
+      tokenizer_make_token(tokenizer, out, Token_Kind_Invalid_Char, cursor,
+                           advance);
+  }
 
-  *out = token;
   buffer->cursor = advance;
   return true;
 }