working on identifiers

3 files changed, 44 insertions, 6 deletions

diff --git a/src/token.cc b/src/token.cc
index d3a787a..bac5a5a 100644
--- a/src/token.cc
+++ b/src/token.cc
@@ -8,7 +8,8 @@
 
 #define TOKEN_KINDS_NOLEX \
   X(Invalid_Char)         \
-  X(Invalid_Literal)
+  X(Invalid_Literal)      \
+  X(Identifier)
 
 #define TOKEN_KINDS_SLEX X(Hash, '#')
 
diff --git a/src/tokenizer.cc b/src/tokenizer.cc
index 26763eb..029d440 100644
--- a/src/tokenizer.cc
+++ b/src/tokenizer.cc
@@ -57,15 +57,31 @@ static Buffer* tokenizer_get_buffer(Tokenizer* tokenizer, usize* cursor) {
   return curr;
 }
 
-static wchar tokenizer_advance(const Tokenizer* tokenizer, usize* offset) {
+[[nodiscard]] static bool tokenizer_advance(const Tokenizer* tokenizer,
+                                            usize* offset, wchar* out) {
   const String text = tokenizer->buffer->content;
+  usize curr_offset = *offset;
 
-  unsigned char c = *text[*offset];
+  assert_ste(curr_offset, text.length);
+  if (curr_offset == text.length) return false;
+
+  unsigned char c = *text[curr_offset];
   u8 nobytes = utf8_nobytes(c);
   if (nobytes > 1) panic("no support for multi-byte chars: %c:%d", c, nobytes);
 
   *offset += nobytes;
-  return (wchar)c;
+  *out = c;
+  return true;
+}
+
+static void tokenizer_lex_identifier(Tokenizer* tokenizer, usize* offset) {
+  assert_neq(tokenizer, nullptr);
+  assert_neq(offset, nullptr);
+
+  wchar c;
+  while (tokenizer_advance(tokenizer, offset, &c))
+    if (!utf8_is_alnum(c) || c == '_') break;
+
 }
 
 bool tokenizer_next(Tokenizer* tokenizer, Token* out) {
@@ -75,9 +91,17 @@ bool tokenizer_next(Tokenizer* tokenizer, Token* out) {
   usize cursor;
   Buffer* buffer = tokenizer_get_buffer(tokenizer, &cursor);
   if (buffer == nullptr) return false;
-
   usize advance = cursor;
-  wchar c = tokenizer_advance(tokenizer, &advance);
+
+  wchar c;
+  (void)tokenizer_advance(tokenizer, &advance,
+                          &c);  // We just checked that we are not at the end
+
+  if (utf8_is_identifier(c)) {
+    tokenizer_lex_identifier(tokenizer, &advance);
+    tokenizer_make_token(tokenizer, out, Token_Kind_Identifier, cursor, advance);
+    goto out;
+  }
 
   switch (c) {
     default:
@@ -85,6 +109,7 @@ bool tokenizer_next(Tokenizer* tokenizer, Token* out) {
                            advance);
   }
 
+out:
   buffer->cursor = advance;
   return true;
 }
diff --git a/src/utf8.cc b/src/utf8.cc
index 255508e..e2da09d 100644
--- a/src/utf8.cc
+++ b/src/utf8.cc
@@ -22,4 +22,16 @@ inline u8 utf8_nobytes(unsigned char c) {
   panic("what even is: %d\n", c);
 }
 
+inline bool utf8_is_identifier(wchar c) {
+  return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z');
+}
+
+inline bool utf8_is_number(wchar c) {
+  return ('0' <= c && c <= '9');
+}
+
+inline bool utf8_is_alnum(wchar c) {
+  return utf8_is_identifier(c) || utf8_is_number(c);
+}
+
 #endif