formatting of tokens

7 files changed, 86 insertions, 43 deletions

diff --git a/src/common.cc b/src/common.cc
index de2f6de..5e84c01 100644
--- a/src/common.cc
+++ b/src/common.cc
@@ -26,7 +26,7 @@ typedef intptr_t isize;
 #define likely(cond) __builtin_expect(!!(cond), 1)
 #define unlikely(cond) __builtin_expect(!!(cond), 0)
 #define containerof(type, member, ptr) \
-    ((type *)((char *)(ptr) - offsetof(type, member)))
+  ((type*)((char*)(ptr) - offsetof(type, member)))
 
 /* error handling sort of */
 [[noreturn]] void panic_impl(const char* file, i32 line, const char* fmt, ...) {
@@ -72,25 +72,24 @@ struct Slice {
   }
 
   const T* operator[](usize index) const {
-    assert(index < this->size());
+    assert(index < this->total_length());
     return this->ptr + index;
   }
 
-  inline usize size() const { return this->length * sizeof(T); }
+  inline usize total_length() const { return this->length * sizeof(T); }
 };
 
-template <typename T>
-bool slice_write(const Slice<T>* slice, FILE* stream) {
-  assert_neq(slice, nullptr);
+typedef Slice<const u8> String;
+
+bool string_write(const String* s, FILE* stream) {
+  assert_neq(s, nullptr);
   assert_neq(stream, nullptr);
 
-  usize rc = fwrite(slice->ptr, sizeof(T), slice->length, stream);
-  if (rc == 0 || slice->size() > rc) return false;
+  i32 rc = fprintf(stream, "%.*s", (i32)s->length, s->ptr);
+  if (unlikely(rc < 0)) return false;
   return true;
 }
 
-typedef Slice<const u8> String;
-
 /* linked list */
 
 struct Link {
diff --git a/src/memory.cc b/src/memory.cc
index 22977d1..720c100 100644
--- a/src/memory.cc
+++ b/src/memory.cc
@@ -16,7 +16,7 @@ struct Allocator {
   Allocator_Deallocate deallocate;
 };
 
-template<typename T>
+template <typename T>
 T* allocate(const Allocator* allocator, usize n) {
   return (T*)allocator->allocate(allocator->self, sizeof(T) * n, alignof(T));
 }
@@ -48,7 +48,7 @@ const Allocator* heap_allocator() {
 template <typename T>
 bool slice_copy(const Allocator* allocator, const Slice<T>* source,
                 Slice<T>* out) {
-  T* new_ptr = allocate(allocator, source->size(), alignof(T));
+  T* new_ptr = allocate(allocator, source->total_length(), alignof(T));
   if (new_ptr == nullptr) return false;
 
   memcpy((u8*)new_ptr, (u8*)source->ptr, source->length);
diff --git a/src/source.cc b/src/source.cc
index a393213..bf00afc 100644
--- a/src/source.cc
+++ b/src/source.cc
@@ -1,19 +1,35 @@
 #ifndef SOURCE_CC
 #define SOURCE_CC
 
+#include <cstdio>
 #include "common.cc"
 #include "memory.cc"
 
-typedef u32 Source_Id;
-
 struct Span {
   String file;
   usize start, end;
 
   Span() : file(), start(0), end(0) {}
-  Span(String file, usize start, usize end) : file(file), start(start), end(end) {}
+  Span(String file, usize start, usize end)
+      : file(file), start(start), end(end) {}
 };
 
+bool span_write(const Span* span, FILE* stream) {
+  assert_neq(span, nullptr);
+  assert_neq(stream, nullptr);
+
+  i32 rc = fprintf(stream, "Span { file: ");
+  if(unlikely(rc < 0)) return false;
+  
+  bool status = string_write(&span->file, stream);
+  if(unlikely(!status)) return false;
+
+  rc = fprintf(stream, ", start: %zu, end: %zu }", span->start, span->end);
+  if(unlikely(rc < 0)) return false;
+
+  return true;
+}
+
 struct Buffer {
   String file;
   String content;
@@ -22,15 +38,17 @@ struct Buffer {
   usize cursor;
 };
 
-bool buffer_init(const Allocator* allocator,
-                 const String* content_in, const String* file_in, Buffer** out) {
+bool buffer_init(const Allocator* allocator, const String* content_in,
+                 const String* file_in, Buffer** out) {
   assert_neq(allocator, nullptr);
   assert_neq(content_in, nullptr);
   assert_neq(file_in, nullptr);
   assert_neq(out, nullptr);
 
+  //TODO: use defer!!
+
   Buffer* buffer = allocate<Buffer>(allocator, 1);
-  if(buffer == nullptr) return false;
+  if (buffer == nullptr) return false;
   memset(buffer, 0, sizeof(*buffer));
 
   String content, file;
@@ -59,8 +77,8 @@ void buffer_deinit(Buffer* buffer) {
   assert_neq(buffer, nullptr);
 
   const Allocator* allocator = buffer->allocator;
-  if(allocator == nullptr) return;
-  
+  if (allocator == nullptr) return;
+
   slice_deallocate(buffer->allocator, &buffer->file);
   slice_deallocate(buffer->allocator, &buffer->content);
 
diff --git a/src/token.cc b/src/token.cc
index 7f38f71..ddc0a68 100644
--- a/src/token.cc
+++ b/src/token.cc
@@ -1,8 +1,9 @@
 #ifndef TOKEN_CC
 #define TOKEN_CC
 
-#include "source.cc"
+#include <cstdio>
 #include "common.cc"
+#include "source.cc"
 
 #define TOKEN_KINDS_NOLEX \
   X(Eof)                  \
@@ -13,7 +14,7 @@
 
 #define TOKEN_KIND(name) Token_Kind_##name
 
-enum Token_Kind {
+enum Token_Kind: u8 {
 #define X(name) TOKEN_KIND(name),
   TOKEN_KINDS_NOLEX
 #undef X
@@ -29,7 +30,30 @@ struct Token {
   Span span;
 
   Token() : kind(Token_Kind_Eof), text(), span() {}
-  Token(Token_Kind kind, String text, Span span) : kind(kind), text(text), span(span) {}
+  Token(Token_Kind kind, String text, Span span)
+      : kind(kind), text(text), span(span) {}
 };
 
+bool token_write(const Token* token, FILE* stream) {
+  assert_neq(token, nullptr);
+  assert_neq(stream, nullptr);
+
+  i32 rc = fprintf(stream, "Token { kind: %d, text: ", token->kind);
+  if(unlikely(rc < 0)) return false;
+
+  bool status = string_write(&token->text, stream);
+  if(unlikely(!status)) return false;
+
+  rc = fprintf(stream, ", span: ");
+  if(unlikely(rc < 0)) return false;
+
+  status = span_write(&token->span, stream);
+  if(unlikely(!status)) return false;
+
+  rc = fprintf(stream, " }");
+  if(unlikely(rc < 0)) return false;
+
+  return true;
+}
+
 #endif
diff --git a/src/tokenizer.cc b/src/tokenizer.cc
index a1a365c..cf8a5fa 100644
--- a/src/tokenizer.cc
+++ b/src/tokenizer.cc
@@ -3,8 +3,8 @@
 
 #include "common.cc"
 #include "source.cc"
-#include "utf8.cc"
 #include "token.cc"
+#include "utf8.cc"
 
 struct Tokenizer {
   Buffer* buffer;
@@ -16,27 +16,29 @@ struct Tokenizer {
 static inline Buffer* tokenizer_get_buffer(Tokenizer* tokenizer) {
   assert_neq(tokenizer, nullptr);
 
-  if(tokenizer->buffer != nullptr) return tokenizer->buffer;
+  if (tokenizer->buffer != nullptr) return tokenizer->buffer;
 
   Buffer* buffer = nullptr;
-  if(!buffer_stack_pop(tokenizer->stack, &buffer)) return nullptr;
+  if (!buffer_stack_pop(tokenizer->stack, &buffer)) return nullptr;
 
   tokenizer->buffer = buffer;
   return buffer;
 }
 
-static inline char tokenizer_advance(const Tokenizer* tokenizer, usize* offset) {
+static inline char tokenizer_advance(const Tokenizer* tokenizer,
+                                     usize* offset) {
   const String text = tokenizer->buffer->content;
 
-  unsigned char c = *text[*offset];  
+  unsigned char c = *text[*offset];
   u8 nobytes = utf8_nobytes(c);
-  if(nobytes > 1) panic("no support for multi-byte chars: %c:%d", c, nobytes);
+  if (nobytes > 1) panic("no support for multi-byte chars: %c:%d", c, nobytes);
 
-  offset += nobytes;
+  *offset += nobytes;
   return c;
 }
 
-static inline String tokenizer_make_lexeme(const Tokenizer* tokenizer, usize start, usize end) {
+static inline String tokenizer_make_lexeme(const Tokenizer* tokenizer,
+                                           usize start, usize end) {
   assert_neq(tokenizer, nullptr);
 
   Buffer* buffer = tokenizer->buffer;
@@ -49,14 +51,14 @@ bool tokenizer_next(Tokenizer* tokenizer, Token* out) {
 
 again:
   Buffer* buffer = tokenizer_get_buffer(tokenizer);
-  if(buffer == nullptr) return false;
+  if (buffer == nullptr) return false;
 
   usize start = buffer->cursor;
-  if(start == buffer->content.length) {
+  if (start == buffer->content.length) {
     tokenizer->buffer = nullptr;
     goto again;
   }
-  
+
   usize offset = start;
   tokenizer_advance(tokenizer, &offset);
 
diff --git a/src/utf8.cc b/src/utf8.cc
index 7360812..255508e 100644
--- a/src/utf8.cc
+++ b/src/utf8.cc
@@ -15,10 +15,10 @@ typedef i32 wchar;
 #define UTF8_4BYTE 0x1E
 
 inline u8 utf8_nobytes(unsigned char c) {
-  if(c < UTF8_1BYTE) return 1;
-  if((c >> UTF8_2SHIFT) == UTF8_2BYTE) return 2;
-  if((c >> UTF8_3SHIFT) == UTF8_3BYTE) return 3;
-  if((c >> UTF8_4SHIFT) == UTF8_4BYTE) return 4;
+  if (c < UTF8_1BYTE) return 1;
+  if ((c >> UTF8_2SHIFT) == UTF8_2BYTE) return 2;
+  if ((c >> UTF8_3SHIFT) == UTF8_3BYTE) return 3;
+  if ((c >> UTF8_4SHIFT) == UTF8_4BYTE) return 4;
   panic("what even is: %d\n", c);
 }
 
diff --git a/src/voidc.cc b/src/voidc.cc
index bba510f..282663e 100644
--- a/src/voidc.cc
+++ b/src/voidc.cc
@@ -1,3 +1,4 @@
+#include <unistd.h>
 #include <cstdio>
 #include <cstdlib>
 
@@ -12,7 +13,6 @@ static const char* SOURCE = R"(
 int main() {
   return EXIT_FAILURE;
 }
-
 )";
 
 static const char* SOURCE_FILE = "source.c";
@@ -32,8 +32,8 @@ int main() {
   Tokenizer tokenizer(&stack);
 
   Token token = {};
-  while(tokenizer_next(&tokenizer, &token)) 
-    fprintf(stdout, "Token(kind: %d, text: %.*s)\n", token.kind, (int)token.text.length, token.text.ptr);
-  
-
+  while (tokenizer_next(&tokenizer, &token)) {
+    token_write(&token, stdout);
+    fprintf(stdout, "\n");
+  }
 }