string handling

1 files changed, 22 insertions, 0 deletions

diff --git a/src/token.cc b/src/token.cc
new file mode 100644
index 0000000..8217138
--- /dev/null
+++ b/src/token.cc
@@ -0,0 +1,22 @@
+#ifndef TOKEN_CC
+#define TOKEN_CC
+
+#include "common.cc"
+
+#define TOKEN_KINDS_NOLEX \
+  X(eof) \
+  X(invalid_char) \
+  X(invalid_literal)
+
+enum token_kind {
+#define X(name) token_kind_##name,
+  TOKEN_KINDS_NOLEX
+#undef X
+};
+
+struct token {
+  token_kind kind;
+  string text;
+};
+
+#endif