From bfe64f7373c858acc37a8a6698afa4fc6b2ad7e0 Mon Sep 17 00:00:00 2001
From: jacob <jacob@cagori.com>
Date: Wed, 3 Apr 2024 23:12:50 -0500
Subject: [PATCH] add BOF json token type

---
 src/json.c | 32 +++++++++++++++++++++++---------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/src/json.c b/src/json.c
index e6a25abe..c501cb83 100644
--- a/src/json.c
+++ b/src/json.c
@@ -32,6 +32,7 @@ enum token_type {
 
     TOKEN_TYPE_NUMBER,
     TOKEN_TYPE_STRING,
+
     TOKEN_TYPE_TRUE,
     TOKEN_TYPE_FALSE,
     TOKEN_TYPE_NULL,
@@ -43,6 +44,7 @@ enum token_type {
     TOKEN_TYPE_CURLY_BRACE_OPEN,
     TOKEN_TYPE_CURLY_BRACE_CLOSE,
 
+    TOKEN_TYPE_BOF,
     TOKEN_TYPE_EOF
 };
 
@@ -76,10 +78,25 @@ GLOBAL READONLY enum token_type g_keyword_types[] = {
     ['n'] = TOKEN_TYPE_NULL
 };
 
+INTERNAL struct token *push_token(struct arena *arena, struct token_list *list)
+{
+    struct token *t = arena_push_zero(arena, struct token);
+    if (!list->token_first) {
+        list->token_first = t;
+    } else {
+        list->token_last->next = t;
+    }
+    list->token_last = t;
+    return t;
+}
+
 INTERNAL struct token_list lex(struct arena *arena, struct string src)
 {
     struct token_list res = { 0 };
 
+    struct token *bof = push_token(arena, &res);
+    bof->type = TOKEN_TYPE_BOF;
+
     u64 pos = 0;
     b32 lexing_done = false;
     while (!lexing_done) {
@@ -99,19 +116,12 @@ INTERNAL struct token_list lex(struct arena *arena, struct string src)
         }
 
         /* Create token */
-        struct token *t = arena_push_zero(arena, struct token);
+        struct token *t = push_token(arena, &res);
         t->start = pos;
 
-        /* Push token to list */
-        if (!res.token_first) {
-            res.token_first = t;
-        } else {
-            res.token_last->next = t;
-        }
-        res.token_last = t;
-
         if (pos >= src.len) {
             t->type = TOKEN_TYPE_EOF;
+            t->next = t;  /* Self reference */
             lexing_done = true;
         } else {
             /* Lex known token types */
@@ -661,6 +671,10 @@ INTERNAL void parse(struct arena *arena, struct parser *p)
     struct token *at = p->at;
     struct string src = p->src;
 
+    if (at->type == TOKEN_TYPE_BOF) {
+        at = at->next;
+    }
+
     /* Depth first stack */
     *arena_push(scratch.arena, struct json *) = root;
     u64 stack_count = 1;