jacob/power_play
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add BOF json token type

Browse Source
This commit is contained in:
jacob 2024-04-03 23:12:50 -05:00
parent 7eb63bc80f
commit bfe64f7373
1 changed files with 23 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
src/json.c
View File

@ -32,6 +32,7 @@ enum token_type {
TOKEN_TYPE_NUMBER, TOKEN_TYPE_NUMBER,
TOKEN_TYPE_STRING, TOKEN_TYPE_STRING,
TOKEN_TYPE_TRUE, TOKEN_TYPE_TRUE,
TOKEN_TYPE_FALSE, TOKEN_TYPE_FALSE,
TOKEN_TYPE_NULL, TOKEN_TYPE_NULL,
@ -43,6 +44,7 @@ enum token_type {
TOKEN_TYPE_CURLY_BRACE_OPEN, TOKEN_TYPE_CURLY_BRACE_OPEN,
TOKEN_TYPE_CURLY_BRACE_CLOSE, TOKEN_TYPE_CURLY_BRACE_CLOSE,
TOKEN_TYPE_BOF,
TOKEN_TYPE_EOF TOKEN_TYPE_EOF
}; };
@ -76,10 +78,25 @@ GLOBAL READONLY enum token_type g_keyword_types[] = {
['n'] = TOKEN_TYPE_NULL ['n'] = TOKEN_TYPE_NULL
}; };
INTERNAL struct token *push_token(struct arena *arena, struct token_list *list)
{
struct token *t = arena_push_zero(arena, struct token);
if (!list->token_first) {
list->token_first = t;
} else {
list->token_last->next = t;
}
list->token_last = t;
return t;
}
INTERNAL struct token_list lex(struct arena *arena, struct string src) INTERNAL struct token_list lex(struct arena *arena, struct string src)
{ {
struct token_list res = { 0 }; struct token_list res = { 0 };
struct token *bof = push_token(arena, &res);
bof->type = TOKEN_TYPE_BOF;
u64 pos = 0; u64 pos = 0;
b32 lexing_done = false; b32 lexing_done = false;
while (!lexing_done) { while (!lexing_done) {
@ -99,19 +116,12 @@ INTERNAL struct token_list lex(struct arena *arena, struct string src)
} }
/* Create token */ /* Create token */
struct token *t = arena_push_zero(arena, struct token); struct token *t = push_token(arena, &res);
t->start = pos; t->start = pos;
/* Push token to list */
if (!res.token_first) {
res.token_first = t;
} else {
res.token_last->next = t;
}
res.token_last = t;
if (pos >= src.len) { if (pos >= src.len) {
t->type = TOKEN_TYPE_EOF; t->type = TOKEN_TYPE_EOF;
t->next = t; /* Self reference */
lexing_done = true; lexing_done = true;
} else { } else {
/* Lex known token types */ /* Lex known token types */
@ -661,6 +671,10 @@ INTERNAL void parse(struct arena *arena, struct parser *p)
struct token *at = p->at; struct token *at = p->at;
struct string src = p->src; struct string src = p->src;
if (at->type == TOKEN_TYPE_BOF) {
at = at->next;
}
/* Depth first stack */ /* Depth first stack */
*arena_push(scratch.arena, struct json *) = root; *arena_push(scratch.arena, struct json *) = root;
u64 stack_count = 1; u64 stack_count = 1;