From 3f2abf5b3edd1171d4d98efb00fe161894c48cda Mon Sep 17 00:00:00 2001 From: jacob Date: Wed, 30 Jul 2025 23:31:46 -0500 Subject: [PATCH] tar layer refactor --- src/gp/gp_core_dx12.c | 10 +-- src/resource/resource_core.c | 4 +- src/resource/resource_core.h | 2 +- src/tar/tar_core.c | 115 +++++++++++++---------------------- src/tar/tar_core.h | 71 ++++++++++++++++++--- 5 files changed, 114 insertions(+), 88 deletions(-) diff --git a/src/gp/gp_core_dx12.c b/src/gp/gp_core_dx12.c index 5b5b9a7b..b86fa37f 100644 --- a/src/gp/gp_core_dx12.c +++ b/src/gp/gp_core_dx12.c @@ -335,7 +335,7 @@ Global struct { struct swapchain *first_free_swapchain; /* Shader bytecode archive */ - struct tar_archive dxc_archive; + TAR_Archive dxc_archive; /* Pipeline cache */ P_Mutex pipelines_mutex; @@ -419,7 +419,7 @@ void gp_startup(void) if (embedded_data.len <= 0) { P_Panic(Lit("No embedded shaders found")); } - G.dxc_archive = tar_parse(G.pipelines_arena, embedded_data, Lit("")); + G.dxc_archive = TAR_ArchiveFromString(G.pipelines_arena, embedded_data, Lit("")); /* Initialize dx12 */ /* TODO: Parallelize phases */ @@ -956,9 +956,9 @@ internal P_JobDef(pipeline_alloc_job, job) String error_str = ZI; - String vs_dxc = desc->vs_dxc.len > 0 ? desc->vs_dxc : tar_get(&G.dxc_archive, CatString(scratch.arena, pipeline_name, Lit(".vs")))->data; - String ps_dxc = desc->ps_dxc.len > 0 ? desc->ps_dxc : tar_get(&G.dxc_archive, CatString(scratch.arena, pipeline_name, Lit(".ps")))->data; - String cs_dxc = desc->cs_dxc.len > 0 ? desc->cs_dxc : tar_get(&G.dxc_archive, CatString(scratch.arena, pipeline_name, Lit(".cs")))->data; + String vs_dxc = desc->vs_dxc.len > 0 ? desc->vs_dxc : TAR_EntryFromName(&G.dxc_archive, CatString(scratch.arena, pipeline_name, Lit(".vs")))->data; + String ps_dxc = desc->ps_dxc.len > 0 ? desc->ps_dxc : TAR_EntryFromName(&G.dxc_archive, CatString(scratch.arena, pipeline_name, Lit(".ps")))->data; + String cs_dxc = desc->cs_dxc.len > 0 ? desc->cs_dxc : TAR_EntryFromName(&G.dxc_archive, CatString(scratch.arena, pipeline_name, Lit(".cs")))->data; if (success && vs_dxc.len > 0 && ps_dxc.len <= 0) { error_str = Lit("Pipeline has vertex shader without pixel shader"); success = 0; diff --git a/src/resource/resource_core.c b/src/resource/resource_core.c index a7d04749..95667c81 100644 --- a/src/resource/resource_core.c +++ b/src/resource/resource_core.c @@ -15,7 +15,7 @@ R_StartupReceipt R_Startup(void) { P_Panic(Lit("No embedded resources found")); } - g->archive = tar_parse(g->arena, embedded_data, Lit("")); + g->archive = TAR_ArchiveFromString(g->arena, embedded_data, Lit("")); #else /* Ensure we have the right working directory */ if (!P_IsDir(Lit("res"))) @@ -36,7 +36,7 @@ R_Resource R_OpenResource(String name) #if RESOURCES_EMBEDDED R_SharedState *g = &R_shared_state; R_Resource result = ZI; - struct tar_entry *entry = tar_get(&g->archive, name); + TAR_Entry *entry = TAR_EntryFromName(&g->archive, name); result._data = entry->data; result._name = entry->file_name; result._exists = entry->valid; diff --git a/src/resource/resource_core.h b/src/resource/resource_core.h index 3211f8ca..fc836bad 100644 --- a/src/resource/resource_core.h +++ b/src/resource/resource_core.h @@ -24,7 +24,7 @@ Struct(R_SharedState) { Arena *arena; #if RESOURCES_EMBEDDED - struct tar_archive archive; + TAR_Archive archive; #endif }; diff --git a/src/tar/tar_core.c b/src/tar/tar_core.c index c05970b0..bb98cfcd 100644 --- a/src/tar/tar_core.c +++ b/src/tar/tar_core.c @@ -1,53 +1,10 @@ -#define ARCHIVE_LOOKUP_TABLE_CAPACITY_FACTOR 2.0 +Readonly TAR_Entry TAR_nil_entry = ZI; -/* File types: - * '0' or (ASCII NUL) Normal file - * '1' Hard link - * '2' Symbolic link - * '3' Character special - * '4' Block special - * '5' Directory - * '6' FIFO - * '7' Contiguous file - * 'g' Global extended header with meta data(POSIX.1 - 2001) - * 'x' Extended header with metadata for the next file in the archive(POSIX.1 - 2001) - * 'A'-'Z' Vendor specific extensions(POSIX.1 - 1988) - */ - -#define TAR_TYPE_FILE '0' -#define TAR_TYPE_DIRECTORY '5' -#define TAR_TYPE_PAX_HEADER_X 'x' -#define TAR_TYPE_PAX_HEADER_G 'g' - -Packed(struct tar_header { - /* Pre-posix */ - u8 file_name[100]; - u8 file_mode[8]; - u8 owner_id[8]; - u8 group_id[8]; - u8 file_size[12]; - u8 last_modified[12]; - u8 checksum[8]; - - /* Both */ - u8 file_type; - u8 linked_file_name[100]; - - /* UStar */ - u8 ustar_indicator[6]; - u8 ustar_version[2]; - u8 owner_user_name[32]; - u8 owner_group_name[32]; - u8 device_major_number[8]; - u8 device_minor_number[8]; - u8 file_name_prefix[155]; - u8 padding[12]; -}); - -internal u64 str_oct_to_u64(String str) +u64 TAR_U64FromOctString(String str) { u64 n = 0; - for (u64 i = 0; i < str.len; ++i) { + for (u64 i = 0; i < str.len; ++i) + { n *= 8; n += (u64)(str.text[i]) - '0'; } @@ -59,27 +16,30 @@ internal u64 str_oct_to_u64(String str) * NOTE: The resulting archive merely points into the supplied tar data, no * copying is done. Accessing the archive assumes that the data string is still valid. */ -struct tar_archive tar_parse(Arena *arena, String data, String prefix) +TAR_Archive TAR_ArchiveFromString(Arena *arena, String data, String prefix) { __prof; - struct tar_archive archive = ZI; + TAR_Archive archive = ZI; BB_Buff bb = BitbuffFromString(data); BB_Reader br = BB_ReaderFromBuffNoDebug(&bb); u64 num_files = 0; - while (BB_NumBytesRemaining(&br) > 1024) { + while (BB_NumBytesRemaining(&br) > 1024) + { - struct tar_header header = ZI; + TAR_Header header = ZI; BB_ReadBytes(&br, StringFromStruct(&header)); - if (!EqString(StringFromArray(header.ustar_indicator), Lit("ustar\0"))) { + if (!EqString(StringFromArray(header.ustar_indicator), Lit("ustar\0"))) + { /* Invalid header */ Assert(0); continue; } - if (header.file_name_prefix[0] != 0) { + if (header.file_name_prefix[0] != 0) + { /* Header file name prefix not supported */ Assert(0); continue; @@ -87,9 +47,10 @@ struct tar_archive tar_parse(Arena *arena, String data, String prefix) String file_size_oct_str = { .len = 11, .text = header.file_size }; - u64 file_size = str_oct_to_u64(file_size_oct_str); + u64 file_size = TAR_U64FromOctString(file_size_oct_str); u8 *file_data_ptr = BB_ReadBytesRaw(&br, file_size); - if (!file_data_ptr) { + if (!file_data_ptr) + { file_size = 0; } String file_data = STRING(file_size, file_data_ptr); @@ -98,23 +59,25 @@ struct tar_archive tar_parse(Arena *arena, String data, String prefix) u64 remaining = (512 - (file_size % 512)) % 512; BB_SeekBytes(&br, remaining); - b32 is_dir = header.file_type == TAR_TYPE_DIRECTORY; - if (!is_dir && header.file_type != TAR_TYPE_FILE) { + b32 is_dir = header.file_type == TAR_FileKind_Directory; + if (!is_dir && header.file_type != TAR_FileKind_File) + { /* Unsupported type */ - Assert(header.file_type == TAR_TYPE_PAX_HEADER_X || - header.file_type == TAR_TYPE_PAX_HEADER_G); + Assert(header.file_type == TAR_FileKind_PaxHeaderX || + header.file_type == TAR_FileKind_PaxHeaderG); continue; } String file_name_cstr = StringFromCstrNoLimit((char *)header.file_name); - if (file_name_cstr.len >= 2) { + if (file_name_cstr.len >= 2) + { /* Chop off './' prefix */ file_name_cstr.len -= 2; file_name_cstr.text += 2; } String file_name = CatString(arena, prefix, file_name_cstr); - struct tar_entry *entry = PushStruct(arena, struct tar_entry); + TAR_Entry *entry = PushStruct(arena, TAR_Entry); entry->valid = 1; entry->is_dir = is_dir; entry->file_name = file_name; @@ -126,8 +89,9 @@ struct tar_archive tar_parse(Arena *arena, String data, String prefix) } /* Build lookup table */ - archive.lookup = InitDict(arena, (u64)((f64)num_files * ARCHIVE_LOOKUP_TABLE_CAPACITY_FACTOR)); - for (struct tar_entry *entry = archive.head; entry; entry = entry->next) { + archive.lookup = InitDict(arena, (u64)((f64)num_files * TAR_ArchiveLookupTableCapacityFactor)); + for (TAR_Entry *entry = archive.head; entry; entry = entry->next) + { u64 hash = HashFnv64(Fnv64Basis, entry->file_name); SetDictValue(arena, archive.lookup, hash, (u64)entry); } @@ -135,20 +99,25 @@ struct tar_archive tar_parse(Arena *arena, String data, String prefix) /* Build hierarchy */ /* NOTE: This is a separate pass because tar entry order is not guaranteed * (IE file entries may be encountered before their parent directory entry) */ - for (struct tar_entry *entry = archive.head; entry; entry = entry->next) { + for (TAR_Entry *entry = archive.head; entry; entry = entry->next) + { /* Enter into hierarchy */ - if (!entry->is_dir) { + if (!entry->is_dir) + { /* Find parent entry */ - struct tar_entry *parent_entry = 0; - for (String parent_dir_name = entry->file_name; parent_dir_name.len > 0; --parent_dir_name.len) { - if (parent_dir_name.text[parent_dir_name.len - 1] == '/') { + TAR_Entry *parent_entry = 0; + for (String parent_dir_name = entry->file_name; parent_dir_name.len > 0; --parent_dir_name.len) + { + if (parent_dir_name.text[parent_dir_name.len - 1] == '/') + { u64 hash = HashFnv64(Fnv64Basis, parent_dir_name); - parent_entry = (struct tar_entry *)DictValueFromHash(archive.lookup, hash); + parent_entry = (TAR_Entry *)DictValueFromHash(archive.lookup, hash); break; } } /* Insert child into parent's list */ - if (parent_entry) { + if (parent_entry) + { entry->next_child = parent_entry->next_child; parent_entry->next_child = entry; } @@ -158,10 +127,10 @@ struct tar_archive tar_parse(Arena *arena, String data, String prefix) return archive; } -Readonly Global struct tar_entry g_nil_tar_entry = ZI; -struct tar_entry *tar_get(struct tar_archive *archive, String name) +Readonly Global TAR_Entry g_nil_tar_entry = ZI; +TAR_Entry *TAR_EntryFromName(TAR_Archive *archive, String name) { u64 hash = HashFnv64(Fnv64Basis, name); - struct tar_entry *lookup = (struct tar_entry *)DictValueFromHash(archive->lookup, hash); + TAR_Entry *lookup = (TAR_Entry *)DictValueFromHash(archive->lookup, hash); return lookup ? lookup : &g_nil_tar_entry; } diff --git a/src/tar/tar_core.h b/src/tar/tar_core.h index af57a512..c2edd414 100644 --- a/src/tar/tar_core.h +++ b/src/tar/tar_core.h @@ -1,17 +1,74 @@ -struct tar_entry { +//////////////////////////////// +//~ Archive types + +#define TAR_ArchiveLookupTableCapacityFactor 2.0 + +Struct(TAR_Entry) +{ b32 valid; String file_name; String data; b32 is_dir; - struct tar_entry *next; - struct tar_entry *next_child; /* If entry is dir, points to first child. Otherwise points to next sibling. */ + TAR_Entry *next; + TAR_Entry *next_child; /* If entry is dir, points to first child. Otherwise points to next sibling. */ }; -struct tar_archive { +Struct(TAR_Archive) +{ Dict *lookup; - struct tar_entry *head; + TAR_Entry *head; }; -struct tar_archive tar_parse(Arena *arena, String data, String prefix); -struct tar_entry *tar_get(struct tar_archive *archive, String name); +extern Readonly TAR_Entry TAR_nil_entry; + +//////////////////////////////// +//~ Header types + +typedef u8 TAR_FileKind; enum +{ + TAR_FileKind_File = '0', + TAR_FileKind_HardLink = '1', + TAR_FileKind_SymLink = '2', + TAR_FileKind_CharacterSpecial = '3', + TAR_FileKind_BlockSpecial = '4', + TAR_FileKind_Directory = '5', + TAR_FileKind_Fifo = '6', + TAR_FileKind_ContiguousFile = '7', + TAR_FileKind_PaxHeaderG = 'g', + TAR_FileKind_PaxHeaderX = 'x' +}; + +Packed(Struct(TAR_Header) +{ + /* Pre-posix */ + u8 file_name[100]; + u8 file_mode[8]; + u8 owner_id[8]; + u8 group_id[8]; + u8 file_size[12]; + u8 last_modified[12]; + u8 checksum[8]; + + /* Both */ + u8 file_type; + u8 linked_file_name[100]; + + /* UStar */ + u8 ustar_indicator[6]; + u8 ustar_version[2]; + u8 owner_user_name[32]; + u8 owner_group_name[32]; + u8 device_major_number[8]; + u8 device_minor_number[8]; + u8 file_name_prefix[155]; + u8 padding[12]; +}); + +//////////////////////////////// +//~ Archive operations + +u64 TAR_U64FromOctString(String str); +TAR_Archive TAR_ArchiveFromString(Arena *arena, String data, String prefix); + +TAR_Entry *TAR_EntryFromName(TAR_Archive *archive, String name);