From a8651f7aa7e7c4eda39f40ade14de64ba610396a Mon Sep 17 00:00:00 2001 From: jacob Date: Fri, 25 Jul 2025 13:43:47 -0500 Subject: [PATCH] upgrade from fxc to dxc --- build.c | 180 ++++- src/app.c | 2 + src/arena.c | 4 +- src/common.h | 24 +- src/config.h | 2 +- src/dxc.cpp | 69 ++ src/dxc.h | 12 + src/gp_dx12.c | 745 +++++++----------- src/inc.c | 7 + src/inc.h | 2 + src/incbin.c | 2 +- src/playback_wasapi.c | 4 + src/prof_tracy.h | 4 +- src/resource.c | 197 +---- src/resource.h | 15 +- src/sh/blit.d | 3 + res/sh/blit.hlsl => src/sh/blit.hlsl_rs | 4 +- {res => src}/sh/common.hlsl | 2 +- src/sh/flood.d | 3 + res/sh/flood.hlsl => src/sh/flood.hlsl_cs | 2 +- src/sh/material.d | 3 + .../material.hlsl => src/sh/material.hlsl_rs | 2 +- {res => src}/sh/sh_common.h | 6 +- src/sh/shade.d | 3 + res/sh/shade.hlsl => src/sh/shade.hlsl_cs | 3 +- src/sh/shape.d | 3 + res/sh/shape.hlsl => src/sh/shape.hlsl_rs | 2 +- src/sh/ui.d | 3 + res/sh/ui.hlsl => src/sh/ui.hlsl_rs | 2 +- src/sprite.c | 12 +- src/string.c | 60 +- src/sys.h | 5 +- src/sys_win32.c | 30 +- src/tar.c | 5 +- src/tar.h | 1 + src/util.h | 2 +- src/watch.c | 245 ++++++ src/watch.h | 15 + 38 files changed, 913 insertions(+), 772 deletions(-) create mode 100644 src/dxc.cpp create mode 100644 src/dxc.h create mode 100644 src/sh/blit.d rename res/sh/blit.hlsl => src/sh/blit.hlsl_rs (97%) rename {res => src}/sh/common.hlsl (96%) create mode 100644 src/sh/flood.d rename res/sh/flood.hlsl => src/sh/flood.hlsl_cs (99%) create mode 100644 src/sh/material.d rename res/sh/material.hlsl => src/sh/material.hlsl_rs (99%) rename {res => src}/sh/sh_common.h (94%) create mode 100644 src/sh/shade.d rename res/sh/shade.hlsl => src/sh/shade.hlsl_cs (99%) create mode 100644 src/sh/shape.d rename res/sh/shape.hlsl => src/sh/shape.hlsl_rs (97%) create mode 100644 src/sh/ui.d rename res/sh/ui.hlsl => src/sh/ui.hlsl_rs (98%) create mode 100644 src/watch.c create mode 100644 src/watch.h diff --git a/build.c b/build.c index 2cd3afe9..e71793cf 100644 --- a/build.c +++ b/build.c @@ -201,15 +201,19 @@ String CleanResultOutput(Arena *arena, String output) typedef struct BuildStepSimpleCommandArg BuildStepSimpleCommandArg; struct BuildStepSimpleCommandArg { String cmd; - AtomicI32 *skip_flag; - AtomicI32 *failure_flag; + Atomic32 *skip_flag; + Atomic32 *failure_flag; + D_Tag delete_file_on_failure; + D_Tag dxc_depfile_dependent; + D_Tag dxc_depfile; }; typedef struct BuildStepMsvcCompileCommandArg BuildStepMsvcCompileCommandArg; struct BuildStepMsvcCompileCommandArg { String cmd; - AtomicI32 *skip_flag; - AtomicI32 *failure_flag; + Atomic32 *skip_flag; + Atomic32 *failure_flag; + D_Tag delete_file_on_failure; D_Tag depfile_dependent; D_Tag output_depfile; D_TagList depfile_force_includes; @@ -221,9 +225,9 @@ void BuildStepSimpleCommand(void *arg_raw) Step *s = arg_raw; BuildStepSimpleCommandArg *arg = s->arg; - AtomicI32 *skip_flag = arg->skip_flag; - AtomicI32 *failure_flag = arg->failure_flag; - if (!skip_flag || !AtomicI32Eval(skip_flag)) { + Atomic32 *skip_flag = arg->skip_flag; + Atomic32 *failure_flag = arg->failure_flag; + if (!skip_flag || !Atomic32Fetch(skip_flag)) { SH_CommandResult result = RunCommand(scratch.arena, arg->cmd); String result_output_cleaned = CleanResultOutput(scratch.arena, result.output); { @@ -238,8 +242,17 @@ void BuildStepSimpleCommand(void *arg_raw) MemoryCopy(s->res_output.text, result_output_cleaned.text, result_output_cleaned.len); } s->res_status = result.error ? StepStatus_Failure : StepStatus_Success; - if (result.error && failure_flag) { - AtomicI32EvalExchange(failure_flag, 1); + if (!D_IsNil(arg->dxc_depfile)) { + String depfile_data = D_DepfileTextFromDxcOutput(scratch.arena, arg->dxc_depfile_dependent, result.output); + D_ClearWrite(arg->dxc_depfile, depfile_data); + } + if (result.error) { + if (failure_flag) { + Atomic32FetchSet(failure_flag, 1); + } + if (!D_IsNil(arg->delete_file_on_failure)) { + D_Delete(arg->delete_file_on_failure); + } } OS_ConditionVariableBroadcast(&s->res_cv); OS_MutexUnlock(&res_lock); @@ -248,7 +261,7 @@ void BuildStepSimpleCommand(void *arg_raw) OS_Lock res_lock = OS_MutexLockE(&s->res_mutex); s->res_status = StepStatus_Skipped; if (failure_flag) { - AtomicI32EvalExchange(failure_flag, 1); + Atomic32FetchSet(failure_flag, 1); } OS_ConditionVariableBroadcast(&s->res_cv); OS_MutexUnlock(&res_lock); @@ -264,9 +277,9 @@ void BuildStepMsvcCompileCommand(void *arg_raw) Step *s = arg_raw; BuildStepMsvcCompileCommandArg *arg = s->arg; - AtomicI32 *skip_flag = arg->skip_flag; - AtomicI32 *failure_flag = arg->failure_flag; - if (!skip_flag || !AtomicI32Eval(skip_flag)) { + Atomic32 *skip_flag = arg->skip_flag; + Atomic32 *failure_flag = arg->failure_flag; + if (!skip_flag || !Atomic32Fetch(skip_flag)) { SH_CommandResult result = RunCommand(scratch.arena, arg->cmd); if (!result.error && !D_IsNil(arg->depfile_dependent) && !D_IsNil(arg->output_depfile)) { String depfile_data = D_DepfileTextFromMsvcOutput(scratch.arena, arg->depfile_dependent, arg->depfile_force_includes, result.output); @@ -285,8 +298,13 @@ void BuildStepMsvcCompileCommand(void *arg_raw) MemoryCopy(s->res_output.text, result_output_cleaned.text, result_output_cleaned.len); } s->res_status = result.error ? StepStatus_Failure : StepStatus_Success; - if (result.error && failure_flag) { - AtomicI32EvalExchange(failure_flag, 1); + if (result.error) { + if (failure_flag) { + Atomic32FetchSet(failure_flag, 1); + } + if (!D_IsNil(arg->delete_file_on_failure)) { + D_Delete(arg->delete_file_on_failure); + } } OS_ConditionVariableBroadcast(&s->res_cv); OS_MutexUnlock(&res_lock); @@ -295,7 +313,7 @@ void BuildStepMsvcCompileCommand(void *arg_raw) OS_Lock res_lock = OS_MutexLockE(&s->res_mutex); s->res_status = StepStatus_Skipped; if (failure_flag) { - AtomicI32EvalExchange(failure_flag, 1); + Atomic32FetchSet(failure_flag, 1); } OS_ConditionVariableBroadcast(&s->res_cv); OS_MutexUnlock(&res_lock); @@ -369,12 +387,16 @@ void OnBuild(StringList cli_args) String build_hash_path = OS_GetAbsPath(&perm, StringF(&perm, Lit("%F/.pp_build_hash"), FmtStr(arg_outdir))); String out_dep_dir_path = OS_GetAbsPath(&perm, StringF(&perm, Lit("%F/dep/"), FmtStr(arg_outdir))); String out_obj_dir_path = OS_GetAbsPath(&perm, StringF(&perm, Lit("%F/obj/"), FmtStr(arg_outdir))); + String out_dxc_dir_path = OS_GetAbsPath(&perm, StringF(&perm, Lit("%F/dxc/"), FmtStr(arg_outdir))); String out_inc_dir_path = OS_GetAbsPath(&perm, StringF(&perm, Lit("%F/inc/"), FmtStr(arg_outdir))); String out_bin_dir_path = OS_GetAbsPath(&perm, StringF(&perm, Lit("%F/bin/"), FmtStr(arg_outdir))); if (!OS_DirExists(out_obj_dir_path)) { OS_CreateDirAtAbsPath(out_obj_dir_path); } + if (!OS_DirExists(out_dxc_dir_path)) { + OS_CreateDirAtAbsPath(out_dxc_dir_path); + } if (!OS_DirExists(out_inc_dir_path)) { OS_CreateDirAtAbsPath(out_inc_dir_path); } @@ -416,6 +438,7 @@ void OnBuild(StringList cli_args) Bool should_embed_in_rc = !!arg_msvc; D_Tag executable_file = D_TagFromPath(&perm, StringF(&perm, Lit("%F/PowerPlay.exe"), FmtStr(out_bin_dir_path)), D_TagKind_File); + D_Tag dxc_dir = D_TagFromPath(&perm, out_dxc_dir_path, D_TagKind_Dir); D_Tag res_dir = D_TagFromPath(&perm, Lit("res"), D_TagKind_Dir); D_Tag icon_file = D_TagFromPath(&perm, Lit("icon.ico"), D_TagKind_File); @@ -436,6 +459,7 @@ void OnBuild(StringList cli_args) StringList link_warnings = { 0 }; StringList link_args = { 0 }; StringList rc_compile_args = { 0 }; + StringList dxc_compile_args = { 0 }; { if (arg_msvc) { /* Msvc */ @@ -449,7 +473,7 @@ void OnBuild(StringList cli_args) String warnings = Lit("/WX /Wall " "/options:strict " - "/wd4820 /wd4201 /wd5220 /wd4514 /wd4244 /wd5045 /wd4242 /wd4061 /wd4189 /wd4723 /wd5246"); + "/wd4820 /wd4201 /wd5220 /wd4514 /wd4244 /wd5045 /wd4242 /wd4061 /wd4189 /wd4723 /wd5246 /wd4324"); StringListAppend(&perm, &compile_warnings, warnings); StringListAppend(&perm, &link_warnings, Lit("/WX")); @@ -495,6 +519,12 @@ void OnBuild(StringList cli_args) StringListAppend(&perm, &link_warnings, warnings); } + /* DXC */ + { + StringListAppend(&perm, &dxc_compile_args, Lit("dxc %F -Fo %F -E %F -T %F -H")); + } + + /* RTC */ if (arg_rtc) { if (!arg_crtlib) { @@ -535,6 +565,7 @@ void OnBuild(StringList cli_args) } else { StringListAppend(&perm, &compile_and_link_args, Lit("-O0")); } + StringListAppend(&perm, &dxc_compile_args, Lit("-Od")); } else { if (arg_msvc) { StringListAppend(&perm, &compile_args, Lit("/O2")); @@ -542,6 +573,7 @@ void OnBuild(StringList cli_args) } else { StringListAppend(&perm, &compile_and_link_args, Lit("-O3 -flto")); } + StringListAppend(&perm, &dxc_compile_args, Lit("-O3")); } /* Debug info */ @@ -552,6 +584,7 @@ void OnBuild(StringList cli_args) } else { StringListAppend(&perm, &compile_and_link_args, Lit("-g")); } + StringListAppend(&perm, &dxc_compile_args, Lit("-Zi -Qembed_debug")); } /* Address sanitizer */ @@ -638,16 +671,106 @@ void OnBuild(StringList cli_args) hist = D_HistFromPath(&perm, hist_path); D_TagList link_files = { 0 }; + /* ========================== * + * Build step: Compile shaders + * ========================== */ + + Atomic32 shader_success_flag = { 0 }; + { + D_TagList src_input_files = { 0 }; + { + D_Tag src_dir = D_TagFromPath(&perm, Lit("src/sh"), D_TagKind_Dir); + D_TagList src_files = D_GetDirContents(&perm, src_dir, 0); + + for (D_TagListNode *n = src_files.first; n; n = n->next) { + D_Tag file = n->tag; + String path = file.full_path; + String name = D_GetName(file); + String extension = StringPathExtension(name); + Bool is_dir = file.kind == D_TagKind_Dir; + Bool is_rs = !is_dir && StringEqual(extension, Lit("hlsl_rs")); + Bool is_cs = !is_dir && StringEqual(extension, Lit("hlsl_cs")); + if (is_rs || is_cs) { + D_TagListAppend(&perm, &src_input_files, file); + } + } + } + + { + AddSyncPoint(); + + String dxc_compile_args_fmt = StringFromStringLists(&perm, Lit(" "), dxc_compile_args); + for (D_TagListNode *n = src_input_files.first; n; n = n->next) { + D_Tag file = n->tag; + String path = file.full_path; + String name = D_GetName(file); + String name_no_extension = StringPathNoExtension(name); + String extension = StringPathExtension(name); + Bool is_rs = StringEqual(extension, Lit("hlsl_rs")); + Bool is_cs = !is_rs && StringEqual(extension, Lit("hlsl_cs")); + for (I32 kind = 0; kind < 3; ++kind) { + String out_file_extension = { 0 }; + String entry = { 0 }; + String profile = { 0 }; + if (kind == 0 && is_rs) { + /* Vertex shader */ + out_file_extension = Lit("dxc_vs"); + entry = Lit("vs"); + profile = Lit("vs_6_6"); + } else if (kind == 1 && is_rs) { + /* Pixel shader */ + out_file_extension = Lit("dxc_ps"); + entry = Lit("ps"); + profile = Lit("ps_6_6"); + } else if (kind == 2 && is_cs) { + /* Compute shader */ + out_file_extension = Lit("dxc_cs"); + entry = Lit("cs"); + profile = Lit("cs_6_6"); + } + if (entry.len > 0) { + D_Tag dep_file; + { + String dep_file_path = StringF(&perm, Lit("%F/%F.%F"), FmtStr(out_dxc_dir_path), FmtStr(name_no_extension), FmtStr(dep_file_extension)); + dep_file = D_TagFromPath(&perm, dep_file_path, D_TagKind_DepFile); + } + + D_Tag dxc_file; + { + String dxc_file_path = StringF(&perm, Lit("%F/%F.%F"), FmtStr(out_dxc_dir_path), FmtStr(name_no_extension), FmtStr(out_file_extension)); + dxc_file = D_TagFromPath(&perm, dxc_file_path, D_TagKind_File); + } + D_AddDependency(&store, dxc_file, file); + D_AddDependency(&store, dxc_file, dep_file); + + if (IsDirty(dxc_file)) { + String step_name = StringF(&perm, Lit("%F -> %F"), FmtStr(name), FmtStr(D_GetName(dxc_file))); + { + BuildStepSimpleCommandArg *bs_arg = ArenaPush(&perm, BuildStepSimpleCommandArg); + bs_arg->cmd = StringF(&perm, dxc_compile_args_fmt, FmtStr(file.full_path), FmtStr(dxc_file.full_path), FmtStr(entry), FmtStr(profile)); + bs_arg->failure_flag = &shader_success_flag; + bs_arg->delete_file_on_failure = dxc_file; + bs_arg->dxc_depfile_dependent = dxc_file; + bs_arg->dxc_depfile = dep_file; + AddStep(step_name, &BuildStepSimpleCommand, bs_arg); + } + } + } + } + } + } + } + /* ========================== * * Build step: Tar archives * ========================== */ - AtomicI32 tar_success_flag = { 0 }; + Atomic32 tar_success_flag = { 0 }; { AddSyncPoint(); D_TagList tar_input_dirs = { 0 }; - + D_TagListAppend(&perm, &tar_input_dirs, dxc_dir); if (should_embed_res_dir) { D_TagListAppend(&perm, &tar_input_dirs, res_dir); } @@ -665,6 +788,7 @@ void OnBuild(StringList cli_args) BuildStepSimpleCommandArg *bs_arg = ArenaPush(&perm, BuildStepSimpleCommandArg); bs_arg->cmd = StringF(&perm, Lit("cd %F && tar cvf %F ."), FmtStr(input_dir.full_path), FmtStr(tar_file.full_path)); bs_arg->failure_flag = &tar_success_flag; + bs_arg->skip_flag = &shader_success_flag; String step_name = StringF(&perm, Lit("%F -> %F"), FmtStr(D_GetName(input_dir)), FmtStr(D_GetName(tar_file))); AddStep(step_name, &BuildStepSimpleCommand, bs_arg); } @@ -675,20 +799,23 @@ void OnBuild(StringList cli_args) * Build step: Compile RC files * ========================== */ - AtomicI32 rc_success_flag = { 0 }; + Atomic32 rc_success_flag = { 0 }; if (PlatformWindows) { AddSyncPoint(); D_Tag rc_input_file = D_TagFromPath(&perm, StringF(&perm, Lit("%F/rc.rc"), FmtStr(out_inc_dir_path)), D_TagKind_File); { D_Tag res_tar_file = D_TagFromPath(&perm, StringF(&perm, Lit("%F/%F.tar"), FmtStr(out_inc_dir_path), FmtStr(D_GetName(res_dir))), D_TagKind_File); + D_Tag dxc_tar_file = D_TagFromPath(&perm, StringF(&perm, Lit("%F/%F.tar"), FmtStr(out_inc_dir_path), FmtStr(D_GetName(dxc_dir))), D_TagKind_File); D_AddDependency(&store, rc_input_file, icon_file); + D_AddDependency(&store, rc_input_file, dxc_tar_file); if (should_embed_in_rc && should_embed_res_dir) { D_AddDependency(&store, rc_input_file, res_tar_file); } if (IsDirty(rc_input_file)) { D_ClearWrite(rc_input_file, Lit("")); D_AppendWrite(rc_input_file, StringF(&perm, Lit("%F %F DISCARDABLE %F\n"), FmtStr(D_GetName(icon_file)), FmtStr(Lit("ICON")), FmtStr(D_GetName(icon_file)))); + D_AppendWrite(rc_input_file, StringF(&perm, Lit("%F %F DISCARDABLE %F\n"), FmtStr(D_GetName(dxc_tar_file)), FmtStr(Lit("RCDATA")), FmtStr(D_GetName(dxc_tar_file)))); if (should_embed_in_rc && should_embed_res_dir) { D_AppendWrite(rc_input_file, StringF(&perm, Lit("%F %F DISCARDABLE %F\n"), FmtStr(D_GetName(res_tar_file)), FmtStr(Lit("RCDATA")), FmtStr(D_GetName(res_tar_file)))); } @@ -716,7 +843,7 @@ void OnBuild(StringList cli_args) * Build step: Compile pch files * ========================== */ - AtomicI32 pch_success_flag = { 0 }; + Atomic32 pch_success_flag = { 0 }; D_TagList depfile_force_includes = { 0 }; { AddSyncPoint(); @@ -767,6 +894,7 @@ void OnBuild(StringList cli_args) bs_arg->output_depfile = dep_file; bs_arg->skip_flag = &rc_success_flag; bs_arg->failure_flag = &pch_success_flag; + bs_arg->delete_file_on_failure = pch_c_file; String step_name = StringF(&perm, Lit("%F -> %F"), FmtStr(D_GetName(pch_header_file)), FmtStr(D_GetName(pch_c_file))); AddStep(step_name, &BuildStepMsvcCompileCommand, bs_arg); } @@ -782,6 +910,7 @@ void OnBuild(StringList cli_args) bs_arg->output_depfile = dep_file; bs_arg->skip_flag = &rc_success_flag; bs_arg->failure_flag = &pch_success_flag; + bs_arg->delete_file_on_failure = pch_cpp_file; String step_name = StringF(&perm, Lit("%F -> %F"), FmtStr(D_GetName(pch_header_file)), FmtStr(D_GetName(pch_cpp_file))); AddStep(step_name, &BuildStepMsvcCompileCommand, bs_arg); } @@ -818,7 +947,7 @@ void OnBuild(StringList cli_args) * Build step: Compile src files * ========================== */ - AtomicI32 src_success_flag = { 0 }; + Atomic32 src_success_flag = { 0 }; { D_TagList src_input_files = { 0 }; { @@ -840,13 +969,15 @@ void OnBuild(StringList cli_args) StringBeginsWith(name, Lit("gp_")) || StringBeginsWith(name, Lit("playback_")) || StringBeginsWith(name, Lit("mp3_")) || - StringBeginsWith(name, Lit("ttf_"))) { + StringBeginsWith(name, Lit("ttf_")) || + StringBeginsWith(name, Lit("dxc"))) { if (PlatformWindows) { ignore = !(StringEqual(name, Lit("sys_win32.c")) || StringEqual(name, Lit("gp_dx12.c")) || StringEqual(name, Lit("playback_wasapi.c")) || StringEqual(name, Lit("mp3_mmf.c")) || - StringEqual(name, Lit("ttf_dwrite.cpp"))); + StringEqual(name, Lit("ttf_dwrite.cpp")) || + StringEqual(name, Lit("dxc.cpp"))); } } else { ignore = 0; @@ -899,6 +1030,7 @@ void OnBuild(StringList cli_args) bs_arg->depfile_force_includes = depfile_force_includes; bs_arg->skip_flag = &pch_success_flag; bs_arg->failure_flag = &src_success_flag; + bs_arg->delete_file_on_failure = obj_file; AddStep(step_name, &BuildStepMsvcCompileCommand, bs_arg); } else { BuildStepSimpleCommandArg *bs_arg = ArenaPush(&perm, BuildStepSimpleCommandArg); diff --git a/src/app.c b/src/app.c index f293070d..1c59b4ee 100644 --- a/src/app.c +++ b/src/app.c @@ -21,6 +21,7 @@ #include "phys.h" #include "host.h" #include "bitbuff.h" +#include "watch.h" GLOBAL struct { struct arena *arena; @@ -278,6 +279,7 @@ void sys_app_startup(struct string args_str) /* Global systems */ resource_startup(); + watch_startup(); gp_startup(); /* Subsystems */ diff --git a/src/arena.c b/src/arena.c index 4c87aeff..89e47742 100644 --- a/src/arena.c +++ b/src/arena.c @@ -54,8 +54,8 @@ void arena_release(struct arena *arena) ASAN_UNPOISON(arena, arena->committed + ARENA_HEADER_SIZE); __prof; __proffree(arena); - gstat_add(GSTAT_MEMORY_COMMITTED, -arena->committed - ARENA_HEADER_SIZE); - gstat_add(GSTAT_MEMORY_RESERVED, -arena->reserved); + gstat_add(GSTAT_MEMORY_COMMITTED, -(i64)(arena->committed - ARENA_HEADER_SIZE)); + gstat_add(GSTAT_MEMORY_RESERVED, -(i64)(arena->reserved)); gstat_add(GSTAT_NUM_ARENAS, -1); sys_memory_release(arena); } diff --git a/src/common.h b/src/common.h index cad50ec5..802d7336 100644 --- a/src/common.h +++ b/src/common.h @@ -119,7 +119,7 @@ extern "C" { * ========================== */ /* Compile time assert */ -#if LANGUAGE_C && (__STDC_VERSION__ < 202311L) +#if COMPILER_MSVC || (LANGUAGE_C && __STDC_VERSION__ < 202311L) # if COMPILER_MSVC # define STATIC_ASSERT3(cond, line) struct STATIC_ASSERT_____##line {int foo[(cond) ? 1 : -1];} # define STATIC_ASSERT2(cond, line) STATIC_ASSERT3(cond, line) @@ -265,7 +265,7 @@ void __asan_unpoison_memory_region(void const volatile *add, size_t); #endif /* alignas */ -#if (LANGUAGE_C && (__STDC_VERSION__ < 202311L)) +#if COMPILER_MSVC || (LANGUAGE_C && __STDC_VERSION__ < 202311L) # if COMPILER_MSVC # define alignas(n) __declspec(align(n)) # else @@ -276,10 +276,14 @@ void __asan_unpoison_memory_region(void const volatile *add, size_t); /* Field macros */ #define FIELD_SIZEOF(type, field) sizeof(((type *)0)->field) -#if COMPILER_MSVC && !defined _CRT_USE_BUILTIN_OFFSETOF -# define offsetof(type, field) ((u64)&(((type *)0)->field)) -#else -# define offsetof(type, field) __builtin_offsetof(type, field) +#if 0 +#if !COMPILER_MSVC +# if !defined _CRT_USE_BUILTIN_OFFSETOF +# define offsetof(type, field) ((u64)&(((type *)0)->field)) +# else +# define offsetof(type, field) __builtin_offsetof(type, field) +# endif +#endif #endif /* Array */ @@ -682,10 +686,10 @@ INLINE f64 clamp_f64(f64 v, f64 min, f64 max) { return v < min ? min : v > max ? #include "prof_tracy.h" -#define PROF_THREAD_GROUP_FIBERS -GIBI(1) -#define PROF_THREAD_GROUP_SCHEDULER -MEBI(3) -#define PROF_THREAD_GROUP_WINDOW -MEBI(2) -#define PROF_THREAD_GROUP_MAIN -MEBI(1) +#define PROF_THREAD_GROUP_FIBERS -(i64)GIBI(1) +#define PROF_THREAD_GROUP_SCHEDULER -(i64)MEBI(3) +#define PROF_THREAD_GROUP_WINDOW -(i64)MEBI(2) +#define PROF_THREAD_GROUP_MAIN -(i64)MEBI(1) #ifdef __cplusplus } diff --git a/src/config.h b/src/config.h index 569dc350..bb6667f2 100644 --- a/src/config.h +++ b/src/config.h @@ -73,7 +73,7 @@ #define COLLIDER_DEBUG_DETAILED 1 #define COLLIDER_DEBUG_DETAILED_DRAW_MENKOWSKI 1 -#define FLOOD_DEBUG 1 +#define FLOOD_DEBUG 0 /* If enabled, bitbuffs will insert/verify magic numbers & length for each read & write */ #define BITBUFF_DEBUG 0 diff --git a/src/dxc.cpp b/src/dxc.cpp new file mode 100644 index 00000000..555994a7 --- /dev/null +++ b/src/dxc.cpp @@ -0,0 +1,69 @@ +extern "C" +{ +#include "dxc.h" +#include "arena.h" +#include "string.h" +} + +#pragma clang diagnostic ignored "-Wlanguage-extension-token" +#include +#include +#include +#include + +#pragma comment(lib, "d3dcompiler") +#pragma comment(lib, "dxcompiler") + +/* https://github.com/microsoft/DirectXShaderCompiler/wiki/Using-dxc.exe-and-dxcompiler.dll */ +struct dxc_compile_result dxc_compile(struct arena *arena, string shader_source, i32 num_args, struct string *args) +{ + __prof; + struct arena_temp scratch = scratch_begin(arena); + struct dxc_compile_result res = ZI; + + wchar_t **wstr_args = arena_push_array(scratch.arena, wchar_t *, num_args); + for (i32 i = 0; i < num_args; ++i) { + wstr_args[i] = wstr_from_string(scratch.arena, args[i]); + } + + DxcBuffer dxc_src_buffer = ZI; + dxc_src_buffer.Ptr = shader_source.text; + dxc_src_buffer.Size = shader_source.len; + dxc_src_buffer.Encoding = DXC_CP_UTF8; + + /* Init compiler */ + CComPtr dxc_utils; + CComPtr dxc_compiler; + CComPtr dxc_include_handler; + DxcCreateInstance(CLSID_DxcUtils, IID_PPV_ARGS(&dxc_utils)); + DxcCreateInstance(CLSID_DxcCompiler, IID_PPV_ARGS(&dxc_compiler)); + dxc_utils->CreateDefaultIncludeHandler(&dxc_include_handler); + + /* Compile */ + CComPtr compile_results = 0; + dxc_compiler->Compile(&dxc_src_buffer, (LPCWSTR *)wstr_args, num_args, dxc_include_handler, IID_PPV_ARGS(&compile_results)); + + /* Copy errors */ + CComPtr dxc_errors = 0; + compile_results->GetOutput(DXC_OUT_ERRORS, IID_PPV_ARGS(&dxc_errors), 0); + if (dxc_errors != 0) { + res.errors = string_copy(arena, STRING(dxc_errors->GetStringLength(), (u8 *)dxc_errors->GetBufferPointer())); + } + + /* Get status */ + HRESULT dxc_hr = 0; + compile_results->GetStatus(&dxc_hr); + res.success = SUCCEEDED(dxc_hr); + + /* Copy shader output */ + if (res.success) { + CComPtr dxc_shader = 0; + compile_results->GetOutput(DXC_OUT_OBJECT, IID_PPV_ARGS(&dxc_shader), 0); + if (dxc_shader != 0) { + res.dxc = string_copy(arena, STRING(dxc_shader->GetBufferSize(), (u8 *)dxc_shader->GetBufferPointer())); + } + } + + scratch_end(scratch); + return res; +} diff --git a/src/dxc.h b/src/dxc.h new file mode 100644 index 00000000..253edbeb --- /dev/null +++ b/src/dxc.h @@ -0,0 +1,12 @@ +#ifndef DXC_H +#define DXC_H + +struct dxc_compile_result { + struct string dxc; + struct string errors; + b32 success; +}; + +struct dxc_compile_result dxc_compile(struct arena *arena, struct string shader_source, i32 num_args, struct string *args); + +#endif diff --git a/src/gp_dx12.c b/src/gp_dx12.c index 8d8ed59c..302d3ce2 100644 --- a/src/gp_dx12.c +++ b/src/gp_dx12.c @@ -5,7 +5,6 @@ #include "string.h" #include "app.h" #include "log.h" -#include "resource.h" #include "atomic.h" #include "util.h" #include "rand.h" @@ -13,10 +12,15 @@ #include "gstat.h" #include "snc.h" #include "ase.h" +#include "resource.h" +#include "tar.h" +#include "inc.h" +#include "dxc.h" +#include "watch.h" /* Include common shader types */ #define SH_CPU 1 -#include "../res/sh/sh_common.h" +#include "sh/sh_common.h" #pragma warning(push, 0) # define UNICODE @@ -32,7 +36,6 @@ #pragma comment(lib, "d3d12") #pragma comment(lib, "dxgi") #pragma comment(lib, "dxguid") -#pragma comment(lib, "d3dcompiler") #if PROFILING_GPU /* For RegOpenKeyEx */ @@ -89,33 +92,31 @@ struct pipeline_rtv_desc { struct pipeline_desc { struct string name; - struct shader_desc cs; - struct shader_desc vs; - struct shader_desc ps; + + /* If a dxc string is set, then it will be used directly instead of looking up dxc from archive using pipeline name */ + struct string vs_dxc; + struct string ps_dxc; + struct string cs_dxc; + D3D12_INPUT_ELEMENT_DESC ia[8]; struct pipeline_rtv_desc rtvs[8]; }; struct pipeline { - b32 success; - - struct arena *arena; struct string name; u64 hash; - - struct pipeline_error *first_error; - struct pipeline_error *last_error; + b32 success; + struct string error; i64 compilation_time_ns; - /* Dict with shader source & included file names as keys */ - struct dict *dependencies; - /* Lock global pipelines mutex when accessing */ i64 refcount; ID3D12PipelineState *pso; ID3D12RootSignature *rootsig; struct pipeline_desc desc; + + struct pipeline *next; }; struct pipeline_error { @@ -323,7 +324,7 @@ struct dx12_upload_job_sig { struct dx12_resource *resource; void *data; }; INTERNAL SYS_JOB_DEF(dx12_upload_job, job); #if RESOURCE_RELOADING -INTERNAL RESOURCE_WATCH_CALLBACK_FUNC_DEF(pipeline_resource_watch_callback, name); +INTERNAL WATCH_CALLBACK_FUNC_DEF(pipeline_watch_callback, name); #endif /* ========================== * @@ -354,9 +355,13 @@ GLOBAL struct { struct arena *swapchains_arena; struct swapchain *first_free_swapchain; + /* Shader bytecode archive */ + struct tar_archive dxc_archive; + /* Pipeline cache */ struct snc_mutex pipelines_mutex; struct arena *pipelines_arena; + struct pipeline *first_free_pipeline; struct dict *pipeline_descs; struct dict *top_pipelines; /* Latest pipelines */ struct dict *top_successful_pipelines; /* Latest pipelines that successfully compiled */ @@ -430,6 +435,13 @@ void gp_startup(void) /* Initialize fenced releases queue */ G.fenced_releases_arena = arena_alloc(GIBI(64)); + /* Initialize embedded shader archive */ + struct string embedded_data = inc_dxc_tar(); + if (embedded_data.len <= 0) { + sys_panic(LIT("No embedded shaders found")); + } + G.dxc_archive = tar_parse(G.pipelines_arena, embedded_data, LIT("")); + /* Initialize dx12 */ /* TODO: Parallelize phases */ dx12_init_device(); @@ -439,7 +451,7 @@ void gp_startup(void) /* Register callbacks */ #if RESOURCE_RELOADING - resource_register_watch_callback(pipeline_resource_watch_callback); + watch_register_callback(pipeline_watch_callback); #endif sys_on_exit(gp_shutdown); @@ -662,10 +674,10 @@ INTERNAL void dx12_init_objects(void) { __profn("Allocate command queues"); struct command_queue_desc params[] = { - { .type = D3D12_COMMAND_LIST_TYPE_DIRECT, .priority = D3D12_COMMAND_QUEUE_PRIORITY_NORMAL, .dbg_name = LIT("Direct queue") }, - { .type = D3D12_COMMAND_LIST_TYPE_COMPUTE, .priority = D3D12_COMMAND_QUEUE_PRIORITY_NORMAL, .dbg_name = LIT("Compute queue") }, - { .type = D3D12_COMMAND_LIST_TYPE_COPY, .priority = D3D12_COMMAND_QUEUE_PRIORITY_HIGH, .dbg_name = LIT("Copy queue") }, - { .type = D3D12_COMMAND_LIST_TYPE_COPY, .priority = D3D12_COMMAND_QUEUE_PRIORITY_NORMAL, .dbg_name = LIT("Background copy queue") } + {.type = D3D12_COMMAND_LIST_TYPE_DIRECT, .priority = D3D12_COMMAND_QUEUE_PRIORITY_NORMAL, .dbg_name = LIT("Direct queue") }, + {.type = D3D12_COMMAND_LIST_TYPE_COMPUTE, .priority = D3D12_COMMAND_QUEUE_PRIORITY_NORMAL, .dbg_name = LIT("Compute queue") }, + {.type = D3D12_COMMAND_LIST_TYPE_COPY, .priority = D3D12_COMMAND_QUEUE_PRIORITY_HIGH, .dbg_name = LIT("Copy queue") }, + {.type = D3D12_COMMAND_LIST_TYPE_COPY, .priority = D3D12_COMMAND_QUEUE_PRIORITY_NORMAL, .dbg_name = LIT("Background copy queue") } }; struct command_queue_alloc_job_sig sig = ZI; sig.descs_in = params; @@ -707,10 +719,6 @@ INTERNAL void dx12_init_pipelines(void) { struct pipeline_desc *desc = arena_push(G.pipelines_arena, struct pipeline_desc); desc->name = LIT("material"); - desc->vs.file = LIT("sh/material.hlsl"); - desc->ps.file = LIT("sh/material.hlsl"); - desc->vs.func = LIT("vs"); - desc->ps.func = LIT("ps"); desc->rtvs[0].format = DXGI_FORMAT_R8G8B8A8_UNORM; desc->rtvs[0].blending = 1; desc->rtvs[1].format = DXGI_FORMAT_R16G16B16A16_FLOAT; @@ -721,26 +729,18 @@ INTERNAL void dx12_init_pipelines(void) { struct pipeline_desc *desc = arena_push(G.pipelines_arena, struct pipeline_desc); desc->name = LIT("flood"); - desc->cs.file = LIT("sh/flood.hlsl"); - desc->cs.func = LIT("cs"); dict_set(G.pipelines_arena, G.pipeline_descs, hash_fnv64(HASH_FNV64_BASIS, desc->name), (u64)desc); } /* Shade pipeline */ { struct pipeline_desc *desc = arena_push(G.pipelines_arena, struct pipeline_desc); desc->name = LIT("shade"); - desc->cs.file = LIT("sh/shade.hlsl"); - desc->cs.func = LIT("cs"); dict_set(G.pipelines_arena, G.pipeline_descs, hash_fnv64(HASH_FNV64_BASIS, desc->name), (u64)desc); } /* Shape pipeline */ { struct pipeline_desc *desc = arena_push(G.pipelines_arena, struct pipeline_desc); desc->name = LIT("shape"); - desc->vs.file = LIT("sh/shape.hlsl"); - desc->ps.file = LIT("sh/shape.hlsl"); - desc->vs.func = LIT("vs"); - desc->ps.func = LIT("ps"); desc->ia[0] = (D3D12_INPUT_ELEMENT_DESC) { "pos", 0, DXGI_FORMAT_R32G32_FLOAT, 0, D3D12_APPEND_ALIGNED_ELEMENT, D3D12_INPUT_CLASSIFICATION_PER_VERTEX_DATA, 0 }; desc->ia[1] = (D3D12_INPUT_ELEMENT_DESC) { "color_srgb", 0, DXGI_FORMAT_R8G8B8A8_UNORM, 0, D3D12_APPEND_ALIGNED_ELEMENT, D3D12_INPUT_CLASSIFICATION_PER_VERTEX_DATA, 0 }; desc->rtvs[0].format = DXGI_FORMAT_R8G8B8A8_UNORM; @@ -751,10 +751,6 @@ INTERNAL void dx12_init_pipelines(void) { struct pipeline_desc *desc = arena_push(G.pipelines_arena, struct pipeline_desc); desc->name = LIT("ui"); - desc->vs.file = LIT("sh/ui.hlsl"); - desc->ps.file = LIT("sh/ui.hlsl"); - desc->vs.func = LIT("vs"); - desc->ps.func = LIT("ps"); desc->rtvs[0].format = DXGI_FORMAT_R8G8B8A8_UNORM; desc->rtvs[0].blending = 1; dict_set(G.pipelines_arena, G.pipeline_descs, hash_fnv64(HASH_FNV64_BASIS, desc->name), (u64)desc); @@ -763,10 +759,6 @@ INTERNAL void dx12_init_pipelines(void) { struct pipeline_desc *desc = arena_push(G.pipelines_arena, struct pipeline_desc); desc->name = LIT("blit"); - desc->vs.file = LIT("sh/blit.hlsl"); - desc->ps.file = LIT("sh/blit.hlsl"); - desc->vs.func = LIT("vs"); - desc->ps.func = LIT("ps"); desc->rtvs[0].format = DXGI_FORMAT_R8G8B8A8_UNORM; desc->rtvs[0].blending = 1; dict_set(G.pipelines_arena, G.pipeline_descs, hash_fnv64(HASH_FNV64_BASIS, desc->name), (u64)desc); @@ -795,12 +787,12 @@ INTERNAL void dx12_init_pipelines(void) struct pipeline *pipeline = pipelines[i]; if (pipeline->success) { logf_success("Successfully compiled pipeline \"%F\" in %F seconds", FMT_STR(pipeline->name), FMT_FLOAT(SECONDS_FROM_NS(pipeline->compilation_time_ns))); - if (pipeline->first_error) { - struct string msg = string_format(scratch.arena, LIT("Warning while compiling pipeline \"%F\":\n%F"), FMT_STR(pipeline->name), FMT_STR(pipeline->first_error->msg)); + if (pipeline->error.len) { + struct string msg = string_format(scratch.arena, LIT("Warning while compiling pipeline \"%F\":\n%F"), FMT_STR(pipeline->name), FMT_STR(pipeline->error)); log_warning(msg); } } else { - struct string error = pipeline->first_error ? pipeline->first_error->msg : LIT("Unknown error"); + struct string error = pipeline->error.len > 0 ? pipeline->error : LIT("Unknown error"); struct string msg = string_format(scratch.arena, LIT("Error initializing pipeline \"%F\":\n\n%F"), FMT_STR(pipeline->name), FMT_STR(error)); log_error(msg); sys_message_box(SYS_MESSAGE_BOX_KIND_WARNING, msg); @@ -880,230 +872,60 @@ INTERNAL void dx12_init_noise(void) * Shader compilation * ========================== */ -struct dx12_include_handler { - ID3DInclude d3d_handler; - ID3DIncludeVtbl vtbl; - struct pipeline *pipeline; - struct snc_mutex pipeline_mutex; - u64 num_open_resources; - struct resource open_resources[1024]; +struct shader_compile_desc { + struct string src; + struct string friendly_name; + struct string entry; + struct string target; }; -INTERNAL HRESULT dx12_include_open(ID3DInclude *d3d_handler, D3D_INCLUDE_TYPE include_type, LPCSTR name_cstr, LPCVOID parent_data, LPCVOID *data_out, UINT *data_len_out) -{ - __prof; - (UNUSED)include_type; - (UNUSED)parent_data; - HRESULT result = E_FAIL; - struct dx12_include_handler *handler = (struct dx12_include_handler *)d3d_handler; - struct string name = string_from_cstr_no_limit((char *)name_cstr); - u64 hash = hash_fnv64(HASH_FNV64_BASIS, name); - - if (handler->num_open_resources >= countof(handler->open_resources)) { - sys_panic(LIT("Dx12 include handler resource oversig")); - } - - struct snc_lock lock = snc_lock_e(&handler->pipeline_mutex); - { - struct pipeline *pipeline = handler->pipeline; - dict_set(pipeline->arena, pipeline->dependencies, hash, 1); - } - snc_unlock(&lock); - - struct resource *res = &handler->open_resources[handler->num_open_resources++]; - *res = resource_open(name); - if (resource_exists(res)) { - struct string data = resource_get_data(res); - *data_out = data.text; - *data_len_out = data.len; - result = S_OK; - } - - return result; -} - -INTERNAL HRESULT dx12_include_close(ID3DInclude *d3d_handler, LPCVOID data) -{ - __prof; - (UNUSED)data; - struct dx12_include_handler *handler = (struct dx12_include_handler *)d3d_handler; - for (u64 i = 0; i < handler->num_open_resources; ++i) { - struct resource *res = &handler->open_resources[i]; - resource_close(res); - } - handler->num_open_resources = 0; - return S_OK; -} - -INTERNAL struct dx12_include_handler *dx12_include_handler_alloc(struct arena *arena, struct pipeline *pipeline) -{ - __prof; - struct dx12_include_handler *handler = arena_push(arena, struct dx12_include_handler); - handler->d3d_handler.lpVtbl = &handler->vtbl; - handler->vtbl.Open = dx12_include_open; - handler->vtbl.Close = dx12_include_close; - handler->pipeline = pipeline; - return handler; -} - -INTERNAL void dx12_include_handler_release(struct dx12_include_handler *handler) -{ - __prof; - for (u64 i = 0; i < handler->num_open_resources; ++i) { - ASSERT(0); /* Resource should have been closed by handler by now */ - struct resource *res = &handler->open_resources[i]; - resource_close(res); - } - handler->num_open_resources = 0; -} - -enum shader_compile_job_kind { - SHADER_COMPILE_TASK_KIND_VS, - SHADER_COMPILE_TASK_KIND_PS, - SHADER_COMPILE_TASK_KIND_CS -}; - -struct shader_compile_job_param { - /* In */ - enum shader_compile_job_kind kind; - struct pipeline *pipeline; - struct shader_desc shader_desc; - struct resource *shader_res; - - /* Out */ +struct shader_compile_result { + i64 elapsed_ns; + struct string dxc; + struct string errors; b32 success; - ID3DBlob *blob; - ID3DBlob *error_blob; - i64 elapsed; }; struct shader_compile_job_sig { - struct shader_compile_job_param **params; + struct arena *arena; + struct shader_compile_desc *descs; + struct shader_compile_result *results; }; -/* TODO: Compile shaders offline w/ dxc for performance & language features */ INTERNAL SYS_JOB_DEF(shader_compile_job, job) { __prof; struct shader_compile_job_sig *sig = job.sig; - struct shader_compile_job_param *param = sig->params[job.id]; - enum shader_compile_job_kind kind = param->kind; - struct pipeline *pipeline = param->pipeline; - struct shader_desc shader_desc = param->shader_desc; - struct resource *shader_res = param->shader_res; + struct arena *arena = sig->arena; + struct shader_compile_desc *desc = &sig->descs[job.id]; + struct shader_compile_result *result = &sig->results[job.id]; - struct arena_temp scratch = scratch_begin_no_conflict(); + struct arena_temp scratch = scratch_begin(arena); { i64 start_ns = sys_time_ns(); - b32 success = 0; - ID3DBlob *blob = 0; - ID3DBlob *error_blob = 0; - - if (resource_exists(shader_res)) { - struct dx12_include_handler *include_handler = dx12_include_handler_alloc(scratch.arena, pipeline); - char *func_cstr = cstr_from_string(scratch.arena, shader_desc.func); - - u32 d3d_compile_flags = D3DCOMPILE_ENABLE_UNBOUNDED_DESCRIPTOR_TABLES; -#if DX12_SHADER_DEBUG - d3d_compile_flags |= D3DCOMPILE_DEBUG | D3DCOMPILE_SKIP_OPTIMIZATION | D3DCOMPILE_ENABLE_STRICTNESS; -#else - d3d_compile_flags |= D3DCOMPILE_OPTIMIZATION_LEVEL3; -#endif - - /* Compile shader */ - { - __profn("Compile shader"); - struct string shader_src = resource_get_data(shader_res); - logf_info("Compiling shader \"%F:%F\"", FMT_STR(shader_desc.file), FMT_STR(shader_desc.func)); - /* Compile shader */ - struct string friendly_name = string_cat(scratch.arena, LIT("res/"), shader_desc.file); - char *friendly_name_cstr = cstr_from_string(scratch.arena, friendly_name); - char *target = 0; - switch (kind) { - case SHADER_COMPILE_TASK_KIND_VS: - { - target = "vs_5_1"; - } break; - - case SHADER_COMPILE_TASK_KIND_PS: - { - target = "ps_5_1"; - } break; - - case SHADER_COMPILE_TASK_KIND_CS: - { - target = "cs_5_1"; - } break; - } - D3D_SHADER_MACRO defines[] = { - { "SH_CPU", "0" }, - { 0, 0 } - }; - HRESULT hr = D3DCompile(shader_src.text, shader_src.len, friendly_name_cstr, defines, (ID3DInclude *)include_handler, func_cstr, target, d3d_compile_flags, 0, &blob, &error_blob); - success = SUCCEEDED(hr); - } - - dx12_include_handler_release(include_handler); + struct dxc_compile_result dxc_result = ZI; + { + __profn("Compile shader"); + logf_info("Compiling shader \"%F:%F\"", FMT_STR(desc->friendly_name), FMT_STR(desc->entry)); + struct string args[] = { + desc->friendly_name, + LIT("-E"), desc->entry, + LIT("-T"), desc->target, + LIT("-D SH_CPU=0"), + LIT("-Zi"), + LIT("-Qembed_debug") + }; + dxc_result = dxc_compile(arena, desc->src, countof(args), args); } + result->success = dxc_result.success; + result->dxc = dxc_result.dxc; + result->errors = dxc_result.errors; + result->elapsed_ns = sys_time_ns() - start_ns; -#if 0 - if (success) { - logf_success("Finished compiling shader \"%F\" in %F seconds", FMT_STR(src_name), FMT_FLOAT(SECONDS_FROM_NS(sys_time_ns() - start_ns))); - } -#endif - - param->success = success; - param->blob = blob; - param->error_blob = error_blob; - param->elapsed = sys_time_ns() - start_ns; } scratch_end(scratch); } -/* ========================== * - * Shader error parser - * ========================== */ - -INTERNAL void parse_pipeline_errors(struct pipeline *pipeline, ID3D10Blob *error_blob) -{ - struct string ignored[] = { - LIT("warning X3557") /* Disabled forced loop unrolling warning */ - }; - - if (error_blob) { - struct arena_temp scratch = scratch_begin_no_conflict(); - { - u64 error_blob_cstr_len = ID3D10Blob_GetBufferSize(error_blob); - char *error_blob_cstr = (char *)ID3D10Blob_GetBufferPointer(error_blob); - struct string error_str = string_copy(scratch.arena, string_from_cstr(error_blob_cstr, error_blob_cstr_len)); - if (string_ends_with(error_str, LIT("\n"))) { - /* Remove trailing newline */ - error_str.len -= 1; - } - if (error_str.len > 0) { - b32 is_ignored = 0; - for (u32 i = 0; i < countof(ignored); ++i) { - if (string_contains(error_str, ignored[i])) { - is_ignored = 1; - break; - } - } - if (!is_ignored) { - struct pipeline_error *error = arena_push(pipeline->arena, struct pipeline_error); - error->msg = string_copy(pipeline->arena, error_str); - if (pipeline->last_error) { - pipeline->last_error->next = error; - } else { - pipeline->first_error = error; - } - pipeline->last_error = error; - } - } - } - scratch_end(scratch); - } -} - /* ========================== * * Pipeline * ========================== */ @@ -1117,15 +939,20 @@ INTERNAL SYS_JOB_DEF(pipeline_alloc_job, job) struct pipeline *pipeline = 0; { - struct arena *pipeline_arena = arena_alloc(MEBI(64)); - pipeline = arena_push(pipeline_arena, struct pipeline); - pipeline->arena = pipeline_arena; - pipelines_out[job.id] = pipeline; + struct snc_lock lock = snc_lock_e(&G.pipelines_mutex); + if (G.first_free_pipeline) { + pipeline = G.first_free_pipeline; + G.first_free_pipeline = pipeline->next; + } else { + pipeline = arena_push_no_zero(G.pipelines_arena, struct pipeline); + } + snc_unlock(&lock); } + MEMZERO_STRUCT(pipeline); + pipelines_out[job.id] = pipeline; pipeline->desc = *desc; - pipeline->name = string_copy(pipeline->arena, desc->name); + pipeline->name = desc->name; pipeline->hash = hash_fnv64(HASH_FNV64_BASIS, pipeline->name); - pipeline->dependencies = dict_init(pipeline->arena, 64); struct arena_temp scratch = scratch_begin_no_conflict(); { @@ -1137,110 +964,74 @@ INTERNAL SYS_JOB_DEF(pipeline_alloc_job, job) struct string error_str = ZI; - b32 has_cs = desc->cs.file.len > 0; - b32 ps_res_is_shared = string_eq(desc->vs.file, desc->ps.file); + struct string vs_dxc = desc->vs_dxc.len > 0 ? desc->vs_dxc : tar_get(&G.dxc_archive, string_cat(scratch.arena, pipeline_name, LIT(".dxc_vs")))->data; + struct string ps_dxc = desc->ps_dxc.len > 0 ? desc->ps_dxc : tar_get(&G.dxc_archive, string_cat(scratch.arena, pipeline_name, LIT(".dxc_ps")))->data; + struct string cs_dxc = desc->cs_dxc.len > 0 ? desc->cs_dxc : tar_get(&G.dxc_archive, string_cat(scratch.arena, pipeline_name, LIT(".dxc_cs")))->data; + if (success && vs_dxc.len > 0 && ps_dxc.len <= 0) { + error_str = LIT("Pipeline has vertex shader without pixel shader"); + success = 0; + } + if (success && vs_dxc.len <= 0 && ps_dxc.len > 0) { + error_str = LIT("Pipeline has pixel shader without vertex shader"); + success = 0; + } + if (success && cs_dxc.len > 0 && (vs_dxc.len > 0 || ps_dxc.len > 0)) { + error_str = LIT("Pipeline has a compute shader with a vertex/pixel shader"); + success = 0; + } + if (success && cs_dxc.len <= 0 && vs_dxc.len <= 0 && ps_dxc.len <= 0) { + error_str = LIT("Pipeline has no shaders"); + success = 0; + } - struct resource cs_res = ZI; - struct resource vs_res = ZI; - struct resource ps_res = ZI; - if (has_cs) { - cs_res = resource_open(desc->cs.file); - if (desc->vs.file.len > 0 || desc->ps.file.len > 0) { - error_str = LIT("Pipeline desc contains both compute and vs/ps shader"); - success = 0; - } - } else { - vs_res = resource_open(desc->vs.file); - ps_res = vs_res; - if (!ps_res_is_shared) { - ps_res = resource_open(desc->ps.file); - } - if (desc->vs.file.len <= 0 || desc->ps.file.len <= 0) { - error_str = LIT("Pipeline desc is missing shaders"); + ID3D10Blob *vs_blob = 0; + ID3D10Blob *ps_blob = 0; + ID3D10Blob *cs_blob = 0; + if (success && vs_dxc.len > 0) { + hr = D3DCreateBlob(vs_dxc.len, &vs_blob); + if (SUCCEEDED(hr)) { + MEMCPY(ID3D10Blob_GetBufferPointer(vs_blob), vs_dxc.text, vs_dxc.len); + } else { + error_str = LIT("Failed to create vertex shader blob"); success = 0; } } - - if (success) { - if (has_cs) { - dict_set(pipeline->arena, pipeline->dependencies, hash_fnv64(HASH_FNV64_BASIS, desc->cs.file), 1); + if (success && ps_dxc.len > 0) { + hr = D3DCreateBlob(ps_dxc.len, &ps_blob); + if (SUCCEEDED(hr)) { + MEMCPY(ID3D10Blob_GetBufferPointer(ps_blob), ps_dxc.text, ps_dxc.len); } else { - dict_set(pipeline->arena, pipeline->dependencies, hash_fnv64(HASH_FNV64_BASIS, desc->vs.file), 1); - dict_set(pipeline->arena, pipeline->dependencies, hash_fnv64(HASH_FNV64_BASIS, desc->ps.file), 1); + error_str = LIT("Failed to create pixel shader blob"); + success = 0; } } - - if (success) { - if (has_cs) { - if (!resource_exists(&cs_res)) { - error_str = string_format(scratch.arena, LIT("Compute shader source \"%F\" not found"), FMT_STR(desc->vs.file)); - success = 0; - } + if (success && cs_dxc.len > 0) { + hr = D3DCreateBlob(cs_dxc.len, &cs_blob); + if (SUCCEEDED(hr)) { + MEMCPY(ID3D10Blob_GetBufferPointer(cs_blob), cs_dxc.text, cs_dxc.len); } else { - if (!resource_exists(&vs_res)) { - error_str = string_format(scratch.arena, LIT("Vertex shader source \"%F\" not found"), FMT_STR(desc->vs.file)); - success = 0; - } else if (!resource_exists(&ps_res)) { - error_str = string_format(scratch.arena, LIT("Pixel shader source \"%F\" not found"), FMT_STR(desc->ps.file)); - success = 0; - } - } - } - - struct shader_compile_job_param vs = ZI; - vs.kind = SHADER_COMPILE_TASK_KIND_VS; - vs.pipeline = pipeline; - vs.shader_desc = desc->vs; - vs.shader_res = &vs_res; - - struct shader_compile_job_param ps = ZI; - ps.kind = SHADER_COMPILE_TASK_KIND_PS; - ps.pipeline = pipeline; - ps.shader_desc = desc->ps; - ps.shader_res = &ps_res; - - struct shader_compile_job_param cs = ZI; - cs.kind = SHADER_COMPILE_TASK_KIND_CS; - cs.pipeline = pipeline; - cs.shader_desc = desc->cs; - cs.shader_res = &cs_res; - - /* Compile shaders */ - if (success) { - if (has_cs) { - struct shader_compile_job_param *params[] = { &cs }; - struct shader_compile_job_sig comp_sig = { .params = params }; - struct snc_counter counter = ZI; - sys_run(countof(params), shader_compile_job, &comp_sig, SYS_POOL_INHERIT, SYS_PRIORITY_INHERIT, &counter); - snc_counter_wait(&counter); - success = cs.success; - } else { - struct shader_compile_job_param *params[] = { &vs, &ps }; - struct shader_compile_job_sig comp_sig = { .params = params }; - struct snc_counter counter = ZI; - sys_run(countof(params), shader_compile_job, &comp_sig, SYS_POOL_INHERIT, SYS_PRIORITY_INHERIT, &counter); - snc_counter_wait(&counter); - success = vs.success && ps.success; + error_str = LIT("Failed to create compute shader blob"); + success = 0; } } /* Get root signature blob * NOTE: This isn't necessary for creating the root signature (since it * could reuse the shader blob), however we'd like to verify that the - * root signature exists and matches between shaders. */ + * root signature exists and matches between vs & ps shaders. */ ID3D10Blob *rootsig_blob = 0; if (success) { __profn("Validate root signatures"); - if (has_cs) { + if (cs_dxc.len > 0) { u32 cs_rootsig_data_len = 0; ID3D10Blob *cs_rootsig_blob = 0; - D3DGetBlobPart(ID3D10Blob_GetBufferPointer(cs.blob), ID3D10Blob_GetBufferSize(cs.blob), D3D_BLOB_ROOT_SIGNATURE, 0, &cs_rootsig_blob); + D3DGetBlobPart(ID3D10Blob_GetBufferPointer(cs_blob), ID3D10Blob_GetBufferSize(cs_blob), D3D_BLOB_ROOT_SIGNATURE, 0, &cs_rootsig_blob); if (cs_rootsig_blob) { cs_rootsig_data_len = ID3D10Blob_GetBufferSize(cs_rootsig_blob); } if (cs_rootsig_data_len == 0) { success = 0; - error_str = LIT("Vertex shader is missing root signature"); + error_str = LIT("Compute shader is missing root signature"); } else { rootsig_blob = cs_rootsig_blob; } @@ -1251,8 +1042,8 @@ INTERNAL SYS_JOB_DEF(pipeline_alloc_job, job) u32 ps_rootsig_data_len = 0; ID3D10Blob *vs_rootsig_blob = 0; ID3D10Blob *ps_rootsig_blob = 0; - D3DGetBlobPart(ID3D10Blob_GetBufferPointer(vs.blob), ID3D10Blob_GetBufferSize(vs.blob), D3D_BLOB_ROOT_SIGNATURE, 0, &vs_rootsig_blob); - D3DGetBlobPart(ID3D10Blob_GetBufferPointer(ps.blob), ID3D10Blob_GetBufferSize(ps.blob), D3D_BLOB_ROOT_SIGNATURE, 0, &ps_rootsig_blob); + D3DGetBlobPart(ID3D10Blob_GetBufferPointer(vs_blob), ID3D10Blob_GetBufferSize(vs_blob), D3D_BLOB_ROOT_SIGNATURE, 0, &vs_rootsig_blob); + D3DGetBlobPart(ID3D10Blob_GetBufferPointer(ps_blob), ID3D10Blob_GetBufferSize(ps_blob), D3D_BLOB_ROOT_SIGNATURE, 0, &ps_rootsig_blob); if (vs_rootsig_blob) { vs_rootsig_data = ID3D10Blob_GetBufferPointer(vs_rootsig_blob); vs_rootsig_data_len = ID3D10Blob_GetBufferSize(vs_rootsig_blob); @@ -1293,14 +1084,12 @@ INTERNAL SYS_JOB_DEF(pipeline_alloc_job, job) /* Create PSO */ ID3D12PipelineState *pso = 0; if (success) { - if (has_cs) { + if (cs_dxc.len > 0) { __profn("Create compute PSO"); D3D12_COMPUTE_PIPELINE_STATE_DESC pso_desc = { 0 }; pso_desc.pRootSignature = rootsig; - if (cs.success) { - pso_desc.CS.pShaderBytecode = ID3D10Blob_GetBufferPointer(cs.blob); - pso_desc.CS.BytecodeLength = ID3D10Blob_GetBufferSize(cs.blob); - } + pso_desc.CS.pShaderBytecode = ID3D10Blob_GetBufferPointer(cs_blob); + pso_desc.CS.BytecodeLength = ID3D10Blob_GetBufferSize(cs_blob); hr = ID3D12Device_CreateComputePipelineState(G.device, &pso_desc, &IID_ID3D12PipelineState, (void **)&pso); } else { __profn("Create graphics PSO"); @@ -1364,14 +1153,10 @@ INTERNAL SYS_JOB_DEF(pipeline_alloc_job, job) /* PSO */ D3D12_GRAPHICS_PIPELINE_STATE_DESC pso_desc = { 0 }; pso_desc.pRootSignature = rootsig; - if (vs.success) { - pso_desc.VS.pShaderBytecode = ID3D10Blob_GetBufferPointer(vs.blob); - pso_desc.VS.BytecodeLength = ID3D10Blob_GetBufferSize(vs.blob); - } - if (ps.success) { - pso_desc.PS.pShaderBytecode = ID3D10Blob_GetBufferPointer(ps.blob); - pso_desc.PS.BytecodeLength = ID3D10Blob_GetBufferSize(ps.blob); - } + pso_desc.VS.pShaderBytecode = ID3D10Blob_GetBufferPointer(vs_blob); + pso_desc.VS.BytecodeLength = ID3D10Blob_GetBufferSize(vs_blob); + pso_desc.PS.pShaderBytecode = ID3D10Blob_GetBufferPointer(ps_blob); + pso_desc.PS.BytecodeLength = ID3D10Blob_GetBufferSize(ps_blob); pso_desc.BlendState = blend_desc; pso_desc.SampleMask = UINT_MAX; pso_desc.RasterizerState = raster_desc; @@ -1398,50 +1183,27 @@ INTERNAL SYS_JOB_DEF(pipeline_alloc_job, job) } /* Parse errors */ - parse_pipeline_errors(pipeline, cs.error_blob); - parse_pipeline_errors(pipeline, vs.error_blob); - parse_pipeline_errors(pipeline, ps.error_blob); - if (!success && pipeline->first_error == 0 && error_str.len == 0) { + if (!success && error_str.len <= 0) { error_str = LIT("Unknown error"); } - if (error_str.len > 0) { - struct pipeline_error *error = arena_push(pipeline->arena, struct pipeline_error); - error->msg = string_copy(pipeline->arena, error_str); - if (pipeline->last_error) { - pipeline->last_error->next = error; - } else { - pipeline->first_error = error; - } - pipeline->last_error = error; - } pipeline->pso = pso; pipeline->rootsig = rootsig; pipeline->compilation_time_ns = sys_time_ns() - start_ns; pipeline->success = success; + pipeline->error = error_str; - if (has_cs) { - resource_close(&cs_res); - } else { - resource_close(&vs_res); - if (!ps_res_is_shared) { - resource_close(&ps_res); - } - } if (rootsig_blob) { ID3D10Blob_Release(rootsig_blob); } - if (vs.blob) { - ID3D10Blob_Release(vs.blob); + if (vs_blob) { + ID3D10Blob_Release(vs_blob); } - if (vs.error_blob) { - ID3D10Blob_Release(vs.error_blob); + if (ps_blob) { + ID3D10Blob_Release(ps_blob); } - if (ps.blob) { - ID3D10Blob_Release(ps.blob); - } - if (ps.error_blob) { - ID3D10Blob_Release(ps.error_blob); + if (cs_blob) { + ID3D10Blob_Release(cs_blob); } } scratch_end(scratch); @@ -1453,7 +1215,12 @@ INTERNAL void pipeline_release_now(struct pipeline *pipeline) if (pipeline->pso) { ID3D12PipelineState_Release(pipeline->pso); } - arena_release(pipeline->arena); + struct snc_lock lock = snc_lock_e(&G.pipelines_mutex); + { + pipeline->next = G.first_free_pipeline; + G.first_free_pipeline = pipeline; + } + snc_unlock(&lock); } /* ========================== * @@ -1509,13 +1276,13 @@ INTERNAL struct pipeline *pipeline_from_name(struct pipeline_scope *scope, struc struct pipeline *res = &g_nil_pipeline; u64 hash = hash_fnv64(HASH_FNV64_BASIS, name); - struct pipeline *tmp = dict_get(scope->refs, hash); + struct pipeline *tmp = (struct pipeline *)dict_get(scope->refs, hash); if (tmp) { res = tmp; } else { { struct snc_lock lock = snc_lock_e(&G.pipelines_mutex); - tmp = dict_get(G.top_successful_pipelines, hash); + tmp = (struct pipeline *)dict_get(G.top_successful_pipelines, hash); if (tmp) { ++tmp->refcount; } @@ -1562,69 +1329,157 @@ INTERNAL void pipeline_register(u64 num_pipelines, struct pipeline **pipelines) } #if RESOURCE_RELOADING -INTERNAL RESOURCE_WATCH_CALLBACK_FUNC_DEF(pipeline_resource_watch_callback, name) +INTERNAL WATCH_CALLBACK_FUNC_DEF(pipeline_watch_callback, name) { __prof; struct arena_temp scratch = scratch_begin_no_conflict(); - /* Find dirty pipelines */ - u64 hash = hash_fnv64(HASH_FNV64_BASIS, name); - u32 num_pipelines = 0; - struct pipeline_desc *pipeline_descs = arena_push_dry(scratch.arena, struct pipeline_desc); - { - struct snc_lock lock = snc_lock_s(&G.pipelines_mutex); - for (struct dict_entry *entry = G.top_pipelines->first; entry; entry = entry->next) { - struct pipeline *pipeline = (struct pipeline *)entry->value; - if (dict_get(pipeline->dependencies, hash) == 1) { - logf_debug("Change detected in shader source file \"%F\", recompiling pipeline \"%F\"", FMT_STR(name), FMT_STR(pipeline->name)); - *arena_push(scratch.arena, struct pipeline_desc) = pipeline->desc; + struct string rs_extension = LIT(".hlsl_rs"); + struct string cs_extension = LIT(".hlsl_cs"); + + b32 is_src = string_starts_with(name, LIT("src/")); + b32 is_rs = is_src && string_ends_with(name, rs_extension); + b32 is_cs = is_src && !is_rs && string_ends_with(name, cs_extension); + b32 success = 0; + + /* Recompile shaders */ + struct string pipeline_name = ZI; + struct string friendly_name = ZI; + i32 num_shaders = 0; + struct shader_compile_desc *shader_descs = 0; + struct shader_compile_result *shader_results = 0; + if (is_rs || is_cs) { + logf_debug("Change detected in shader source file \"%F\", recompiling...", FMT_STR(name)); + success = 1; + struct sys_file file = sys_file_open_read_wait(name); + struct string data = sys_file_read_all(scratch.arena, file); + { + friendly_name = name; + struct string_array split = string_split(scratch.arena, friendly_name, LIT("src/")); + friendly_name = split.count > 0 ? string_cat(scratch.arena, LIT("src/"), split.strings[split.count - 1]) : friendly_name; + } + { + pipeline_name = name; + struct string_array split = string_split(scratch.arena, pipeline_name, LIT("/")); + pipeline_name = split.count > 0 ? split.strings[split.count - 1] : pipeline_name; + split = string_split(scratch.arena, pipeline_name, LIT(".")); + pipeline_name = split.count > 1 ? split.strings[split.count - 2] : pipeline_name; + } + { + struct shader_compile_job_sig sig = ZI; + sig.arena = scratch.arena; + if (is_rs) { + num_shaders = 2; + shader_descs = arena_push_array(scratch.arena, struct shader_compile_desc, num_shaders); + shader_results = arena_push_array(scratch.arena, struct shader_compile_result, num_shaders); + sig.descs = shader_descs; + sig.results = shader_results; + sig.descs[0].src = data; + sig.descs[0].friendly_name = friendly_name; + sig.descs[0].entry = LIT("vs"); + sig.descs[0].target = LIT("vs_6_6"); + sig.descs[1].src = data; + sig.descs[1].friendly_name = friendly_name; + sig.descs[1].entry = LIT("ps"); + sig.descs[1].target = LIT("ps_6_6"); + } else if (is_cs) { + num_shaders = 1; + shader_descs = arena_push_array(scratch.arena, struct shader_compile_desc, num_shaders); + shader_results = arena_push_array(scratch.arena, struct shader_compile_result, num_shaders); + sig.descs = shader_descs; + sig.results = shader_results; + sig.descs[0].src = data; + sig.descs[0].friendly_name = friendly_name; + sig.descs[0].entry = LIT("cs"); + sig.descs[0].target = LIT("cs_6_6"); + } + { + struct snc_counter counter = ZI; + sys_run(num_shaders, shader_compile_job, &sig, SYS_POOL_INHERIT, SYS_PRIORITY_INHERIT, &counter); + snc_counter_wait(&counter); + } + } + sys_file_close(file); + } + + + for (i32 i = 0; i < num_shaders; ++i) { + struct shader_compile_desc *desc = &shader_descs[i]; + struct shader_compile_result *result = &shader_results[i]; + if (result->success) { + logf_success("Finished compiling shader \"%F:%F\" in %F seconds", FMT_STR(desc->friendly_name), FMT_STR(desc->entry), FMT_FLOAT(SECONDS_FROM_NS(result->elapsed_ns))); + if (result->errors.len > 0) { + struct string msg = result->errors; + log_warning(msg); + } + } else { + struct string msg = result->errors; + log_error(msg); + success = 0; + } + } + + if (success) { + /* Create pipeline descs */ + u32 num_pipelines = 0; + struct pipeline_desc *pipeline_descs = arena_push_dry(scratch.arena, struct pipeline_desc); + for (struct dict_entry *entry = G.pipeline_descs->first; entry; entry = entry->next) { + struct pipeline_desc *pipeline_desc = (struct pipeline_desc *)entry->value; + struct pipeline_desc new_pipeline_desc = *pipeline_desc; + if (string_eq(pipeline_desc->name, pipeline_name)) { + if (is_rs) { + new_pipeline_desc.vs_dxc = shader_results[0].dxc; + new_pipeline_desc.ps_dxc = shader_results[1].dxc; + } else if (is_cs) { + new_pipeline_desc.cs_dxc = shader_results[0].dxc; + } + *arena_push_no_zero(scratch.arena, struct pipeline_desc) = new_pipeline_desc; ++num_pipelines; } } - snc_unlock(&lock); - } - /* Recompile dirty pipelines */ - if (num_pipelines > 0) { - __profn("Compile dirty pipelines"); - struct pipeline **pipelines = arena_push_array(scratch.arena, struct pipeline *, num_pipelines); - { - struct pipeline_alloc_job_sig sig = ZI; - sig.descs_in = pipeline_descs; - sig.pipelines_out = pipelines; - struct snc_counter counter = ZI; - sys_run(num_pipelines, pipeline_alloc_job, &sig, SYS_POOL_INHERIT, SYS_PRIORITY_INHERIT, &counter); - snc_counter_wait(&counter); - } - { - struct snc_lock lock = snc_lock_s(&G.pipelines_mutex); - for (u32 i = 0; i < num_pipelines; ++i) { - struct pipeline *pipeline = pipelines[i]; - if (pipeline->success) { - logf_success("Successfully compiled pipeline \"%F\" in %F seconds", FMT_STR(pipeline->name), FMT_FLOAT(SECONDS_FROM_NS(pipeline->compilation_time_ns))); - if (pipeline->first_error) { - struct string msg = string_format(scratch.arena, LIT("Warning while compiling pipeline \"%F\":\n%F"), FMT_STR(pipeline->name), FMT_STR(pipeline->first_error->msg)); - log_warning(msg); - } - } else { - { - struct string error = pipeline->first_error ? pipeline->first_error->msg : LIT("Unknown error"); - struct string msg = string_format(scratch.arena, LIT("Error compiling pipeline \"%F\":\n%F"), FMT_STR(pipeline->name), FMT_STR(error)); - log_error(msg); - } - struct pipeline *old_pipeline = dict_get(G.top_successful_pipelines, pipeline->hash); - if (!old_pipeline) { - /* If no previously successful pipeline exists, then show a message box rather than logging since logs may not be visible to user */ - struct string error = pipeline->first_error ? pipeline->first_error->msg : LIT("Unknown error"); - struct string msg = string_format(scratch.arena, LIT("Error compiling pipeline \"%F\":\n\n%F"), FMT_STR(pipeline->name), FMT_STR(error)); - sys_message_box(SYS_MESSAGE_BOX_KIND_WARNING, msg); - } - - } + /* Recompile dirty pipelines */ + if (num_pipelines > 0) { + __profn("Compile dirty pipelines"); + struct pipeline **pipelines = arena_push_array(scratch.arena, struct pipeline *, num_pipelines); + { + struct pipeline_alloc_job_sig sig = ZI; + sig.descs_in = pipeline_descs; + sig.pipelines_out = pipelines; + struct snc_counter counter = ZI; + sys_run(num_pipelines, pipeline_alloc_job, &sig, SYS_POOL_INHERIT, SYS_PRIORITY_INHERIT, &counter); + snc_counter_wait(&counter); } - snc_unlock(&lock); + { + struct snc_lock lock = snc_lock_s(&G.pipelines_mutex); + for (u32 i = 0; i < num_pipelines; ++i) { + struct pipeline *pipeline = pipelines[i]; + if (pipeline->success) { + logf_success("Successfully compiled pipeline \"%F\" in %F seconds", FMT_STR(pipeline->name), FMT_FLOAT(SECONDS_FROM_NS(pipeline->compilation_time_ns))); + if (pipeline->error.len > 0) { + struct string msg = string_format(scratch.arena, LIT("Warning while compiling pipeline \"%F\":\n%F"), FMT_STR(pipeline->name), FMT_STR(pipeline->error)); + log_warning(msg); + } + } else { + { + struct string error = pipeline->error.len > 0 ? pipeline->error : LIT("Unknown error"); + struct string msg = string_format(scratch.arena, LIT("Error compiling pipeline \"%F\":\n%F"), FMT_STR(pipeline->name), FMT_STR(error)); + log_error(msg); + } + struct pipeline *old_pipeline = (struct pipeline *)dict_get(G.top_successful_pipelines, pipeline->hash); + if (!old_pipeline) { + /* If no previously successful pipeline exists, then show a message box rather than logging since logs may not be visible to user */ + struct string error = pipeline->error.len > 0 ? pipeline->error : LIT("Unknown error"); + struct string msg = string_format(scratch.arena, LIT("Error compiling pipeline \"%F\":\n\n%F"), FMT_STR(pipeline->name), FMT_STR(error)); + sys_message_box(SYS_MESSAGE_BOX_KIND_WARNING, msg); + } + + } + } + snc_unlock(&lock); + } + pipeline_register(num_pipelines, pipelines); } - pipeline_register(num_pipelines, pipelines); } scratch_end(scratch); @@ -2262,7 +2117,7 @@ INTERNAL struct command_buffer *command_list_push_buffer(struct command_list *cl { u64 group_hash = command_buffer_hash_from_size(size); struct dict_entry *cb_group_entry = dict_ensure_entry(G.command_buffers_arena, G.command_buffers_dict, group_hash); - cb_group = cb_group_entry->value; + cb_group = (struct command_buffer_group *)cb_group_entry->value; if (!cb_group) { /* Create group */ cb_group = arena_push(G.command_buffers_arena, struct command_buffer_group); @@ -2386,10 +2241,10 @@ struct gp_resource *gp_texture_alloc(enum gp_texture_format format, u32 flags, s sys_panic(LIT("Tried to create texture with dimension <= 0")); } LOCAL_PERSIST const DXGI_FORMAT formats[] = { - [GP_TEXTURE_FORMAT_R8_UNORM] = DXGI_FORMAT_R8_UNORM, - [GP_TEXTURE_FORMAT_R8G8B8A8_UNORM] = DXGI_FORMAT_R8G8B8A8_UNORM, - [GP_TEXTURE_FORMAT_R8G8B8A8_UNORM_SRGB] = DXGI_FORMAT_R8G8B8A8_UNORM_SRGB, - [GP_TEXTURE_FORMAT_R16G16B16A16_FLOAT] = DXGI_FORMAT_R16G16B16A16_FLOAT + [GP_TEXTURE_FORMAT_R8_UNORM] = DXGI_FORMAT_R8_UNORM, + [GP_TEXTURE_FORMAT_R8G8B8A8_UNORM] = DXGI_FORMAT_R8G8B8A8_UNORM, + [GP_TEXTURE_FORMAT_R8G8B8A8_UNORM_SRGB] = DXGI_FORMAT_R8G8B8A8_UNORM_SRGB, + [GP_TEXTURE_FORMAT_R16G16B16A16_FLOAT] = DXGI_FORMAT_R16G16B16A16_FLOAT }; DXGI_FORMAT dxgi_format = ZI; @@ -3288,10 +3143,10 @@ struct gp_resource *gp_run_render(struct gp_render_sig *render_sig, struct gp_re constants.flags = sh_uint_from_u32(shade_flags); constants.tex_width = sh_uint_from_u32(render_size.x); constants.tex_height = sh_uint_from_u32(render_size.y); - constants.frame_seed = sh_uint4_from_u32((u32)rand_u64_from_state(&sig->rand), - (u32)rand_u64_from_state(&sig->rand), - (u32)rand_u64_from_state(&sig->rand), - (u32)rand_u64_from_state(&sig->rand)); + constants.frame_seed = sh_uint4_from_u32((u32)(rand_u64_from_state(&sig->rand) & 0xFFFFFFFF), + (u32)(rand_u64_from_state(&sig->rand) & 0xFFFFFFFF), + (u32)(rand_u64_from_state(&sig->rand) & 0xFFFFFFFF), + (u32)(rand_u64_from_state(&sig->rand) & 0xFFFFFFFF)); constants.frame_index = sh_uint_from_u32(sig->frame_index); constants.camera_offset = sh_float2_from_v2(world_to_render_xf.og); constants.albedo_tex_urid = sh_uint_from_u32(sig->albedo->srv_descriptor->index); @@ -3357,7 +3212,7 @@ struct gp_resource *gp_run_render(struct gp_render_sig *render_sig, struct gp_re constants.projection = sh_float4x4_from_mat4x4(blit_vp_matrix); constants.flags = sh_uint_from_u32(SH_BLIT_FLAG_TONE_MAP | SH_BLIT_FLAG_GAMMA_CORRECT); constants.exposure = sh_float_from_f32(2.0); - constants.gamma = sh_float_from_f32(2.2); + constants.gamma = sh_float_from_f32((f32)2.2); constants.tex_urid = sh_uint_from_u32(sig->shade_read->uav_descriptor->index); /* Set parameters */ diff --git a/src/inc.c b/src/inc.c index 21a4664b..6ab43a69 100644 --- a/src/inc.c +++ b/src/inc.c @@ -13,3 +13,10 @@ struct string inc_res_tar(void) return INCBIN_GET(res_tar); } #endif + + +INCBIN_INCLUDE(dxc_tar, INCBIN_DIR "dxc.tar"); +struct string inc_dxc_tar(void) +{ + return INCBIN_GET(dxc_tar); +} diff --git a/src/inc.h b/src/inc.h index 8cba4b5a..c14138ed 100644 --- a/src/inc.h +++ b/src/inc.h @@ -5,4 +5,6 @@ struct string inc_res_tar(void); #endif +struct string inc_dxc_tar(void); + #endif diff --git a/src/incbin.c b/src/incbin.c index 98d3f9ad..f8b79dd0 100644 --- a/src/incbin.c +++ b/src/incbin.c @@ -5,7 +5,7 @@ * ========================== */ #include "incbin.h" -#include "scratch.h" +#include "arena.h" #include "string.h" #include "atomic.h" #include "intrinsics.h" diff --git a/src/playback_wasapi.c b/src/playback_wasapi.c index 1b96f1bf..98a05cdc 100644 --- a/src/playback_wasapi.c +++ b/src/playback_wasapi.c @@ -16,6 +16,10 @@ #define COBJMACROS #define WIN32_LEAN_AND_MEAN #define UNICODE +#define WINVER 0x0A00 +#define _WIN32_WINNT 0x0A00 +#define NTDDI_WIN11_DT 0x0C0A0000 +#define NTDDI_VERSION 0x0A000000 #include #include #include diff --git a/src/prof_tracy.h b/src/prof_tracy.h index febf2c1c..9a7e0b97 100644 --- a/src/prof_tracy.h +++ b/src/prof_tracy.h @@ -1,12 +1,12 @@ #ifndef PROF_H #define PROF_H +#if PROFILING + #if COMPILER_MSVC # error "MSVC not supported for profiling (cleanup attributes are required for profiling markup)" #endif -#if PROFILING - #define PROFILING_SYSTEM_TRACE 0 #define PROFILING_CAPTURE_FRAME_IMAGE 0 #define PROFILING_LOCKS 0 diff --git a/src/resource.c b/src/resource.c index 9ff74486..397e58db 100644 --- a/src/resource.c +++ b/src/resource.c @@ -21,33 +21,12 @@ GLOBAL struct { #if RESOURCES_EMBEDDED struct tar_archive archive; #endif - -#if RESOURCE_RELOADING - struct sys_watch *watch; - struct atomic32 watch_shutdown; - struct snc_counter watch_jobs_counter; - - struct snc_mutex watch_dispatcher_mutex; - struct arena *watch_dispatcher_info_arena; - struct sys_watch_info_list watch_dispatcher_info_list; - struct snc_cv watch_dispatcher_cv; - - struct snc_mutex watch_callbacks_mutex; - resource_watch_callback *watch_callbacks[64]; - u64 num_watch_callbacks; -#endif } G = ZI, DEBUG_ALIAS(G, G_resource); /* ========================== * * Startup * ========================== */ -#if RESOURCE_RELOADING -INTERNAL SYS_JOB_DEF(resource_watch_monitor_job, _); -INTERNAL SYS_JOB_DEF(resource_watch_dispatcher_job, _); -INTERNAL SYS_EXIT_FUNC(resource_shutdown); -#endif - struct resource_startup_receipt resource_startup(void) { __prof; @@ -66,17 +45,6 @@ struct resource_startup_receipt resource_startup(void) } #endif -#if RESOURCE_RELOADING - G.watch = sys_watch_alloc(LIT("res")); - - G.watch_dispatcher_info_arena = arena_alloc(GIBI(64)); - - sys_run(1, resource_watch_monitor_job, 0, SYS_POOL_FLOATING, SYS_PRIORITY_LOW, &G.watch_jobs_counter); - sys_run(1, resource_watch_dispatcher_job, 0, SYS_POOL_BACKGROUND, SYS_PRIORITY_LOW, &G.watch_jobs_counter); - sys_on_exit(&resource_shutdown); -#endif - - return (struct resource_startup_receipt) { 0 }; } @@ -90,11 +58,9 @@ struct resource resource_open(struct string name) #if RESOURCES_EMBEDDED struct resource res = ZI; struct tar_entry *entry = tar_get(&G.archive, name); - if (entry) { - res._data = entry->data; - res._name = entry->file_name; - res._exists = 1; - } + res._data = entry->data; + res._name = entry->file_name; + res._exists = entry->valid; return res; #else struct resource res = ZI; @@ -146,160 +112,3 @@ void resource_close(struct resource *res_ptr) sys_file_close(res_ptr->_file); } #endif - -/* ========================== * - * Watch - * ========================== */ - -#if RESOURCE_RELOADING - -INTERNAL SYS_EXIT_FUNC(resource_shutdown) -{ - __prof; - atomic32_fetch_set(&G.watch_shutdown, 1); - - { - struct snc_lock lock = snc_lock_e(&G.watch_dispatcher_mutex); - snc_cv_signal(&G.watch_dispatcher_cv, I32_MAX); - sys_watch_wake(G.watch); - snc_unlock(&lock); - } - snc_counter_wait(&G.watch_jobs_counter); -} - -void resource_register_watch_callback(resource_watch_callback *callback) -{ - struct snc_lock lock = snc_lock_e(&G.watch_callbacks_mutex); - { - if (G.num_watch_callbacks < countof(G.watch_callbacks)) { - G.watch_callbacks[G.num_watch_callbacks++] = callback; - } else { - sys_panic(LIT("Max resource watch callbacks reached")); - } - } - snc_unlock(&lock); -} - -INTERNAL SYS_JOB_DEF(resource_watch_monitor_job, _) -{ - (UNUSED)_; - struct arena_temp scratch = scratch_begin_no_conflict(); - - while (!atomic32_fetch(&G.watch_shutdown)) { - struct arena_temp temp = arena_temp_begin(scratch.arena); - struct sys_watch_info_list res = sys_watch_read_wait(temp.arena, G.watch); - if (res.first && !atomic32_fetch(&G.watch_shutdown)) { - struct snc_lock lock = snc_lock_e(&G.watch_dispatcher_mutex); - { - struct sys_watch_info_list list_part = sys_watch_info_copy(G.watch_dispatcher_info_arena, res); - if (G.watch_dispatcher_info_list.last) { - G.watch_dispatcher_info_list.last->next = list_part.first; - list_part.first->prev = G.watch_dispatcher_info_list.last; - G.watch_dispatcher_info_list.last = list_part.last; - } else { - G.watch_dispatcher_info_list = list_part; - } - } - snc_cv_signal(&G.watch_dispatcher_cv, I32_MAX); - snc_unlock(&lock); - } - arena_temp_end(temp); - } - - scratch_end(scratch); -} - -/* NOTE: We separate the responsibilities of monitoring directory changes - * & dispatching watch callbacks into two separate jobs so that we can delay - * the dispatch, allowing for deduplication of file modification notifications. */ - -#define WATCH_DISPATCHER_DELAY_SECONDS 0.050 -#define WATCH_DISPATCHER_DEDUP_DICT_BINS 128 - -struct resource_watch_callback_job_sig { - struct string name; - resource_watch_callback **callbacks; -}; - -INTERNAL SYS_JOB_DEF(resource_watch_callback_job, job) -{ - __prof; - struct resource_watch_callback_job_sig *sig = job.sig; - struct string name = sig->name; - resource_watch_callback *callback = sig->callbacks[job.id]; - callback(name); -} - -INTERNAL SYS_JOB_DEF(resource_watch_dispatcher_job, _) -{ - (UNUSED)_; - struct arena_temp scratch = scratch_begin_no_conflict(); - - struct snc_lock watch_dispatcher_lock = snc_lock_e(&G.watch_dispatcher_mutex); - while (!atomic32_fetch(&G.watch_shutdown)) { - snc_cv_wait(&G.watch_dispatcher_cv, &watch_dispatcher_lock); - if (!atomic32_fetch(&G.watch_shutdown) && G.watch_dispatcher_info_arena->pos > 0) { - __profn("Dispatch resource watch callbacks"); - /* Unlock and sleep a bit so duplicate events pile up */ - { - __profn("Delay"); - snc_unlock(&watch_dispatcher_lock); - sys_wait(0, 0, 0, NS_FROM_SECONDS(WATCH_DISPATCHER_DELAY_SECONDS)); - watch_dispatcher_lock = snc_lock_e(&G.watch_dispatcher_mutex); - } - if (!atomic32_fetch(&G.watch_shutdown)) { - struct arena_temp temp = arena_temp_begin(scratch.arena); - - /* Pull watch info from queue */ - struct sys_watch_info_list watch_info_list = sys_watch_info_copy(temp.arena, G.watch_dispatcher_info_list); - MEMZERO_STRUCT(&G.watch_dispatcher_info_list); - arena_reset(G.watch_dispatcher_info_arena); - - /* Build callbacks array */ - u64 num_callbacks = 0; - resource_watch_callback **callbacks = 0; - struct snc_lock callbacks_lock = snc_lock_s(&G.watch_callbacks_mutex); - { - num_callbacks = G.num_watch_callbacks; - callbacks = arena_push_array_no_zero(temp.arena, resource_watch_callback *, num_callbacks); - for (u64 i = 0; i < num_callbacks; ++i) { - callbacks[i] = G.watch_callbacks[i]; - } - } - snc_unlock(&callbacks_lock); - - /* Unlock and run callbacks */ - snc_unlock(&watch_dispatcher_lock); - { - struct dict *dedup_dict = dict_init(temp.arena, WATCH_DISPATCHER_DEDUP_DICT_BINS); - for (struct sys_watch_info *info = watch_info_list.first; info; info = info->next) { - __profn("Dispatch"); - /* Do not run callbacks for the same file more than once */ - b32 skip = 0; - u64 hash = hash_fnv64(HASH_FNV64_BASIS, info->name); - if (dict_get(dedup_dict, hash) == 1) { - skip = 1; - } else { - dict_set(temp.arena, dedup_dict, hash, 1); - } - if (!skip) { - struct resource_watch_callback_job_sig sig = ZI; - sig.name = info->name; - sig.callbacks = callbacks; - struct snc_counter counter = ZI; - sys_run(num_callbacks, resource_watch_callback_job, &sig, SYS_POOL_BACKGROUND, SYS_PRIORITY_LOW, &counter); - snc_counter_wait(&counter); - } - } - } - watch_dispatcher_lock = snc_lock_e(&G.watch_dispatcher_mutex); - - arena_temp_end(temp); - } - } - } - - scratch_end(scratch); -} - -#endif diff --git a/src/resource.h b/src/resource.h index b5dd1858..86d6c36b 100644 --- a/src/resource.h +++ b/src/resource.h @@ -3,7 +3,7 @@ #include "sys.h" -#define RESOURCE_NAME_LEN_MAX 255 +#define RESOURCE_NAME_LEN_MAX 256 /* A resource contains data that can be retrieved globally by name. * If enabled during compilation, resource data is embedded in the @@ -54,17 +54,4 @@ struct resource resource_open(struct string name); #define resource_get_name(res_ptr) STRING((res_ptr)->_name_len, (res_ptr)->_name_text) #endif -/* ========================== * - * Watch - * ========================== */ - -#define RESOURCE_WATCH_CALLBACK_FUNC_DEF(func_name, arg_name) void func_name(struct string arg_name) -typedef RESOURCE_WATCH_CALLBACK_FUNC_DEF(resource_watch_callback, name); - -#if RESOURCE_RELOADING - void resource_register_watch_callback(resource_watch_callback *callback); -#else - #define resource_register_watch_callback(callback) -#endif - #endif diff --git a/src/sh/blit.d b/src/sh/blit.d new file mode 100644 index 00000000..d62e2cd8 --- /dev/null +++ b/src/sh/blit.d @@ -0,0 +1,3 @@ +C:\Users\Jacob\Home\dev\repos\power_play\build\clang-developer-debug\dxc\blit.dxc_ps: C:\Users\Jacob\Home\dev\repos\power_play\src\sh\blit.hlsl_rs \ + C:\Users\Jacob\Home\dev\repos\power_play\src\sh/common.hlsl \ + C:\Users\Jacob\Home\dev\repos\power_play\src\sh/sh_common.h diff --git a/res/sh/blit.hlsl b/src/sh/blit.hlsl_rs similarity index 97% rename from res/sh/blit.hlsl rename to src/sh/blit.hlsl_rs index caf18a58..99a9717c 100644 --- a/res/sh/blit.hlsl +++ b/src/sh/blit.hlsl_rs @@ -1,4 +1,4 @@ -#include "sh/common.hlsl" +#include "common.hlsl" /* ========================== * * Root signature @@ -54,7 +54,7 @@ SH_ENTRY(ROOTSIG) struct vs_output vs(struct vs_input input) * Tone map * ========================== */ -/* ACES approximation by Krzysztof Narkowicz +/* ACES approximation by Krzysztof Narkowicz * https://knarkowicz.wordpress.com/2016/01/06/aces-filmic-tone-mapping-curve/ */ INLINE float3 tone_map(float3 v) { diff --git a/res/sh/common.hlsl b/src/sh/common.hlsl similarity index 96% rename from res/sh/common.hlsl rename to src/sh/common.hlsl index cdb2487b..61119ae7 100644 --- a/res/sh/common.hlsl +++ b/src/sh/common.hlsl @@ -1,4 +1,4 @@ -#include "sh/sh_common.h" +#include "sh_common.h" #define TAU 6.28318530718 #define PI 3.14159265359 diff --git a/src/sh/flood.d b/src/sh/flood.d new file mode 100644 index 00000000..3adc62bf --- /dev/null +++ b/src/sh/flood.d @@ -0,0 +1,3 @@ +C:\Users\Jacob\Home\dev\repos\power_play\build\clang-developer-debug\dxc\flood.dxc_cs: C:\Users\Jacob\Home\dev\repos\power_play\src\sh\flood.hlsl_cs \ + C:\Users\Jacob\Home\dev\repos\power_play\src\sh/common.hlsl \ + C:\Users\Jacob\Home\dev\repos\power_play\src\sh/sh_common.h diff --git a/res/sh/flood.hlsl b/src/sh/flood.hlsl_cs similarity index 99% rename from res/sh/flood.hlsl rename to src/sh/flood.hlsl_cs index 80b3a578..38fc3d8b 100644 --- a/res/sh/flood.hlsl +++ b/src/sh/flood.hlsl_cs @@ -1,4 +1,4 @@ -#include "sh/common.hlsl" +#include "common.hlsl" /* ========================== * * Root signature diff --git a/src/sh/material.d b/src/sh/material.d new file mode 100644 index 00000000..39249f99 --- /dev/null +++ b/src/sh/material.d @@ -0,0 +1,3 @@ +C:\Users\Jacob\Home\dev\repos\power_play\build\clang-developer-debug\dxc\material.dxc_ps: C:\Users\Jacob\Home\dev\repos\power_play\src\sh\material.hlsl_rs \ + C:\Users\Jacob\Home\dev\repos\power_play\src\sh/common.hlsl \ + C:\Users\Jacob\Home\dev\repos\power_play\src\sh/sh_common.h diff --git a/res/sh/material.hlsl b/src/sh/material.hlsl_rs similarity index 99% rename from res/sh/material.hlsl rename to src/sh/material.hlsl_rs index 11b00384..e7285a00 100644 --- a/res/sh/material.hlsl +++ b/src/sh/material.hlsl_rs @@ -1,4 +1,4 @@ -#include "sh/common.hlsl" +#include "common.hlsl" /* ========================== * * Root signature diff --git a/res/sh/sh_common.h b/src/sh/sh_common.h similarity index 94% rename from res/sh/sh_common.h rename to src/sh/sh_common.h index d69b44fc..f75f6567 100644 --- a/res/sh/sh_common.h +++ b/src/sh/sh_common.h @@ -4,9 +4,9 @@ #define SH_DECL(t, n) struct CAT(sh_, t) n #define SH_DECLS(t, n) SH_DECL(t, n) #define SH_ENTRY(rootsig) static -#define SH_ASSERT_ROOT_CONST(s, n) STATIC_ASSERT(sizeof(s) % 16 == 0); /* Root constant struct should pad to 16 byte alignment */ \ - STATIC_ASSERT((sizeof(s) / 4) == n); /* Root constant struct size should match the specified 32-bit-constant count */ \ - STATIC_ASSERT((sizeof(s) <= 256)) /* Root constant struct can only fit 64 DWORDS */ +#define SH_ASSERT_ROOT_CONST(s, n) STATIC_ASSERT((sizeof(s) % 16 == 0) && /* Root constant struct should pad to 16 byte alignment */ \ + ((sizeof(s) / 4) == n) && /* Root constant struct size should match the specified 32-bit-constant count */ \ + (sizeof(s) <= 256)) /* Root constant struct can only fit 64 DWORDS */ struct sh_uint { u32 v; }; INLINE struct sh_uint sh_uint_from_u32(u32 v) diff --git a/src/sh/shade.d b/src/sh/shade.d new file mode 100644 index 00000000..1a48e9c7 --- /dev/null +++ b/src/sh/shade.d @@ -0,0 +1,3 @@ +C:\Users\Jacob\Home\dev\repos\power_play\build\clang-developer-debug\dxc\shade.dxc_cs: C:\Users\Jacob\Home\dev\repos\power_play\src\sh\shade.hlsl_cs \ + C:\Users\Jacob\Home\dev\repos\power_play\src\sh/common.hlsl \ + C:\Users\Jacob\Home\dev\repos\power_play\src\sh/sh_common.h diff --git a/res/sh/shade.hlsl b/src/sh/shade.hlsl_cs similarity index 99% rename from res/sh/shade.hlsl rename to src/sh/shade.hlsl_cs index 18641be0..1ed4d0ec 100644 --- a/res/sh/shade.hlsl +++ b/src/sh/shade.hlsl_cs @@ -1,4 +1,4 @@ -#include "sh/common.hlsl" +#include "common.hlsl" /* ========================== * * Root signature @@ -69,7 +69,6 @@ INLINE float3 get_light_in_dir(uint2 ray_start, float2 ray_dir) } } } - return result; } diff --git a/src/sh/shape.d b/src/sh/shape.d new file mode 100644 index 00000000..fdc1eadb --- /dev/null +++ b/src/sh/shape.d @@ -0,0 +1,3 @@ +C:\Users\Jacob\Home\dev\repos\power_play\build\clang-developer-debug\dxc\shape.dxc_ps: C:\Users\Jacob\Home\dev\repos\power_play\src\sh\shape.hlsl_rs \ + C:\Users\Jacob\Home\dev\repos\power_play\src\sh/common.hlsl \ + C:\Users\Jacob\Home\dev\repos\power_play\src\sh/sh_common.h diff --git a/res/sh/shape.hlsl b/src/sh/shape.hlsl_rs similarity index 97% rename from res/sh/shape.hlsl rename to src/sh/shape.hlsl_rs index 47729c5f..8907b85c 100644 --- a/res/sh/shape.hlsl +++ b/src/sh/shape.hlsl_rs @@ -1,4 +1,4 @@ -#include "sh/common.hlsl" +#include "common.hlsl" /* ========================== * * Root signature diff --git a/src/sh/ui.d b/src/sh/ui.d new file mode 100644 index 00000000..87e07c54 --- /dev/null +++ b/src/sh/ui.d @@ -0,0 +1,3 @@ +C:\Users\Jacob\Home\dev\repos\power_play\build\clang-developer-debug\dxc\ui.dxc_ps: C:\Users\Jacob\Home\dev\repos\power_play\src\sh\ui.hlsl_rs \ + C:\Users\Jacob\Home\dev\repos\power_play\src\sh/common.hlsl \ + C:\Users\Jacob\Home\dev\repos\power_play\src\sh/sh_common.h diff --git a/res/sh/ui.hlsl b/src/sh/ui.hlsl_rs similarity index 98% rename from res/sh/ui.hlsl rename to src/sh/ui.hlsl_rs index 480e5755..fb0463f0 100644 --- a/res/sh/ui.hlsl +++ b/src/sh/ui.hlsl_rs @@ -1,4 +1,4 @@ -#include "sh/common.hlsl" +#include "common.hlsl" /* ========================== * * Root signature diff --git a/src/sprite.c b/src/sprite.c index 902d2284..e0e7f904 100644 --- a/src/sprite.c +++ b/src/sprite.c @@ -11,6 +11,7 @@ #include "math.h" #include "rand.h" #include "snc.h" +#include "watch.h" /* The evictor will begin evicting once cache usage is > threshold. * It will entries until the budget has shrunk < target. */ @@ -205,7 +206,7 @@ INTERNAL SYS_JOB_DEF(sprite_load_job, arg); INTERNAL SYS_JOB_DEF(sprite_evictor_job, _); #if RESOURCE_RELOADING -INTERNAL RESOURCE_WATCH_CALLBACK_FUNC_DEF(sprite_resource_watch_callback, info); +INTERNAL WATCH_CALLBACK_FUNC_DEF(sprite_watch_callback, info); #endif struct sprite_startup_receipt sprite_startup(void) @@ -249,7 +250,7 @@ struct sprite_startup_receipt sprite_startup(void) sys_run(1, sprite_evictor_job, 0, SYS_POOL_BACKGROUND, SYS_PRIORITY_LOW, &G.shutdown_counter); sys_on_exit(&sprite_shutdown); - resource_register_watch_callback(&sprite_resource_watch_callback); + watch_register_callback(&sprite_watch_callback); return (struct sprite_startup_receipt) { 0 }; } @@ -1173,10 +1174,15 @@ INTERNAL void reload_if_exists(struct sprite_scope *scope, struct sprite_tag tag } } -INTERNAL RESOURCE_WATCH_CALLBACK_FUNC_DEF(sprite_resource_watch_callback, name) +INTERNAL WATCH_CALLBACK_FUNC_DEF(sprite_watch_callback, name) { struct sprite_scope *scope = sprite_scope_begin(); + if (string_starts_with(name, LIT("res/"))) { + name.len -= LIT("res/").len; + name.text += LIT("res/").len; + } + struct sprite_tag tag = sprite_tag_from_path(name); for (enum cache_entry_kind kind = 0; kind < NUM_CACHE_ENTRY_KINDS; ++kind) { reload_if_exists(scope, tag, kind); diff --git a/src/string.c b/src/string.c index 42c94318..683c9884 100644 --- a/src/string.c +++ b/src/string.c @@ -225,44 +225,38 @@ struct string string_cat(struct arena *arena, struct string str1, struct string return new_str; } -/* `arena` is where pieces items will be allocated. These strings point - * into the existing supplied string and do not allocate any new text. */ +/* `arena` is where pieces will be allocated. These strings point + * into the existing string and do not allocate any new text. */ struct string_array string_split(struct arena *arena, struct string str, struct string delim) { - struct string_array pieces = { - .count = 0, - .strings = arena_push_dry(arena, struct string) - }; + struct string_array pieces = ZI; + pieces.strings = arena_push_dry(arena, struct string); + i64 piece_start = 0; + for (i64 i = 0; i < (i64)str.len - (i64)delim.len; ++i) { + struct string cmp = ZI; + cmp.text = &str.text[i]; + cmp.len = min_i64(str.len - i, delim.len); - struct string piece = { - .len = 0, - .text = str.text - }; - - for (u64 i = 0; i <= str.len - delim.len; ++i) { - /* Clamp comparison string so we don't overflow. */ - struct string comp_str = { - .len = delim.len, - .text = &str.text[i] - }; - - b32 is_delimiter = string_eq(comp_str, delim); - b32 is_end = i == str.len - 1; - - if (!is_delimiter || is_end) { - ++piece.len; - } - - if (is_delimiter || is_end) { - /* Delimiter found */ - struct string *piece_pushed = arena_push_no_zero(arena, struct string); - *piece_pushed = piece; - ++pieces.count; - piece.text = piece.text + piece.len + delim.len; - piece.len = 0; + b32 is_delimiter = string_eq(cmp, delim); + if (is_delimiter) { + struct string piece = ZI; + piece.text = &str.text[piece_start]; + piece.len = i - piece_start; + i += delim.len; + piece_start = i; + if (piece.len > 0) { + *arena_push_no_zero(arena, struct string) = piece; + ++pieces.count; + } } } - + if (piece_start < (i64)str.len) { + struct string piece = ZI; + piece.text = &str.text[piece_start]; + piece.len = str.len - piece_start; + *arena_push_no_zero(arena, struct string) = piece; + ++pieces.count; + } return pieces; } diff --git a/src/sys.h b/src/sys.h index 443cdb41..80e06683 100644 --- a/src/sys.h +++ b/src/sys.h @@ -8,8 +8,7 @@ struct snc_counter; * ========================== */ /* Futex-like wait & wake */ - -void sys_wait(void *addr, void *cmp, u32 size, i64 timeout_ns); +void sys_wait(volatile void *addr, void *cmp, u32 size, i64 timeout_ns); void sys_wake(void *addr, i32 count); /* ========================== * @@ -232,8 +231,6 @@ struct sys_watch_info_list sys_watch_read_wait(struct arena *arena, struct sys_w void sys_watch_wake(struct sys_watch *dw); -struct sys_watch_info_list sys_watch_info_copy(struct arena *arena, struct sys_watch_info_list src); - /* ========================== * * Window * ========================== */ diff --git a/src/sys_win32.c b/src/sys_win32.c index ea04bffc..77744cad 100644 --- a/src/sys_win32.c +++ b/src/sys_win32.c @@ -481,7 +481,7 @@ INTERNAL b32 thread_try_release(struct thread *thread, f32 timeout_seconds) HANDLE handle = t->handle; if (handle) { /* Wait for thread to stop */ - DWORD timeout_ms = (timeout_seconds == F32_INFINITY) ? INFINITE : math_round_to_int(timeout_seconds * 1000); + DWORD timeout_ms = (timeout_seconds > 10000000) ? INFINITE : math_round_to_int(timeout_seconds * 1000); DWORD wait_res = WaitForSingleObject(handle, timeout_ms); if (wait_res == WAIT_OBJECT_0) { /* Release thread */ @@ -524,7 +524,7 @@ INTERNAL void thread_wait_release(struct thread *thread) * Wait / wake * ========================== */ -void sys_wait(void *addr, void *cmp, u32 size, i64 timeout_ns) +void sys_wait(volatile void *addr, void *cmp, u32 size, i64 timeout_ns) { struct fiber *fiber = fiber_from_id(sys_current_fiber_id()); i16 parent_id = fiber->parent_id; @@ -541,7 +541,7 @@ void sys_wait(void *addr, void *cmp, u32 size, i64 timeout_ns) job_fiber_yield(fiber, fiber_from_id(parent_id)); } else { i32 timeout_ms = 0; - if (timeout_ns == I64_MAX) { + if (timeout_ns > 10000000000000000ll) { timeout_ms = INFINITE; } else if (timeout_ns != 0) { timeout_ms = timeout_ns / 1000000; @@ -1258,7 +1258,7 @@ INTERNAL THREAD_DEF(job_worker_entry, worker_ctx_arg) tm_unlock(&G.wait_lists_arena_lock); } MEMZERO_STRUCT(wait_addr_list); - wait_addr_list->value = wait_addr; + wait_addr_list->value = (u64)wait_addr; if (wait_addr_bin->last_wait_list) { wait_addr_bin->last_wait_list->next_in_bin = wait_addr_list; wait_addr_list->prev_in_bin = wait_addr_bin->last_wait_list; @@ -1268,7 +1268,7 @@ INTERNAL THREAD_DEF(job_worker_entry, worker_ctx_arg) wait_addr_bin->last_wait_list = wait_addr_list; } /* Insert fiber into wait addr list */ - job_fiber->wait_addr = wait_addr; + job_fiber->wait_addr = (u64)wait_addr; if (wait_addr_list->last_waiter) { fiber_from_id(wait_addr_list->last_waiter)->next_addr_waiter = job_fiber_id; job_fiber->prev_addr_waiter = wait_addr_list->last_waiter; @@ -2107,26 +2107,6 @@ void sys_watch_wake(struct sys_watch *dw) SetEvent(w32_watch->wake_handle); } -struct sys_watch_info_list sys_watch_info_copy(struct arena *arena, struct sys_watch_info_list src_list) -{ - struct sys_watch_info_list dst_list = ZI; - for (struct sys_watch_info *src = src_list.first; src; src = src->next) { - struct sys_watch_info *dst = arena_push(arena, struct sys_watch_info); - dst->kind = src->kind; - dst->name = string_copy(arena, src->name); - if (dst_list.last) { - dst_list.last->next = dst; - dst->prev = dst_list.last; - dst_list.last = dst; - } else { - dst_list.first = dst; - dst_list.last = dst; - } - dst_list.count = src_list.count; - } - return dst_list; -} - /* ========================== * * Window * ========================== */ diff --git a/src/tar.c b/src/tar.c index 0c8c0326..3ca91d72 100644 --- a/src/tar.c +++ b/src/tar.c @@ -120,6 +120,7 @@ struct tar_archive tar_parse(struct arena *arena, struct string data, struct str struct string file_name = string_cat(arena, prefix, file_name_cstr); struct tar_entry *entry = arena_push(arena, struct tar_entry); + entry->valid = 1; entry->is_dir = is_dir; entry->file_name = file_name; entry->data = file_data; @@ -162,8 +163,10 @@ struct tar_archive tar_parse(struct arena *arena, struct string data, struct str return archive; } +READONLY GLOBAL struct tar_entry g_nil_tar_entry = ZI; struct tar_entry *tar_get(struct tar_archive *archive, struct string name) { u64 hash = hash_fnv64(HASH_FNV64_BASIS, name); - return (struct tar_entry *)dict_get(archive->lookup, hash); + struct tar_entry *lookup = (struct tar_entry *)dict_get(archive->lookup, hash); + return lookup ? lookup : &g_nil_tar_entry; } diff --git a/src/tar.h b/src/tar.h index 7a9b9c2b..1b9aef94 100644 --- a/src/tar.h +++ b/src/tar.h @@ -4,6 +4,7 @@ #include "util.h" struct tar_entry { + b32 valid; struct string file_name; struct string data; diff --git a/src/util.h b/src/util.h index 70c13e1d..10a46bb4 100644 --- a/src/util.h +++ b/src/util.h @@ -266,7 +266,7 @@ INLINE void sleep_precise(i64 sleep_time_ns) __prof; i64 big_sleep = sys_current_scheduler_period_ns(); - i64 tolerance = big_sleep * 0.5; + i64 tolerance = (f64)big_sleep * 0.5; //i64 tolerance = 1000000000; i64 now_ns = sys_time_ns(); diff --git a/src/watch.c b/src/watch.c new file mode 100644 index 00000000..8cf7a74a --- /dev/null +++ b/src/watch.c @@ -0,0 +1,245 @@ +#include "watch.h" + +#if RESOURCE_RELOADING + +#include "sys.h" +#include "snc.h" +#include "arena.h" +#include "atomic.h" +#include "util.h" +#include "string.h" + +struct watch_event { + struct string name; + struct watch_event *next; +}; + +GLOBAL struct { + struct sys_watch *watch; + struct atomic32 watch_shutdown; + struct snc_counter watch_jobs_counter; + + struct snc_mutex watch_dispatcher_mutex; + struct arena *watch_events_arena; + struct watch_event *first_watch_event; + struct watch_event *last_watch_event; + struct snc_cv watch_dispatcher_cv; + + struct snc_mutex watch_callbacks_mutex; + watch_callback *watch_callbacks[64]; + u64 num_watch_callbacks; +} G = ZI, DEBUG_ALIAS(G, G_watch); + +/* ========================== * + * Startup + * ========================== */ + +INTERNAL SYS_JOB_DEF(watch_monitor_job, _); +INTERNAL SYS_JOB_DEF(watch_dispatcher_job, _); +INTERNAL SYS_EXIT_FUNC(watch_shutdown); + +void watch_startup(void) +{ + G.watch = sys_watch_alloc(LIT("./")); + + G.watch_events_arena = arena_alloc(GIBI(64)); + + sys_run(1, watch_monitor_job, 0, SYS_POOL_FLOATING, SYS_PRIORITY_LOW, &G.watch_jobs_counter); + sys_run(1, watch_dispatcher_job, 0, SYS_POOL_BACKGROUND, SYS_PRIORITY_LOW, &G.watch_jobs_counter); + sys_on_exit(&watch_shutdown); +} + +/* ========================== * + * Watch + * ========================== */ + +INTERNAL SYS_EXIT_FUNC(watch_shutdown) +{ + __prof; + atomic32_fetch_set(&G.watch_shutdown, 1); + + { + struct snc_lock lock = snc_lock_e(&G.watch_dispatcher_mutex); + snc_cv_signal(&G.watch_dispatcher_cv, I32_MAX); + sys_watch_wake(G.watch); + snc_unlock(&lock); + } + snc_counter_wait(&G.watch_jobs_counter); +} + +void watch_register_callback(watch_callback *callback) +{ + struct snc_lock lock = snc_lock_e(&G.watch_callbacks_mutex); + { + if (G.num_watch_callbacks < countof(G.watch_callbacks)) { + G.watch_callbacks[G.num_watch_callbacks++] = callback; + } else { + sys_panic(LIT("Max resource watch callbacks reached")); + } + } + snc_unlock(&lock); +} + +INTERNAL SYS_JOB_DEF(watch_monitor_job, _) +{ + (UNUSED)_; + struct arena_temp scratch = scratch_begin_no_conflict(); + + struct string ignored[] = { + LIT(".vs"), + LIT(".git") + }; + + while (!atomic32_fetch(&G.watch_shutdown)) { + struct arena_temp temp = arena_temp_begin(scratch.arena); + struct sys_watch_info_list info_list = sys_watch_read_wait(temp.arena, G.watch); + if (info_list.first && !atomic32_fetch(&G.watch_shutdown)) { + struct snc_lock lock = snc_lock_e(&G.watch_dispatcher_mutex); + { + for (struct sys_watch_info *info = info_list.first; info; info = info->next) { + struct string name_src = info->name; + b32 ignore = 0; + for (u32 i = 0; i < countof(ignored); ++i) { + if (string_starts_with(name_src, ignored[i])) { + ignore = 1; + break; + } + } + if (!ignore) { + struct watch_event *e = arena_push(G.watch_events_arena, struct watch_event); + e->name = string_copy(G.watch_events_arena, name_src); + if (G.last_watch_event) { + G.last_watch_event->next = e; + } else { + G.first_watch_event = e; + } + G.last_watch_event = e; + } + } + } + snc_cv_signal(&G.watch_dispatcher_cv, I32_MAX); + snc_unlock(&lock); + } + arena_temp_end(temp); + } + + scratch_end(scratch); +} + +/* NOTE: We separate the responsibilities of monitoring directory changes + * & dispatching watch callbacks into two separate jobs so that we can delay + * the dispatch, allowing for deduplication of file modification notifications. */ + +#define WATCH_DISPATCHER_DELAY_SECONDS 0.050 +#define WATCH_DISPATCHER_DEDUP_DICT_BINS 128 + +struct watch_callback_job_sig { + struct string name; + watch_callback **callbacks; +}; + +INTERNAL SYS_JOB_DEF(watch_callback_job, job) +{ + __prof; + struct watch_callback_job_sig *sig = job.sig; + struct string name = sig->name; + watch_callback *callback = sig->callbacks[job.id]; + callback(name); +} + +INTERNAL SYS_JOB_DEF(watch_dispatcher_job, _) +{ + (UNUSED)_; + + b32 shutdown = 0; + while (!shutdown) { + { + struct arena_temp scratch = scratch_begin_no_conflict(); + struct watch_event *first_watch_event = 0; + struct watch_event *last_watch_event = 0; + + /* Delay so that duplicate events pile up */ + { + __profn("Delay"); + sys_wait(0, 0, 0, NS_FROM_SECONDS(WATCH_DISPATCHER_DELAY_SECONDS)); + } + + /* Pull watch events from queue */ + { + struct snc_lock lock = snc_lock_e(&G.watch_dispatcher_mutex); + for (struct watch_event *src_event = G.first_watch_event; src_event; src_event = src_event->next) { + struct watch_event *e = arena_push(scratch.arena, struct watch_event); + e->name = string_copy(scratch.arena, src_event->name); + if (last_watch_event) { + last_watch_event->next = e; + } else { + first_watch_event = e; + } + last_watch_event = e; + } + G.first_watch_event = 0; + G.last_watch_event = 0; + arena_reset(G.watch_events_arena); + snc_unlock(&lock); + } + + /* Build callbacks array */ + u64 num_callbacks = 0; + watch_callback **callbacks = 0; + struct snc_lock callbacks_lock = snc_lock_s(&G.watch_callbacks_mutex); + { + num_callbacks = G.num_watch_callbacks; + callbacks = arena_push_array_no_zero(scratch.arena, watch_callback *, num_callbacks); + for (u64 i = 0; i < num_callbacks; ++i) { + callbacks[i] = G.watch_callbacks[i]; + } + } + snc_unlock(&callbacks_lock); + + /* Run callbacks */ + { + struct dict *dedup_dict = dict_init(scratch.arena, WATCH_DISPATCHER_DEDUP_DICT_BINS); + for (struct watch_event *e = first_watch_event; e; e = e->next) { + __profn("Dispatch"); + /* Do not run callbacks for the same file more than once */ + b32 skip = 0; + u64 hash = hash_fnv64(HASH_FNV64_BASIS, e->name); + if (dict_get(dedup_dict, hash) == 1) { + skip = 1; + } else { + dict_set(scratch.arena, dedup_dict, hash, 1); + } + if (!skip) { + struct watch_callback_job_sig sig = ZI; + sig.name = e->name; + sig.callbacks = callbacks; + struct snc_counter counter = ZI; + sys_run(num_callbacks, watch_callback_job, &sig, SYS_POOL_BACKGROUND, SYS_PRIORITY_LOW, &counter); + snc_counter_wait(&counter); + } + } + } + + scratch_end(scratch); + } + + /* Wait for event */ + struct snc_lock lock = snc_lock_s(&G.watch_dispatcher_mutex); + { + shutdown = atomic32_fetch(&G.watch_shutdown); + while (!shutdown && !G.first_watch_event) { + snc_cv_wait(&G.watch_dispatcher_cv, &lock); + shutdown = atomic32_fetch(&G.watch_shutdown); + } + } + snc_unlock(&lock); + } +} + +#else /* RESOURCE_RELOADING */ + +void watch_startup(void) +{ +} + +#endif /* RESOURCE_RELOADING */ diff --git a/src/watch.h b/src/watch.h new file mode 100644 index 00000000..d116ada1 --- /dev/null +++ b/src/watch.h @@ -0,0 +1,15 @@ +#ifndef WATCH_H +#define WATCH_H + +#define WATCH_CALLBACK_FUNC_DEF(func_name, arg_name) void func_name(struct string arg_name) +typedef WATCH_CALLBACK_FUNC_DEF(watch_callback, name); + +void watch_startup(void); + +#if RESOURCE_RELOADING +void watch_register_callback(watch_callback *callback); +#else +#define watch_register_callback(callback) +#endif + +#endif