jacob/power_play
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

compile shaders & embed resource dirs in parallel

Browse Source
This commit is contained in:
jacob 2025-12-07 09:05:46 -06:00
parent 32002c75bb
commit 9329d41c88
5 changed files with 341 additions and 337 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/base/base_win32/base_win32_wave.c
View File

@ -38,6 +38,16 @@ void DispatchWave(String name, u32 num_lanes, WaveLaneEntryFunc *entry, void *ud
{ {
Arena *perm = PermArena(); Arena *perm = PermArena();
/* Catch high lane count to prevent OS crash while debugging */
u32 max_lanes = 4096;
if (num_lanes <= 0 || num_lanes > max_lanes)
{
Panic(StringF(perm,
"Tried to create a wave with %F lanes (valid range is [1, %F])",
FmtUint(num_lanes),
FmtUint(max_lanes)));
}
WaveCtx *wave_ctx = PushStruct(perm, WaveCtx); WaveCtx *wave_ctx = PushStruct(perm, WaveCtx);
wave_ctx->lanes_count = num_lanes; wave_ctx->lanes_count = num_lanes;
wave_ctx->udata = udata; wave_ctx->udata = udata;

610
src/meta/meta.c
View File

@ -1,18 +1,18 @@
#include "meta.h" #include "meta.h"
Atomic32 g_status = ZI; BuildData build = ZI;
//////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////
//~ Helpers //~ Helpers
i32 GetBuildStatus(void) i32 GetBuildStatus(void)
{ {
return Atomic32Fetch(&g_status); return Atomic32Fetch(&build.status);
} }
void SetBuildStatus(i32 code) void SetBuildStatus(i32 code)
{ {
Atomic32FetchTestSet(&g_status, 0, code); Atomic32FetchTestSet(&build.status, 0, code);
} }
void EchoLine(String msg) void EchoLine(String msg)
@ -94,6 +94,151 @@ String StringFromMetaErrors(Arena *arena, M_ErrorList errors)
return result; return result;
} }
EmbedObj Embed(String store_name, String dir_path)
{
Arena *perm = PermArena();
EmbedObj result = ZI;
/* Generate resource archive contents */
String arc_contents = ZI;
{
StringList files = ZI;
F_FilesFromDir(perm, &files, dir_path, F_IterFlag_Recurse);
Struct(EntryNode)
{
EntryNode *next;
String entry_name;
String file_name;
};
EntryNode *first_entry = 0;
EntryNode *last_entry = 0;
u64 entries_count = 0;
{
for (StringListNode *file_node = files.first; file_node; file_node = file_node->next)
{
String file_name = file_node->s;
if (F_IsFile(file_name))
{
String entry_name = file_name;
if (entry_name.len > (dir_path.len + 1))
{
entry_name.len -= dir_path.len + 1;
entry_name.text += dir_path.len + 1;
}
entry_name = StringF(perm, "%F/%F", FmtString(store_name), FmtString(entry_name));
for (u64 i = 0; i < entry_name.len; ++i)
{
if (entry_name.text[i] == '\\')
{
entry_name.text[i] = '/';
}
}
EntryNode *en = PushStruct(perm, EntryNode);
en->entry_name = entry_name;
en->file_name = file_name;
SllQueuePush(first_entry, last_entry, en);
++entries_count;
}
}
}
{
BB_Buff bb = BB_AcquireBuff(Gibi(2));
BB_Writer bw = BB_WriterFromBuff(&bb);
/* Write magic */
BB_WriteUBits(&bw, ResourceEmbeddedMagic, 64);
/* Write header */
BB_WriteUBits(&bw, entries_count, 64);
/* Reserve entries space */
u64 entry_size = 8 /* Name start */
+ 8 /* Name end */
+ 8 /* Data start */
+ 8; /* Data end */
u8 *entries_start = BB_GetWrittenRaw(&bw) + BB_GetNumBytesWritten(&bw);
u64 entries_size = entry_size * entries_count;
String entries_str = STRING(entries_size, entries_start);
BB_WriteSeekBytes(&bw, entries_size);
BB_Buff entries_bb = BB_BuffFromString(entries_str);
BB_Writer entries_bw = BB_WriterFromBuff(&entries_bb);
/* Write entries */
for (EntryNode *en = first_entry; en; en = en->next)
{
/* TODO: Copy file data directly into archive file */
String file_data = F_DataFromFile(perm, en->file_name);
/* Write name */
BB_WriteAlignBytes(&bw, 64);
u64 name_start = BB_GetNumBytesWritten(&bw) + 1;
BB_WriteString(&bw, en->entry_name);
u64 name_len = BB_GetNumBytesWritten(&bw) - name_start;
/* Write data */
BB_WriteAlignBytes(&bw, 64);
/* FIXME: Why no +1 here? */
u64 data_start = BB_GetNumBytesWritten(&bw);
BB_WriteBytes(&bw, file_data);
u64 data_len = BB_GetNumBytesWritten(&bw) - data_start;
/* Write entry */
BB_WriteUBits(&entries_bw, name_start, 64);
BB_WriteUBits(&entries_bw, name_len, 64);
BB_WriteUBits(&entries_bw, data_start, 64);
BB_WriteUBits(&entries_bw, data_len, 64);
}
arc_contents.len = BB_GetNumBytesWritten(&bw);
arc_contents.text = BB_GetWrittenRaw(&bw);
}
}
/* Write archive to file */
String arc_path = StringF(perm, "%F.arc", FmtString(store_name));
F_ClearWrite(arc_path, arc_contents);
/* Generate object file */
if (IsPlatformWindows)
{
/* Generate RC file */
String rc_out_file = StringF(perm, "%F.rc", FmtString(store_name));
{
RandState rs = ZI;
StringList rc_out_lines = ZI;
String arc_file_cp = F_GetFullCrossPlatform(perm, arc_path);
String line = StringF(perm, "%F_%F RCDATA \"%F\"", FmtString(Lit(Stringize(W32_EmbeddedDataPrefix))), FmtHex(RandU64FromState(&rs)), FmtString(arc_file_cp));
PushStringToList(perm, &rc_out_lines, line);
/* Write to file */
String rc_out = StringFromList(perm, rc_out_lines, Lit("\n"));
F_ClearWrite(rc_out_file, rc_out);
}
/* Compile RC file */
result.obj_file = StringF(perm, "%F.res", FmtString(store_name));
{
result.obj_file, FmtString(F_GetFull(perm, rc_out_file));
String cmd = StringF(perm, "rc.exe -nologo -fo %F %F", FmtString(result.obj_file), FmtString(F_GetFull(perm, rc_out_file)));
OS_CommandResult cmd_result = OS_RunCommand(perm, cmd);
String cmd_output = TrimWhitespace(cmd_result.output);
result.output = cmd_output;
result.return_code = cmd_result.code;
}
SetBuildStatus(result.return_code);
}
else
{
/* TODO: Compile object files using .incbin on non-windows platforms */
Panic(Lit("Embedded resource path not implemented for this platform"));
}
return result;
}
//////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////
//~ Build //~ Build
@ -152,9 +297,8 @@ void BuildEntryPoint(WaveLaneCtx *lane)
Struct(CmdLineArgs) Struct(CmdLineArgs)
{ {
String leaf_layer_name; String leaf_layer_name;
} cmdline = ZI; };
CmdLineArgs cmdline = ZI;
if (lane->idx == 0)
{ {
String layer_name = ZI; String layer_name = ZI;
CommandlineArg arg = CommandlineArgFromName(Lit("layer")); CommandlineArg arg = CommandlineArgFromName(Lit("layer"));
@ -168,13 +312,15 @@ void BuildEntryPoint(WaveLaneCtx *lane)
} }
if (layer_name.len == 0) if (layer_name.len == 0)
{ {
EchoLine(Lit("No layer supplied, assuming \"pp\" build")); if (lane->idx == 0) EchoLine(Lit("No layer supplied, assuming \"pp\" build"));
layer_name = Lit("pp"); layer_name = Lit("pp");
} }
cmdline.leaf_layer_name = layer_name; cmdline.leaf_layer_name = layer_name;
EchoLine(StringF(perm, "Building \"%F\"", FmtString(cmdline.leaf_layer_name))); if (lane->idx == 0)
{
if (lane->idx == 0) EchoLine(StringF(perm, "Building layer \"%F\"", FmtString(cmdline.leaf_layer_name)));
}
} }
WaveSyncBroadcast(lane, 0, &cmdline);
////////////////////////////// //////////////////////////////
//- Generate compiler params //- Generate compiler params
@ -269,48 +415,30 @@ void BuildEntryPoint(WaveLaneCtx *lane)
} }
////////////////////////////// //////////////////////////////
//- Prep //- Phase 1/3: Prep
/*
* Phase 1/3: Prep (narrow)
* - Parse layers
* - Generate final C file
* - Generate final HLSL file
* - Generate resource dirs info
*
* Phase 2/3: Compile (wide)
* - Compile C
* - Compile & embed shaders
* - Embed resource dirs
*
* Phase 3/3: Link (narrow)
* - Link
*/
/* TODO: Dispatch OS commands asynchronously */
String shader_store_name = Lit("ShadersStore"); String shader_store_name = Lit("ShadersStore");
String c_out_file = F_GetFull(perm, StringF(perm, "%F_gen_c.c", FmtString(cmdline.leaf_layer_name))); String c_out_file = F_GetFull(perm, StringF(perm, "%F_gen_c.c", FmtString(cmdline.leaf_layer_name)));
String gpu_out_file = F_GetFull(perm, StringF(perm, "%F_gen_gpu.hlsl", FmtString(cmdline.leaf_layer_name))); String gpu_out_file = F_GetFull(perm, StringF(perm, "%F_gen_gpu.hlsl", FmtString(cmdline.leaf_layer_name)));
/*
* Phase 1
* - Parse layers
*
* Phase 2
* - Generate final C file
* - Generate final HLSL file
* - Generate resource archive info
*
* Phase 3
* - C compiler process
* - HLSL compiler processes
*
* Phase 4
* - Generate shader resource archive
* - Generate embedded resource archives
*
* Phase 5
* - Shader resource compilation process
* - Embedded resource compilation processes
*
* Phase 6
*/
/* TODO: Dispatch OS processes asynchronously rather than synchronously waiting on each lane */
BuildData *build = 0;
if (lane->idx == 0)
{
build = PushStruct(perm, BuildData);
}
WaveSyncBroadcast(lane, 0, &build);
//////////////////////////////
//- Parse meta layer info into build data
if (lane->idx == 0 && GetBuildStatus() == 0) if (lane->idx == 0 && GetBuildStatus() == 0)
{ {
//- Parse layers //- Parse layers
@ -326,9 +454,9 @@ void BuildEntryPoint(WaveLaneCtx *lane)
/* Flatten */ /* Flatten */
StringList starting_layer_names = ZI; StringList starting_layer_names = ZI;
PushStringToList(perm, &starting_layer_names, cmdline.leaf_layer_name); PushStringToList(perm, &starting_layer_names, cmdline.leaf_layer_name);
build->flattened = M_GetFlattenedEntries(perm, parsed, starting_layer_names); build.layers_parse = M_FlattenEntries(perm, parsed, starting_layer_names);
SetBuildStatus(build->flattened.errors.count > 0); SetBuildStatus(build.layers_parse.errors.count > 0);
} }
//- Generate C file //- Generate C file
@ -339,7 +467,7 @@ void BuildEntryPoint(WaveLaneCtx *lane)
StringList c_include_lines = ZI; StringList c_include_lines = ZI;
StringList c_startup_lines = ZI; StringList c_startup_lines = ZI;
{ {
for (M_Entry *entry = build->flattened.first; entry->valid; entry = entry->next) for (M_Entry *entry = build.layers_parse.first; entry->valid; entry = entry->next)
{ {
M_EntryKind kind = entry->kind; M_EntryKind kind = entry->kind;
M_Token *entry_tok = entry->name_token; M_Token *entry_tok = entry->name_token;
@ -366,12 +494,12 @@ void BuildEntryPoint(WaveLaneCtx *lane)
else else
{ {
String err = StringF(perm, "Directory '%F' not found", FmtString(full)); String err = StringF(perm, "Directory '%F' not found", FmtString(full));
M_PushError(perm, &build->cgen.errors, arg1_tok, err); M_PushError(perm, &build.c_parse.errors, arg1_tok, err);
} }
} }
else else
{ {
M_PushError(perm, &build->cgen.errors, entry_tok, Lit("Expected resource store & directory name")); M_PushError(perm, &build.c_parse.errors, entry_tok, Lit("Expected resource store & directory name"));
} }
} break; } break;
case M_EntryKind_VertexShader: case M_EntryKind_VertexShader:
@ -391,7 +519,7 @@ void BuildEntryPoint(WaveLaneCtx *lane)
} }
else else
{ {
M_PushError(perm, &build->cgen.errors, entry_tok, Lit("Expected shader name")); M_PushError(perm, &build.c_parse.errors, entry_tok, Lit("Expected shader name"));
} }
} break; } break;
case M_EntryKind_IncludeC: case M_EntryKind_IncludeC:
@ -410,12 +538,12 @@ void BuildEntryPoint(WaveLaneCtx *lane)
else else
{ {
String err = StringF(perm, "File '%F' not found", FmtString(full)); String err = StringF(perm, "File '%F' not found", FmtString(full));
M_PushError(perm, &build->cgen.errors, arg0_tok, err); M_PushError(perm, &build.c_parse.errors, arg0_tok, err);
} }
} }
else else
{ {
M_PushError(perm, &build->cgen.errors, entry_tok, Lit("Expected file name")); M_PushError(perm, &build.c_parse.errors, entry_tok, Lit("Expected file name"));
} }
} break; } break;
case M_EntryKind_Startup: case M_EntryKind_Startup:
@ -428,13 +556,13 @@ void BuildEntryPoint(WaveLaneCtx *lane)
} }
else else
{ {
M_PushError(perm, &build->cgen.errors, entry_tok, Lit("Expected startup function name")); M_PushError(perm, &build.c_parse.errors, entry_tok, Lit("Expected startup function name"));
} }
} break; } break;
} }
} }
} }
if (build->cgen.errors.count == 0) if (build.c_parse.errors.count == 0)
{ {
StringList c_out_lines = ZI; StringList c_out_lines = ZI;
PushStringToList(perm, &c_out_lines, Lit("// Auto generated file")); PushStringToList(perm, &c_out_lines, Lit("// Auto generated file"));
@ -493,12 +621,13 @@ void BuildEntryPoint(WaveLaneCtx *lane)
F_ClearWrite(c_out_file, c_out); F_ClearWrite(c_out_file, c_out);
} }
SetBuildStatus(build->cgen.errors.count > 0); SetBuildStatus(build.c_parse.errors.count > 0);
} }
//- Generate HLSL file //- Generate HLSL file
{ {
/* Clear shader store */ /* Clear shader store */
/* TODO: Move to separate artifacts dir that gets cleared, including archive files */
OS_Mkdir(shader_store_name); OS_Mkdir(shader_store_name);
{ {
/* Remove all old shaders */ /* Remove all old shaders */
@ -524,7 +653,7 @@ void BuildEntryPoint(WaveLaneCtx *lane)
{ {
StringList gpu_include_lines = ZI; StringList gpu_include_lines = ZI;
{ {
for (M_Entry *entry = build->flattened.first; entry->valid; entry = entry->next) for (M_Entry *entry = build.layers_parse.first; entry->valid; entry = entry->next)
{ {
M_EntryKind kind = entry->kind; M_EntryKind kind = entry->kind;
M_Token *entry_tok = entry->name_token; M_Token *entry_tok = entry->name_token;
@ -549,12 +678,12 @@ void BuildEntryPoint(WaveLaneCtx *lane)
else else
{ {
String err = StringF(perm, "File '%F' not found", FmtString(full)); String err = StringF(perm, "File '%F' not found", FmtString(full));
M_PushError(perm, &build->gpugen.errors, arg0_tok, err); M_PushError(perm, &build.gpu_parse.errors, arg0_tok, err);
} }
} }
else else
{ {
M_PushError(perm, &build->gpugen.errors, entry_tok, Lit("Expected file name")); M_PushError(perm, &build.gpu_parse.errors, entry_tok, Lit("Expected file name"));
} }
} break; } break;
case M_EntryKind_VertexShader: case M_EntryKind_VertexShader:
@ -571,18 +700,18 @@ void BuildEntryPoint(WaveLaneCtx *lane)
ShaderEntry *e = PushStruct(perm, ShaderEntry); ShaderEntry *e = PushStruct(perm, ShaderEntry);
e->kind = shader_kind; e->kind = shader_kind;
e->name = shader_name; e->name = shader_name;
SllQueuePush(build->gpugen.first_shader_entry, build->gpugen.last_shader_entry, e); SllQueuePush(build.gpu_parse.first_shader_entry, build.gpu_parse.last_shader_entry, e);
++build->gpugen.shader_entries_count; ++build.gpu_parse.shader_entries_count;
} }
else else
{ {
M_PushError(perm, &build->gpugen.errors, entry_tok, Lit("Expected shader name")); M_PushError(perm, &build.gpu_parse.errors, entry_tok, Lit("Expected shader name"));
} }
} break; } break;
} }
} }
} }
if (build->gpugen.errors.count == 0) if (build.gpu_parse.errors.count == 0)
{ {
StringList gpu_out_lines = ZI; StringList gpu_out_lines = ZI;
PushStringToList(perm, &gpu_out_lines, Lit("// Auto generated file")); PushStringToList(perm, &gpu_out_lines, Lit("// Auto generated file"));
@ -610,16 +739,13 @@ void BuildEntryPoint(WaveLaneCtx *lane)
} }
} }
build->gpucomps.count = build->gpugen.shader_entries_count; SetBuildStatus(build.gpu_parse.errors.count > 0);
build->gpucomps.array = PushStructs(perm, GpuComp, build->gpucomps.count);
SetBuildStatus(build->gpugen.errors.count > 0);
} }
//- Generate archive info //- Generate archive info
{ {
/* Gather archives from embedded dirs */ /* Push embedded archive dirs */
for (M_Entry *entry = build->flattened.first; entry->valid; entry = entry->next) for (M_Entry *entry = build.layers_parse.first; entry->valid; entry = entry->next)
{ {
M_EntryKind kind = entry->kind; M_EntryKind kind = entry->kind;
M_Token *entry_tok = entry->name_token; M_Token *entry_tok = entry->name_token;
@ -639,82 +765,90 @@ void BuildEntryPoint(WaveLaneCtx *lane)
String full = F_GetFullCrossPlatform(perm, StringF(perm, "%F/%F", FmtString(token_parent_dir), FmtString(arg_dir))); String full = F_GetFullCrossPlatform(perm, StringF(perm, "%F/%F", FmtString(token_parent_dir), FmtString(arg_dir)));
if (F_IsDir(full)) if (F_IsDir(full))
{ {
ArcInfoEntry *arc = PushStruct(perm, ArcInfoEntry); ResDir *rd = PushStruct(perm, ResDir);
arc->store_name = store_name; rd->store_name = store_name;
arc->dir_path = full; rd->dir_path = full;
arc->out_path = StringF(perm, "%F.arc", FmtString(store_name)); SllQueuePush(build.res_dir_parse.first_res_dir, build.res_dir_parse.last_res_dir, rd);
SllQueuePush(build->arcinfogen.first_arc_entry, build->arcinfogen.last_arc_entry, arc); ++build.res_dir_parse.res_dirs_count;
++build->arcinfogen.arc_entries_count;
} }
else else
{ {
String err = StringF(perm, "Directory '%F' not found", FmtString(full)); String err = StringF(perm, "Directory '%F' not found", FmtString(full));
M_PushError(perm, &build->arcinfogen.errors, arg1_tok, err); M_PushError(perm, &build.res_dir_parse.errors, arg1_tok, err);
} }
} }
else else
{ {
M_PushError(perm, &build->arcinfogen.errors, entry_tok, Lit("Expected resource store & directory name")); M_PushError(perm, &build.res_dir_parse.errors, entry_tok, Lit("Expected resource store & directory name"));
} }
} break; } break;
} }
} }
/* Gather shader archive */ SetBuildStatus(build.res_dir_parse.errors.count > 0);
{
ArcInfoEntry *arc = PushStruct(perm, ArcInfoEntry);
arc->store_name = shader_store_name;
arc->dir_path = F_GetFullCrossPlatform(perm, shader_store_name);
arc->out_path = StringF(perm, "%F.arc", FmtString(shader_store_name));
SllQueuePush(build->arcinfogen.first_arc_entry, build->arcinfogen.last_arc_entry, arc);
++build->arcinfogen.arc_entries_count;
} }
build->rescomps.count = build->arcinfogen.arc_entries_count; //- Prep objs
build->rescomps.array = PushStructs(perm, ResComp, build->rescomps.count); {
/* Gpu objs */
build.gpu_objs.count = build.gpu_parse.shader_entries_count;
build.gpu_objs.array = PushStructs(perm, GpuObj, build.gpu_objs.count);
SetBuildStatus(build->arcinfogen.errors.count > 0); /* Embed objs */
build.embed_objs.count += build.res_dir_parse.res_dirs_count;
if (build.gpu_parse.shader_entries_count > 0)
{
build.embed_objs.count += 1;
}
build.embed_objs.array = PushStructs(perm, EmbedObj, build.embed_objs.count);
} }
} }
WaveSync(lane); WaveSync(lane);
////////////////////////////// //////////////////////////////
//- Compile //- Phase 2/3: Compile
if (GetBuildStatus() == 0)
{ {
u64 task_idx = 0; u64 task_idx = 0;
u32 embed_idx = 0;
//- Compile C //- Compile C
{ {
if (lane->idx == WaveLaneIdxFromTaskIdx(lane, task_idx++)) if (lane->idx == WaveLaneIdxFromTaskIdx(lane, task_idx++) && GetBuildStatus() == 0)
{ {
build->ccomp.obj_file = StringF(perm, "%F_gen_c.obj", FmtString(cmdline.leaf_layer_name)); build.c_obj.obj_file = StringF(perm, "%F_gen_c.obj", FmtString(cmdline.leaf_layer_name));
String cmd = StringF(perm, String cmd = StringF(perm,
"cl.exe /c %F -Fo:%F %F %F %F %F", "cl.exe /c %F -Fo:%F %F %F %F %F",
FmtString(c_out_file), FmtString(c_out_file),
FmtString(build->ccomp.obj_file), FmtString(build.c_obj.obj_file),
FmtString(StringFromList(perm, cp.flags_msvc, Lit(" "))), FmtString(StringFromList(perm, cp.flags_msvc, Lit(" "))),
FmtString(StringFromList(perm, cp.compiler_only_flags_msvc, Lit(" "))), FmtString(StringFromList(perm, cp.compiler_only_flags_msvc, Lit(" "))),
FmtString(StringFromList(perm, cp.warnings_msvc, Lit(" "))), FmtString(StringFromList(perm, cp.warnings_msvc, Lit(" "))),
FmtString(StringFromList(perm, cp.defs, Lit(" ")))); FmtString(StringFromList(perm, cp.defs, Lit(" "))));
OS_CommandResult cmd_result = OS_RunCommand(perm, cmd); OS_CommandResult cmd_result = OS_RunCommand(perm, cmd);
String cmd_output = TrimWhitespace(cmd_result.output); String cmd_output = TrimWhitespace(cmd_result.output);
build->ccomp.output = cmd_output; build.c_obj.output = cmd_output;
build->ccomp.return_code = cmd_result.code; build.c_obj.return_code = cmd_result.code;
SetBuildStatus(build->ccomp.return_code); /* Ignore MSVC file-name echo */
if (MatchString(TrimWhitespace(build.c_obj.output), F_GetFileName(c_out_file)))
{
build.c_obj.output = Zstr;
}
SetBuildStatus(build.c_obj.return_code);
} }
} }
//- Compile shaders //- Compile shaders
{ {
u32 gpucomp_idx = 0; u32 gpu_obj_idx = 0;
for (ShaderEntry *e = build->gpugen.first_shader_entry; e; e = e->next) for (ShaderEntry *e = build.gpu_parse.first_shader_entry; e; e = e->next)
{ {
if (lane->idx == WaveLaneIdxFromTaskIdx(lane, task_idx++)) if (lane->idx == WaveLaneIdxFromTaskIdx(lane, task_idx++) && GetBuildStatus() == 0)
{ {
GpuComp *gpucomp = &build->gpucomps.array[gpucomp_idx]; String shader_name = e->name;
GpuObj *gpu_obj = &build.gpu_objs.array[gpu_obj_idx];
String out_file = StringF(perm, "%F/%F", FmtString(shader_store_name), FmtString(e->name)); String out_file = StringF(perm, "%F/%F", FmtString(shader_store_name), FmtString(e->name));
String target = e->kind == ShaderEntryKind_VS ? Lit("vs_6_6") String target = e->kind == ShaderEntryKind_VS ? Lit("vs_6_6")
: e->kind == ShaderEntryKind_PS ? Lit("ps_6_6") : e->kind == ShaderEntryKind_PS ? Lit("ps_6_6")
@ -730,11 +864,36 @@ void BuildEntryPoint(WaveLaneCtx *lane)
FmtString(StringFromList(perm, cp.flags_dxc, Lit(" ")))); FmtString(StringFromList(perm, cp.flags_dxc, Lit(" "))));
OS_CommandResult cmd_result = OS_RunCommand(perm, compile_cmd); OS_CommandResult cmd_result = OS_RunCommand(perm, compile_cmd);
gpucomp->output = cmd_result.output; gpu_obj->output = cmd_result.output;
gpucomp->return_code = cmd_result.code; gpu_obj->return_code = cmd_result.code;
SetBuildStatus(gpucomp->return_code);
SetBuildStatus(gpu_obj->return_code);
/* Final shader compilation embeds shader archive */
u32 finished_count = Atomic32FetchAdd(&build.gpu_objs.finished_count, 1) + 1;
if (finished_count == build.gpu_objs.count)
{
EmbedObj *embed = &build.embed_objs.array[embed_idx];
*embed = Embed(shader_store_name, shader_store_name);
} }
++gpucomp_idx; }
++gpu_obj_idx;
}
}
++embed_idx; /* Shader archive embed */
//- Embed resources
{
for (ResDir *rd = build.res_dir_parse.first_res_dir; rd; rd = rd->next)
{
if (lane->idx == WaveLaneIdxFromTaskIdx(lane, task_idx++) && GetBuildStatus() == 0)
{
String dir_path = rd->dir_path;
String store_name = rd->store_name;
EmbedObj *embed = &build.embed_objs.array[embed_idx];
*embed = Embed(store_name, dir_path);
}
++embed_idx;
} }
} }
} }
@ -742,177 +901,7 @@ void BuildEntryPoint(WaveLaneCtx *lane)
WaveSync(lane); WaveSync(lane);
////////////////////////////// //////////////////////////////
//- Generate resource archives //- Phase 3/3: Link
if (GetBuildStatus() == 0)
{
u32 task_idx = 0;
for (ArcInfoEntry *entry = build->arcinfogen.first_arc_entry; entry; entry = entry->next)
{
if (lane->idx == WaveLaneIdxFromTaskIdx(lane, task_idx++))
{
String dir_path = entry->dir_path;
String store_name = entry->store_name;
String arc_path = entry->out_path;
Struct(EntryNode)
{
EntryNode *next;
String entry_name;
String file_name;
};
EntryNode *first_entry = 0;
EntryNode *last_entry = 0;
u64 entries_count = 0;
StringList files = ZI;
F_FilesFromDir(perm, &files, dir_path, F_IterFlag_Recurse);
for (StringListNode *file_node = files.first; file_node; file_node = file_node->next)
{
String file_name = file_node->s;
if (F_IsFile(file_name))
{
String entry_name = file_name;
if (entry_name.len > (dir_path.len + 1))
{
entry_name.len -= dir_path.len + 1;
entry_name.text += dir_path.len + 1;
}
entry_name = StringF(perm, "%F/%F", FmtString(store_name), FmtString(entry_name));
for (u64 i = 0; i < entry_name.len; ++i)
{
if (entry_name.text[i] == '\\')
{
entry_name.text[i] = '/';
}
}
EntryNode *en = PushStruct(perm, EntryNode);
en->entry_name = entry_name;
en->file_name = file_name;
SllQueuePush(first_entry, last_entry, en);
++entries_count;
}
}
String arc_contents = ZI;
{
BB_Buff bb = BB_AcquireBuff(Gibi(2));
BB_Writer bw = BB_WriterFromBuff(&bb);
/* Write magic */
BB_WriteUBits(&bw, ResourceEmbeddedMagic, 64);
/* Write header */
BB_WriteUBits(&bw, entries_count, 64);
/* Reserve entries space */
u64 entry_size = 8 /* Name start */
+ 8 /* Name end */
+ 8 /* Data start */
+ 8; /* Data end */
u8 *entries_start = BB_GetWrittenRaw(&bw) + BB_GetNumBytesWritten(&bw);
u64 entries_size = entry_size * entries_count;
String entries_str = STRING(entries_size, entries_start);
BB_WriteSeekBytes(&bw, entries_size);
BB_Buff entries_bb = BB_BuffFromString(entries_str);
BB_Writer entries_bw = BB_WriterFromBuff(&entries_bb);
/* Write entries */
for (EntryNode *en = first_entry; en; en = en->next)
{
/* TODO: Copy file data directly into archive file */
String file_data = F_DataFromFile(perm, en->file_name);
/* Write name */
BB_WriteAlignBytes(&bw, 64);
u64 name_start = BB_GetNumBytesWritten(&bw) + 1;
BB_WriteString(&bw, en->entry_name);
u64 name_len = BB_GetNumBytesWritten(&bw) - name_start;
/* Write data */
BB_WriteAlignBytes(&bw, 64);
/* FIXME: Why no +1 here? */
u64 data_start = BB_GetNumBytesWritten(&bw);
BB_WriteBytes(&bw, file_data);
u64 data_len = BB_GetNumBytesWritten(&bw) - data_start;
/* Write entry */
BB_WriteUBits(&entries_bw, name_start, 64);
BB_WriteUBits(&entries_bw, name_len, 64);
BB_WriteUBits(&entries_bw, data_start, 64);
BB_WriteUBits(&entries_bw, data_len, 64);
}
arc_contents.len = BB_GetNumBytesWritten(&bw);
arc_contents.text = BB_GetWrittenRaw(&bw);
}
/* Write arc file */
F_ClearWrite(arc_path, arc_contents);
}
}
}
WaveSync(lane);
//////////////////////////////
//- Compile resource archives
if (GetBuildStatus() == 0)
{
if (IsPlatformWindows)
{
i32 rescomp_idx = 0;
for (ArcInfoEntry *entry = build->arcinfogen.first_arc_entry; entry; entry = entry->next)
{
if (lane->idx == WaveLaneIdxFromTaskIdx(lane, rescomp_idx))
{
ResComp *rescomp = &build->rescomps.array[rescomp_idx];
String arc_path = entry->out_path;
/* Generate RC file */
String rc_out_file = StringF(perm, "%F.rc", FmtString(entry->store_name));
{
RandState rs = ZI;
StringList rc_out_lines = ZI;
String arc_file_cp = F_GetFullCrossPlatform(perm, arc_path);
String line = StringF(perm, "%F_%F RCDATA \"%F\"", FmtString(Lit(Stringize(W32_EmbeddedDataPrefix))), FmtHex(RandU64FromState(&rs)), FmtString(arc_file_cp));
PushStringToList(perm, &rc_out_lines, line);
/* Write to file */
String rc_out = StringFromList(perm, rc_out_lines, Lit("\n"));
F_ClearWrite(rc_out_file, rc_out);
}
/* Compile RC file */
rescomp->obj_file = StringF(perm, "%F.res", FmtString(entry->store_name));
{
String cmd = StringF(perm, "rc.exe -nologo -fo %F %F", FmtString(rescomp->obj_file), FmtString(F_GetFull(perm, rc_out_file)));
OS_CommandResult cmd_result = OS_RunCommand(perm, cmd);
String cmd_output = TrimWhitespace(cmd_result.output);
rescomp->output = cmd_output;
rescomp->return_code = cmd_result.code;
}
SetBuildStatus(rescomp->return_code);
}
++rescomp_idx;
}
}
else
{
/* TODO: Compile object files using .incbin on non-windows platforms */
Panic(Lit("Embedded format not implemented for this platform"));
}
}
//////////////////////////////
//- Link
WaveSync(lane);
if (lane->idx == 0 && GetBuildStatus() == 0) if (lane->idx == 0 && GetBuildStatus() == 0)
{ {
@ -929,11 +918,11 @@ void BuildEntryPoint(WaveLaneCtx *lane)
String obj_files_str = ZI; String obj_files_str = ZI;
{ {
StringList obj_files = ZI; StringList obj_files = ZI;
PushStringToList(perm, &obj_files, build->ccomp.obj_file); PushStringToList(perm, &obj_files, build.c_obj.obj_file);
for (u64 rescomp_idx = 0; rescomp_idx < build->rescomps.count; ++rescomp_idx) for (u64 embed_obj_idx = 0; embed_obj_idx < build.embed_objs.count; ++embed_obj_idx)
{ {
ResComp *rescomp = &build->rescomps.array[rescomp_idx]; EmbedObj *embed_obj = &build.embed_objs.array[embed_obj_idx];
PushStringToList(perm, &obj_files, rescomp->obj_file); PushStringToList(perm, &obj_files, embed_obj->obj_file);
} }
obj_files_str = StringFromList(perm, obj_files, Lit(" ")); obj_files_str = StringFromList(perm, obj_files, Lit(" "));
} }
@ -945,49 +934,54 @@ void BuildEntryPoint(WaveLaneCtx *lane)
FmtString(StringFromList(perm, cp.flags_msvc, Lit(" "))), FmtString(StringFromList(perm, cp.flags_msvc, Lit(" "))),
FmtString(StringFromList(perm, cp.linker_only_flags_msvc, Lit(" ")))); FmtString(StringFromList(perm, cp.linker_only_flags_msvc, Lit(" "))));
OS_CommandResult result = OS_RunCommand(perm, cmd); OS_CommandResult result = OS_RunCommand(perm, cmd);
build->link.output = TrimWhitespace(result.output); build.link.output = TrimWhitespace(result.output);
build->link.return_code = result.code; build.link.return_code = result.code;
i64 link_elapsed_ns = TimeNs() - start_ns; i64 link_elapsed_ns = TimeNs() - start_ns;
// EchoLine(StringF(perm, ">>>>> Linked in %Fs", FmtFloat(SecondsFromNs(link_elapsed_ns)))); // EchoLine(StringF(perm, ">>>>> Linked in %Fs", FmtFloat(SecondsFromNs(link_elapsed_ns))));
SetBuildStatus(build->link.return_code); SetBuildStatus(build.link.return_code);
} }
WaveSync(lane);
////////////////////////////// //////////////////////////////
//- Process output & errors //- Display build results
if (lane->idx == 0) if (lane->idx == 0)
{ {
String gpucomp_output = ZI; String gpu_obj_output = ZI;
{ {
StringList gpucomp_outputs = ZI; GpuObj *disp_obj = 0;
for (u32 gpucomp_idx = 0; gpucomp_idx < build->gpucomps.count; ++gpucomp_idx) for (u32 gpu_obj_idx = 0; gpu_obj_idx < build.gpu_objs.count; ++gpu_obj_idx)
{ {
GpuComp *gpucomp = &build->gpucomps.array[gpucomp_idx]; GpuObj *gpu_obj = &build.gpu_objs.array[gpu_obj_idx];
PushStringToList(perm, &gpucomp_outputs, gpucomp->output); if (!disp_obj || TrimWhitespace(disp_obj->output).len == 0 || disp_obj->return_code != 0)
}
gpucomp_output = StringFromList(perm, gpucomp_outputs, Lit("\n"));
}
String rescomp_output = ZI;
{ {
StringList rescomp_outputs = ZI; disp_obj = gpu_obj;
for (u32 rescomp_idx = 0; rescomp_idx < build->rescomps.count; ++rescomp_idx) }
}
if (disp_obj)
{ {
ResComp *rescomp = &build->rescomps.array[rescomp_idx]; gpu_obj_output = TrimWhitespace(disp_obj->output);
PushStringToList(perm, &rescomp_outputs, rescomp->output);
} }
rescomp_output = StringFromList(perm, rescomp_outputs, Lit("\n"));
} }
EchoLineOrNothing(StringFromMetaErrors(perm, build->flattened.errors)); String embed_obj_output = ZI;
EchoLineOrNothing(StringFromMetaErrors(perm, build->cgen.errors)); {
EchoLineOrNothing(StringFromMetaErrors(perm, build->gpugen.errors)); StringList embed_obj_outputs = ZI;
EchoLineOrNothing(build->ccomp.output); for (u32 embed_obj_idx = 0; embed_obj_idx < build.embed_objs.count; ++embed_obj_idx)
EchoLineOrNothing(gpucomp_output); {
EchoLineOrNothing(StringFromMetaErrors(perm, build->arcinfogen.errors)); EmbedObj *embed_obj = &build.embed_objs.array[embed_obj_idx];
EchoLineOrNothing(rescomp_output); PushStringToList(perm, &embed_obj_outputs, embed_obj->output);
EchoLineOrNothing(build->link.output); }
embed_obj_output = StringFromList(perm, embed_obj_outputs, Lit("\n"));
}
EchoLineOrNothing(StringFromMetaErrors(perm, build.layers_parse.errors));
EchoLineOrNothing(StringFromMetaErrors(perm, build.c_parse.errors));
EchoLineOrNothing(StringFromMetaErrors(perm, build.gpu_parse.errors));
EchoLineOrNothing(build.c_obj.output);
EchoLineOrNothing(gpu_obj_output);
EchoLineOrNothing(StringFromMetaErrors(perm, build.res_dir_parse.errors));
EchoLineOrNothing(embed_obj_output);
EchoLineOrNothing(build.link.output);
} }
////////////////////////////// //////////////////////////////

70
src/meta/meta.h
View File

@ -56,7 +56,16 @@
#include "meta_lay.c" #include "meta_lay.c"
//////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////
//~ Build types //~ Helper types
Struct(LineCol)
{
i64 line;
i64 col;
};
////////////////////////////////////////////////////////////
//~ Build state types
Struct(CompilerParams) Struct(CompilerParams)
{ {
@ -91,21 +100,20 @@ Struct(ShaderEntry)
String name; String name;
}; };
Struct(ArcInfoEntry) Struct(ResDir)
{ {
ArcInfoEntry *next; ResDir *next;
String dir_path; String dir_path;
String store_name; String store_name;
String out_path;
}; };
Struct(GpuComp) Struct(GpuObj)
{ {
String output; String output;
i32 return_code; i32 return_code;
}; };
Struct(ResComp) Struct(EmbedObj)
{ {
String obj_file; String obj_file;
String output; String output;
@ -114,12 +122,14 @@ Struct(ResComp)
Struct(BuildData) Struct(BuildData)
{ {
M_Layer flattened; Atomic32 status;
M_Layer layers_parse;
struct struct
{ {
M_ErrorList errors; M_ErrorList errors;
} cgen; } c_parse;
struct struct
{ {
@ -127,34 +137,35 @@ Struct(BuildData)
ShaderEntry *first_shader_entry; ShaderEntry *first_shader_entry;
ShaderEntry *last_shader_entry; ShaderEntry *last_shader_entry;
u64 shader_entries_count; u64 shader_entries_count;
} gpugen; } gpu_parse;
struct
{
M_ErrorList errors;
ResDir *first_res_dir;
ResDir *last_res_dir;
u64 res_dirs_count;
} res_dir_parse;
struct struct
{ {
String obj_file; String obj_file;
String output; String output;
i32 return_code; i32 return_code;
} ccomp; } c_obj;
struct struct
{ {
GpuComp *array; GpuObj *array;
u32 count; u32 count;
} gpucomps; Atomic32 finished_count;
} gpu_objs;
struct struct
{ {
M_ErrorList errors; EmbedObj *array;
ArcInfoEntry *first_arc_entry;
ArcInfoEntry *last_arc_entry;
u64 arc_entries_count;
} arcinfogen;
struct
{
ResComp *array;
u32 count; u32 count;
} rescomps; } embed_objs;
struct struct
{ {
@ -163,19 +174,7 @@ Struct(BuildData)
} link; } link;
}; };
//////////////////////////////////////////////////////////// extern BuildData build;
//~ Helper types
Struct(LineCol)
{
i64 line;
i64 col;
};
////////////////////////////////////////////////////////////
//~ Globals
extern Atomic32 g_status;
//////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////
//~ Helpers //~ Helpers
@ -186,6 +185,7 @@ void EchoLine(String msg);
void EchoLineOrNothing(String msg); void EchoLineOrNothing(String msg);
LineCol LineColFromPos(String data, i64 pos); LineCol LineColFromPos(String data, i64 pos);
String StringFromMetaErrors(Arena *arena, M_ErrorList errors); String StringFromMetaErrors(Arena *arena, M_ErrorList errors);
EmbedObj Embed(String store_name, String dir_path);
//////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////
//~ Build //~ Build

2
src/meta/meta_lay.c
View File

@ -327,7 +327,7 @@ M_LayerList M_LayersFromTokenFiles(Arena *arena, M_TokenFileList lexed)
//////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////
//~ Flatten operations //~ Flatten operations
M_Layer M_GetFlattenedEntries(Arena *arena, M_LayerList unflattened, StringList starting_layer_names) M_Layer M_FlattenEntries(Arena *arena, M_LayerList unflattened, StringList starting_layer_names)
{ {
TempArena scratch = BeginScratch(arena); TempArena scratch = BeginScratch(arena);
M_Layer result = M_NilLayer; M_Layer result = M_NilLayer;

2
src/meta/meta_lay.h
View File

@ -151,4 +151,4 @@ M_LayerList M_LayersFromTokenFiles(Arena *arena, M_TokenFileList token_files);
//////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////
//~ Flatten operations //~ Flatten operations
M_Layer M_GetFlattenedEntries(Arena *arena, M_LayerList unflattened, StringList starting_layer_names); M_Layer M_FlattenEntries(Arena *arena, M_LayerList unflattened, StringList starting_layer_names);