jacob/power_play
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

compile shaders & embed resource dirs in parallel

Browse Source
This commit is contained in:
jacob 2025-12-07 09:05:46 -06:00
parent 32002c75bb
commit 9329d41c88
5 changed files with 341 additions and 337 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/base/base_win32/base_win32_wave.c
View File

@ -38,6 +38,16 @@ void DispatchWave(String name, u32 num_lanes, WaveLaneEntryFunc *entry, void *ud
{
Arena *perm = PermArena();
/* Catch high lane count to prevent OS crash while debugging */
u32 max_lanes = 4096;
if (num_lanes <= 0 || num_lanes > max_lanes)
{
Panic(StringF(perm,
"Tried to create a wave with %F lanes (valid range is [1, %F])",
FmtUint(num_lanes),
FmtUint(max_lanes)));
}
WaveCtx *wave_ctx = PushStruct(perm, WaveCtx);
wave_ctx->lanes_count = num_lanes;
wave_ctx->udata = udata;

610
src/meta/meta.c
View File

@ -1,18 +1,18 @@
#include "meta.h"
Atomic32 g_status = ZI;
BuildData build = ZI;
////////////////////////////////////////////////////////////
//~ Helpers
i32 GetBuildStatus(void)
{
return Atomic32Fetch(&g_status);
return Atomic32Fetch(&build.status);
}
void SetBuildStatus(i32 code)
{
Atomic32FetchTestSet(&g_status, 0, code);
Atomic32FetchTestSet(&build.status, 0, code);
}
void EchoLine(String msg)
@ -94,6 +94,151 @@ String StringFromMetaErrors(Arena *arena, M_ErrorList errors)
return result;
}
EmbedObj Embed(String store_name, String dir_path)
{
Arena *perm = PermArena();
EmbedObj result = ZI;
/* Generate resource archive contents */
String arc_contents = ZI;
{
StringList files = ZI;
F_FilesFromDir(perm, &files, dir_path, F_IterFlag_Recurse);
Struct(EntryNode)
{
EntryNode *next;
String entry_name;
String file_name;
};
EntryNode *first_entry = 0;
EntryNode *last_entry = 0;
u64 entries_count = 0;
{
for (StringListNode *file_node = files.first; file_node; file_node = file_node->next)
{
String file_name = file_node->s;
if (F_IsFile(file_name))
{
String entry_name = file_name;
if (entry_name.len > (dir_path.len + 1))
{
entry_name.len -= dir_path.len + 1;
entry_name.text += dir_path.len + 1;
}
entry_name = StringF(perm, "%F/%F", FmtString(store_name), FmtString(entry_name));
for (u64 i = 0; i < entry_name.len; ++i)
{
if (entry_name.text[i] == '\\')
{
entry_name.text[i] = '/';
}
}
EntryNode *en = PushStruct(perm, EntryNode);
en->entry_name = entry_name;
en->file_name = file_name;
SllQueuePush(first_entry, last_entry, en);
++entries_count;
}
}
}
{
BB_Buff bb = BB_AcquireBuff(Gibi(2));
BB_Writer bw = BB_WriterFromBuff(&bb);
/* Write magic */
BB_WriteUBits(&bw, ResourceEmbeddedMagic, 64);
/* Write header */
BB_WriteUBits(&bw, entries_count, 64);
/* Reserve entries space */
u64 entry_size = 8 /* Name start */
+ 8 /* Name end */
+ 8 /* Data start */
+ 8; /* Data end */
u8 *entries_start = BB_GetWrittenRaw(&bw) + BB_GetNumBytesWritten(&bw);
u64 entries_size = entry_size * entries_count;
String entries_str = STRING(entries_size, entries_start);
BB_WriteSeekBytes(&bw, entries_size);
BB_Buff entries_bb = BB_BuffFromString(entries_str);
BB_Writer entries_bw = BB_WriterFromBuff(&entries_bb);
/* Write entries */
for (EntryNode *en = first_entry; en; en = en->next)
{
/* TODO: Copy file data directly into archive file */
String file_data = F_DataFromFile(perm, en->file_name);
/* Write name */
BB_WriteAlignBytes(&bw, 64);
u64 name_start = BB_GetNumBytesWritten(&bw) + 1;
BB_WriteString(&bw, en->entry_name);
u64 name_len = BB_GetNumBytesWritten(&bw) - name_start;
/* Write data */
BB_WriteAlignBytes(&bw, 64);
/* FIXME: Why no +1 here? */
u64 data_start = BB_GetNumBytesWritten(&bw);
BB_WriteBytes(&bw, file_data);
u64 data_len = BB_GetNumBytesWritten(&bw) - data_start;
/* Write entry */
BB_WriteUBits(&entries_bw, name_start, 64);
BB_WriteUBits(&entries_bw, name_len, 64);
BB_WriteUBits(&entries_bw, data_start, 64);
BB_WriteUBits(&entries_bw, data_len, 64);
}
arc_contents.len = BB_GetNumBytesWritten(&bw);
arc_contents.text = BB_GetWrittenRaw(&bw);
}
}
/* Write archive to file */
String arc_path = StringF(perm, "%F.arc", FmtString(store_name));
F_ClearWrite(arc_path, arc_contents);
/* Generate object file */
if (IsPlatformWindows)
{
/* Generate RC file */
String rc_out_file = StringF(perm, "%F.rc", FmtString(store_name));
{
RandState rs = ZI;
StringList rc_out_lines = ZI;
String arc_file_cp = F_GetFullCrossPlatform(perm, arc_path);
String line = StringF(perm, "%F_%F RCDATA \"%F\"", FmtString(Lit(Stringize(W32_EmbeddedDataPrefix))), FmtHex(RandU64FromState(&rs)), FmtString(arc_file_cp));
PushStringToList(perm, &rc_out_lines, line);
/* Write to file */
String rc_out = StringFromList(perm, rc_out_lines, Lit("\n"));
F_ClearWrite(rc_out_file, rc_out);
}
/* Compile RC file */
result.obj_file = StringF(perm, "%F.res", FmtString(store_name));
{
result.obj_file, FmtString(F_GetFull(perm, rc_out_file));
String cmd = StringF(perm, "rc.exe -nologo -fo %F %F", FmtString(result.obj_file), FmtString(F_GetFull(perm, rc_out_file)));
OS_CommandResult cmd_result = OS_RunCommand(perm, cmd);
String cmd_output = TrimWhitespace(cmd_result.output);
result.output = cmd_output;
result.return_code = cmd_result.code;
}
SetBuildStatus(result.return_code);
}
else
{
/* TODO: Compile object files using .incbin on non-windows platforms */
Panic(Lit("Embedded resource path not implemented for this platform"));
}
return result;
}
////////////////////////////////////////////////////////////
//~ Build
@ -152,9 +297,8 @@ void BuildEntryPoint(WaveLaneCtx *lane)
Struct(CmdLineArgs)
{
String leaf_layer_name;
} cmdline = ZI;
if (lane->idx == 0)
};
CmdLineArgs cmdline = ZI;
{
String layer_name = ZI;
CommandlineArg arg = CommandlineArgFromName(Lit("layer"));
@ -168,13 +312,15 @@ void BuildEntryPoint(WaveLaneCtx *lane)
}
if (layer_name.len == 0)
{
EchoLine(Lit("No layer supplied, assuming \"pp\" build"));
if (lane->idx == 0) EchoLine(Lit("No layer supplied, assuming \"pp\" build"));
layer_name = Lit("pp");
}
cmdline.leaf_layer_name = layer_name;
EchoLine(StringF(perm, "Building \"%F\"", FmtString(cmdline.leaf_layer_name)));
if (lane->idx == 0)
{
if (lane->idx == 0) EchoLine(StringF(perm, "Building layer \"%F\"", FmtString(cmdline.leaf_layer_name)));
}
}
WaveSyncBroadcast(lane, 0, &cmdline);
//////////////////////////////
//- Generate compiler params
@ -269,48 +415,30 @@ void BuildEntryPoint(WaveLaneCtx *lane)
}
//////////////////////////////
//- Prep
//- Phase 1/3: Prep
/*
* Phase 1/3: Prep (narrow)
* - Parse layers
* - Generate final C file
* - Generate final HLSL file
* - Generate resource dirs info
*
* Phase 2/3: Compile (wide)
* - Compile C
* - Compile & embed shaders
* - Embed resource dirs
*
* Phase 3/3: Link (narrow)
* - Link
*/
/* TODO: Dispatch OS commands asynchronously */
String shader_store_name = Lit("ShadersStore");
String c_out_file = F_GetFull(perm, StringF(perm, "%F_gen_c.c", FmtString(cmdline.leaf_layer_name)));
String gpu_out_file = F_GetFull(perm, StringF(perm, "%F_gen_gpu.hlsl", FmtString(cmdline.leaf_layer_name)));
/*
* Phase 1
* - Parse layers
*
* Phase 2
* - Generate final C file
* - Generate final HLSL file
* - Generate resource archive info
*
* Phase 3
* - C compiler process
* - HLSL compiler processes
*
* Phase 4
* - Generate shader resource archive
* - Generate embedded resource archives
*
* Phase 5
* - Shader resource compilation process
* - Embedded resource compilation processes
*
* Phase 6
*/
/* TODO: Dispatch OS processes asynchronously rather than synchronously waiting on each lane */
BuildData *build = 0;
if (lane->idx == 0)
{
build = PushStruct(perm, BuildData);
}
WaveSyncBroadcast(lane, 0, &build);
//////////////////////////////
//- Parse meta layer info into build data
if (lane->idx == 0 && GetBuildStatus() == 0)
{
//- Parse layers
@ -326,9 +454,9 @@ void BuildEntryPoint(WaveLaneCtx *lane)
/* Flatten */
StringList starting_layer_names = ZI;
PushStringToList(perm, &starting_layer_names, cmdline.leaf_layer_name);
build->flattened = M_GetFlattenedEntries(perm, parsed, starting_layer_names);
build.layers_parse = M_FlattenEntries(perm, parsed, starting_layer_names);
SetBuildStatus(build->flattened.errors.count > 0);
SetBuildStatus(build.layers_parse.errors.count > 0);
}
//- Generate C file
@ -339,7 +467,7 @@ void BuildEntryPoint(WaveLaneCtx *lane)
StringList c_include_lines = ZI;
StringList c_startup_lines = ZI;
{
for (M_Entry *entry = build->flattened.first; entry->valid; entry = entry->next)
for (M_Entry *entry = build.layers_parse.first; entry->valid; entry = entry->next)
{
M_EntryKind kind = entry->kind;
M_Token *entry_tok = entry->name_token;
@ -366,12 +494,12 @@ void BuildEntryPoint(WaveLaneCtx *lane)
else
{
String err = StringF(perm, "Directory '%F' not found", FmtString(full));
M_PushError(perm, &build->cgen.errors, arg1_tok, err);
M_PushError(perm, &build.c_parse.errors, arg1_tok, err);
}
}
else
{
M_PushError(perm, &build->cgen.errors, entry_tok, Lit("Expected resource store & directory name"));
M_PushError(perm, &build.c_parse.errors, entry_tok, Lit("Expected resource store & directory name"));
}
} break;
case M_EntryKind_VertexShader:
@ -391,7 +519,7 @@ void BuildEntryPoint(WaveLaneCtx *lane)
}
else
{
M_PushError(perm, &build->cgen.errors, entry_tok, Lit("Expected shader name"));
M_PushError(perm, &build.c_parse.errors, entry_tok, Lit("Expected shader name"));
}
} break;
case M_EntryKind_IncludeC:
@ -410,12 +538,12 @@ void BuildEntryPoint(WaveLaneCtx *lane)
else
{
String err = StringF(perm, "File '%F' not found", FmtString(full));
M_PushError(perm, &build->cgen.errors, arg0_tok, err);
M_PushError(perm, &build.c_parse.errors, arg0_tok, err);
}
}
else
{
M_PushError(perm, &build->cgen.errors, entry_tok, Lit("Expected file name"));
M_PushError(perm, &build.c_parse.errors, entry_tok, Lit("Expected file name"));
}
} break;
case M_EntryKind_Startup:
@ -428,13 +556,13 @@ void BuildEntryPoint(WaveLaneCtx *lane)
}
else
{
M_PushError(perm, &build->cgen.errors, entry_tok, Lit("Expected startup function name"));
M_PushError(perm, &build.c_parse.errors, entry_tok, Lit("Expected startup function name"));
}
} break;
}
}
}
if (build->cgen.errors.count == 0)
if (build.c_parse.errors.count == 0)
{
StringList c_out_lines = ZI;
PushStringToList(perm, &c_out_lines, Lit("// Auto generated file"));
@ -493,12 +621,13 @@ void BuildEntryPoint(WaveLaneCtx *lane)
F_ClearWrite(c_out_file, c_out);
}
SetBuildStatus(build->cgen.errors.count > 0);
SetBuildStatus(build.c_parse.errors.count > 0);
}
//- Generate HLSL file
{
/* Clear shader store */
/* TODO: Move to separate artifacts dir that gets cleared, including archive files */
OS_Mkdir(shader_store_name);
{
/* Remove all old shaders */
@ -524,7 +653,7 @@ void BuildEntryPoint(WaveLaneCtx *lane)
{
StringList gpu_include_lines = ZI;
{
for (M_Entry *entry = build->flattened.first; entry->valid; entry = entry->next)
for (M_Entry *entry = build.layers_parse.first; entry->valid; entry = entry->next)
{
M_EntryKind kind = entry->kind;
M_Token *entry_tok = entry->name_token;
@ -549,12 +678,12 @@ void BuildEntryPoint(WaveLaneCtx *lane)
else
{
String err = StringF(perm, "File '%F' not found", FmtString(full));
M_PushError(perm, &build->gpugen.errors, arg0_tok, err);
M_PushError(perm, &build.gpu_parse.errors, arg0_tok, err);
}
}
else
{
M_PushError(perm, &build->gpugen.errors, entry_tok, Lit("Expected file name"));
M_PushError(perm, &build.gpu_parse.errors, entry_tok, Lit("Expected file name"));
}
} break;
case M_EntryKind_VertexShader:
@ -571,18 +700,18 @@ void BuildEntryPoint(WaveLaneCtx *lane)
ShaderEntry *e = PushStruct(perm, ShaderEntry);
e->kind = shader_kind;
e->name = shader_name;
SllQueuePush(build->gpugen.first_shader_entry, build->gpugen.last_shader_entry, e);
++build->gpugen.shader_entries_count;
SllQueuePush(build.gpu_parse.first_shader_entry, build.gpu_parse.last_shader_entry, e);
++build.gpu_parse.shader_entries_count;
}
else
{
M_PushError(perm, &build->gpugen.errors, entry_tok, Lit("Expected shader name"));
M_PushError(perm, &build.gpu_parse.errors, entry_tok, Lit("Expected shader name"));
}
} break;
}
}
}
if (build->gpugen.errors.count == 0)
if (build.gpu_parse.errors.count == 0)
{
StringList gpu_out_lines = ZI;
PushStringToList(perm, &gpu_out_lines, Lit("// Auto generated file"));
@ -610,16 +739,13 @@ void BuildEntryPoint(WaveLaneCtx *lane)
}
}
build->gpucomps.count = build->gpugen.shader_entries_count;
build->gpucomps.array = PushStructs(perm, GpuComp, build->gpucomps.count);
SetBuildStatus(build->gpugen.errors.count > 0);
SetBuildStatus(build.gpu_parse.errors.count > 0);
}
//- Generate archive info
{
/* Gather archives from embedded dirs */
for (M_Entry *entry = build->flattened.first; entry->valid; entry = entry->next)
/* Push embedded archive dirs */
for (M_Entry *entry = build.layers_parse.first; entry->valid; entry = entry->next)
{
M_EntryKind kind = entry->kind;
M_Token *entry_tok = entry->name_token;
@ -639,82 +765,90 @@ void BuildEntryPoint(WaveLaneCtx *lane)
String full = F_GetFullCrossPlatform(perm, StringF(perm, "%F/%F", FmtString(token_parent_dir), FmtString(arg_dir)));
if (F_IsDir(full))
{
ArcInfoEntry *arc = PushStruct(perm, ArcInfoEntry);
arc->store_name = store_name;
arc->dir_path = full;
arc->out_path = StringF(perm, "%F.arc", FmtString(store_name));
SllQueuePush(build->arcinfogen.first_arc_entry, build->arcinfogen.last_arc_entry, arc);
++build->arcinfogen.arc_entries_count;
ResDir *rd = PushStruct(perm, ResDir);
rd->store_name = store_name;
rd->dir_path = full;
SllQueuePush(build.res_dir_parse.first_res_dir, build.res_dir_parse.last_res_dir, rd);
++build.res_dir_parse.res_dirs_count;
}
else
{
String err = StringF(perm, "Directory '%F' not found", FmtString(full));
M_PushError(perm, &build->arcinfogen.errors, arg1_tok, err);
M_PushError(perm, &build.res_dir_parse.errors, arg1_tok, err);
}
}
else
{
M_PushError(perm, &build->arcinfogen.errors, entry_tok, Lit("Expected resource store & directory name"));
M_PushError(perm, &build.res_dir_parse.errors, entry_tok, Lit("Expected resource store & directory name"));
}
} break;
}
}
/* Gather shader archive */
{
ArcInfoEntry *arc = PushStruct(perm, ArcInfoEntry);
arc->store_name = shader_store_name;
arc->dir_path = F_GetFullCrossPlatform(perm, shader_store_name);
arc->out_path = StringF(perm, "%F.arc", FmtString(shader_store_name));
SllQueuePush(build->arcinfogen.first_arc_entry, build->arcinfogen.last_arc_entry, arc);
++build->arcinfogen.arc_entries_count;
SetBuildStatus(build.res_dir_parse.errors.count > 0);
}
build->rescomps.count = build->arcinfogen.arc_entries_count;
build->rescomps.array = PushStructs(perm, ResComp, build->rescomps.count);
//- Prep objs
{
/* Gpu objs */
build.gpu_objs.count = build.gpu_parse.shader_entries_count;
build.gpu_objs.array = PushStructs(perm, GpuObj, build.gpu_objs.count);
SetBuildStatus(build->arcinfogen.errors.count > 0);
/* Embed objs */
build.embed_objs.count += build.res_dir_parse.res_dirs_count;
if (build.gpu_parse.shader_entries_count > 0)
{
build.embed_objs.count += 1;
}
build.embed_objs.array = PushStructs(perm, EmbedObj, build.embed_objs.count);
}
}
WaveSync(lane);
//////////////////////////////
//- Compile
//- Phase 2/3: Compile
if (GetBuildStatus() == 0)
{
u64 task_idx = 0;
u32 embed_idx = 0;
//- Compile C
{
if (lane->idx == WaveLaneIdxFromTaskIdx(lane, task_idx++))
if (lane->idx == WaveLaneIdxFromTaskIdx(lane, task_idx++) && GetBuildStatus() == 0)
{
build->ccomp.obj_file = StringF(perm, "%F_gen_c.obj", FmtString(cmdline.leaf_layer_name));
build.c_obj.obj_file = StringF(perm, "%F_gen_c.obj", FmtString(cmdline.leaf_layer_name));
String cmd = StringF(perm,
"cl.exe /c %F -Fo:%F %F %F %F %F",
FmtString(c_out_file),
FmtString(build->ccomp.obj_file),
FmtString(build.c_obj.obj_file),
FmtString(StringFromList(perm, cp.flags_msvc, Lit(" "))),
FmtString(StringFromList(perm, cp.compiler_only_flags_msvc, Lit(" "))),
FmtString(StringFromList(perm, cp.warnings_msvc, Lit(" "))),
FmtString(StringFromList(perm, cp.defs, Lit(" "))));
OS_CommandResult cmd_result = OS_RunCommand(perm, cmd);
String cmd_output = TrimWhitespace(cmd_result.output);
build->ccomp.output = cmd_output;
build->ccomp.return_code = cmd_result.code;
build.c_obj.output = cmd_output;
build.c_obj.return_code = cmd_result.code;
SetBuildStatus(build->ccomp.return_code);
/* Ignore MSVC file-name echo */
if (MatchString(TrimWhitespace(build.c_obj.output), F_GetFileName(c_out_file)))
{
build.c_obj.output = Zstr;
}
SetBuildStatus(build.c_obj.return_code);
}
}
//- Compile shaders
{
u32 gpucomp_idx = 0;
for (ShaderEntry *e = build->gpugen.first_shader_entry; e; e = e->next)
u32 gpu_obj_idx = 0;
for (ShaderEntry *e = build.gpu_parse.first_shader_entry; e; e = e->next)
{
if (lane->idx == WaveLaneIdxFromTaskIdx(lane, task_idx++))
if (lane->idx == WaveLaneIdxFromTaskIdx(lane, task_idx++) && GetBuildStatus() == 0)
{
GpuComp *gpucomp = &build->gpucomps.array[gpucomp_idx];
String shader_name = e->name;
GpuObj *gpu_obj = &build.gpu_objs.array[gpu_obj_idx];
String out_file = StringF(perm, "%F/%F", FmtString(shader_store_name), FmtString(e->name));
String target = e->kind == ShaderEntryKind_VS ? Lit("vs_6_6")
: e->kind == ShaderEntryKind_PS ? Lit("ps_6_6")
@ -730,11 +864,36 @@ void BuildEntryPoint(WaveLaneCtx *lane)
FmtString(StringFromList(perm, cp.flags_dxc, Lit(" "))));
OS_CommandResult cmd_result = OS_RunCommand(perm, compile_cmd);
gpucomp->output = cmd_result.output;
gpucomp->return_code = cmd_result.code;
SetBuildStatus(gpucomp->return_code);
gpu_obj->output = cmd_result.output;
gpu_obj->return_code = cmd_result.code;
SetBuildStatus(gpu_obj->return_code);
/* Final shader compilation embeds shader archive */
u32 finished_count = Atomic32FetchAdd(&build.gpu_objs.finished_count, 1) + 1;
if (finished_count == build.gpu_objs.count)
{
EmbedObj *embed = &build.embed_objs.array[embed_idx];
*embed = Embed(shader_store_name, shader_store_name);
}
++gpucomp_idx;
}
++gpu_obj_idx;
}
}
++embed_idx; /* Shader archive embed */
//- Embed resources
{
for (ResDir *rd = build.res_dir_parse.first_res_dir; rd; rd = rd->next)
{
if (lane->idx == WaveLaneIdxFromTaskIdx(lane, task_idx++) && GetBuildStatus() == 0)
{
String dir_path = rd->dir_path;
String store_name = rd->store_name;
EmbedObj *embed = &build.embed_objs.array[embed_idx];
*embed = Embed(store_name, dir_path);
}
++embed_idx;
}
}
}
@ -742,177 +901,7 @@ void BuildEntryPoint(WaveLaneCtx *lane)
WaveSync(lane);
//////////////////////////////
//- Generate resource archives
if (GetBuildStatus() == 0)
{
u32 task_idx = 0;
for (ArcInfoEntry *entry = build->arcinfogen.first_arc_entry; entry; entry = entry->next)
{
if (lane->idx == WaveLaneIdxFromTaskIdx(lane, task_idx++))
{
String dir_path = entry->dir_path;
String store_name = entry->store_name;
String arc_path = entry->out_path;
Struct(EntryNode)
{
EntryNode *next;
String entry_name;
String file_name;
};
EntryNode *first_entry = 0;
EntryNode *last_entry = 0;
u64 entries_count = 0;
StringList files = ZI;
F_FilesFromDir(perm, &files, dir_path, F_IterFlag_Recurse);
for (StringListNode *file_node = files.first; file_node; file_node = file_node->next)
{
String file_name = file_node->s;
if (F_IsFile(file_name))
{
String entry_name = file_name;
if (entry_name.len > (dir_path.len + 1))
{
entry_name.len -= dir_path.len + 1;
entry_name.text += dir_path.len + 1;
}
entry_name = StringF(perm, "%F/%F", FmtString(store_name), FmtString(entry_name));
for (u64 i = 0; i < entry_name.len; ++i)
{
if (entry_name.text[i] == '\\')
{
entry_name.text[i] = '/';
}
}
EntryNode *en = PushStruct(perm, EntryNode);
en->entry_name = entry_name;
en->file_name = file_name;
SllQueuePush(first_entry, last_entry, en);
++entries_count;
}
}
String arc_contents = ZI;
{
BB_Buff bb = BB_AcquireBuff(Gibi(2));
BB_Writer bw = BB_WriterFromBuff(&bb);
/* Write magic */
BB_WriteUBits(&bw, ResourceEmbeddedMagic, 64);
/* Write header */
BB_WriteUBits(&bw, entries_count, 64);
/* Reserve entries space */
u64 entry_size = 8 /* Name start */
+ 8 /* Name end */
+ 8 /* Data start */
+ 8; /* Data end */
u8 *entries_start = BB_GetWrittenRaw(&bw) + BB_GetNumBytesWritten(&bw);
u64 entries_size = entry_size * entries_count;
String entries_str = STRING(entries_size, entries_start);
BB_WriteSeekBytes(&bw, entries_size);
BB_Buff entries_bb = BB_BuffFromString(entries_str);
BB_Writer entries_bw = BB_WriterFromBuff(&entries_bb);
/* Write entries */
for (EntryNode *en = first_entry; en; en = en->next)
{
/* TODO: Copy file data directly into archive file */
String file_data = F_DataFromFile(perm, en->file_name);
/* Write name */
BB_WriteAlignBytes(&bw, 64);
u64 name_start = BB_GetNumBytesWritten(&bw) + 1;
BB_WriteString(&bw, en->entry_name);
u64 name_len = BB_GetNumBytesWritten(&bw) - name_start;
/* Write data */
BB_WriteAlignBytes(&bw, 64);
/* FIXME: Why no +1 here? */
u64 data_start = BB_GetNumBytesWritten(&bw);
BB_WriteBytes(&bw, file_data);
u64 data_len = BB_GetNumBytesWritten(&bw) - data_start;
/* Write entry */
BB_WriteUBits(&entries_bw, name_start, 64);
BB_WriteUBits(&entries_bw, name_len, 64);
BB_WriteUBits(&entries_bw, data_start, 64);
BB_WriteUBits(&entries_bw, data_len, 64);
}
arc_contents.len = BB_GetNumBytesWritten(&bw);
arc_contents.text = BB_GetWrittenRaw(&bw);
}
/* Write arc file */
F_ClearWrite(arc_path, arc_contents);
}
}
}
WaveSync(lane);
//////////////////////////////
//- Compile resource archives
if (GetBuildStatus() == 0)
{
if (IsPlatformWindows)
{
i32 rescomp_idx = 0;
for (ArcInfoEntry *entry = build->arcinfogen.first_arc_entry; entry; entry = entry->next)
{
if (lane->idx == WaveLaneIdxFromTaskIdx(lane, rescomp_idx))
{
ResComp *rescomp = &build->rescomps.array[rescomp_idx];
String arc_path = entry->out_path;
/* Generate RC file */
String rc_out_file = StringF(perm, "%F.rc", FmtString(entry->store_name));
{
RandState rs = ZI;
StringList rc_out_lines = ZI;
String arc_file_cp = F_GetFullCrossPlatform(perm, arc_path);
String line = StringF(perm, "%F_%F RCDATA \"%F\"", FmtString(Lit(Stringize(W32_EmbeddedDataPrefix))), FmtHex(RandU64FromState(&rs)), FmtString(arc_file_cp));
PushStringToList(perm, &rc_out_lines, line);
/* Write to file */
String rc_out = StringFromList(perm, rc_out_lines, Lit("\n"));
F_ClearWrite(rc_out_file, rc_out);
}
/* Compile RC file */
rescomp->obj_file = StringF(perm, "%F.res", FmtString(entry->store_name));
{
String cmd = StringF(perm, "rc.exe -nologo -fo %F %F", FmtString(rescomp->obj_file), FmtString(F_GetFull(perm, rc_out_file)));
OS_CommandResult cmd_result = OS_RunCommand(perm, cmd);
String cmd_output = TrimWhitespace(cmd_result.output);
rescomp->output = cmd_output;
rescomp->return_code = cmd_result.code;
}
SetBuildStatus(rescomp->return_code);
}
++rescomp_idx;
}
}
else
{
/* TODO: Compile object files using .incbin on non-windows platforms */
Panic(Lit("Embedded format not implemented for this platform"));
}
}
//////////////////////////////
//- Link
WaveSync(lane);
//- Phase 3/3: Link
if (lane->idx == 0 && GetBuildStatus() == 0)
{
@ -929,11 +918,11 @@ void BuildEntryPoint(WaveLaneCtx *lane)
String obj_files_str = ZI;
{
StringList obj_files = ZI;
PushStringToList(perm, &obj_files, build->ccomp.obj_file);
for (u64 rescomp_idx = 0; rescomp_idx < build->rescomps.count; ++rescomp_idx)
PushStringToList(perm, &obj_files, build.c_obj.obj_file);
for (u64 embed_obj_idx = 0; embed_obj_idx < build.embed_objs.count; ++embed_obj_idx)
{
ResComp *rescomp = &build->rescomps.array[rescomp_idx];
PushStringToList(perm, &obj_files, rescomp->obj_file);
EmbedObj *embed_obj = &build.embed_objs.array[embed_obj_idx];
PushStringToList(perm, &obj_files, embed_obj->obj_file);
}
obj_files_str = StringFromList(perm, obj_files, Lit(" "));
}
@ -945,49 +934,54 @@ void BuildEntryPoint(WaveLaneCtx *lane)
FmtString(StringFromList(perm, cp.flags_msvc, Lit(" "))),
FmtString(StringFromList(perm, cp.linker_only_flags_msvc, Lit(" "))));
OS_CommandResult result = OS_RunCommand(perm, cmd);
build->link.output = TrimWhitespace(result.output);
build->link.return_code = result.code;
build.link.output = TrimWhitespace(result.output);
build.link.return_code = result.code;
i64 link_elapsed_ns = TimeNs() - start_ns;
// EchoLine(StringF(perm, ">>>>> Linked in %Fs", FmtFloat(SecondsFromNs(link_elapsed_ns))));
SetBuildStatus(build->link.return_code);
SetBuildStatus(build.link.return_code);
}
WaveSync(lane);
//////////////////////////////
//- Process output & errors
//- Display build results
if (lane->idx == 0)
{
String gpucomp_output = ZI;
String gpu_obj_output = ZI;
{
StringList gpucomp_outputs = ZI;
for (u32 gpucomp_idx = 0; gpucomp_idx < build->gpucomps.count; ++gpucomp_idx)
GpuObj *disp_obj = 0;
for (u32 gpu_obj_idx = 0; gpu_obj_idx < build.gpu_objs.count; ++gpu_obj_idx)
{
GpuComp *gpucomp = &build->gpucomps.array[gpucomp_idx];
PushStringToList(perm, &gpucomp_outputs, gpucomp->output);
}
gpucomp_output = StringFromList(perm, gpucomp_outputs, Lit("\n"));
}
String rescomp_output = ZI;
GpuObj *gpu_obj = &build.gpu_objs.array[gpu_obj_idx];
if (!disp_obj || TrimWhitespace(disp_obj->output).len == 0 || disp_obj->return_code != 0)
{
StringList rescomp_outputs = ZI;
for (u32 rescomp_idx = 0; rescomp_idx < build->rescomps.count; ++rescomp_idx)
disp_obj = gpu_obj;
}
}
if (disp_obj)
{
ResComp *rescomp = &build->rescomps.array[rescomp_idx];
PushStringToList(perm, &rescomp_outputs, rescomp->output);
gpu_obj_output = TrimWhitespace(disp_obj->output);
}
rescomp_output = StringFromList(perm, rescomp_outputs, Lit("\n"));
}
EchoLineOrNothing(StringFromMetaErrors(perm, build->flattened.errors));
EchoLineOrNothing(StringFromMetaErrors(perm, build->cgen.errors));
EchoLineOrNothing(StringFromMetaErrors(perm, build->gpugen.errors));
EchoLineOrNothing(build->ccomp.output);
EchoLineOrNothing(gpucomp_output);
EchoLineOrNothing(StringFromMetaErrors(perm, build->arcinfogen.errors));
EchoLineOrNothing(rescomp_output);
EchoLineOrNothing(build->link.output);
String embed_obj_output = ZI;
{
StringList embed_obj_outputs = ZI;
for (u32 embed_obj_idx = 0; embed_obj_idx < build.embed_objs.count; ++embed_obj_idx)
{
EmbedObj *embed_obj = &build.embed_objs.array[embed_obj_idx];
PushStringToList(perm, &embed_obj_outputs, embed_obj->output);
}
embed_obj_output = StringFromList(perm, embed_obj_outputs, Lit("\n"));
}
EchoLineOrNothing(StringFromMetaErrors(perm, build.layers_parse.errors));
EchoLineOrNothing(StringFromMetaErrors(perm, build.c_parse.errors));
EchoLineOrNothing(StringFromMetaErrors(perm, build.gpu_parse.errors));
EchoLineOrNothing(build.c_obj.output);
EchoLineOrNothing(gpu_obj_output);
EchoLineOrNothing(StringFromMetaErrors(perm, build.res_dir_parse.errors));
EchoLineOrNothing(embed_obj_output);
EchoLineOrNothing(build.link.output);
}
//////////////////////////////

70
src/meta/meta.h
View File

@ -56,7 +56,16 @@
#include "meta_lay.c"
////////////////////////////////////////////////////////////
//~ Build types
//~ Helper types
Struct(LineCol)
{
i64 line;
i64 col;
};
////////////////////////////////////////////////////////////
//~ Build state types
Struct(CompilerParams)
{
@ -91,21 +100,20 @@ Struct(ShaderEntry)
String name;
};
Struct(ArcInfoEntry)
Struct(ResDir)
{
ArcInfoEntry *next;
ResDir *next;
String dir_path;
String store_name;
String out_path;
};
Struct(GpuComp)
Struct(GpuObj)
{
String output;
i32 return_code;
};
Struct(ResComp)
Struct(EmbedObj)
{
String obj_file;
String output;
@ -114,12 +122,14 @@ Struct(ResComp)
Struct(BuildData)
{
M_Layer flattened;
Atomic32 status;
M_Layer layers_parse;
struct
{
M_ErrorList errors;
} cgen;
} c_parse;
struct
{
@ -127,34 +137,35 @@ Struct(BuildData)
ShaderEntry *first_shader_entry;
ShaderEntry *last_shader_entry;
u64 shader_entries_count;
} gpugen;
} gpu_parse;
struct
{
M_ErrorList errors;
ResDir *first_res_dir;
ResDir *last_res_dir;
u64 res_dirs_count;
} res_dir_parse;
struct
{
String obj_file;
String output;
i32 return_code;
} ccomp;
} c_obj;
struct
{
GpuComp *array;
GpuObj *array;
u32 count;
} gpucomps;
Atomic32 finished_count;
} gpu_objs;
struct
{
M_ErrorList errors;
ArcInfoEntry *first_arc_entry;
ArcInfoEntry *last_arc_entry;
u64 arc_entries_count;
} arcinfogen;
struct
{
ResComp *array;
EmbedObj *array;
u32 count;
} rescomps;
} embed_objs;
struct
{
@ -163,19 +174,7 @@ Struct(BuildData)
} link;
};
////////////////////////////////////////////////////////////
//~ Helper types
Struct(LineCol)
{
i64 line;
i64 col;
};
////////////////////////////////////////////////////////////
//~ Globals
extern Atomic32 g_status;
extern BuildData build;
////////////////////////////////////////////////////////////
//~ Helpers
@ -186,6 +185,7 @@ void EchoLine(String msg);
void EchoLineOrNothing(String msg);
LineCol LineColFromPos(String data, i64 pos);
String StringFromMetaErrors(Arena *arena, M_ErrorList errors);
EmbedObj Embed(String store_name, String dir_path);
////////////////////////////////////////////////////////////
//~ Build

2
src/meta/meta_lay.c
View File

@ -327,7 +327,7 @@ M_LayerList M_LayersFromTokenFiles(Arena *arena, M_TokenFileList lexed)
////////////////////////////////////////////////////////////
//~ Flatten operations
M_Layer M_GetFlattenedEntries(Arena *arena, M_LayerList unflattened, StringList starting_layer_names)
M_Layer M_FlattenEntries(Arena *arena, M_LayerList unflattened, StringList starting_layer_names)
{
TempArena scratch = BeginScratch(arena);
M_Layer result = M_NilLayer;

2
src/meta/meta_lay.h
View File

@ -151,4 +151,4 @@ M_LayerList M_LayersFromTokenFiles(Arena *arena, M_TokenFileList token_files);
////////////////////////////////////////////////////////////
//~ Flatten operations
M_Layer M_GetFlattenedEntries(Arena *arena, M_LayerList unflattened, StringList starting_layer_names);
M_Layer M_FlattenEntries(Arena *arena, M_LayerList unflattened, StringList starting_layer_names);