#include "meta.h" M_Ctx M = Zi; //////////////////////////////////////////////////////////// //~ Helpers i32 M_GetBuildStatus(void) { return Atomic32Fetch(&M.status); } void M_SetBuildStatus(i32 code) { Atomic32FetchTestSet(&M.status, 0, code); } void M_EchoLine(String msg) { TempArena scratch = BeginScratchNoConflict(); { String msg_w_newline = StringF(scratch.arena, "%F\n", FmtString(msg)); Echo(msg_w_newline); } EndScratch(scratch); } void M_EchoLineOrNothing(String msg) { String trimmed_msg = TrimWhitespace(msg); if (trimmed_msg.len > 0) { M_EchoLine(trimmed_msg); } } M_LineCol M_LineColFromPos(String data, i64 pos) { TempArena scratch = BeginScratchNoConflict(); M_LineCol result = Zi; for (u64 cur = 0; cur < data.len && cur <= (u64)pos; ++cur) { u8 c = data.text[cur]; if (c == '\n') { ++result.line; result.col = 0; } else if (c != '\r') { ++result.col; } } result.line += 1; EndScratch(scratch); return result; } String M_StringFromMetaErrors(Arena *arena, M_ErrorList errors) { TempArena scratch = BeginScratch(arena); StringList error_strings = Zi; for (M_Error *e = errors.first; e; e = e->next) { M_Token *token = e->token; String token_file = token->file->name; String token_file_data = token->file->data; if (token_file.len > 0) { i64 token_pos = -1; if ( token->s.len > 0 && token_file_data.len > 0 && token->s.text > token_file_data.text && token->s.text < (token_file_data.text + token_file_data.len) ) { token_pos = token->s.text - token_file_data.text; } M_LineCol line_col = Zi; if (token_pos >= 0) { line_col = M_LineColFromPos(token_file_data, token_pos); } String formatted = StringF( scratch.arena, "%F:%F:%F: error: %F", FmtString(token_file), FmtSint(line_col.line), FmtSint(line_col.col), FmtString(e->msg) ); PushStringToList(scratch.arena, &error_strings, formatted); } else { PushStringToList(scratch.arena, &error_strings, StringF(scratch.arena, "error: %F", FmtString(e->msg))); } } String result = StringFromList(arena, error_strings, Lit("\n")); EndScratch(scratch); return result; } M_EmbedObj M_Embed(String store_name, String dir_path) { Arena *perm = PermArena(); M_EmbedObj result = Zi; // Generate resource archive contents u64 store_hash = HashString(store_name); String arc_contents = Zi; { StringList files = Zi; F_FilesFromDir(perm, &files, dir_path, F_IterFlag_Recurse); Struct(EntryNode) { EntryNode *next; String entry_name; String file_name; }; EntryNode *first_entry = 0; EntryNode *last_entry = 0; u64 entries_count = 0; for (StringListNode *file_node = files.first; file_node; file_node = file_node->next) { String file_name = file_node->s; if (F_IsFile(file_name)) { String entry_name = file_name; if (entry_name.len > (dir_path.len + 1)) { entry_name.len -= dir_path.len + 1; entry_name.text += dir_path.len + 1; } for (u64 i = 0; i < entry_name.len; ++i) { if (entry_name.text[i] == '\\') { entry_name.text[i] = '/'; } } EntryNode *en = PushStruct(perm, EntryNode); en->entry_name = entry_name; en->file_name = file_name; SllQueuePush(first_entry, last_entry, en); ++entries_count; } } // TODO: Cache dynamic bitbuffs? BB_Buff bb = BB_AcquireDynamicBuff(Gibi(2)); BB_Writer bbw = BB_WriterFromBuff(&bb); // Write magic BB_WriteUBits(&bbw, ResourceEmbeddedMagic, 64); // Write header BB_WriteUBits(&bbw, entries_count, 64); // Reserve entries space u64 entry_size = 0 + 8 // Store hash + 8 // Name start + 8 // Name end + 8 // Data start + 8; // Data end u8 *entries_start = BB_GetWrittenRaw(&bbw) + BB_GetNumBytesWritten(&bbw); u64 entries_size = entry_size * entries_count; String entries_str = STRING(entries_size, entries_start); BB_WriteSeekBytes(&bbw, entries_size); BB_Buff entries_bb = BB_BuffFromString(entries_str); BB_Writer entries_bw = BB_WriterFromBuff(&entries_bb); // Write entries for (EntryNode *en = first_entry; en; en = en->next) { // TODO: Copy file data directly into archive file String file_data = F_DataFromFile(perm, en->file_name); // Write name BB_WriteAlignBytes(&bbw, 64); u64 name_start = BB_GetNumBytesWritten(&bbw) + 1; BB_WriteString(&bbw, en->entry_name); u64 name_len = BB_GetNumBytesWritten(&bbw) - name_start; // Write data BB_WriteAlignBytes(&bbw, 64); // FIXME: Why no +1 here? u64 data_start = BB_GetNumBytesWritten(&bbw); BB_WriteBytes(&bbw, file_data); u64 data_len = BB_GetNumBytesWritten(&bbw) - data_start; // Write entry BB_WriteUBits(&entries_bw, store_hash, 64); BB_WriteUBits(&entries_bw, name_start, 64); BB_WriteUBits(&entries_bw, name_len, 64); BB_WriteUBits(&entries_bw, data_start, 64); BB_WriteUBits(&entries_bw, data_len, 64); } arc_contents.len = BB_GetNumBytesWritten(&bbw); arc_contents.text = BB_GetWrittenRaw(&bbw); } // Write archive to file String arc_path = StringF(perm, "%F.arc", FmtString(store_name)); F_ClearWrite(arc_path, arc_contents); // Generate object file if (M.cmdline.target_platform == M_PlatformKind_Windows) { // Generate RC file String rc_out_file = StringF(perm, "%F.rc", FmtString(store_name)); { RandState rs = Zi; StringList rc_out_lines = Zi; String arc_file_cp = F_GetFullCrossPlatform(perm, arc_path); String line = StringF(perm, "%F_%F RCDATA \"%F\"", FmtString(Lit(Stringize(W32_EmbeddedDataPrefix))), FmtHex(RandU64FromState(&rs)), FmtString(arc_file_cp)); PushStringToList(perm, &rc_out_lines, line); // Write to file String rc_out = StringFromList(perm, rc_out_lines, Lit("\n")); F_ClearWrite(rc_out_file, rc_out); } // Compile RC file result.obj_file = StringF(perm, "%F.res", FmtString(store_name)); { result.obj_file, FmtString(F_GetFull(perm, rc_out_file)); String cmd = StringF(perm, "rc.exe -nologo -fo %F %F", FmtString(result.obj_file), FmtString(F_GetFull(perm, rc_out_file))); OS_CommandResult cmd_result = OS_RunCommand(perm, cmd); String cmd_output = TrimWhitespace(cmd_result.output); result.output = cmd_output; result.return_code = cmd_result.code; } M_SetBuildStatus(result.return_code); } else { // TODO: Generate object files using .incbin on non-windows platforms Panic(Lit("Resource embedding not implemented for this platform")); } return result; } //////////////////////////////////////////////////////////// //~ Build void M_BuildEntryPoint(WaveLaneCtx *lane) { Arena *perm = PermArena(); ////////////////////////////// //- Dirty check // Return rebuild code if metaprogram is dirty if (lane->idx == 0) { // Read old metahash u64 old_metahash = 0; if (F_IsFile(Lit("metahash.dat"))) { String hashes_str = F_DataFromFile(perm, Lit("metahash.dat")); if (hashes_str.len == sizeof(old_metahash)) { CopyBytes(&old_metahash, hashes_str.text, sizeof(old_metahash)); } OS_Rm(Lit("metahash.dat")); } // Compute new metahash u64 new_metahash = 0; { StringList check_files = Zi; F_FilesFromDir(perm, &check_files, Lit("../src/base"), F_IterFlag_Recurse); F_FilesFromDir(perm, &check_files, Lit("../src/meta"), F_IterFlag_Recurse); PushStringToList(perm, &check_files, Lit("../src/config.h")); for (StringListNode *n = check_files.first; n; n = n->next) { String file = n->s; new_metahash = MixU64s(new_metahash, OS_LastWriteTimestampFromPath(file)); new_metahash = HashStringEx(new_metahash , file); } } // Exit if metaprogram needs recompilation if (old_metahash == 0 || old_metahash == new_metahash) { F_ClearWrite(Lit("metahash.dat"), StringFromStruct(&new_metahash)); } else { M_EchoLine(Lit("Metaprogram is dirty")); ExitNow(M_RebuildCode); } } ////////////////////////////// //- Init command line // Layer name { CommandlineArg arg = CommandlineArgFromName(Lit("layer")); String layer_name = Zi; if (arg.name.len != 0) { layer_name = arg.value; } else { layer_name = StringFromCommandlineIdx(1); } if (layer_name.len == 0) { layer_name = Lit("pp"); if (lane->idx == 0) { M_EchoLine(Lit("No layer specified, assuming \"pp\" build")); } } M.cmdline.leaf_layer_name = layer_name; } // Platform { CommandlineArg arg = CommandlineArgFromName(Lit("platform")); if (MatchString(arg.value, Lit("windows"))) { M.cmdline.target_platform = M_PlatformKind_Windows; } else if (MatchString(arg.value, Lit("macos"))) { M.cmdline.target_platform = M_PlatformKind_Mac; } else if (MatchString(arg.value, Lit("linux"))) { M.cmdline.target_platform = M_PlatformKind_Linux; } else { if (IsPlatformWindows) { M.cmdline.target_platform = M_PlatformKind_Windows; } else if (IsPlatformMac) { M.cmdline.target_platform = M_PlatformKind_Mac; } else { M.cmdline.target_platform = M_PlatformKind_Linux; } } } // Compiler { CommandlineArg arg = CommandlineArgFromName(Lit("compiler")); if (MatchString(arg.value, Lit("msvc"))) { M.cmdline.target_compiler = M_CompilerKind_Msvc; } else if (MatchString(arg.value, Lit("clang"))) { M.cmdline.target_compiler = M_CompilerKind_Clang; } else { if (IsPlatformWindows) { M.cmdline.target_compiler = M_CompilerKind_Msvc; } else { M.cmdline.target_compiler = M_CompilerKind_Clang; } } } // Release/Debug { CommandlineArg arg = CommandlineArgFromName(Lit("release")); if (arg.exists) { M.cmdline.release = 1; } } if (lane->idx == 0) { if (M.cmdline.target_compiler == M_CompilerKind_Msvc) { M_EchoLine(Lit("[Msvc]")); } else if (M.cmdline.target_compiler == M_CompilerKind_Clang) { M_EchoLine(Lit("[Clang]")); } if (M.cmdline.release) { M_EchoLine(Lit("[Release build]")); } else { M_EchoLine(Lit("[Debug build]")); } M_EchoLine(StringF(perm, "Building layer \"%F\"", FmtString(M.cmdline.leaf_layer_name))); } ////////////////////////////// //- Generate compiler params M_CompilerParams cp = Zi; { //- Common { PushStringToList(perm, &cp.defs, Lit("-DIsConsoleApp=0")); PushStringToList(perm, &cp.defs, Lit("-DIsRtcEnabled=1")); PushStringToList(perm, &cp.defs, Lit("-DIsAsanEnabled=0")); PushStringToList(perm, &cp.defs, Lit("-DIsDebinfoEnabled=1")); PushStringToList(perm, &cp.defs, Lit("-DIsDeveloperModeEnabled=1")); PushStringToList(perm, &cp.defs, Lit("-DIsUnoptimized=1")); PushStringToList(perm, &cp.defs, Lit("-DIsTestingEnabled=0")); PushStringToList(perm, &cp.defs, Lit("-DIsHotSwappingEnabled=1")); PushStringToList(perm, &cp.defs, StringF(perm, "-DDefaultAppName=%F", FmtString(M.cmdline.leaf_layer_name))); } //- Msvc { PushStringToList(perm, &cp.flags_msvc, Lit("-INCREMENTAL:NO")); PushStringToList(perm, &cp.flags_msvc, Lit("-nologo")); PushStringToList(perm, &cp.compiler_only_flags_msvc, Lit("-diagnostics:column")); if (M.cmdline.release) { PushStringToList(perm, &cp.compiler_only_flags_msvc, Lit("-MT")); PushStringToList(perm, &cp.compiler_only_flags_msvc, Lit("-O2")); } else { PushStringToList(perm, &cp.compiler_only_flags_msvc, Lit("-MTd")); PushStringToList(perm, &cp.compiler_only_flags_msvc, Lit("-Od")); } // TODO: Export debug info separately for release builds PushStringToList(perm, &cp.flags_msvc, Lit("-DEBUG:FULL")); PushStringToList(perm, &cp.compiler_only_flags_msvc, Lit("-Z7")); // Enable warnings PushStringToList(perm, &cp.warnings_msvc, Lit("-W4")); PushStringToList(perm, &cp.warnings_msvc, Lit("-WX")); // PushStringToList(perm, &cp.warnings_msvc, Lit("-we4013")); // function undefined; assuming extern returning int PushStringToList(perm, &cp.warnings_msvc, Lit("-we4668")); // X is not defined as a preprocessor macro, replacing with '0' for '#if/#elif' // Disable warnings PushStringToList(perm, &cp.warnings_msvc, Lit("-wd4244")); // 'function': conversion from 'int' to 'f32', possible loss of data PushStringToList(perm, &cp.warnings_msvc, Lit("-wd4201")); // nonstandard extension used: nameless struct/union PushStringToList(perm, &cp.warnings_msvc, Lit("-wd4324")); // structure was padded due to alignment specifier PushStringToList(perm, &cp.warnings_msvc, Lit("-wd4100")); // unreferenced parameter PushStringToList(perm, &cp.warnings_msvc, Lit("-wd4101")); // unreferenced local variable PushStringToList(perm, &cp.warnings_msvc, Lit("-wd4189")); // local variable is initialized but not referenced PushStringToList(perm, &cp.warnings_msvc, Lit("-wd4200")); // nonstandard extension used: zero-sized array in struct/union PushStringToList(perm, &cp.warnings_msvc, Lit("-wd4702")); // unreachable code PushStringToList(perm, &cp.warnings_msvc, Lit("-wd4305")); // 'initializing': truncation from 'double' to 'f32' // PushStringToList(perm, &cp.warnings_msvc, Lit("-wd4127")); // conditional expression is constant // PushStringToList(perm, &cp.warnings_msvc, Lit("-wd4820")); // bytes padding added after data member // PushStringToList(perm, &cp.warnings_msvc, Lit("-wd4464")); // relative include path contains '..' // PushStringToList(perm, &cp.warnings_msvc, Lit("-wd4061")); // enumerator is not explicitly handled by a case label // PushStringToList(perm, &cp.warnings_msvc, Lit("-wd4242")); // conversion from X to Y, possible loss of data // PushStringToList(perm, &cp.warnings_msvc, Lit("-wd4388")); // signed/unsigned mismatch // PushStringToList(perm, &cp.warnings_msvc, Lit("-wd5045")); // Compiler will insert Spectre mitigation for memory load if /Qspectre switch specified } //- Clang { PushStringToList(perm, &cp.compiler_only_flags_clang, Lit("-std=c99")); PushStringToList(perm, &cp.compiler_only_flags_clang, Lit("-fno-finite-loops")); PushStringToList(perm, &cp.compiler_only_flags_clang, Lit("-fno-strict-aliasing")); PushStringToList(perm, &cp.compiler_only_flags_clang, Lit("-msse4.2")); // TODO: Export debug info separately for release builds PushStringToList(perm, &cp.compiler_only_flags_clang, Lit("-g -gcodeview")); if (M.cmdline.release) { PushStringToList(perm, &cp.compiler_only_flags_clang, Lit("-O2")); } else { PushStringToList(perm, &cp.compiler_only_flags_clang, Lit("-O0")); } if (M.cmdline.target_platform == M_PlatformKind_Windows) { if (M.cmdline.release) { PushStringToList(perm, &cp.compiler_only_flags_clang, Lit("-fms-runtime-lib=static_dbg")); } else { PushStringToList(perm, &cp.compiler_only_flags_clang, Lit("-fms-runtime-lib=static")); } } // Enable warnings PushStringToList(perm, &cp.warnings_clang, Lit("-Wall")); PushStringToList(perm, &cp.warnings_clang, Lit("-Werror")); PushStringToList(perm, &cp.warnings_clang, Lit("-Wframe-larger-than=1048575")); PushStringToList(perm, &cp.warnings_clang, Lit("-Wmissing-prototypes")); PushStringToList(perm, &cp.warnings_clang, Lit("-Wmissing-declarations")); PushStringToList(perm, &cp.warnings_clang, Lit("-Wimplicit-fallthrough")); // Disable warnings PushStringToList(perm, &cp.warnings_clang, Lit("-Wno-initializer-overrides")); PushStringToList(perm, &cp.warnings_clang, Lit("-Wno-microsoft-enum-forward-reference")); PushStringToList(perm, &cp.warnings_clang, Lit("-Wno-microsoft-anon-tag")); PushStringToList(perm, &cp.warnings_clang, Lit("-Wno-incompatible-function-pointer-types")); PushStringToList(perm, &cp.warnings_clang, Lit("-Wno-missing-braces")); PushStringToList(perm, &cp.warnings_clang, Lit("-Wno-unused-value")); PushStringToList(perm, &cp.warnings_clang, Lit("-Wno-unused-variable")); PushStringToList(perm, &cp.warnings_clang, Lit("-Wno-unused-but-set-variable")); PushStringToList(perm, &cp.warnings_clang, Lit("-Wno-switch")); } //- Dxc { PushStringToList(perm, &cp.flags_dxc, Lit("-O3")); PushStringToList(perm, &cp.flags_dxc, Lit("-HV 202x")); // 202x makes numeric literals less weird // TODO: Export debug info separately for release builds PushStringToList(perm, &cp.flags_dxc, Lit("-Zi -Qembed_debug")); // Enable warnings PushStringToList(perm, &cp.warnings_dxc, Lit("-Wall")); PushStringToList(perm, &cp.warnings_dxc, Lit("-Werror")); PushStringToList(perm, &cp.warnings_dxc, Lit("-Wshadow")); // Disable warnings PushStringToList(perm, &cp.warnings_dxc, Lit("-Wno-local-type-template-args")); PushStringToList(perm, &cp.warnings_dxc, Lit("-Wno-unused-variable")); PushStringToList(perm, &cp.warnings_dxc, Lit("-Wno-unused-local-typedef")); PushStringToList(perm, &cp.warnings_dxc, Lit("-Wno-conversion")); PushStringToList(perm, &cp.warnings_dxc, Lit("-Wno-switch")); } } ////////////////////////////// //- Phase 1/3: Prep // Build phases: // // Phase 1/3: Prep (narrow) // - Parse layers // - Generate final C file // - Generate final HLSL file // - Generate resource dirs info // // Phase 2/3: Compile (wide) // - Compile C // - Compile & embed shaders // - M_Embed resource dirs // // Phase 3/3: Link (narrow) // - Link // TODO: Dispatch OS commands asynchronously String shader_store_name = Lit("ShadersStore"); String c_out_file = F_GetFull(perm, StringF(perm, "%F_gen.c", FmtString(M.cmdline.leaf_layer_name))); String gpu_out_file = F_GetFull(perm, StringF(perm, "%F_gen.g", FmtString(M.cmdline.leaf_layer_name))); u64 shader_store_hash = HashString(shader_store_name); if (lane->idx == 0 && M_GetBuildStatus() == 0) { //- Parse layers { // Lex StringList src_dirs = Zi; PushStringToList(perm, &src_dirs, Lit("../src")); M_TokenFileList lexed = M_TokensFromSrcDirs(perm, src_dirs); // Parse M_LayerList parsed = M_LayersFromTokenFiles(perm, lexed); // Flatten StringList starting_layer_names = Zi; PushStringToList(perm, &starting_layer_names, M.cmdline.leaf_layer_name); M.layers_parse = M_FlattenEntries(perm, parsed, starting_layer_names); M_SetBuildStatus(M.layers_parse.errors.count > 0); } //- Generate C file { StringList c_store_lines = Zi; StringList c_shader_lines = Zi; StringList c_include_lines = Zi; StringList c_bootstrap_lines = Zi; { for (M_Entry *entry = M.layers_parse.first; entry->valid; entry = entry->next) { M_EntryKind kind = entry->kind; M_Token *entry_tok = entry->name_token; M_Token *arg0_tok = entry->arg_tokens[0]; M_Token *arg1_tok = entry->arg_tokens[1]; switch (kind) { default: break; case M_EntryKind_EmbedDir: { if (arg0_tok->valid && arg1_tok->valid) { String store_name = arg0_tok->s; String token_file = arg1_tok->file->name; String token_parent_dir = F_GetParentDir(token_file); String arg_dir = arg1_tok->s; String full = F_GetFullCrossPlatform(perm, StringF(perm, "%F/%F", FmtString(token_parent_dir), FmtString(arg_dir))); if (F_IsDir(full)) { u64 store_hash = HashString(store_name); String line = StringF(perm, "ResourceStore %F = { 0x%F };", FmtString(store_name), FmtHex(store_hash)); PushStringToList(perm, &c_store_lines, line); } else { String err = StringF(perm, "Directory '%F' not found", FmtString(full)); M_PushError(perm, &M.c_parse.errors, arg1_tok, err); } } else { M_PushError(perm, &M.c_parse.errors, entry_tok, Lit("Expected resource store & directory name")); } } break; case M_EntryKind_VertexShader: case M_EntryKind_PixelShader: case M_EntryKind_ComputeShader: { if (arg0_tok->valid) { String shader_type = kind == M_EntryKind_VertexShader ? Lit("VertexShader") : kind == M_EntryKind_PixelShader ? Lit("PixelShader") : kind == M_EntryKind_ComputeShader ? Lit("ComputeShader") : Lit(""); String shader_name = arg0_tok->s; u64 shader_resource_hash = HashStringEx(shader_store_hash, StringF(perm, "%F.dxil", FmtString(shader_name))); String line = StringF(perm, "%F %F = { 0x%F };", FmtString(shader_type), FmtString(shader_name), FmtHex(shader_resource_hash)); PushStringToList(perm, &c_shader_lines, line); } else { M_PushError(perm, &M.c_parse.errors, entry_tok, Lit("Expected shader name")); } } break; case M_EntryKind_IncludeC: { if (arg0_tok->valid) { String token_file = arg0_tok->file->name; String token_parent_dir = F_GetParentDir(token_file); String arg_file = arg0_tok->s; String full = F_GetFull(perm, StringF(perm, "%F/%F", FmtString(token_parent_dir), FmtString(arg_file))); if (F_IsFile(full)) { String line = StringF(perm, "#include \"%F\"", FmtString(full)); PushStringToList(perm, &c_include_lines, line); } else { String err = StringF(perm, "File '%F' not found", FmtString(full)); M_PushError(perm, &M.c_parse.errors, arg0_tok, err); } } else { M_PushError(perm, &M.c_parse.errors, entry_tok, Lit("Expected file name")); } } break; case M_EntryKind_Bootstrap: { if (arg0_tok->valid) { String bootstrap = arg0_tok->s; String line = StringF(perm, " %F();", FmtString(bootstrap)); PushStringToList(perm, &c_bootstrap_lines, line); } else { M_PushError(perm, &M.c_parse.errors, entry_tok, Lit("Expected bootstrap function name")); } } break; } } } if (M.c_parse.errors.count == 0) { StringList c_out_lines = Zi; PushStringToList(perm, &c_out_lines, Lit("// Auto generated file")); // Include base layer { String base_inc_path = F_GetFull(perm, Lit("../src/base/base_inc.h")); PushStringToList(perm, &c_out_lines, Lit("")); PushStringToList(perm, &c_out_lines, Lit("//- Base layer includes")); PushStringToList(perm, &c_out_lines, StringF(perm, "#include \"%F\"", FmtString(base_inc_path))); } // Define resource stores if (c_store_lines.count > 0) { PushStringToList(perm, &c_out_lines, Lit("")); PushStringToList(perm, &c_out_lines, Lit("//- Resource stores")); for (StringListNode *n = c_store_lines.first; n; n = n->next) { PushStringToList(perm, &c_out_lines, n->s); } } // Define shaders if (c_shader_lines.count > 0) { PushStringToList(perm, &c_out_lines, Lit("")); PushStringToList(perm, &c_out_lines, Lit("//- Shaders")); for (StringListNode *n = c_shader_lines.first; n; n = n->next) { PushStringToList(perm, &c_out_lines, n->s); } } // Include dependency layers if (c_include_lines.count > 0) { PushStringToList(perm, &c_out_lines, Lit("")); PushStringToList(perm, &c_out_lines, Lit("//- Dependency graph includes")); for (StringListNode *n = c_include_lines.first; n; n = n->next) { PushStringToList(perm, &c_out_lines, n->s); } } // Define BootstrapLayers { PushStringToList(perm, &c_out_lines, Lit("")); PushStringToList(perm, &c_out_lines, Lit("//- Bootstrap")); PushStringToList(perm, &c_out_lines, Lit("void BootstrapLayers(void)")); PushStringToList(perm, &c_out_lines, Lit("{")); for (StringListNode *n = c_bootstrap_lines.first; n; n = n->next) { PushStringToList(perm, &c_out_lines, n->s); } PushStringToList(perm, &c_out_lines, Lit("}")); } // Write to file PushStringToList(perm, &c_out_lines, Lit("")); String c_out = StringFromList(perm, c_out_lines, Lit("\n")); F_ClearWrite(c_out_file, c_out); } M_SetBuildStatus(M.c_parse.errors.count > 0); } //- Generate HLSL file { // Clear shader store // TODO: Move to separate artifacts dir that gets cleared, including archive files OS_Mkdir(shader_store_name); { // Remove all old shaders StringList files = Zi; F_FilesFromDir(perm, &files, shader_store_name, F_IterFlag_None); for (StringListNode *n = files.first; n; n = n->next) { String file = n->s; // Safety check to prevent non-shader files from being removed if (StringEndsWith(file, Lit(".dxil"))) { OS_Rm(n->s); } else { // Unexpected file in shader store Assert(0); } } } // Generate GPU file & shader entries { StringList gpu_include_lines = Zi; { for (M_Entry *entry = M.layers_parse.first; entry->valid; entry = entry->next) { M_EntryKind kind = entry->kind; M_Token *entry_tok = entry->name_token; M_Token *arg0_tok = entry->arg_tokens[0]; M_Token *arg1_tok = entry->arg_tokens[1]; switch (kind) { default: break; case M_EntryKind_IncludeG: { if (arg0_tok->valid) { String token_file = arg0_tok->file->name; String token_parent_dir = F_GetParentDir(token_file); String arg_file = arg0_tok->s; String full = F_GetFull(perm, StringF(perm, "%F/%F", FmtString(token_parent_dir), FmtString(arg_file))); if (F_IsFile(full)) { String line = StringF(perm, "#include \"%F\"", FmtString(full)); PushStringToList(perm, &gpu_include_lines, line); } else { String err = StringF(perm, "File '%F' not found", FmtString(full)); M_PushError(perm, &M.gpu_parse.errors, arg0_tok, err); } } else { M_PushError(perm, &M.gpu_parse.errors, entry_tok, Lit("Expected file name")); } } break; case M_EntryKind_VertexShader: case M_EntryKind_PixelShader: case M_EntryKind_ComputeShader: { if (arg0_tok->valid) { M_ShaderEntryKind shader_kind = kind == M_EntryKind_VertexShader ? ShaderEntryKind_VS : kind == M_EntryKind_PixelShader ? ShaderEntryKind_PS : kind == M_EntryKind_ComputeShader ? ShaderEntryKind_CS : ShaderEntryKind_VS; String shader_name = arg0_tok->s; M_ShaderEntry *e = PushStruct(perm, M_ShaderEntry); e->kind = shader_kind; e->name = shader_name; SllQueuePush(M.gpu_parse.first_shader_entry, M.gpu_parse.last_shader_entry, e); ++M.gpu_parse.shader_entries_count; } else { M_PushError(perm, &M.gpu_parse.errors, entry_tok, Lit("Expected shader name")); } } break; } } } if (M.gpu_parse.errors.count == 0) { StringList gpu_out_lines = Zi; PushStringToList(perm, &gpu_out_lines, Lit("// Auto generated file")); // Include base layer { String base_inc_path = F_GetFull(perm, Lit("../src/base/base_inc.h")); PushStringToList(perm, &gpu_out_lines, Lit("")); PushStringToList(perm, &gpu_out_lines, Lit("//- Base layer includes")); PushStringToList(perm, &gpu_out_lines, StringF(perm, "#include \"%F\"", FmtString(base_inc_path))); } // Include dependency layers if (gpu_out_lines.count > 0) { PushStringToList(perm, &gpu_out_lines, Lit("")); PushStringToList(perm, &gpu_out_lines, Lit("//- Dependency graph includes")); for (StringListNode *n = gpu_include_lines.first; n; n = n->next) { PushStringToList(perm, &gpu_out_lines, n->s); } } // Write to file PushStringToList(perm, &gpu_out_lines, Lit("")); String c_out = StringFromList(perm, gpu_out_lines, Lit("\n")); F_ClearWrite(gpu_out_file, c_out); } } M_SetBuildStatus(M.gpu_parse.errors.count > 0); } //- Generate archive info { // Push embedded archive dirs for (M_Entry *entry = M.layers_parse.first; entry->valid; entry = entry->next) { M_EntryKind kind = entry->kind; M_Token *entry_tok = entry->name_token; M_Token *arg0_tok = entry->arg_tokens[0]; M_Token *arg1_tok = entry->arg_tokens[1]; switch (kind) { default: break; case M_EntryKind_EmbedDir: { if (arg0_tok->valid && arg1_tok->valid) { String store_name = arg0_tok->s; String token_file = arg1_tok->file->name; String token_parent_dir = F_GetParentDir(token_file); String arg_dir = arg1_tok->s; String full = F_GetFullCrossPlatform(perm, StringF(perm, "%F/%F", FmtString(token_parent_dir), FmtString(arg_dir))); if (F_IsDir(full)) { M_ResDir *rd = PushStruct(perm, M_ResDir); rd->store_name = store_name; rd->dir_path = full; SllQueuePush(M.res_dir_parse.first_res_dir, M.res_dir_parse.last_res_dir, rd); ++M.res_dir_parse.res_dirs_count; } else { String err = StringF(perm, "Directory '%F' not found", FmtString(full)); M_PushError(perm, &M.res_dir_parse.errors, arg1_tok, err); } } else { M_PushError(perm, &M.res_dir_parse.errors, entry_tok, Lit("Expected resource store & directory name")); } } break; } } M_SetBuildStatus(M.res_dir_parse.errors.count > 0); } //- Prep obj arrays { // Gpu objs M.gpu_objs.count = M.gpu_parse.shader_entries_count; M.gpu_objs.array = PushStructs(perm, M_GpuObj, M.gpu_objs.count); // M_Embed objs M.embed_objs.count += M.res_dir_parse.res_dirs_count; if (M.gpu_parse.shader_entries_count > 0) { M.embed_objs.count += 1; } M.embed_objs.array = PushStructs(perm, M_EmbedObj, M.embed_objs.count); } } WaveSync(lane); ////////////////////////////// //- Phase 2/3: Compile { u64 task_idx = 0; u32 embed_idx = 0; //- Compile C { if (lane->idx == WaveLaneIdxFromTaskIdx(lane, task_idx++) && M_GetBuildStatus() == 0) { M.c_obj.obj_file = StringF(perm, "%F_gen_c.obj", FmtString(M.cmdline.leaf_layer_name)); String cmd = Zi; if (M.cmdline.target_compiler == M_CompilerKind_Msvc) { cmd = StringF( perm, "cl.exe /c %F -Fo:%F %F %F %F %F", FmtString(c_out_file), FmtString(M.c_obj.obj_file), FmtString(StringFromList(perm, cp.flags_msvc, Lit(" "))), FmtString(StringFromList(perm, cp.compiler_only_flags_msvc, Lit(" "))), FmtString(StringFromList(perm, cp.warnings_msvc, Lit(" "))), FmtString(StringFromList(perm, cp.defs, Lit(" "))) ); } else { cmd = StringF( perm, "clang.exe -c %F -o %F %F %F %F %F", FmtString(c_out_file), FmtString(M.c_obj.obj_file), FmtString(StringFromList(perm, cp.flags_clang, Lit(" "))), FmtString(StringFromList(perm, cp.compiler_only_flags_clang, Lit(" "))), FmtString(StringFromList(perm, cp.warnings_clang, Lit(" "))), FmtString(StringFromList(perm, cp.defs, Lit(" "))) ); } OS_CommandResult cmd_result = OS_RunCommand(perm, cmd); String cmd_output = TrimWhitespace(cmd_result.output); M.c_obj.output = cmd_output; M.c_obj.return_code = cmd_result.code; // Ignore MSVC file-name echo if (MatchString(TrimWhitespace(M.c_obj.output), F_GetFileName(c_out_file))) { M.c_obj.output = Zstr; } M_SetBuildStatus(M.c_obj.return_code); } } //- Compile shaders { u32 gpu_obj_idx = 0; for (M_ShaderEntry *e = M.gpu_parse.first_shader_entry; e; e = e->next) { if (lane->idx == WaveLaneIdxFromTaskIdx(lane, task_idx++) && M_GetBuildStatus() == 0) { String shader_name = e->name; M_GpuObj *gpu_obj = &M.gpu_objs.array[gpu_obj_idx]; String out_file = StringF(perm, "%F/%F.dxil", FmtString(shader_store_name), FmtString(e->name)); String target = ( e->kind == ShaderEntryKind_VS ? Lit("vs_6_6") : e->kind == ShaderEntryKind_PS ? Lit("ps_6_6") : Lit("cs_6_6") ); String compile_cmd = StringF( perm, "dxc.exe -T %F -E %F -Fo %F %F %F %F %F", FmtString(target), FmtString(e->name), FmtString(out_file), FmtString(gpu_out_file), FmtString(StringFromList(perm, cp.defs, Lit(" "))), FmtString(StringFromList(perm, cp.flags_dxc, Lit(" "))), FmtString(StringFromList(perm, cp.warnings_dxc, Lit(" "))) ); OS_CommandResult cmd_result = OS_RunCommand(perm, compile_cmd); gpu_obj->name = shader_name; gpu_obj->output = cmd_result.output; gpu_obj->return_code = cmd_result.code; M_SetBuildStatus(gpu_obj->return_code); // Final shader compilation lane embeds shader archive u32 finished_count = Atomic32FetchAdd(&M.gpu_objs.finished_count, 1) + 1; if (finished_count == M.gpu_objs.count && M_GetBuildStatus() == 0) { M_EmbedObj *embed = &M.embed_objs.array[embed_idx]; *embed = M_Embed(shader_store_name, shader_store_name); } } ++gpu_obj_idx; } if (M.gpu_parse.shader_entries_count > 0) { ++embed_idx; } } //- M_Embed resource dirs { for (M_ResDir *rd = M.res_dir_parse.first_res_dir; rd; rd = rd->next) { if (lane->idx == WaveLaneIdxFromTaskIdx(lane, task_idx++) && M_GetBuildStatus() == 0) { String dir_path = rd->dir_path; String store_name = rd->store_name; M_EmbedObj *embed = &M.embed_objs.array[embed_idx]; *embed = M_Embed(store_name, dir_path); } ++embed_idx; } } } WaveSync(lane); ////////////////////////////// //- Phase 3/3: Link if (lane->idx == 0 && M_GetBuildStatus() == 0) { String exe_file = StringF(perm, "%F.exe", FmtString(M.cmdline.leaf_layer_name)); { // Wait for exe to become writable (wait for program to close) f32 timeout = 1.0; OS_File file = OS_OpenFile(exe_file, OS_FileFlag_Write, NsFromSeconds(timeout)); OS_CloseFile(file); } i64 start_ns = TimeNs(); String obj_files_str = Zi; { StringList obj_files = Zi; PushStringToList(perm, &obj_files, M.c_obj.obj_file); for (u64 embed_obj_idx = 0; embed_obj_idx < M.embed_objs.count; ++embed_obj_idx) { M_EmbedObj *embed_obj = &M.embed_objs.array[embed_obj_idx]; PushStringToList(perm, &obj_files, embed_obj->obj_file); } obj_files_str = StringFromList(perm, obj_files, Lit(" ")); } String cmd = Zi; if (M.cmdline.target_compiler == M_CompilerKind_Msvc) { cmd = StringF( perm, "link.exe %F /OUT:%F %F %F", FmtString(obj_files_str), FmtString(exe_file), FmtString(StringFromList(perm, cp.flags_msvc, Lit(" "))), FmtString(StringFromList(perm, cp.linker_only_flags_msvc, Lit(" "))) ); } else { cmd = StringF( perm, "lld-link.exe %F /OUT:%F %F %F", FmtString(obj_files_str), FmtString(exe_file), FmtString(StringFromList(perm, cp.flags_clang, Lit(" "))), FmtString(StringFromList(perm, cp.linker_only_flags_clang, Lit(" "))) ); } OS_CommandResult result = OS_RunCommand(perm, cmd); M.link.output = TrimWhitespace(result.output); M.link.return_code = result.code; i64 link_elapsed_ns = TimeNs() - start_ns; // M_EchoLine(StringF(perm, ">>>>> Linked in %Fs", FmtFloat(SecondsFromNs(link_elapsed_ns)))); M_SetBuildStatus(M.link.return_code); } ////////////////////////////// //- Display build results if (lane->idx == 0) { String gpu_obj_output = Zi; { b32 ok = 1; StringList success_gpu_obj_outputs = Zi; StringList error_gpu_obj_outputs = Zi; for (u32 gpu_obj_idx = 0; gpu_obj_idx < M.gpu_objs.count; ++gpu_obj_idx) { M_GpuObj *gpu_obj = &M.gpu_objs.array[gpu_obj_idx]; String output = TrimWhitespace(gpu_obj->output); b32 obj_errored = gpu_obj->return_code != 0; if (obj_errored) { ok = 0; } if (output.len > 0) { String msg = output; if (!StringContains(msg, Lit("In file"))) { // If error message is missing "In file" then it may have // failed to even find the entry point, meaning we should // include the name of the shader in the error message for // clarification. msg = StringF( perm, "Error compiling shader \"%F\"\n%F", FmtString(gpu_obj->name), FmtString(output) ); } if (obj_errored) { if (error_gpu_obj_outputs.count == 0) { PushStringToList(perm, &error_gpu_obj_outputs, msg); } } else { PushStringToList(perm, &success_gpu_obj_outputs, msg); } } } if (ok) { gpu_obj_output = StringFromList(perm, success_gpu_obj_outputs, Lit("\n")); } else { gpu_obj_output = StringFromList(perm, error_gpu_obj_outputs, Lit("\n")); if (TrimWhitespace(gpu_obj_output).len == 0) { gpu_obj_output = Lit("Unknown error compiling shaders"); } } } String embed_obj_output = Zi; { StringList embed_obj_outputs = Zi; for (u32 embed_obj_idx = 0; embed_obj_idx < M.embed_objs.count; ++embed_obj_idx) { M_EmbedObj *embed_obj = &M.embed_objs.array[embed_obj_idx]; PushStringToList(perm, &embed_obj_outputs, embed_obj->output); } embed_obj_output = StringFromList(perm, embed_obj_outputs, Lit("\n")); } M_EchoLineOrNothing(M_StringFromMetaErrors(perm, M.layers_parse.errors)); M_EchoLineOrNothing(M_StringFromMetaErrors(perm, M.c_parse.errors)); M_EchoLineOrNothing(M_StringFromMetaErrors(perm, M.gpu_parse.errors)); M_EchoLineOrNothing(M.c_obj.output); M_EchoLineOrNothing(gpu_obj_output); M_EchoLineOrNothing(M_StringFromMetaErrors(perm, M.res_dir_parse.errors)); M_EchoLineOrNothing(embed_obj_output); M_EchoLineOrNothing(M.link.output); } ////////////////////////////// //- Exit if (lane->idx == 0) { M_EchoLine(StringF(perm, "Runtime: %Fs", FmtFloat(SecondsFromNs(TimeNs()), .p = 3))); ExitNow(M_GetBuildStatus()); } } //////////////////////////////////////////////////////////// //~ @hookimpl Bootstrap layers void BootstrapLayers(void) { OS_Bootstrap(); CpuTopologyInfo cpu_info = GetCpuTopologyInfo(); i32 meta_lanes_count = cpu_info.num_logical_cores - 1; DispatchWave(Lit("Meta"), MaxI32(meta_lanes_count, 1), M_BuildEntryPoint, 0); }