diff --git a/build.c b/build.c index 5c42e2a0..da242dc4 100644 --- a/build.c +++ b/build.c @@ -1,6 +1,6 @@ #include "buildit.h" -typedef struct WorkerSharedState WorkerSharedState; +typedef struct StepList StepList; /* ========================== * * Globals @@ -11,7 +11,7 @@ Bool force_rebuild = false; Arena arena = { 0 }; D_Store store = { 0 }; D_Hist hist = { 0 }; -WorkerSharedState *worker_shared = { 0 }; +StepList *sl = { 0 }; /* ========================== * * Util @@ -38,11 +38,8 @@ Bool IsDirty(D_Tag tag) * Step * ========================== */ -typedef struct Step Step; typedef struct StepList StepList; -typedef void StepFunc(Step *step); - typedef enum StepStatus { StepStatus_None, StepStatus_Success, @@ -52,9 +49,7 @@ typedef enum StepStatus { typedef struct Step Step; struct Step { String name; - StepFunc *func; void *arg; - StepList *sl; OS_Mutex res_mutex; OS_ConditionVariable res_cv; @@ -62,10 +57,6 @@ struct Step { String res_output; Step *next; - Step *prev; - - Step *next_unqueued; - Step *prev_unqueued; }; typedef struct StepList StepList; @@ -73,163 +64,41 @@ struct StepList { OS_Mutex mutex; Arena arena; + T_Queue tq; + Step *first; Step *last; - Size count; - - Step *first_unqueued; - Step *last_unqueued; - - StepList *next; + I64 count; }; -StepList StepListAlloc(void) +void AddStep(String name, T_TaskFunc *func, void *arg) { - StepList sl = { 0 }; - sl.mutex = OS_MutexAlloc(); - sl.arena = ArenaAlloc(Gigabyte(64)); - return sl; -} - -Step *DequeueNextStep(StepList *sl) -{ - Step *res = 0; - OS_MutexLockW(&sl->mutex); - { - res = sl->first_unqueued; - if (res) { - DllRemoveNP(sl->first_unqueued, sl->last_unqueued, res, next_unqueued, prev_unqueued); - } - } - OS_MutexUnlockW(&sl->mutex); - return res; -} - -void AddStep(StepList *sl, String name, StepFunc *func, void *arg) -{ - OS_MutexLockW(&sl->mutex); + OS_MutexLockE(&sl->mutex); { Step *s = ArenaPush(&sl->arena, Step); - s->name = StringCopy(&sl->arena, name); - s->func = func; - s->arg = arg; - s->sl = sl; s->res_mutex = OS_MutexAlloc(); s->res_cv = OS_ConditionVariableAlloc(); - DllPushBack(sl->first, sl->last, s); - DllPushBackNP(sl->first_unqueued, sl->last_unqueued, s, next_unqueued, prev_unqueued); + s->name = name; + s->arg = arg; + SllQueuePush(sl->first, sl->last, s); ++sl->count; + T_AddTask(&sl->tq, func, s); } - OS_MutexUnlockW(&sl->mutex); + OS_MutexUnlockE(&sl->mutex); } -/* ========================== * - * Worker - * ========================== */ - -typedef struct Worker Worker; - -typedef struct WorkerSharedState WorkerSharedState; -struct WorkerSharedState { - OS_Mutex mutex; - OS_ConditionVariable cv; - Arena arena; - - StepList *first_step_list; - StepList *last_step_list; - - Bool shutdown; - Worker *first_worker; - I64 worker_count; -}; - -typedef struct Worker Worker; -struct Worker { - OS_Thread thread; - Worker *next; -}; - -OS_ThreadEntryPointFuncDef(WorkerEntryPoint, arg) +void AddSyncPoint(void) { - I64 id = (I64)arg; + OS_MutexLockE(&sl->mutex); { - TempArena scratch = ScratchBeginNoConflict(); - String name = StringF(scratch.arena, Lit("Buildit worker #%F"), FmtI64Zfill(id, 2)); - OS_SetThreadName(name); - ScratchEnd(scratch); - } - - while (!worker_shared->shutdown) { - OS_MutexLockW(&worker_shared->mutex); - { - if (!worker_shared->shutdown && !worker_shared->first_step_list) { - OS_ConditionVariableWaitW(&worker_shared->cv, &worker_shared->mutex); - } - - if (!worker_shared->shutdown && worker_shared->first_step_list) { - StepList *sl = worker_shared->first_step_list; - Step *s = DequeueNextStep(sl); - if (s) { - OS_MutexUnlockW(&worker_shared->mutex); - { - s->func(s); - } - OS_MutexLockW(&worker_shared->mutex); - } else { - /* Wait for all steps in old step list to finish, then pop the list */ - for (Step *check_step = sl->first; check_step; check_step = check_step->next) { - OS_MutexLockW(&check_step->res_mutex); - { - while (check_step->res_status == StepStatus_None) { - OS_ConditionVariableWaitW(&check_step->res_cv, &check_step->res_mutex); - } - } - OS_MutexUnlockW(&check_step->res_mutex); - } - SllQueuePop(worker_shared->first_step_list, worker_shared->last_step_list); - } - } - } - OS_MutexUnlockW(&worker_shared->mutex); + T_AddSyncPoint(&sl->tq); } + OS_MutexUnlockE(&sl->mutex); } -void StartWorkers(I64 count) +void ExecuteSteps(void) { - OS_MutexLockW(&worker_shared->mutex); - { - for (I64 i = 0; i < count; ++i) { - Worker *w = ArenaPush(&worker_shared->arena, Worker); - w->thread = OS_ThreadAlloc(&WorkerEntryPoint, (void *)worker_shared->worker_count); - SllStackPush(worker_shared->first_worker, w); - ++worker_shared->worker_count; - } - } - OS_MutexUnlockW(&worker_shared->mutex); -} - -void ShutdownWorkers(void) -{ - OS_MutexLockW(&worker_shared->mutex); - { - worker_shared->shutdown = true; - OS_ConditionVariableBroadcast(&worker_shared->cv); - } - OS_MutexUnlockW(&worker_shared->mutex); - - for (Worker *w = worker_shared->first_worker; w; w = w->next) { - OS_ThreadWaitRelease(&w->thread); - } -} - -void AddStepListToWorkQueue(StepList *sl) -{ - OS_MutexLockW(&worker_shared->mutex); - { - SllQueuePush(worker_shared->first_step_list, worker_shared->last_step_list, sl); - OS_ConditionVariableBroadcast(&worker_shared->cv); - } - OS_MutexUnlockW(&worker_shared->mutex); + T_Execute(&sl->tq); } /* ========================== * @@ -249,35 +118,37 @@ struct BuildStepMsvcCompileCommandArg { D_TagList depfile_force_includes; }; -void BuildStepSimpleCommand(Step *s) +void BuildStepSimpleCommand(void *arg_raw) { TempArena scratch = ScratchBeginNoConflict(); + Step *s = arg_raw; BuildStepSimpleCommandArg *arg = s->arg; SH_CommandResult result = SH_RunCommandCaptureOutput(scratch.arena, arg->cmd, true); - OS_MutexLockW(&s->res_mutex); + OS_MutexLockE(&s->res_mutex); { if (result.output.len > 0) { - OS_MutexLockW(&s->sl->mutex); + OS_MutexLockE(&sl->mutex); { - s->res_output.text = ArenaPushArrayNoZero(&s->sl->arena, Byte, result.output.len); + s->res_output.text = ArenaPushArrayNoZero(&sl->arena, Byte, result.output.len); } - OS_MutexUnlockW(&s->sl->mutex); + OS_MutexUnlockE(&sl->mutex); s->res_output.len = result.output.len; MemoryCopy(s->res_output.text, result.output.text, result.output.len); } s->res_status = result.error ? StepStatus_Failure : StepStatus_Success; OS_ConditionVariableBroadcast(&s->res_cv); } - OS_MutexUnlockW(&s->res_mutex); + OS_MutexUnlockE(&s->res_mutex); ScratchEnd(scratch); } -void BuildStepMsvcCompileCommand(Step *s) +void BuildStepMsvcCompileCommand(void *arg_raw) { TempArena scratch = ScratchBeginNoConflict(); + Step *s = arg_raw; BuildStepMsvcCompileCommandArg *arg = s->arg; SH_CommandResult result = SH_RunCommandCaptureOutput(scratch.arena, arg->cmd, true); @@ -287,21 +158,21 @@ void BuildStepMsvcCompileCommand(Step *s) D_ClearWrite(arg->output_depfile, depfile_data); } - OS_MutexLockW(&s->res_mutex); + OS_MutexLockE(&s->res_mutex); { if (result.output.len > 0) { - OS_MutexLockW(&s->sl->mutex); + OS_MutexLockE(&sl->mutex); { - s->res_output.text = ArenaPushArrayNoZero(&s->sl->arena, Byte, result.output.len); + s->res_output.text = ArenaPushArrayNoZero(&sl->arena, Byte, result.output.len); } - OS_MutexUnlockW(&s->sl->mutex); + OS_MutexUnlockE(&sl->mutex); s->res_output.len = result.output.len; MemoryCopy(s->res_output.text, result.output.text, result.output.len); } s->res_status = result.error ? StepStatus_Failure : StepStatus_Success; OS_ConditionVariableBroadcast(&s->res_cv); } - OS_MutexUnlockW(&s->res_mutex); + OS_MutexUnlockE(&s->res_mutex); ScratchEnd(scratch); } @@ -312,19 +183,17 @@ void BuildStepMsvcCompileCommand(Step *s) void OnBuild(StringList cli_args) { - arena = ArenaAlloc(Gigabyte(64)); - store = D_StoreAlloc(); - worker_shared = ArenaPush(&arena, WorkerSharedState); - - /* ========================== * - * Setup workers - * ========================== */ - - worker_shared->mutex = OS_MutexAlloc(); - worker_shared->arena = ArenaAlloc(Gigabyte(64)); - I64 worker_count = OS_GetProcessorCount(); - StartWorkers(worker_count); + T_StartupWorkers(worker_count); + + arena = ArenaAlloc(Gigabyte(64)); + + store = D_StoreAlloc(); + + sl = ArenaPush(&arena, StepList); + sl->arena = ArenaAlloc(Gigabyte(64)); + sl->mutex = OS_MutexAlloc(); + sl->tq = T_QueueAlloc(); /* ========================== * * Read args @@ -643,16 +512,10 @@ void OnBuild(StringList cli_args) } /* ========================== * - * Build + * Begin build * ========================== */ hist = D_HistFromPath(&arena, hist_path); - - StepList tar_build_steps = StepListAlloc(); - StepList rc_and_pch_build_steps = StepListAlloc(); - StepList comp_build_steps = StepListAlloc(); - StepList link_build_steps = StepListAlloc(); - D_TagList link_files = { 0 }; /* ========================== * @@ -660,6 +523,8 @@ void OnBuild(StringList cli_args) * ========================== */ { + AddSyncPoint(); + D_TagList tar_input_dirs = { 0 }; { D_TagListAppend(&arena, &tar_input_dirs, shaders_dir); @@ -681,7 +546,7 @@ void OnBuild(StringList cli_args) BuildStepSimpleCommandArg *bs_arg = ArenaPush(&arena, BuildStepSimpleCommandArg); bs_arg->cmd = StringF(&arena, Lit("cd %F && tar cvf %F ."), FmtStr(input_dir.full_path), FmtStr(tar_file.full_path)); String step_name = StringF(&arena, Lit("%F -> %F"), FmtStr(D_GetName(input_dir)), FmtStr(D_GetName(tar_file))); - AddStep(&tar_build_steps, step_name, &BuildStepSimpleCommand, bs_arg); + AddStep(step_name, &BuildStepSimpleCommand, bs_arg); } } } @@ -691,6 +556,8 @@ void OnBuild(StringList cli_args) * ========================== */ if (PlatformWindows) { + AddSyncPoint(); + D_Tag rc_input_file = D_TagFromPath(&arena, StringF(&arena, Lit("%F/rc.rc"), FmtStr(out_inc_dir_path)), D_TagKind_File); { D_Tag shaders_tar_file = D_TagFromPath(&arena, StringF(&arena, Lit("%F/%F.tar"), FmtStr(out_inc_dir_path), FmtStr(D_GetName(shaders_dir))), D_TagKind_File); @@ -722,7 +589,7 @@ void OnBuild(StringList cli_args) BuildStepSimpleCommandArg *bs_arg = ArenaPush(&arena, BuildStepSimpleCommandArg); bs_arg->cmd = StringF(&arena, rc_compile_args_fmt, FmtStr(rc_res_file.full_path), FmtStr(rc_input_file.full_path)); String step_name = StringF(&arena, Lit("%F -> %F"), FmtStr(D_GetName(rc_input_file)), FmtStr(D_GetName(rc_res_file))); - AddStep(&rc_and_pch_build_steps, step_name, &BuildStepSimpleCommand, bs_arg); + AddStep(step_name, &BuildStepSimpleCommand, bs_arg); } D_TagListAppend(&arena, &link_files, rc_res_file); @@ -735,6 +602,8 @@ void OnBuild(StringList cli_args) D_TagList depfile_force_includes = { 0 }; { + AddSyncPoint(); + D_Tag pch_header_file = D_TagFromPath(&arena, Lit("src/common.h"), D_TagKind_File); D_Tag dep_file; { @@ -778,7 +647,7 @@ void OnBuild(StringList cli_args) BuildStepSimpleCommandArg *bs_arg = ArenaPush(&arena, BuildStepSimpleCommandArg); bs_arg->cmd = StringF(&arena, pch_c_compile_args_fmt, FmtStr(pch_header_file.full_path), FmtStr(pch_c_src_gen_file.full_path)); String step_name = StringF(&arena, Lit("%F -> %F"), FmtStr(D_GetName(pch_header_file)), FmtStr(D_GetName(pch_c_file))); - AddStep(&rc_and_pch_build_steps, step_name, &BuildStepSimpleCommand, bs_arg); + AddStep(step_name, &BuildStepSimpleCommand, bs_arg); } /* Cpp */ @@ -789,7 +658,7 @@ void OnBuild(StringList cli_args) BuildStepSimpleCommandArg *bs_arg = ArenaPush(&arena, BuildStepSimpleCommandArg); bs_arg->cmd = StringF(&arena, pch_cpp_compile_args_fmt, FmtStr(pch_header_file.full_path), FmtStr(pch_cpp_src_gen_file.full_path)); String step_name = StringF(&arena, Lit("%F -> %F"), FmtStr(D_GetName(pch_header_file)), FmtStr(D_GetName(pch_cpp_file))); - AddStep(&rc_and_pch_build_steps, step_name, &BuildStepSimpleCommand, bs_arg); + AddStep(step_name, &BuildStepSimpleCommand, bs_arg); } } else { StringListAppend(&arena, &c_compile_args, StringF(&arena, Lit("-include-pch %F"), FmtStr(pch_c_file.full_path))); @@ -800,22 +669,22 @@ void OnBuild(StringList cli_args) /* C */ if (IsDirty(pch_c_file)) { - String step_name = StringF(&arena, Lit("%F -> %F"), FmtStr(D_GetName(pch_header_file)), FmtStr(D_GetName(pch_c_file))); BuildStepMsvcCompileCommandArg *bs_arg = ArenaPush(&arena, BuildStepMsvcCompileCommandArg); bs_arg->cmd = StringF(&arena, pch_c_compile_args_fmt, FmtStr(pch_header_file.full_path), FmtStr(pch_c_file.full_path)); bs_arg->depfile_dependent = pch_header_file; bs_arg->output_depfile = dep_file; - AddStep(&rc_and_pch_build_steps, step_name, &BuildStepMsvcCompileCommand, bs_arg); + String step_name = StringF(&arena, Lit("%F -> %F"), FmtStr(D_GetName(pch_header_file)), FmtStr(D_GetName(pch_c_file))); + AddStep(step_name, &BuildStepMsvcCompileCommand, bs_arg); } /* Cpp */ if (IsDirty(pch_cpp_file)) { - String step_name = StringF(&arena, Lit("%F -> %F"), FmtStr(D_GetName(pch_header_file)), FmtStr(D_GetName(pch_cpp_file))); BuildStepMsvcCompileCommandArg *bs_arg = ArenaPush(&arena, BuildStepMsvcCompileCommandArg); bs_arg->cmd = StringF(&arena, pch_cpp_compile_args_fmt, FmtStr(pch_header_file.full_path), FmtStr(pch_cpp_file.full_path)); bs_arg->depfile_dependent = pch_header_file; bs_arg->output_depfile = dep_file; - AddStep(&rc_and_pch_build_steps, step_name, &BuildStepMsvcCompileCommand, bs_arg); + String step_name = StringF(&arena, Lit("%F -> %F"), FmtStr(D_GetName(pch_header_file)), FmtStr(D_GetName(pch_cpp_file))); + AddStep(step_name, &BuildStepMsvcCompileCommand, bs_arg); } } } @@ -873,6 +742,8 @@ void OnBuild(StringList cli_args) } { + AddSyncPoint(); + String c_compile_args_fmt = StringFromStringLists(&arena, Lit(" "), c_compile_args, compile_warnings, compile_and_link_args, compile_args); String cpp_compile_args_fmt = StringFromStringLists(&arena, Lit(" "), cpp_compile_args, compile_warnings, compile_and_link_args, compile_args); for (D_TagListNode *n = src_input_files.first; n; n = n->next) { @@ -908,11 +779,11 @@ void OnBuild(StringList cli_args) bs_arg->depfile_dependent = obj_file; bs_arg->output_depfile = dep_file; bs_arg->depfile_force_includes = depfile_force_includes; - AddStep(&comp_build_steps, step_name, &BuildStepMsvcCompileCommand, bs_arg); + AddStep(step_name, &BuildStepMsvcCompileCommand, bs_arg); } else { BuildStepSimpleCommandArg *bs_arg = ArenaPush(&arena, BuildStepSimpleCommandArg); bs_arg->cmd = StringF(&arena, comp_cmd_fmt, FmtStr(file.full_path), FmtStr(obj_file.full_path)); - AddStep(&comp_build_steps, step_name, &BuildStepSimpleCommand, bs_arg); + AddStep(step_name, &BuildStepSimpleCommand, bs_arg); } } @@ -926,6 +797,8 @@ void OnBuild(StringList cli_args) * ========================== */ { + AddSyncPoint(); + String link_files_str = { 0 }; { StringList link_files_quoted_list = { 0 }; @@ -937,60 +810,38 @@ void OnBuild(StringList cli_args) } link_files_str = StringFromStringList(&arena, Lit(" "), link_files_quoted_list); } - if (link_files_str.len > 0) { + if (link_files_str.len > 0 && IsDirty(executable_file)) { String link_args_fmt = StringFromStringLists(&arena, Lit(" "), link_args, link_warnings, compile_and_link_args); BuildStepSimpleCommandArg *bs_arg = ArenaPush(&arena, BuildStepSimpleCommandArg); bs_arg->cmd = StringF(&arena, link_args_fmt, FmtStr(link_files_str), FmtStr(executable_file.full_path)); String step_name = Lit("Linking"); - AddStep(&link_build_steps, step_name, &BuildStepSimpleCommand, bs_arg); + AddStep(step_name, &BuildStepSimpleCommand, bs_arg); } } - /* ========================== * - * Execute build steps - * ========================== */ - - /* TODO: Cleanup (testing parallel stuff) */ + /* ========================== * + * Execute build steps + * ========================== */ Bool success = true; - if (IsDirty(executable_file)) { - AddStepListToWorkQueue(&tar_build_steps); - AddStepListToWorkQueue(&rc_and_pch_build_steps); - AddStepListToWorkQueue(&comp_build_steps); - AddStepListToWorkQueue(&link_build_steps); + I64 step_count = sl->count; + if (step_count > 0) { + ExecuteSteps(); - I64 step_count = 0; - step_count += tar_build_steps.count; - step_count += rc_and_pch_build_steps.count; - step_count += comp_build_steps.count; - step_count += link_build_steps.count; - - Step *s = tar_build_steps.first; - { - if (!s) { - s = rc_and_pch_build_steps.first; - if (!s) { - s = comp_build_steps.first; - if (!s) { - s = link_build_steps.first; - } - } - - } - } + Step *s = sl->first; I64 step_i = 0; while (s) { ++step_i; SH_PrintF(Lit("[%F/%F] %F\n"), FmtI64(step_i), FmtI64(step_count), FmtStr(s->name)); - OS_MutexLockR(&s->res_mutex); + OS_MutexLockS(&s->res_mutex); { while (s->res_status == StepStatus_None) { - OS_ConditionVariableWaitR(&s->res_cv, &s->res_mutex); + OS_ConditionVariableWaitS(&s->res_cv, &s->res_mutex); } } - OS_MutexUnlockR(&s->res_mutex); + OS_MutexUnlockS(&s->res_mutex); String output = s->res_output; if (s->res_status != StepStatus_Success) { @@ -1000,20 +851,6 @@ void OnBuild(StringList cli_args) break; } - { - if (s == tar_build_steps.last) { - s = rc_and_pch_build_steps.first; - if (s) goto skip_cont; - } - if (s == rc_and_pch_build_steps.last) { - s = comp_build_steps.first; - if (s) goto skip_cont; - } - if (s == comp_build_steps.last) { - s = link_build_steps.first; - if (s) goto skip_cont; - } - } s = s->next; skip_cont: continue; @@ -1027,17 +864,6 @@ void OnBuild(StringList cli_args) OS_Exit(1); } - /* ========================== * - * Write hist file - * ========================== */ - + T_ShutdownWorkers(); D_WriteStoreToHistFile(&store, hist_path); - - ShutdownWorkers(); - -#if 0 -#if Rtc - getchar(); -#endif -#endif }