use buildit task system

This commit is contained in:
jacob 2024-06-20 16:02:31 -05:00
parent d571a9647d
commit 0039afa32e

320
build.c
View File

@ -1,6 +1,6 @@
#include "buildit.h"
typedef struct WorkerSharedState WorkerSharedState;
typedef struct StepList StepList;
/* ========================== *
* Globals
@ -11,7 +11,7 @@ Bool force_rebuild = false;
Arena arena = { 0 };
D_Store store = { 0 };
D_Hist hist = { 0 };
WorkerSharedState *worker_shared = { 0 };
StepList *sl = { 0 };
/* ========================== *
* Util
@ -38,11 +38,8 @@ Bool IsDirty(D_Tag tag)
* Step
* ========================== */
typedef struct Step Step;
typedef struct StepList StepList;
typedef void StepFunc(Step *step);
typedef enum StepStatus {
StepStatus_None,
StepStatus_Success,
@ -52,9 +49,7 @@ typedef enum StepStatus {
typedef struct Step Step;
struct Step {
String name;
StepFunc *func;
void *arg;
StepList *sl;
OS_Mutex res_mutex;
OS_ConditionVariable res_cv;
@ -62,10 +57,6 @@ struct Step {
String res_output;
Step *next;
Step *prev;
Step *next_unqueued;
Step *prev_unqueued;
};
typedef struct StepList StepList;
@ -73,163 +64,41 @@ struct StepList {
OS_Mutex mutex;
Arena arena;
T_Queue tq;
Step *first;
Step *last;
Size count;
Step *first_unqueued;
Step *last_unqueued;
StepList *next;
I64 count;
};
StepList StepListAlloc(void)
void AddStep(String name, T_TaskFunc *func, void *arg)
{
StepList sl = { 0 };
sl.mutex = OS_MutexAlloc();
sl.arena = ArenaAlloc(Gigabyte(64));
return sl;
}
Step *DequeueNextStep(StepList *sl)
{
Step *res = 0;
OS_MutexLockW(&sl->mutex);
{
res = sl->first_unqueued;
if (res) {
DllRemoveNP(sl->first_unqueued, sl->last_unqueued, res, next_unqueued, prev_unqueued);
}
}
OS_MutexUnlockW(&sl->mutex);
return res;
}
void AddStep(StepList *sl, String name, StepFunc *func, void *arg)
{
OS_MutexLockW(&sl->mutex);
OS_MutexLockE(&sl->mutex);
{
Step *s = ArenaPush(&sl->arena, Step);
s->name = StringCopy(&sl->arena, name);
s->func = func;
s->arg = arg;
s->sl = sl;
s->res_mutex = OS_MutexAlloc();
s->res_cv = OS_ConditionVariableAlloc();
DllPushBack(sl->first, sl->last, s);
DllPushBackNP(sl->first_unqueued, sl->last_unqueued, s, next_unqueued, prev_unqueued);
s->name = name;
s->arg = arg;
SllQueuePush(sl->first, sl->last, s);
++sl->count;
T_AddTask(&sl->tq, func, s);
}
OS_MutexUnlockW(&sl->mutex);
OS_MutexUnlockE(&sl->mutex);
}
/* ========================== *
* Worker
* ========================== */
typedef struct Worker Worker;
typedef struct WorkerSharedState WorkerSharedState;
struct WorkerSharedState {
OS_Mutex mutex;
OS_ConditionVariable cv;
Arena arena;
StepList *first_step_list;
StepList *last_step_list;
Bool shutdown;
Worker *first_worker;
I64 worker_count;
};
typedef struct Worker Worker;
struct Worker {
OS_Thread thread;
Worker *next;
};
OS_ThreadEntryPointFuncDef(WorkerEntryPoint, arg)
void AddSyncPoint(void)
{
I64 id = (I64)arg;
OS_MutexLockE(&sl->mutex);
{
TempArena scratch = ScratchBeginNoConflict();
String name = StringF(scratch.arena, Lit("Buildit worker #%F"), FmtI64Zfill(id, 2));
OS_SetThreadName(name);
ScratchEnd(scratch);
}
while (!worker_shared->shutdown) {
OS_MutexLockW(&worker_shared->mutex);
{
if (!worker_shared->shutdown && !worker_shared->first_step_list) {
OS_ConditionVariableWaitW(&worker_shared->cv, &worker_shared->mutex);
}
if (!worker_shared->shutdown && worker_shared->first_step_list) {
StepList *sl = worker_shared->first_step_list;
Step *s = DequeueNextStep(sl);
if (s) {
OS_MutexUnlockW(&worker_shared->mutex);
{
s->func(s);
}
OS_MutexLockW(&worker_shared->mutex);
} else {
/* Wait for all steps in old step list to finish, then pop the list */
for (Step *check_step = sl->first; check_step; check_step = check_step->next) {
OS_MutexLockW(&check_step->res_mutex);
{
while (check_step->res_status == StepStatus_None) {
OS_ConditionVariableWaitW(&check_step->res_cv, &check_step->res_mutex);
}
}
OS_MutexUnlockW(&check_step->res_mutex);
}
SllQueuePop(worker_shared->first_step_list, worker_shared->last_step_list);
}
}
}
OS_MutexUnlockW(&worker_shared->mutex);
T_AddSyncPoint(&sl->tq);
}
OS_MutexUnlockE(&sl->mutex);
}
void StartWorkers(I64 count)
void ExecuteSteps(void)
{
OS_MutexLockW(&worker_shared->mutex);
{
for (I64 i = 0; i < count; ++i) {
Worker *w = ArenaPush(&worker_shared->arena, Worker);
w->thread = OS_ThreadAlloc(&WorkerEntryPoint, (void *)worker_shared->worker_count);
SllStackPush(worker_shared->first_worker, w);
++worker_shared->worker_count;
}
}
OS_MutexUnlockW(&worker_shared->mutex);
}
void ShutdownWorkers(void)
{
OS_MutexLockW(&worker_shared->mutex);
{
worker_shared->shutdown = true;
OS_ConditionVariableBroadcast(&worker_shared->cv);
}
OS_MutexUnlockW(&worker_shared->mutex);
for (Worker *w = worker_shared->first_worker; w; w = w->next) {
OS_ThreadWaitRelease(&w->thread);
}
}
void AddStepListToWorkQueue(StepList *sl)
{
OS_MutexLockW(&worker_shared->mutex);
{
SllQueuePush(worker_shared->first_step_list, worker_shared->last_step_list, sl);
OS_ConditionVariableBroadcast(&worker_shared->cv);
}
OS_MutexUnlockW(&worker_shared->mutex);
T_Execute(&sl->tq);
}
/* ========================== *
@ -249,35 +118,37 @@ struct BuildStepMsvcCompileCommandArg {
D_TagList depfile_force_includes;
};
void BuildStepSimpleCommand(Step *s)
void BuildStepSimpleCommand(void *arg_raw)
{
TempArena scratch = ScratchBeginNoConflict();
Step *s = arg_raw;
BuildStepSimpleCommandArg *arg = s->arg;
SH_CommandResult result = SH_RunCommandCaptureOutput(scratch.arena, arg->cmd, true);
OS_MutexLockW(&s->res_mutex);
OS_MutexLockE(&s->res_mutex);
{
if (result.output.len > 0) {
OS_MutexLockW(&s->sl->mutex);
OS_MutexLockE(&sl->mutex);
{
s->res_output.text = ArenaPushArrayNoZero(&s->sl->arena, Byte, result.output.len);
s->res_output.text = ArenaPushArrayNoZero(&sl->arena, Byte, result.output.len);
}
OS_MutexUnlockW(&s->sl->mutex);
OS_MutexUnlockE(&sl->mutex);
s->res_output.len = result.output.len;
MemoryCopy(s->res_output.text, result.output.text, result.output.len);
}
s->res_status = result.error ? StepStatus_Failure : StepStatus_Success;
OS_ConditionVariableBroadcast(&s->res_cv);
}
OS_MutexUnlockW(&s->res_mutex);
OS_MutexUnlockE(&s->res_mutex);
ScratchEnd(scratch);
}
void BuildStepMsvcCompileCommand(Step *s)
void BuildStepMsvcCompileCommand(void *arg_raw)
{
TempArena scratch = ScratchBeginNoConflict();
Step *s = arg_raw;
BuildStepMsvcCompileCommandArg *arg = s->arg;
SH_CommandResult result = SH_RunCommandCaptureOutput(scratch.arena, arg->cmd, true);
@ -287,21 +158,21 @@ void BuildStepMsvcCompileCommand(Step *s)
D_ClearWrite(arg->output_depfile, depfile_data);
}
OS_MutexLockW(&s->res_mutex);
OS_MutexLockE(&s->res_mutex);
{
if (result.output.len > 0) {
OS_MutexLockW(&s->sl->mutex);
OS_MutexLockE(&sl->mutex);
{
s->res_output.text = ArenaPushArrayNoZero(&s->sl->arena, Byte, result.output.len);
s->res_output.text = ArenaPushArrayNoZero(&sl->arena, Byte, result.output.len);
}
OS_MutexUnlockW(&s->sl->mutex);
OS_MutexUnlockE(&sl->mutex);
s->res_output.len = result.output.len;
MemoryCopy(s->res_output.text, result.output.text, result.output.len);
}
s->res_status = result.error ? StepStatus_Failure : StepStatus_Success;
OS_ConditionVariableBroadcast(&s->res_cv);
}
OS_MutexUnlockW(&s->res_mutex);
OS_MutexUnlockE(&s->res_mutex);
ScratchEnd(scratch);
}
@ -312,19 +183,17 @@ void BuildStepMsvcCompileCommand(Step *s)
void OnBuild(StringList cli_args)
{
arena = ArenaAlloc(Gigabyte(64));
store = D_StoreAlloc();
worker_shared = ArenaPush(&arena, WorkerSharedState);
/* ========================== *
* Setup workers
* ========================== */
worker_shared->mutex = OS_MutexAlloc();
worker_shared->arena = ArenaAlloc(Gigabyte(64));
I64 worker_count = OS_GetProcessorCount();
StartWorkers(worker_count);
T_StartupWorkers(worker_count);
arena = ArenaAlloc(Gigabyte(64));
store = D_StoreAlloc();
sl = ArenaPush(&arena, StepList);
sl->arena = ArenaAlloc(Gigabyte(64));
sl->mutex = OS_MutexAlloc();
sl->tq = T_QueueAlloc();
/* ========================== *
* Read args
@ -643,16 +512,10 @@ void OnBuild(StringList cli_args)
}
/* ========================== *
* Build
* Begin build
* ========================== */
hist = D_HistFromPath(&arena, hist_path);
StepList tar_build_steps = StepListAlloc();
StepList rc_and_pch_build_steps = StepListAlloc();
StepList comp_build_steps = StepListAlloc();
StepList link_build_steps = StepListAlloc();
D_TagList link_files = { 0 };
/* ========================== *
@ -660,6 +523,8 @@ void OnBuild(StringList cli_args)
* ========================== */
{
AddSyncPoint();
D_TagList tar_input_dirs = { 0 };
{
D_TagListAppend(&arena, &tar_input_dirs, shaders_dir);
@ -681,7 +546,7 @@ void OnBuild(StringList cli_args)
BuildStepSimpleCommandArg *bs_arg = ArenaPush(&arena, BuildStepSimpleCommandArg);
bs_arg->cmd = StringF(&arena, Lit("cd %F && tar cvf %F ."), FmtStr(input_dir.full_path), FmtStr(tar_file.full_path));
String step_name = StringF(&arena, Lit("%F -> %F"), FmtStr(D_GetName(input_dir)), FmtStr(D_GetName(tar_file)));
AddStep(&tar_build_steps, step_name, &BuildStepSimpleCommand, bs_arg);
AddStep(step_name, &BuildStepSimpleCommand, bs_arg);
}
}
}
@ -691,6 +556,8 @@ void OnBuild(StringList cli_args)
* ========================== */
if (PlatformWindows) {
AddSyncPoint();
D_Tag rc_input_file = D_TagFromPath(&arena, StringF(&arena, Lit("%F/rc.rc"), FmtStr(out_inc_dir_path)), D_TagKind_File);
{
D_Tag shaders_tar_file = D_TagFromPath(&arena, StringF(&arena, Lit("%F/%F.tar"), FmtStr(out_inc_dir_path), FmtStr(D_GetName(shaders_dir))), D_TagKind_File);
@ -722,7 +589,7 @@ void OnBuild(StringList cli_args)
BuildStepSimpleCommandArg *bs_arg = ArenaPush(&arena, BuildStepSimpleCommandArg);
bs_arg->cmd = StringF(&arena, rc_compile_args_fmt, FmtStr(rc_res_file.full_path), FmtStr(rc_input_file.full_path));
String step_name = StringF(&arena, Lit("%F -> %F"), FmtStr(D_GetName(rc_input_file)), FmtStr(D_GetName(rc_res_file)));
AddStep(&rc_and_pch_build_steps, step_name, &BuildStepSimpleCommand, bs_arg);
AddStep(step_name, &BuildStepSimpleCommand, bs_arg);
}
D_TagListAppend(&arena, &link_files, rc_res_file);
@ -735,6 +602,8 @@ void OnBuild(StringList cli_args)
D_TagList depfile_force_includes = { 0 };
{
AddSyncPoint();
D_Tag pch_header_file = D_TagFromPath(&arena, Lit("src/common.h"), D_TagKind_File);
D_Tag dep_file;
{
@ -778,7 +647,7 @@ void OnBuild(StringList cli_args)
BuildStepSimpleCommandArg *bs_arg = ArenaPush(&arena, BuildStepSimpleCommandArg);
bs_arg->cmd = StringF(&arena, pch_c_compile_args_fmt, FmtStr(pch_header_file.full_path), FmtStr(pch_c_src_gen_file.full_path));
String step_name = StringF(&arena, Lit("%F -> %F"), FmtStr(D_GetName(pch_header_file)), FmtStr(D_GetName(pch_c_file)));
AddStep(&rc_and_pch_build_steps, step_name, &BuildStepSimpleCommand, bs_arg);
AddStep(step_name, &BuildStepSimpleCommand, bs_arg);
}
/* Cpp */
@ -789,7 +658,7 @@ void OnBuild(StringList cli_args)
BuildStepSimpleCommandArg *bs_arg = ArenaPush(&arena, BuildStepSimpleCommandArg);
bs_arg->cmd = StringF(&arena, pch_cpp_compile_args_fmt, FmtStr(pch_header_file.full_path), FmtStr(pch_cpp_src_gen_file.full_path));
String step_name = StringF(&arena, Lit("%F -> %F"), FmtStr(D_GetName(pch_header_file)), FmtStr(D_GetName(pch_cpp_file)));
AddStep(&rc_and_pch_build_steps, step_name, &BuildStepSimpleCommand, bs_arg);
AddStep(step_name, &BuildStepSimpleCommand, bs_arg);
}
} else {
StringListAppend(&arena, &c_compile_args, StringF(&arena, Lit("-include-pch %F"), FmtStr(pch_c_file.full_path)));
@ -800,22 +669,22 @@ void OnBuild(StringList cli_args)
/* C */
if (IsDirty(pch_c_file)) {
String step_name = StringF(&arena, Lit("%F -> %F"), FmtStr(D_GetName(pch_header_file)), FmtStr(D_GetName(pch_c_file)));
BuildStepMsvcCompileCommandArg *bs_arg = ArenaPush(&arena, BuildStepMsvcCompileCommandArg);
bs_arg->cmd = StringF(&arena, pch_c_compile_args_fmt, FmtStr(pch_header_file.full_path), FmtStr(pch_c_file.full_path));
bs_arg->depfile_dependent = pch_header_file;
bs_arg->output_depfile = dep_file;
AddStep(&rc_and_pch_build_steps, step_name, &BuildStepMsvcCompileCommand, bs_arg);
String step_name = StringF(&arena, Lit("%F -> %F"), FmtStr(D_GetName(pch_header_file)), FmtStr(D_GetName(pch_c_file)));
AddStep(step_name, &BuildStepMsvcCompileCommand, bs_arg);
}
/* Cpp */
if (IsDirty(pch_cpp_file)) {
String step_name = StringF(&arena, Lit("%F -> %F"), FmtStr(D_GetName(pch_header_file)), FmtStr(D_GetName(pch_cpp_file)));
BuildStepMsvcCompileCommandArg *bs_arg = ArenaPush(&arena, BuildStepMsvcCompileCommandArg);
bs_arg->cmd = StringF(&arena, pch_cpp_compile_args_fmt, FmtStr(pch_header_file.full_path), FmtStr(pch_cpp_file.full_path));
bs_arg->depfile_dependent = pch_header_file;
bs_arg->output_depfile = dep_file;
AddStep(&rc_and_pch_build_steps, step_name, &BuildStepMsvcCompileCommand, bs_arg);
String step_name = StringF(&arena, Lit("%F -> %F"), FmtStr(D_GetName(pch_header_file)), FmtStr(D_GetName(pch_cpp_file)));
AddStep(step_name, &BuildStepMsvcCompileCommand, bs_arg);
}
}
}
@ -873,6 +742,8 @@ void OnBuild(StringList cli_args)
}
{
AddSyncPoint();
String c_compile_args_fmt = StringFromStringLists(&arena, Lit(" "), c_compile_args, compile_warnings, compile_and_link_args, compile_args);
String cpp_compile_args_fmt = StringFromStringLists(&arena, Lit(" "), cpp_compile_args, compile_warnings, compile_and_link_args, compile_args);
for (D_TagListNode *n = src_input_files.first; n; n = n->next) {
@ -908,11 +779,11 @@ void OnBuild(StringList cli_args)
bs_arg->depfile_dependent = obj_file;
bs_arg->output_depfile = dep_file;
bs_arg->depfile_force_includes = depfile_force_includes;
AddStep(&comp_build_steps, step_name, &BuildStepMsvcCompileCommand, bs_arg);
AddStep(step_name, &BuildStepMsvcCompileCommand, bs_arg);
} else {
BuildStepSimpleCommandArg *bs_arg = ArenaPush(&arena, BuildStepSimpleCommandArg);
bs_arg->cmd = StringF(&arena, comp_cmd_fmt, FmtStr(file.full_path), FmtStr(obj_file.full_path));
AddStep(&comp_build_steps, step_name, &BuildStepSimpleCommand, bs_arg);
AddStep(step_name, &BuildStepSimpleCommand, bs_arg);
}
}
@ -926,6 +797,8 @@ void OnBuild(StringList cli_args)
* ========================== */
{
AddSyncPoint();
String link_files_str = { 0 };
{
StringList link_files_quoted_list = { 0 };
@ -937,12 +810,12 @@ void OnBuild(StringList cli_args)
}
link_files_str = StringFromStringList(&arena, Lit(" "), link_files_quoted_list);
}
if (link_files_str.len > 0) {
if (link_files_str.len > 0 && IsDirty(executable_file)) {
String link_args_fmt = StringFromStringLists(&arena, Lit(" "), link_args, link_warnings, compile_and_link_args);
BuildStepSimpleCommandArg *bs_arg = ArenaPush(&arena, BuildStepSimpleCommandArg);
bs_arg->cmd = StringF(&arena, link_args_fmt, FmtStr(link_files_str), FmtStr(executable_file.full_path));
String step_name = Lit("Linking");
AddStep(&link_build_steps, step_name, &BuildStepSimpleCommand, bs_arg);
AddStep(step_name, &BuildStepSimpleCommand, bs_arg);
}
}
@ -950,47 +823,25 @@ void OnBuild(StringList cli_args)
* Execute build steps
* ========================== */
/* TODO: Cleanup (testing parallel stuff) */
Bool success = true;
if (IsDirty(executable_file)) {
AddStepListToWorkQueue(&tar_build_steps);
AddStepListToWorkQueue(&rc_and_pch_build_steps);
AddStepListToWorkQueue(&comp_build_steps);
AddStepListToWorkQueue(&link_build_steps);
I64 step_count = sl->count;
if (step_count > 0) {
ExecuteSteps();
I64 step_count = 0;
step_count += tar_build_steps.count;
step_count += rc_and_pch_build_steps.count;
step_count += comp_build_steps.count;
step_count += link_build_steps.count;
Step *s = tar_build_steps.first;
{
if (!s) {
s = rc_and_pch_build_steps.first;
if (!s) {
s = comp_build_steps.first;
if (!s) {
s = link_build_steps.first;
}
}
}
}
Step *s = sl->first;
I64 step_i = 0;
while (s) {
++step_i;
SH_PrintF(Lit("[%F/%F] %F\n"), FmtI64(step_i), FmtI64(step_count), FmtStr(s->name));
OS_MutexLockR(&s->res_mutex);
OS_MutexLockS(&s->res_mutex);
{
while (s->res_status == StepStatus_None) {
OS_ConditionVariableWaitR(&s->res_cv, &s->res_mutex);
OS_ConditionVariableWaitS(&s->res_cv, &s->res_mutex);
}
}
OS_MutexUnlockR(&s->res_mutex);
OS_MutexUnlockS(&s->res_mutex);
String output = s->res_output;
if (s->res_status != StepStatus_Success) {
@ -1000,20 +851,6 @@ void OnBuild(StringList cli_args)
break;
}
{
if (s == tar_build_steps.last) {
s = rc_and_pch_build_steps.first;
if (s) goto skip_cont;
}
if (s == rc_and_pch_build_steps.last) {
s = comp_build_steps.first;
if (s) goto skip_cont;
}
if (s == comp_build_steps.last) {
s = link_build_steps.first;
if (s) goto skip_cont;
}
}
s = s->next;
skip_cont:
continue;
@ -1027,17 +864,6 @@ void OnBuild(StringList cli_args)
OS_Exit(1);
}
/* ========================== *
* Write hist file
* ========================== */
T_ShutdownWorkers();
D_WriteStoreToHistFile(&store, hist_path);
ShutdownWorkers();
#if 0
#if Rtc
getchar();
#endif
#endif
}